Prometheus metrics

api/api.go

@@ -13,6 +13,28 @@ import (
 	"github.com/hound-search/hound/config"
 	"github.com/hound-search/hound/index"
 	"github.com/hound-search/hound/searcher"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	// Overall timer to assess the overall latency the user sees (as we run many queries in parallel)
+	searchRequestDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name: "hound_api_search_all_duration_seconds",
+		Help: "API searchAll latency in seconds",
+	})
+
+	// Metrics to assess how often/fast/efficiently each index is accessed
+	searchIndexDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name: "hound_api_search_idx_duration_seconds",
+		Help: "Seach single index latency in seconds",
+	}, []string{"index"})
+	searchIndexFilesOpened = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "hound_api_search_idx_files_opened",
+		Help:    "Seach single index files opened",
+		Buckets: prometheus.ExponentialBuckets(1, 2, 10),
+	}, []string{"index"})
 )
 
 const (
@@ -63,13 +85,20 @@ func searchAll(
 
 	startedAt := time.Now()
 
+	timer := prometheus.NewTimer(searchRequestDuration)
+	defer timer.ObserveDuration()
+
 	n := len(repos)
 
 	// use a buffered channel to avoid routine leaks on errs.
 	ch := make(chan *searchResponse, n)
 	for _, repo := range repos {
 		go func(repo string) {
+			// We do Prometheus metrics here, as the Searcher doesn't know it's own name
+			t := prometheus.NewTimer(searchIndexDuration.WithLabelValues(repo))
+			defer t.ObserveDuration()
 			fms, err := idx[repo].Search(query, opts)
+			searchIndexFilesOpened.WithLabelValues(repo).Observe(float64(fms.FilesOpened))
 			ch <- &searchResponse{repo, fms, err}
 		}(repo)
 	}
@@ -89,7 +118,7 @@ func searchAll(
 		*filesOpened += r.res.FilesOpened
 	}
 
-	*duration = int(time.Now().Sub(startedAt).Seconds() * 1000)  //nolint
+	*duration = int(time.Now().Sub(startedAt).Seconds() * 1000) //nolint
 
 	return res, nil
 }
@@ -245,7 +274,6 @@ func Setup(m *http.ServeMux, idx map[string]*searcher.Searcher) {
 					fmt.Errorf("Push updates are not enabled for repository %s", repo),
 					http.StatusForbidden)
 				return
-
 			}
 		}
 
@@ -262,7 +290,7 @@ func Setup(m *http.ServeMux, idx map[string]*searcher.Searcher) {
 
 		type Webhook struct {
 			Repository struct {
-				Name string
+				Name      string
 				Full_name string
 			}
 		}
@@ -272,7 +300,7 @@ func Setup(m *http.ServeMux, idx map[string]*searcher.Searcher) {
 		err := json.NewDecoder(r.Body).Decode(&h)
 
 		if err != nil {
-		   writeError(w,
+			writeError(w,
 				errors.New(http.StatusText(http.StatusBadRequest)),
 				http.StatusBadRequest)
 			return

cmds/houndd/main.go

@@ -5,7 +5,6 @@ import (
 	"flag"
 	"fmt"
 	"log"
-	"net/http"
 	"os"
 	"os/exec"
 	"os/signal"
@@ -15,10 +14,8 @@ import (
 	"syscall"
 
 	"github.com/blang/semver"
-	"github.com/hound-search/hound/api"
 	"github.com/hound-search/hound/config"
 	"github.com/hound-search/hound/searcher"
-	"github.com/hound-search/hound/ui"
 	"github.com/hound-search/hound/web"
 )
 
@@ -98,23 +95,6 @@ func makeTemplateData(cfg *config.Config) (interface{}, error) { //nolint
 	return &data, nil
 }
 
-func runHttp( //nolint
-	addr string,
-	dev bool,
-	cfg *config.Config,
-	idx map[string]*searcher.Searcher) error {
-	m := http.DefaultServeMux
-
-	h, err := ui.Content(dev, cfg)
-	if err != nil {
-		return err
-	}
-
-	m.Handle("/", h)
-	api.Setup(m, idx)
-	return http.ListenAndServe(addr, m)
-}
-
 // TODO: Automatically increment this when building a release
 func getVersion() semver.Version {
 	return semver.Version{

config/config.go

@@ -17,6 +17,7 @@ const (
 	defaultBaseUrl               = "{url}/blob/{rev}/{path}{anchor}"
 	defaultAnchor                = "#L{line}"
 	defaultHealthCheckURI        = "/healthz"
+	defaultMetricsURI            = "/metrics"
 )
 
 type UrlPattern struct {
@@ -61,6 +62,7 @@ type Config struct {
 	Repos                 map[string]*Repo          `json:"repos"`
 	MaxConcurrentIndexers int                       `json:"max-concurrent-indexers"`
 	HealthCheckURI        string                    `json:"health-check-uri"`
+	MetricsURI            string                    `json:"metrics-uri"`
 	VCSConfigMessages     map[string]*SecretMessage `json:"vcs-config"`
 }
 
@@ -128,6 +130,9 @@ func initConfig(c *Config) error {
 	if c.HealthCheckURI == "" {
 		c.HealthCheckURI = defaultHealthCheckURI
 	}
+	if c.MetricsURI == "" {
+		c.MetricsURI = defaultMetricsURI
+	}
 
 	return mergeVCSConfigs(c)
 }

docs/api.md

@@ -0,0 +1,34 @@
+API
+---
+
+## `/` serves the web app
+
+## `/api/v1/repos` returns list of indexed repositories
+
+## `/api/v1/search` performs searches
+
+For example using cURL to search for `foobar` in Hound:
+
+```console
+https://hound.examplle.com/api/v1/search?repos=Hound&q=foobar
+...
+```
+
+## `/api/v1/excludes` returns list of excluded files
+
+## `/api/v1/update` and `/api/v1/github-webhook` hook for updating repositories
+
+## `/healthz` health-check
+
+As configured by `health-check-uri` in the configuration. Defaults to `healthz`.
+
+## `/metrics` Prometheus metrics
+
+Returns the default Prometheus Go-metrics and some custom values on number and
+duration of search requests.
+
+To get the number of searches as a five-minute running average:
+
+```
+rate(hound_api_search_all_duration_seconds_count[5m])
+```

docs/config-options.md

@@ -15,6 +15,7 @@ ConfigOption | Description | Default Values
 :------ | :----- | :-----
 max-concurrent-indexers | defines the total number of indexers required to be used for indexing code | 2
 health-check-uri |  health check url for hound | `/healthz`
+metrics-uri |  Prometheus metrics URI | `/metrics`
 dbpath | absolute file path where the `config.json` file exists| `data`
 title | Title used for the application | Hound
 url-pattern | composed of base url and anchor values in form of key value pairs | n/a

go.mod

@@ -4,5 +4,5 @@ go 1.13
 
 require (
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/go-bindata/go-bindata v3.1.2+incompatible // indirect
+	github.com/prometheus/client_golang v1.13.0
 )