feat(store/github): expose github store metrics

Currently exposes rate limit status towards the GitHub API. These
metrics are polled from the GitHub API on regular intervals. These
requests are exempted from the general rate limits.

The worker for polling this status is not started until the first call
to `ReloadCache`.

https://docs.github.com/en/rest/rate-limit/rate-limit?apiVersion=2022-11-28#get-rate-limit-status-for-the-authenticated-user

go.mod

@@ -12,7 +12,7 @@ toolchain go1.21.4
 require (
 	github.com/aws/aws-sdk-go v1.50.25
 	github.com/go-chi/chi/v5 v5.0.12
-	github.com/google/go-github/v43 v43.0.0
+	github.com/google/go-github/v59 v59.0.0
 	github.com/hashicorp/go-version v1.6.0
 	github.com/matryer/is v1.4.1
 	github.com/migueleliasweb/go-github-mock v0.0.23
@@ -27,7 +27,6 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/go-github/v59 v59.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -38,7 +37,6 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
-	golang.org/x/crypto v0.19.0 // indirect
 	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

go.sum

@@ -17,8 +17,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-github/v43 v43.0.0 h1:y+GL7LIsAIF2NZlJ46ZoC/D1W1ivZasT0lnWHMYPZ+U=
-github.com/google/go-github/v43 v43.0.0/go.mod h1:ZkTvvmCXBvsfPpTHXnH/d2hP9Y0cTbvN9kr5xqyXOIc=
 github.com/google/go-github/v59 v59.0.0 h1:7h6bgpF5as0YQLLkEiVqpgtJqjimMYhBkD4jT5aN3VA=
 github.com/google/go-github/v59 v59.0.0/go.mod h1:rJU4R0rQHFVFDOkqGWxfLNo6vEk4dv40oDjhV/gH6wM=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
@@ -68,8 +66,6 @@ go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=

pkg/store/github/github.go

@@ -10,8 +10,9 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
-	"github.com/google/go-github/v43/github"
+	"github.com/google/go-github/v59/github"
 	goversion "github.com/hashicorp/go-version"
 	"github.com/nrkno/terraform-registry/pkg/core"
 	"github.com/prometheus/client_golang/prometheus"
@@ -27,9 +28,10 @@ type GitHubStore struct {
 	// Topic to filter repositories by. Leave empty for all.
 	topicFilter string
 
-	client *github.Client
-	cache  map[string][]*core.ModuleVersion
-	mut    sync.RWMutex
+	client  *github.Client
+	cache   map[string][]*core.ModuleVersion
+	mut     sync.RWMutex
+	metrics metrics
 
 	logger *zap.Logger
 }
@@ -50,6 +52,17 @@ func NewGitHubStore(ownerFilter, topicFilter, accessToken string, logger *zap.Lo
 		client:      github.NewClient(c),
 		cache:       make(map[string][]*core.ModuleVersion),
 		logger:      logger,
+		metrics: metrics{
+			rateLimitCoreLimit:             newGauge("rate_limit_core_limit"),
+			rateLimitCoreRemaining:         newGauge("rate_limit_core_remaining"),
+			rateLimitCoreResetTimestamp:    newGauge("rate_limit_core_reset_timestamp"),
+			rateLimitSearchLimit:           newGauge("rate_limit_search_limit"),
+			rateLimitSearchRemaining:       newGauge("rate_limit_search_remaining"),
+			rateLimitSearchResetTimestamp:  newGauge("rate_limit_search_reset_timestamp"),
+			rateLimitGraphQLLimit:          newGauge("rate_limit_graphql_limit"),
+			rateLimitGraphQLRemaining:      newGauge("rate_limit_graphql_remaining"),
+			rateLimitGraphQLResetTimestamp: newGauge("rate_limit_graphql_reset_timestamp"),
+		},
 	}
 }
 
@@ -88,7 +101,18 @@ func (s *GitHubStore) GetModuleVersion(ctx context.Context, namespace, name, pro
 // ReloadCache queries the GitHub API and reloads the local cache of module versions.
 // Should be called at least once after initialisation and proably on regular
 // intervals afterwards to keep cache up-to-date.
+// This method also starts the background worker for polling rate limit status of the GitHub API on first invocation.
 func (s *GitHubStore) ReloadCache(ctx context.Context) error {
+	// Start rate limit worker on first invocation
+	sync.OnceFunc(func() {
+		go func() {
+			for {
+				s.updateMetrics(context.Background())
+				time.Sleep(15 * time.Second)
+			}
+		}()
+	})()
+
 	repos, err := s.searchRepositories(ctx)
 	if err != nil {
 		return err
@@ -202,6 +226,56 @@ func (s *GitHubStore) searchRepositories(ctx context.Context) ([]*github.Reposit
 	return allRepos, nil
 }
 
+type metrics struct {
+	rateLimitCoreLimit             prometheus.Gauge
+	rateLimitCoreRemaining         prometheus.Gauge
+	rateLimitCoreResetTimestamp    prometheus.Gauge
+	rateLimitSearchLimit           prometheus.Gauge
+	rateLimitSearchRemaining       prometheus.Gauge
+	rateLimitSearchResetTimestamp  prometheus.Gauge
+	rateLimitGraphQLLimit          prometheus.Gauge
+	rateLimitGraphQLRemaining      prometheus.Gauge
+	rateLimitGraphQLResetTimestamp prometheus.Gauge
+}
+
+// newGauge is a helper function for creating gauges with a common namespace and subsystem.
+func newGauge(name string) prometheus.Gauge {
+	return prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "terraform_registry",
+		Subsystem: "store_github",
+		Name:      name,
+	})
+}
+
+// updateMetrics updates all metrics that needs polling.
+func (s *GitHubStore) updateMetrics(ctx context.Context) {
+	ratel, _, err := s.client.RateLimit.Get(ctx)
+	if err != nil {
+		s.logger.Warn("failed to get rate limit status", zap.Errors("err", []error{err}))
+	} else {
+		s.metrics.rateLimitCoreLimit.Set(float64(ratel.Core.Limit))
+		s.metrics.rateLimitCoreRemaining.Set(float64(ratel.Core.Remaining))
+		s.metrics.rateLimitCoreResetTimestamp.Set(float64(ratel.Core.Reset.Unix()))
+		s.metrics.rateLimitSearchLimit.Set(float64(ratel.Search.Limit))
+		s.metrics.rateLimitSearchRemaining.Set(float64(ratel.Search.Remaining))
+		s.metrics.rateLimitSearchResetTimestamp.Set(float64(ratel.Search.Reset.Unix()))
+		s.metrics.rateLimitGraphQLLimit.Set(float64(ratel.GraphQL.Limit))
+		s.metrics.rateLimitGraphQLRemaining.Set(float64(ratel.GraphQL.Remaining))
+		s.metrics.rateLimitGraphQLResetTimestamp.Set(float64(ratel.GraphQL.Reset.Unix()))
+	}
+}
+
+// Metrics returns a registry with metrics for this store.
 func (s *GitHubStore) Metrics() []prometheus.Collector {
-	return nil
+	return []prometheus.Collector{
+		s.metrics.rateLimitCoreLimit,
+		s.metrics.rateLimitCoreRemaining,
+		s.metrics.rateLimitCoreResetTimestamp,
+		s.metrics.rateLimitSearchLimit,
+		s.metrics.rateLimitSearchRemaining,
+		s.metrics.rateLimitSearchResetTimestamp,
+		s.metrics.rateLimitGraphQLLimit,
+		s.metrics.rateLimitGraphQLRemaining,
+		s.metrics.rateLimitGraphQLResetTimestamp,
+	}
 }

pkg/store/github/github_test.go

@@ -10,7 +10,7 @@ import (
 	"net/http"
 	"testing"
 
-	"github.com/google/go-github/v43/github"
+	"github.com/google/go-github/v59/github"
 	"github.com/matryer/is"
 	"github.com/migueleliasweb/go-github-mock/src/mock"
 	"github.com/nrkno/terraform-registry/pkg/core"