Add single scrape API route

- Don’t run schedules scrapes in development
- Add authenticator middleware for internal routes
- Improve Makefile

Makefile

@@ -9,11 +9,13 @@ install:
 test:
 	go test ./... -v
 
-image:
-	docker build -t $(IMAGE_NAME) .
-	docker tag $(IMAGE_NAME):latest $(IMAGE_NAME):$(VERSION_DOCKER)
+image-push-staging:
+	docker build -t $(IMAGE_NAME):staging .
+	docker push $(IMAGE_NAME):staging
 
 image-push:
+	docker build -t $(IMAGE_NAME):latest .
+	docker tag $(IMAGE_NAME):latest $(IMAGE_NAME):$(VERSION_DOCKER)
 	docker push $(IMAGE_NAME):latest
 	docker push $(IMAGE_NAME):$(VERSION_DOCKER)
 

README.md

@@ -26,17 +26,26 @@ Returns the feed based on a given plugin and output format. That's the URL you s
 - `plugin`: The name of the plugin as returned by `String()`
 - `format`: The format the feed should be returned in, can be `rss`, `atom` or `json`. By default it's RSS.
 
+**POST /feed/{plugin}/refresh** (_Authentication required_)
+
+Route to trigger a refresh for a given plugins, this runs a single scrape of the given plugin. 
+
 **GET /metrics**
 
 Returns the exported Prometheus metrics.
 
+**API Authentication**
+
+Is done through a query parameter (`auth_token`), it's configured via the `API_TOKEN` environment variable.
+
 ## Configuration and Operation
 
 ### Environment
 
 The following environment variables are available, they all have sensible defaults and don't need to be set explicity.
 
 - `REFRESH_INTERVAL`: The interval in which feeds get rescraped in minutes (Default: 15)
+- `API_TOKEN`: A user defined token that is used to protect sensitive API routes (Default: `changeme`)
 - `ENVIRONMENT`: Environment can be `prod` or `develop`. `develop` sets the loglevel to `info` (Default: `develop`)
 - `PORT`: Port that Feedbridge is running on (Default: `8080`)
 

api/http.go

@@ -31,9 +31,26 @@ func NewHandler(s service) *chi.Mux {
 		r.Get("/{plugin}/{format}", getFeedHandler(s))
 	})
 
+	r.Group(func(r chi.Router) {
+		r.Use(s.authenticator)
+		r.Post("/{plugin}/refresh", refreshFeedHandler(s))
+	})
+
 	return r
 }
 
+// authenticator checks if the users sends the correct token to access the internal routes
+func (s *service) authenticator(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		q := r.URL.Query()
+		if q.Get("auth_token") == "" || s.cfg.APIToken == "" || q.Get("auth_token") != s.cfg.APIToken {
+			http.Error(w, http.StatusText(401), 401)
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
 // getFeedHandler returns a feed in a specified format
 func getFeedHandler(s service) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
@@ -79,3 +96,19 @@ func getPluginListHandler(s service) http.HandlerFunc {
 		w.Write(b)
 	}
 }
+
+// refreshFeedHandler returns a list of all available plugins
+func refreshFeedHandler(s service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		plugin := chi.URLParam(r, "plugin")
+		if plugin == "" {
+			http.Error(w, errors.New("plugin not allowed to be empty").Error(), http.StatusInternalServerError)
+			return
+		}
+		if err := s.RefreshFeed(plugin); err != nil {
+			http.Error(w, errors.New("there was an error listing the plugins").Error(), http.StatusInternalServerError)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}
+}

api/service.go

@@ -3,7 +3,9 @@ package api
 import (
 	"fmt"
 
+	"github.com/dewey/feedbridge/config"
 	"github.com/dewey/feedbridge/plugin"
+	"github.com/dewey/feedbridge/runner"
 	"github.com/dewey/feedbridge/store"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
@@ -16,20 +18,27 @@ type Service interface {
 
 	// ListFeeds lists all available feed plugins
 	ListFeeds() []string
+
+	// RefreshFeed triggers a single plugin single scrape to force-refresh a feed based on the plugin name
+	RefreshFeed(technicalName string) error
 }
 
 type service struct {
 	l                 log.Logger
+	cfg               config.Config
 	storageRepository store.StorageRepository
 	pluginRepository  plugin.Repository
+	runner            *runner.Runner
 }
 
 // NewService initializes a new API service
-func NewService(l log.Logger, sr store.StorageRepository, pr plugin.Repository) *service {
+func NewService(l log.Logger, cfg config.Config, sr store.StorageRepository, pr plugin.Repository, r *runner.Runner) *service {
 	return &service{
 		l:                 l,
+		cfg:               cfg,
 		storageRepository: sr,
 		pluginRepository:  pr,
+		runner:            r,
 	}
 }
 
@@ -58,3 +67,14 @@ func (s *service) ListFeeds() []plugin.PluginMetadata {
 	}
 	return pp
 }
+
+func (s *service) RefreshFeed(technicalName string) error {
+	cp, err := s.pluginRepository.Find(technicalName)
+	if err != nil {
+		return err
+	}
+	if err := s.runner.StartSingle(cp); err != nil {
+		return err
+	}
+	return nil
+}

config/config.go

@@ -7,6 +7,7 @@ type Config struct {
 	CacheExpiredPurge int    `env:"CACHE_EXPIRED_PURGE" envDefault:"60"`
 	StorageBackend    string `env:"STORAGE_BACKEND" envDefault:"memory"`
 	StoragePath       string `env:"STORAGE_PATH" envDefault:"/feedbridge-data"`
+	APIToken          string `env:"API_TOKEN" envDefault:"feedbridge"`
 	Environment       string `env:"ENVIRONMENT" envDefault:"develop"`
 	Port              int    `env:"PORT" envDefault:"8080"`
 }

docs/Releasing.md

@@ -9,7 +9,14 @@ Makefile like that: `VERSION=0.1.4 make release`.
 Then run the following steps. Make sure the Github Token (`GITHUB_TOKEN`) is set for goreleaser.
 
 ```
-feedbridge|master⚡ ⇒ make release
-feedbridge|master⚡ ⇒ make image
+feedbridge|master⚡ ⇒ GITHUB_TOKEN="" VERSION=0.1.x make release
 feedbridge|master⚡ ⇒ make image-push
+```
+
+# Releasing a staging version
+
+This just pushes a new image with local development version, this one is usually running on https://beta.feedbridge.notmyhostna.me
+
+```
+feedbridge|master⚡ ⇒ make image-push-staging
 ```

main.go

@@ -65,9 +65,12 @@ func main() {
 	}
 
 	runner := runner.NewRunner(l, pluginRepo, storageRepo, cfg.RefreshInterval)
-	go runner.Start()
+	// No scheduled scrapes running in development, use the refresh route to test plugins
+	if cfg.Environment != "develop" {
+		go runner.Start()
+	}
 
-	apiService := api.NewService(l, storageRepo, pluginRepo)
+	apiService := api.NewService(l, cfg, storageRepo, pluginRepo, runner)
 
 	templates := packr.NewBox("./ui/templates")
 	assets := packr.NewBox("./ui/assets")

runner/runner.go

@@ -27,6 +27,11 @@ type Runner struct {
 	ticker               *time.Ticker
 }
 
+// scrapeConfig contains information about a scrape
+type scrapeConfig struct {
+	Type string
+}
+
 // NewRunner initializes a new runner to run plugins
 func NewRunner(l log.Logger, pluginRepo plugin.Repository, storageRepo store.StorageRepository, checkIntervalMinutes int) *Runner {
 	return &Runner{
@@ -76,8 +81,9 @@ func (r *Runner) Start() {
 			go func(cp plugin.Plugin) {
 				defer wg.Done()
 				start := time.Now()
-				level.Info(log.With(r.l, "plugin", cp.Info().TechnicalName)).Log("msg", "scrape started")
-				ss, err := r.runPlugin(cp)
+				ss, err := r.runPlugin(cp, scrapeConfig{
+					Type: "full",
+				})
 				if err != nil {
 					level.Error(r.l).Log("err", err)
 					return
@@ -86,14 +92,26 @@ func (r *Runner) Start() {
 				duration := time.Since(start)
 				scrapesDurationHistogram.WithLabelValues(cp.Info().TechnicalName).Observe(duration.Seconds())
 				pluginItemsScraped.WithLabelValues(cp.Info().TechnicalName).Set(float64(ss.Items))
-				level.Info(log.With(r.l, "plugin", cp.Info().TechnicalName)).Log("msg", "scrape finished", "feed_items", ss.Items)
 			}(cp)
 		}
 		wg.Wait()
 	}
 }
 
-func (r *Runner) runPlugin(cp plugin.Plugin) (scrape.Statistic, error) {
+// StartSingle runs a single plugin once
+func (r *Runner) StartSingle(cp plugin.Plugin) error {
+	cfg := scrapeConfig{
+		Type: "single",
+	}
+	_, err := r.runPlugin(cp, cfg)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (r *Runner) runPlugin(cp plugin.Plugin, cfg scrapeConfig) (scrape.Statistic, error) {
+	level.Info(log.With(r.l, "plugin", cp.Info().TechnicalName, "scrape_type", cfg.Type)).Log("msg", "scrape started")
 	f, err := cp.Run()
 	if err != nil {
 		return scrape.Statistic{}, err
@@ -128,5 +146,6 @@ func (r *Runner) runPlugin(cp plugin.Plugin) (scrape.Statistic, error) {
 		return scrape.Statistic{}, err
 	}
 	r.StorageRepository.Save(fmt.Sprintf("json_%s", cp.Info().TechnicalName), json)
+	level.Info(log.With(r.l, "plugin", cp.Info().TechnicalName, "scrape_type", cfg.Type)).Log("msg", "scrape finished", "feed_items", len(f.Items))
 	return scrape.Statistic{Items: len(f.Items)}, nil
 }