diff --git a/Makefile b/Makefile index 2a19b04..a88633d 100644 --- a/Makefile +++ b/Makefile @@ -9,11 +9,13 @@ install: test: go test ./... -v -image: - docker build -t $(IMAGE_NAME) . - docker tag $(IMAGE_NAME):latest $(IMAGE_NAME):$(VERSION_DOCKER) +image-push-staging: + docker build -t $(IMAGE_NAME):staging . + docker push $(IMAGE_NAME):staging image-push: + docker build -t $(IMAGE_NAME):latest . + docker tag $(IMAGE_NAME):latest $(IMAGE_NAME):$(VERSION_DOCKER) docker push $(IMAGE_NAME):latest docker push $(IMAGE_NAME):$(VERSION_DOCKER) diff --git a/README.md b/README.md index 65fb1d3..be886fe 100644 --- a/README.md +++ b/README.md @@ -26,10 +26,18 @@ Returns the feed based on a given plugin and output format. That's the URL you s - `plugin`: The name of the plugin as returned by `String()` - `format`: The format the feed should be returned in, can be `rss`, `atom` or `json`. By default it's RSS. +**POST /feed/{plugin}/refresh** (_Authentication required_) + +Route to trigger a refresh for a given plugins, this runs a single scrape of the given plugin. + **GET /metrics** Returns the exported Prometheus metrics. +**API Authentication** + +Is done through a query parameter (`auth_token`), it's configured via the `API_TOKEN` environment variable. + ## Configuration and Operation ### Environment @@ -37,6 +45,7 @@ Returns the exported Prometheus metrics. The following environment variables are available, they all have sensible defaults and don't need to be set explicity. - `REFRESH_INTERVAL`: The interval in which feeds get rescraped in minutes (Default: 15) +- `API_TOKEN`: A user defined token that is used to protect sensitive API routes (Default: `changeme`) - `ENVIRONMENT`: Environment can be `prod` or `develop`. `develop` sets the loglevel to `info` (Default: `develop`) - `PORT`: Port that Feedbridge is running on (Default: `8080`) diff --git a/api/http.go b/api/http.go index 999bf34..70f3659 100644 --- a/api/http.go +++ b/api/http.go @@ -31,9 +31,26 @@ func NewHandler(s service) *chi.Mux { r.Get("/{plugin}/{format}", getFeedHandler(s)) }) + r.Group(func(r chi.Router) { + r.Use(s.authenticator) + r.Post("/{plugin}/refresh", refreshFeedHandler(s)) + }) + return r } +// authenticator checks if the users sends the correct token to access the internal routes +func (s *service) authenticator(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + if q.Get("auth_token") == "" || s.cfg.APIToken == "" || q.Get("auth_token") != s.cfg.APIToken { + http.Error(w, http.StatusText(401), 401) + return + } + next.ServeHTTP(w, r) + }) +} + // getFeedHandler returns a feed in a specified format func getFeedHandler(s service) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -79,3 +96,19 @@ func getPluginListHandler(s service) http.HandlerFunc { w.Write(b) } } + +// refreshFeedHandler returns a list of all available plugins +func refreshFeedHandler(s service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + plugin := chi.URLParam(r, "plugin") + if plugin == "" { + http.Error(w, errors.New("plugin not allowed to be empty").Error(), http.StatusInternalServerError) + return + } + if err := s.RefreshFeed(plugin); err != nil { + http.Error(w, errors.New("there was an error listing the plugins").Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + } +} diff --git a/api/service.go b/api/service.go index 4483c74..0f916ed 100644 --- a/api/service.go +++ b/api/service.go @@ -3,7 +3,9 @@ package api import ( "fmt" + "github.com/dewey/feedbridge/config" "github.com/dewey/feedbridge/plugin" + "github.com/dewey/feedbridge/runner" "github.com/dewey/feedbridge/store" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -16,20 +18,27 @@ type Service interface { // ListFeeds lists all available feed plugins ListFeeds() []string + + // RefreshFeed triggers a single plugin single scrape to force-refresh a feed based on the plugin name + RefreshFeed(technicalName string) error } type service struct { l log.Logger + cfg config.Config storageRepository store.StorageRepository pluginRepository plugin.Repository + runner *runner.Runner } // NewService initializes a new API service -func NewService(l log.Logger, sr store.StorageRepository, pr plugin.Repository) *service { +func NewService(l log.Logger, cfg config.Config, sr store.StorageRepository, pr plugin.Repository, r *runner.Runner) *service { return &service{ l: l, + cfg: cfg, storageRepository: sr, pluginRepository: pr, + runner: r, } } @@ -58,3 +67,14 @@ func (s *service) ListFeeds() []plugin.PluginMetadata { } return pp } + +func (s *service) RefreshFeed(technicalName string) error { + cp, err := s.pluginRepository.Find(technicalName) + if err != nil { + return err + } + if err := s.runner.StartSingle(cp); err != nil { + return err + } + return nil +} diff --git a/config/config.go b/config/config.go index 1837056..94703e7 100644 --- a/config/config.go +++ b/config/config.go @@ -7,6 +7,7 @@ type Config struct { CacheExpiredPurge int `env:"CACHE_EXPIRED_PURGE" envDefault:"60"` StorageBackend string `env:"STORAGE_BACKEND" envDefault:"memory"` StoragePath string `env:"STORAGE_PATH" envDefault:"/feedbridge-data"` + APIToken string `env:"API_TOKEN" envDefault:"feedbridge"` Environment string `env:"ENVIRONMENT" envDefault:"develop"` Port int `env:"PORT" envDefault:"8080"` } diff --git a/docs/Releasing.md b/docs/Releasing.md index f0e075a..108df82 100644 --- a/docs/Releasing.md +++ b/docs/Releasing.md @@ -9,7 +9,14 @@ Makefile like that: `VERSION=0.1.4 make release`. Then run the following steps. Make sure the Github Token (`GITHUB_TOKEN`) is set for goreleaser. ``` -feedbridge|master⚡ ⇒ make release -feedbridge|master⚡ ⇒ make image +feedbridge|master⚡ ⇒ GITHUB_TOKEN="" VERSION=0.1.x make release feedbridge|master⚡ ⇒ make image-push +``` + +# Releasing a staging version + +This just pushes a new image with local development version, this one is usually running on https://beta.feedbridge.notmyhostna.me + +``` +feedbridge|master⚡ ⇒ make image-push-staging ``` \ No newline at end of file diff --git a/main.go b/main.go index b3cb05f..fe06dae 100644 --- a/main.go +++ b/main.go @@ -65,9 +65,12 @@ func main() { } runner := runner.NewRunner(l, pluginRepo, storageRepo, cfg.RefreshInterval) - go runner.Start() + // No scheduled scrapes running in development, use the refresh route to test plugins + if cfg.Environment != "develop" { + go runner.Start() + } - apiService := api.NewService(l, storageRepo, pluginRepo) + apiService := api.NewService(l, cfg, storageRepo, pluginRepo, runner) templates := packr.NewBox("./ui/templates") assets := packr.NewBox("./ui/assets") diff --git a/runner/runner.go b/runner/runner.go index c767da0..ec0ad08 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -27,6 +27,11 @@ type Runner struct { ticker *time.Ticker } +// scrapeConfig contains information about a scrape +type scrapeConfig struct { + Type string +} + // NewRunner initializes a new runner to run plugins func NewRunner(l log.Logger, pluginRepo plugin.Repository, storageRepo store.StorageRepository, checkIntervalMinutes int) *Runner { return &Runner{ @@ -76,8 +81,9 @@ func (r *Runner) Start() { go func(cp plugin.Plugin) { defer wg.Done() start := time.Now() - level.Info(log.With(r.l, "plugin", cp.Info().TechnicalName)).Log("msg", "scrape started") - ss, err := r.runPlugin(cp) + ss, err := r.runPlugin(cp, scrapeConfig{ + Type: "full", + }) if err != nil { level.Error(r.l).Log("err", err) return @@ -86,14 +92,26 @@ func (r *Runner) Start() { duration := time.Since(start) scrapesDurationHistogram.WithLabelValues(cp.Info().TechnicalName).Observe(duration.Seconds()) pluginItemsScraped.WithLabelValues(cp.Info().TechnicalName).Set(float64(ss.Items)) - level.Info(log.With(r.l, "plugin", cp.Info().TechnicalName)).Log("msg", "scrape finished", "feed_items", ss.Items) }(cp) } wg.Wait() } } -func (r *Runner) runPlugin(cp plugin.Plugin) (scrape.Statistic, error) { +// StartSingle runs a single plugin once +func (r *Runner) StartSingle(cp plugin.Plugin) error { + cfg := scrapeConfig{ + Type: "single", + } + _, err := r.runPlugin(cp, cfg) + if err != nil { + return err + } + return nil +} + +func (r *Runner) runPlugin(cp plugin.Plugin, cfg scrapeConfig) (scrape.Statistic, error) { + level.Info(log.With(r.l, "plugin", cp.Info().TechnicalName, "scrape_type", cfg.Type)).Log("msg", "scrape started") f, err := cp.Run() if err != nil { return scrape.Statistic{}, err @@ -128,5 +146,6 @@ func (r *Runner) runPlugin(cp plugin.Plugin) (scrape.Statistic, error) { return scrape.Statistic{}, err } r.StorageRepository.Save(fmt.Sprintf("json_%s", cp.Info().TechnicalName), json) + level.Info(log.With(r.l, "plugin", cp.Info().TechnicalName, "scrape_type", cfg.Type)).Log("msg", "scrape finished", "feed_items", len(f.Items)) return scrape.Statistic{Items: len(f.Items)}, nil }