diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f67c60..eb44942 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased +### Added +- Generate a package listing for sub-paths + that match a subset of the known packages. + ## [1.4.0] ### Added - Publish a Docker image to GitHub Container Registry. diff --git a/go.mod b/go.mod index ee31bf0..d86bef0 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/stretchr/testify v1.8.4 golang.org/x/net v0.15.0 gopkg.in/yaml.v3 v3.0.1 + pgregory.net/rapid v1.1.0 ) require ( diff --git a/go.sum b/go.sum index e4ac672..d3d2117 100644 --- a/go.sum +++ b/go.sum @@ -16,3 +16,5 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw= +pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/handler.go b/handler.go index 18dd1f4..2540a98 100644 --- a/handler.go +++ b/handler.go @@ -1,10 +1,11 @@ package main import ( + "errors" "fmt" "html/template" "net/http" - "sort" + "path" "strings" "go.uber.org/sally/templates" @@ -17,112 +18,131 @@ var ( template.New("package.html").Parse(templates.Package)) ) -// CreateHandler creates a Sally http.Handler -func CreateHandler(config *Config) http.Handler { - mux := http.NewServeMux() +// Handler handles inbound HTTP requests. +// +// It provides the following endpoints: +// +// GET / +// Index page listing all packages. +// GET / +// Package page for the given package. +// GET / +// Page listing packages under the given directory, +// assuming that there's no package with the given name. +// GET // +// Package page for the given subpackage. +type Handler struct { + pkgs pathTree[*sallyPackage] +} - pkgs := make([]packageInfo, 0, len(config.Packages)) +// CreateHandler builds a new handler +// with the provided package configuration. +func CreateHandler(config *Config) *Handler { + var pkgs pathTree[*sallyPackage] for name, pkg := range config.Packages { - handler := newPackageHandler(config, name, pkg) - // Double-register so that "/foo" - // does not redirect to "/foo/" with a 300. - mux.Handle("/"+name, handler) - mux.Handle("/"+name+"/", handler) - - pkgs = append(pkgs, packageInfo{ + baseURL := config.URL + if pkg.URL != "" { + // Package-specific override for the base URL. + baseURL = pkg.URL + } + modulePath := path.Join(baseURL, name) + docURL := "https://" + path.Join(config.Godoc.Host, modulePath) + + pkgs.Set(name, &sallyPackage{ Desc: pkg.Desc, - ImportPath: handler.canonicalURL, - GitURL: handler.gitURL, - GodocHome: handler.godocHost + "/" + handler.canonicalURL, + ModulePath: modulePath, + DocURL: docURL, + GitURL: pkg.Repo, }) } - sort.Slice(pkgs, func(i, j int) bool { - return pkgs[i].ImportPath < pkgs[j].ImportPath - }) - mux.Handle("/", &indexHandler{pkgs: pkgs}) - - return mux -} -type indexHandler struct { - pkgs []packageInfo + return &Handler{ + pkgs: pkgs, + } } -type packageInfo struct { - Desc string // package description - ImportPath string // canonical import path - GitURL string // URL of the Git repository - GodocHome string // documentation home URL -} +var _ http.Handler = (*Handler)(nil) -func (h *indexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // Index handler only supports '/'. - // ServeMux will call us for any '/foo' that is not a known package. - if r.Method != http.MethodGet || r.URL.Path != "/" { - http.NotFound(w, r) - return - } +type sallyPackage struct { + // Canonical import path for the package. + ModulePath string - data := struct{ Packages []packageInfo }{ - Packages: h.pkgs, - } - if err := indexTemplate.Execute(w, data); err != nil { - http.Error(w, err.Error(), 500) - } -} + // Description of the package, if any. + Desc string -type packageHandler struct { - // Hostname of the godoc server, e.g. "godoc.org". - godocHost string + // URL at which documentation for the package can be found. + DocURL string - // Name of the package relative to the vanity base URL. - // For example, "zap" for "go.uber.org/zap". - name string + // URL at which the Git repository is hosted. + GitURL string +} - // Path at which the Git repository is hosted. - // For example, "github.com/uber-go/zap". - gitURL string +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if err := h.serveHTTP(w, r); err != nil { + if herr := new(httpError); errors.As(err, &herr) { + http.Error(w, herr.Message, herr.Code) + } else { + http.Error(w, err.Error(), 500) + } + } +} - // Canonical import path for the package. - canonicalURL string +// httpError indicates that an HTTP error occurred. +// +// The caller will write the error code and message to the response. +type httpError struct { + Code int // HTTP status code + Message string // error message } -func newPackageHandler(cfg *Config, name string, pkg PackageConfig) *packageHandler { - baseURL := cfg.URL - if pkg.URL != "" { - baseURL = pkg.URL +func httpErrorf(code int, format string, args ...interface{}) error { + return &httpError{ + Code: code, + Message: fmt.Sprintf(format, args...), } - canonicalURL := fmt.Sprintf("%s/%s", baseURL, name) +} - return &packageHandler{ - godocHost: cfg.Godoc.Host, - name: name, - canonicalURL: canonicalURL, - gitURL: pkg.Repo, - } +func (e *httpError) Error() string { + return fmt.Sprintf("status %d: %s", e.Code, e.Message) } -func (h *packageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { +// serveHTTP is similar to ServeHTTP, except it returns an error. +// +// If it returns an httpError, +// the caller will write the error code and message to the response. +func (h *Handler) serveHTTP(w http.ResponseWriter, r *http.Request) error { if r.Method != http.MethodGet { - http.NotFound(w, r) - return + return httpErrorf(http.StatusNotFound, "method %q not allowed", r.Method) } - // Extract the relative path to subpackages, if any. - // "/foo/bar" => "/bar" - // "/foo" => "" - relPath := strings.TrimPrefix(r.URL.Path, "/"+h.name) - - data := struct { - Repo string - CanonicalURL string - GodocURL string - }{ - Repo: h.gitURL, - CanonicalURL: h.canonicalURL, - GodocURL: fmt.Sprintf("https://%s/%s%s", h.godocHost, h.canonicalURL, relPath), + path := strings.TrimPrefix(strings.TrimSuffix(r.URL.Path, "/"), "/") + + if pkg, suffix, ok := h.pkgs.Lookup(path); ok { + return h.servePackage(w, pkg, suffix) } - if err := packageTemplate.Execute(w, data); err != nil { - http.Error(w, err.Error(), 500) + return h.serveIndex(w, path, h.pkgs.ListByPath(path)) +} + +func (h *Handler) servePackage(w http.ResponseWriter, pkg *sallyPackage, suffix string) error { + return packageTemplate.Execute(w, + struct { + ModulePath string + GitURL string + DocURL string + }{ + ModulePath: pkg.ModulePath, + GitURL: pkg.GitURL, + DocURL: pkg.DocURL + suffix, + }) +} + +func (h *Handler) serveIndex(w http.ResponseWriter, path string, pkgs []*sallyPackage) error { + if len(pkgs) == 0 { + return httpErrorf(http.StatusNotFound, "no packages found under path: %s", path) } + + return indexTemplate.Execute(w, + struct{ Packages []*sallyPackage }{ + Packages: pkgs, + }) } diff --git a/handler_test.go b/handler_test.go index 370eb45..4de4c3f 100644 --- a/handler_test.go +++ b/handler_test.go @@ -23,6 +23,10 @@ packages: url: go.uberalt.org repo: github.com/uber-go/zap description: A fast, structured logging library. + net/metrics: + repo: github.com/yarpc/metrics + net/something: + repo: github.com/yarpc/something ` @@ -34,6 +38,19 @@ func TestIndex(t *testing.T) { assert.Contains(t, body, "github.com/thriftrw/thriftrw-go") assert.Contains(t, body, "github.com/yarpc/yarpc-go") assert.Contains(t, body, "A fast, structured logging library.") + assert.Contains(t, body, "github.com/yarpc/metrics") + assert.Contains(t, body, "github.com/yarpc/something") +} + +func TestSubindex(t *testing.T) { + rr := CallAndRecord(t, config, "/net") + assert.Equal(t, 200, rr.Code) + + body := rr.Body.String() + assert.NotContains(t, body, "github.com/thriftrw/thriftrw-go") + assert.NotContains(t, body, "github.com/yarpc/yarpc-go") + assert.Contains(t, body, "github.com/yarpc/metrics") + assert.Contains(t, body, "github.com/yarpc/something") } func TestPackageShouldExist(t *testing.T) { @@ -55,7 +72,7 @@ func TestPackageShouldExist(t *testing.T) { func TestNonExistentPackageShould404(t *testing.T) { rr := CallAndRecord(t, config, "/nonexistent") AssertResponse(t, rr, 404, ` -404 page not found +no packages found under path: nonexistent `) } @@ -66,10 +83,10 @@ func TestTrailingSlash(t *testing.T) { - + - Nothing to see here. Please move along. + Nothing to see here. Please move along. `) diff --git a/path_tree.go b/path_tree.go new file mode 100644 index 0000000..b6c4855 --- /dev/null +++ b/path_tree.go @@ -0,0 +1,120 @@ +package main + +import ( + "slices" + "strings" +) + +// pathTree holds values in a tree-like hierarchy +// defined by /-separate paths (e.g. import paths). +// +// It's expensive to add items to the tree, +// but lookups are fast. +// +// Its zero value is a valid empty tree. +type pathTree[T any] struct { + // We track two representations of the tree: + // + // 1. A list of key-value pairs, sorted by key. + // 2. A map of keys to values. + // + // (1) is used for fast scanning of all descendants of a path. + // + // (2) is used for fast lookups of specific paths. + + // paths is a list of all paths in the tree + // that have a value set for them. + paths []string // sorted + + // values[i] is the value for paths[i] + values []T + + // byPath is a map of all paths in the tree + // to their corresponding values. + byPath map[string]T // path => node +} + +// Set sets the value for path to value. +// If path already has a value, it is overwritten. +func (t *pathTree[T]) Set(path string, value T) { + if t.byPath == nil { + t.byPath = make(map[string]T) + } + + t.byPath[path] = value + idx, ok := slices.BinarySearch(t.paths, path) + if ok { + // t.paths[idx] already contains path. + t.values[idx] = value + } else { + t.paths = slices.Insert(t.paths, idx, path) + t.values = slices.Insert(t.values, idx, value) + } +} + +// Lookup retrieves the value for the given path. +// +// If the path doesn't have an explicit value set, +// the value for the closest ancestor with a value is returned. +// +// Suffix is the remaining, unmatched part of path. +// It has a leading '/' if the path wasn't an exact match. +// +// If no value is set for the path or its ancestors, +// ok is false. +func (t *pathTree[T]) Lookup(path string) (value T, suffix string, ok bool) { + idx := len(path) + for idx > 0 { + if value, ok := t.byPath[path[:idx]]; ok { + return value, path[idx:], true + } + + // No match. Trim the last path component. + // "foo/bar" => "foo" + // "foo" => "" + idx = strings.LastIndexByte(path[:idx], '/') + } + + return value, "", false +} + +// ListByPath returns all descendants of path in the tree, +// including the path itself if it exists. +// +// The values are returned in an unspecified order. +func (t *pathTree[T]) ListByPath(path string) []T { + start, end := t.rangeOf(path) + if start == end { + return nil + } + + descendants := make([]T, end-start) + for i := start; i < end; i++ { + descendants[i-start] = t.values[i] + } + return descendants +} + +func (t *pathTree[T]) rangeOf(path string) (start, end int) { + if len(path) == 0 { + return 0, len(t.paths) + } + + start, _ = slices.BinarySearch(t.paths, path) + for idx := start; idx < len(t.paths); idx++ { + if descends(path, t.paths[idx]) { + continue // path is an ancestor of p + } + + // End of matching sequences. + // The next path is not a descendant of path. + return start, idx + } + + // All paths following start are descendants of path. + return start, len(t.paths) +} + +func descends(from, to string) bool { + return to == from || (strings.HasPrefix(to, from) && to[len(from)] == '/') +} diff --git a/path_tree_test.go b/path_tree_test.go new file mode 100644 index 0000000..e5303de --- /dev/null +++ b/path_tree_test.go @@ -0,0 +1,304 @@ +package main + +import ( + "fmt" + "math/rand" + "slices" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "pgregory.net/rapid" +) + +func TestPathTree_empty(t *testing.T) { + var tree pathTree[int] + + _, _, ok := tree.Lookup("") + assert.False(t, ok) + + _, _, ok = tree.Lookup("foo") + assert.False(t, ok) + + assert.Empty(t, tree.ListByPath("")) +} + +func TestPathTree(t *testing.T) { + var tree pathTree[int] + mustHave := func(path string, want int, wantSuffix string) { + t.Helper() + + v, suffix, ok := tree.Lookup(path) + require.True(t, ok, "path %q", path) + assert.Equal(t, v, want, "path %q", path) + assert.Equal(t, wantSuffix, suffix, "path %q", path) + } + + mustNotHave := func(path string) { + t.Helper() + + _, _, ok := tree.Lookup(path) + require.False(t, ok, "path %q", path) + } + + mustList := func(path string, want ...int) { + t.Helper() + slices.Sort(want) + + got := tree.ListByPath(path) + slices.Sort(got) + + assert.Equal(t, want, got, "path %q", path) + } + + tree.Set("foo", 10) + t.Run("single", func(t *testing.T) { + mustHave("foo", 10, "") + mustHave("foo/bar", 10, "/bar") + mustHave("foo/bar/baz", 10, "/bar/baz") + mustNotHave("") + mustNotHave("bar") + mustNotHave("bar/baz") + + t.Run("list", func(t *testing.T) { + mustList("", 10) + mustList("foo", 10) + mustList("foo/bar") + }) + }) + + // Override a descendant value. + t.Run("descendant", func(t *testing.T) { + tree.Set("foo/bar", 20) + mustHave("foo", 10, "") + mustHave("foo/bar", 20, "") + mustHave("foo/bar/baz", 20, "/baz") + + t.Run("list", func(t *testing.T) { + mustList("", 10, 20) + mustList("foo", 10, 20) + mustList("foo/bar", 20) + mustList("foo/bar/baz") + }) + }) + + // Add a sibling. + t.Run("sibling", func(t *testing.T) { + tree.Set("bar", 30) + mustHave("bar", 30, "") + mustHave("bar/baz", 30, "/baz") + + t.Run("list", func(t *testing.T) { + mustList("", 10, 20, 30) + mustList("foo", 10, 20) + mustList("bar", 30) + mustList("bar/baz") + }) + }) + + // Replace an existing value. + t.Run("replace", func(t *testing.T) { + tree.Set("bar", 40) + mustHave("bar", 40, "") + mustHave("bar/baz", 40, "/baz") + + t.Run("list", func(t *testing.T) { + mustList("", 10, 20, 40) + mustList("foo", 10, 20) + mustList("bar", 40) + mustList("bar/baz") + }) + }) +} + +func TestPathTreeRapid(t *testing.T) { + pathGen := rapid.StringMatching(`[a-z]+(/[a-z]+)*`) + valueGen := rapid.Int() + + rapid.Check(t, func(t *rapid.T) { + var tree pathTree[int] + + // Exact lookup table. + exact := make(map[string]int) + var paths []string // known paths + + knownPathGen := rapid.Deferred(func() *rapid.Generator[string] { + return rapid.SampledFrom(paths) + }) + + drawKnownPath := func(t *rapid.T) string { + if len(paths) == 0 { + t.Skip() + } + return knownPathGen.Draw(t, "knownPath") + } + + t.Repeat(map[string]func(*rapid.T){ + "Set": func(t *rapid.T) { + path := pathGen.Draw(t, "path") + if _, ok := exact[path]; ok { + // Already set. + // Overwrite will handle this. + t.Skip() + } + + value := valueGen.Draw(t, "value") + tree.Set(path, value) + exact[path] = value + paths = append(paths, path) + }, + "Overwrite": func(t *rapid.T) { + path := drawKnownPath(t) + value := valueGen.Draw(t, "value") + + tree.Set(path, value) + exact[path] = value + // paths already contains path. + }, + "ExactLookup": func(t *rapid.T) { + if len(paths) == 0 { + t.Skip() + } + + path := drawKnownPath(t) + want := exact[path] + + got, suffix, ok := tree.Lookup(path) + assert.True(t, ok, "path %q", path) + assert.Equal(t, want, got, "path %q", path) + assert.Empty(t, suffix, "path %q", path) + }, + "DescendantLookup": func(t *rapid.T) { + parentPath := drawKnownPath(t) + want := exact[parentPath] + + var childPath string + for { + childPath = "/" + pathGen.Draw(t, "childPath") + if _, ok := exact[parentPath+childPath]; !ok { + break // found a unique child path + } + } + + path := parentPath + childPath + got, suffix, ok := tree.Lookup(path) + assert.True(t, ok, "path %q", path) + assert.Equal(t, want, got, "path %q", path) + assert.Equal(t, childPath, suffix, "path %q", path) + }, + "ListAll": func(t *rapid.T) { + var want []int + for _, v := range exact { + want = append(want, v) + } + slices.Sort(want) + + got := tree.ListByPath("") + slices.Sort(got) + + assert.Equal(t, want, got) + }, + "ListSubset": func(t *rapid.T) { + path := drawKnownPath(t) + + var want []int + for p, v := range exact { + if descends(path, p) { + want = append(want, v) + } + } + slices.Sort(want) + + got := tree.ListByPath(path) + slices.Sort(got) + + if len(want) == 0 { + // Guard against nil != empty. + assert.Empty(t, got, "path %q", path) + } else { + assert.Equal(t, want, got, "path %q", path) + } + }, + }) + }) +} + +func BenchmarkPathTreeDeep(b *testing.B) { + depths := []int{10, 100} + widths := []int{10, 100} + for _, depth := range depths { + b.Run(fmt.Sprintf("depth=%d", depth), func(b *testing.B) { + for _, width := range widths { + b.Run(fmt.Sprintf("width=%d", width), func(b *testing.B) { + benchmarkPathTree(b, depth, width) + }) + } + }) + } +} + +func benchmarkPathTree(b *testing.B, Depth, Width int) { + var ( + tree pathTree[int] + depthpb strings.Builder + ) + paths := make([]string, 0, Depth*Width) + for i := 0; i < Depth; i++ { + if depthpb.Len() > 0 { + depthpb.WriteByte('/') + } + depthpb.WriteString("a") + + depthPath := depthpb.String() + for j := 0; j < Width; j++ { + path := depthPath + "/" + strconv.Itoa(j) + paths = append(paths, path) + tree.Set(path, i+i) + } + } + + b.ResetTimer() + + b.Run("LookupExact", func(b *testing.B) { + path := paths[rand.Intn(len(paths))] + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, _, ok := tree.Lookup(path) + require.True(b, ok) + } + }) + }) + + b.Run("LookupDescendant", func(b *testing.B) { + path := paths[rand.Intn(len(paths))] + strings.Repeat("/xyz", 10) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, _, ok := tree.Lookup(path) + require.True(b, ok) + } + }) + }) + + b.Run("ListSubtree", func(b *testing.B) { + path := paths[rand.Intn(len(paths))] + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if len(tree.ListByPath(path)) == 0 { + b.Fatal("unexpected empty list") + } + } + }) + }) + + b.Run("ListAll", func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if len(tree.ListByPath("")) == 0 { + b.Fatal("unexpected empty list") + } + } + }) + }) +} diff --git a/sally.yaml b/sally.yaml index c9a3af8..3055d87 100644 --- a/sally.yaml +++ b/sally.yaml @@ -6,3 +6,5 @@ packages: description: A customizable implementation of Thrift. yarpc: repo: github.com/yarpc/yarpc-go + net/metrics: + repo: github.com/yarpc/metrics diff --git a/templates/index.html b/templates/index.html index fa061f1..7ffc1a6 100644 --- a/templates/index.html +++ b/templates/index.html @@ -29,15 +29,15 @@
Package: - {{ .ImportPath }} + {{ .ModulePath }}
Source: {{ .GitURL }}
- - Go Reference + + Go Reference
diff --git a/templates/package.html b/templates/package.html index a85ff77..bd56c38 100644 --- a/templates/package.html +++ b/templates/package.html @@ -1,10 +1,10 @@ - - + + - Nothing to see here. Please move along. + Nothing to see here. Please move along.