Skip to content

Commit

Permalink
feat: add option to persist rule index
Browse files Browse the repository at this point in the history
  • Loading branch information
jbedard committed Aug 21, 2024
1 parent be4f484 commit 40a700a
Show file tree
Hide file tree
Showing 3 changed files with 148 additions and 25 deletions.
44 changes: 41 additions & 3 deletions cmd/gazelle/fix-update.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ type updateConfig struct {
patchBuffer bytes.Buffer
print0 bool
profile profiler

// EXPERIMENTAL: caching of the rule index across runs
ruleIndexFile string
}

type emitFunc func(c *config.Config, f *rule.File) error
Expand Down Expand Up @@ -94,6 +97,7 @@ func (ucr *updateConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *conf
fs.StringVar(&ucr.memProfile, "memprofile", "", "write memory profile to `file`")
fs.Var(&gzflag.MultiFlag{Values: &ucr.knownImports}, "known_import", "import path for which external resolution is skipped (can specify multiple times)")
fs.StringVar(&ucr.repoConfigPath, "repo_config", "", "file where Gazelle should load repository configuration. Defaults to WORKSPACE.")
fs.StringVar(&uc.ruleIndexFile, "indexdb", "", "EXPERIMENTAL: file to cache the rule index")
}

func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error {
Expand All @@ -110,6 +114,9 @@ func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) erro
if uc.patchPath != "" && !filepath.IsAbs(uc.patchPath) {
uc.patchPath = filepath.Join(c.WorkDir, uc.patchPath)
}
if uc.ruleIndexFile != "" && !filepath.IsAbs(uc.ruleIndexFile) {
uc.ruleIndexFile = filepath.Join(c.WorkDir, uc.ruleIndexFile)
}
p, err := newProfiler(ucr.cpuProfile, ucr.memProfile)
if err != nil {
return err
Expand Down Expand Up @@ -311,15 +318,38 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
}
}()

walkMode := uc.walkMode

updateRels := walk.NewUpdateFilter(c.RepoRoot, uc.dirs, uc.walkMode)
preindexed := false

// Load the rule index file if it exists.
if c.IndexLibraries && uc.ruleIndexFile != "" {
// Do not load index entries from directories that are being updated.
indexLoaded, err := ruleIndex.LoadIndex(uc.ruleIndexFile, updateRels.ShouldReIndex)
if err != nil {
log.Printf("Failed to load index file %s: %v", uc.ruleIndexFile, err)
} else if indexLoaded {
// Drop "visit all" since indexing has been loaded from disk.
if walkMode == walk.VisitAllUpdateSubdirsMode {
walkMode = walk.UpdateSubdirsMode
} else if walkMode == walk.VisitAllUpdateDirsMode {
walkMode = walk.UpdateDirsMode
}

preindexed = true
}
}

var errorsFromWalk []error
walk.Walk(c, cexts, uc.dirs, uc.walkMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) {
walk.Walk(c, cexts, uc.dirs, walkMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) {
// If this file is ignored or if Gazelle was not asked to update this
// directory, just index the build file and move on.
if !update {
for _, repl := range c.KindMap {
mrslv.MappedKind(rel, repl)
}
if c.IndexLibraries && f != nil {
if c.IndexLibraries && f != nil && (!preindexed || updateRels.ShouldReIndex(rel)) {
for _, r := range f.Rules {
ruleIndex.AddRule(c, r, f)
}
Expand Down Expand Up @@ -446,7 +476,7 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
})

// Add library rules to the dependency resolution table.
if c.IndexLibraries {
if c.IndexLibraries && (!preindexed || updateRels.ShouldReIndex(rel)) {
for _, r := range f.Rules {
ruleIndex.AddRule(c, r, f)
}
Expand Down Expand Up @@ -475,6 +505,14 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
// Finish building the index for dependency resolution.
ruleIndex.Finish()

// Persist the index for future runs.
if c.IndexLibraries && uc.ruleIndexFile != "" {
err := ruleIndex.SaveIndex(uc.ruleIndexFile)
if err != nil {
fmt.Printf("Failed to save index file %s: %v", uc.ruleIndexFile, err)
}
}

// Resolve dependencies.
rc, cleanupRc := repo.NewRemoteCache(uc.repos)
defer func() {
Expand Down
63 changes: 63 additions & 0 deletions resolve/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ limitations under the License.
package resolve

import (
"encoding/json"
"io"
"log"
"os"

"github.com/bazelbuild/bazel-gazelle/config"
"github.com/bazelbuild/bazel-gazelle/label"
Expand Down Expand Up @@ -145,6 +148,66 @@ func NewRuleIndex(mrslv func(ruleKind, pkgRel string) Resolver, exts ...interfac
}
}

func (ix *RuleIndex) LoadIndex(indexDbPath string, isPkgExcluded func(string) bool) (bool, error) {
if ix.indexed {
log.Fatal("LoadIndex called after Finish")
}

indexDbFile, err := os.Open(indexDbPath)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
defer indexDbFile.Close()

indexDbContent, err := io.ReadAll(indexDbFile)
if err != nil {
return false, err
}

// TODO: read & verify index version

var rules []*ruleRecord

err = json.Unmarshal(indexDbContent, &rules)
if err != nil {
return false, err
}

ix.rules = make([]*ruleRecord, 0, len(rules))
for _, r := range rules {
if !isPkgExcluded(r.Label.Pkg) {
ix.rules = append(ix.rules, r)
}
}

return true, nil
}

func (ix *RuleIndex) SaveIndex(indexDbPath string) error {
// TODO: write index version

indexDbContent, err := json.Marshal(ix.rules)
if err != nil {
return err
}

indexDbFile, err := os.OpenFile(indexDbPath, os.O_RDWR|os.O_CREATE, 0666)
if err != nil {
return err
}
defer indexDbFile.Close()

_, err = indexDbFile.Write(indexDbContent)
if err != nil {
return err
}

return nil
}

// AddRule adds a rule r to the index. The rule will only be indexed if there
// is a known resolver for the rule's kind and Resolver.Imports returns a
// non-nil slice.
Expand Down
66 changes: 44 additions & 22 deletions walk/walk.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ func Walk(c *config.Config, cexts []config.Configurer, dirs []string, mode Mode,
}
}

updateRels := buildUpdateRelMap(c.RepoRoot, dirs)
updateRels := NewUpdateFilter(c.RepoRoot, dirs, mode)

var visit func(*config.Config, string, string, bool)
visit = func(c *config.Config, dir, rel string, updateParent bool) {
Expand Down Expand Up @@ -162,32 +162,38 @@ func Walk(c *config.Config, cexts []config.Configurer, dirs []string, mode Mode,
}
}

shouldUpdate := shouldUpdate(rel, mode, updateParent, updateRels)
shouldUpdate := updateRels.shouldUpdate(rel, updateParent)
for _, sub := range subdirs {
if subRel := path.Join(rel, sub); shouldVisit(subRel, mode, shouldUpdate, updateRels) {
if subRel := path.Join(rel, sub); updateRels.shouldVisit(subRel, shouldUpdate) {
visit(c, filepath.Join(dir, sub), subRel, shouldUpdate)
}
}

update := !haveError && !wc.ignore && shouldUpdate
if shouldCall(rel, mode, updateParent, updateRels) {
if updateRels.shouldCall(rel, updateParent) {
genFiles := findGenFiles(wc, f)
wf(dir, rel, c, update, f, subdirs, regularFiles, genFiles)
}
}
visit(c, c.RepoRoot, "", false)
}

// buildUpdateRelMap builds a table of prefixes, used to determine which
// An UpdateFilter tracks which directories need to be updated
type UpdateFilter struct {
mode Mode

// map from slash-separated paths relative to the
// root directory ("" for the root itself) to a boolean indicating whether
// the directory should be updated.
updateRels map[string]bool
}

// NewUpdateFilter builds a table of prefixes, used to determine which
// directories to update and visit.
//
// root and dirs must be absolute, canonical file paths. Each entry in dirs
// must be a subdirectory of root. The caller is responsible for checking this.
//
// buildUpdateRelMap returns a map from slash-separated paths relative to the
// root directory ("" for the root itself) to a boolean indicating whether
// the directory should be updated.
func buildUpdateRelMap(root string, dirs []string) map[string]bool {
func NewUpdateFilter(root string, dirs []string, mode Mode) *UpdateFilter {
relMap := make(map[string]bool)
for _, dir := range dirs {
rel, _ := filepath.Rel(root, dir)
Expand All @@ -210,42 +216,58 @@ func buildUpdateRelMap(root string, dirs []string) map[string]bool {
i = next + 1
}
}
return relMap
return &UpdateFilter{mode, relMap}
}

func (u *UpdateFilter) ShouldReIndex(rel string) bool {
if rel == "." {
rel = ""
}

if should, found := u.updateRels[rel]; found {
return should
}

if rel != "" && (u.mode == UpdateSubdirsMode || u.mode == VisitAllUpdateSubdirsMode) {
return u.ShouldReIndex(path.Dir(rel))
}

return false
}

// shouldCall returns true if Walk should call the callback in the
// directory rel.
func shouldCall(rel string, mode Mode, updateParent bool, updateRels map[string]bool) bool {
switch mode {
func (u *UpdateFilter) shouldCall(rel string, updateParent bool) bool {
switch u.mode {
case VisitAllUpdateSubdirsMode, VisitAllUpdateDirsMode:
return true
case UpdateSubdirsMode:
return updateParent || updateRels[rel]
return updateParent || u.updateRels[rel]
default: // UpdateDirsMode
return updateRels[rel]
return u.updateRels[rel]
}
}

// shouldUpdate returns true if Walk should pass true to the callback's update
// parameter in the directory rel. This indicates the build file should be
// updated.
func shouldUpdate(rel string, mode Mode, updateParent bool, updateRels map[string]bool) bool {
if (mode == VisitAllUpdateSubdirsMode || mode == UpdateSubdirsMode) && updateParent {
func (u *UpdateFilter) shouldUpdate(rel string, updateParent bool) bool {
if (u.mode == VisitAllUpdateSubdirsMode || u.mode == UpdateSubdirsMode) && updateParent {
return true
}
return updateRels[rel]
return u.updateRels[rel]
}

// shouldVisit returns true if Walk should visit the subdirectory rel.
func shouldVisit(rel string, mode Mode, updateParent bool, updateRels map[string]bool) bool {
switch mode {
func (u *UpdateFilter) shouldVisit(rel string, updateParent bool) bool {
switch u.mode {
case VisitAllUpdateSubdirsMode, VisitAllUpdateDirsMode:
return true
case UpdateSubdirsMode:
_, ok := updateRels[rel]
_, ok := u.updateRels[rel]
return ok || updateParent
default: // UpdateDirsMode
_, ok := updateRels[rel]
_, ok := u.updateRels[rel]
return ok
}
}
Expand Down

0 comments on commit 40a700a

Please sign in to comment.