From 29586680c0609a34d729d0be5678da9ca99ed58b Mon Sep 17 00:00:00 2001 From: Simon Dassow Date: Tue, 1 Oct 2024 17:41:52 +0200 Subject: [PATCH] Add flag for dry-run and to update existing translations, clean up and document, make verbose mode more useful, and split into smaller pieces --- cmd/fyne/internal/commands/translate.go | 243 +++++++++++++++--------- 1 file changed, 156 insertions(+), 87 deletions(-) diff --git a/cmd/fyne/internal/commands/translate.go b/cmd/fyne/internal/commands/translate.go index 3a8383075f..b47fcfcada 100644 --- a/cmd/fyne/internal/commands/translate.go +++ b/cmd/fyne/internal/commands/translate.go @@ -7,6 +7,7 @@ import ( "go/ast" "go/parser" "go/token" + "io" "io/fs" "os" "path/filepath" @@ -24,17 +25,27 @@ func Translate() *cli.Command { &cli.BoolFlag{ Name: "verbose", Aliases: []string{"v"}, - Usage: "Shows files that are being scanned etc.", + Usage: "Show files that are being scanned etc.", + }, + &cli.BoolFlag{ + Name: "update", + Aliases: []string{"u"}, + Usage: "Update existing translations (use with care).", + }, + &cli.BoolFlag{ + Name: "dry-run", + Aliases: []string{"n"}, + Usage: "Scan without storing the results.", }, &cli.StringFlag{ Name: "sourceDir", - Aliases: []string{"src"}, + Aliases: []string{"d"}, Usage: "Directory to scan recursively for go files.", Value: ".", }, &cli.StringFlag{ Name: "translationsFile", - Aliases: []string{"file"}, + Aliases: []string{"f"}, Usage: "File to read from and write translations to.", Value: "translations/en.json", }, @@ -44,121 +55,155 @@ func Translate() *cli.Command { translationsFile := ctx.String("translationsFile") files := ctx.Args().Slice() + // without any argument find all .go files in the source directory if len(files) == 0 { - err := filepath.Walk(sourceDir, func(path string, fi fs.FileInfo, err error) error { - if err != nil { - return err - } - if fi.IsDir() { - return nil - } - - if !fi.Mode().IsRegular() { - return nil - } - - if filepath.Ext(path) != ".go" { - return nil - } - - files = append(files, path) - - return nil - }) + sources, err := findFilesExt(sourceDir, ".go") if err != nil { return err } + files = sources } - if ctx.Bool("verbose") { - fmt.Printf("files: %v\n", files) - } + // update the translation file by scanning the given source files + return updateTranslationsFile(ctx, translationsFile, files) + }, + } +} - translations := make(map[string]interface{}) +// Recursively walk the given directory and return all files with the matching extension +func findFilesExt(dir, ext string) ([]string, error) { + files := []string{} + err := filepath.Walk(dir, func(path string, fi fs.FileInfo, err error) error { + if err != nil { + return err + } - // get current translations - f, err := os.Open(translationsFile) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - defer f.Close() + if filepath.Ext(path) != ext || fi.IsDir() || !fi.Mode().IsRegular() { + return nil + } - if f != nil { - dec := json.NewDecoder(f) - if err := dec.Decode(&translations); err != nil { - return err - } - } + files = append(files, path) - // update translations hash - if err := updateTranslations(translations, files); err != nil { - return err - } + return nil + }) + return files, err +} - // serialize in readable format for humas to change - b, err := json.MarshalIndent(translations, "", "\t") - if err != nil { - return err - } +// Create or add to translations file by scanning the given files for translation calls +func updateTranslationsFile(ctx *cli.Context, file string, files []string) error { + translations := make(map[string]interface{}) - if ctx.Bool("verbose") { - fmt.Printf("%s\n", string(b)) - } + // try to get current translations first + f, err := os.Open(file) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + defer f.Close() - if len(translations) == 0 { - fmt.Println("No translations found") - return nil - } + // only try to parse translations that exist + if f != nil { + dec := json.NewDecoder(f) + if err := dec.Decode(&translations); err != nil { + return err + } + } - // use temporary file to do atomic change - nf, err := os.CreateTemp(filepath.Dir(translationsFile), filepath.Base(translationsFile)+"-*") - if err != nil { - return err - } + if ctx.Bool("verbose") { + fmt.Fprintf(os.Stderr, "scanning files: %v\n", files) + } - n, err := nf.Write(b) - if err != nil { - return err - } - if n != len(b) { - return err - } + // update translations hash + if err := updateTranslationsHash(ctx, translations, files); err != nil { + return err + } - if err := nf.Chmod(0644); err != nil { - return err - } - nf.Close() - if err := os.Rename(nf.Name(), translationsFile); err != nil { - return err - } + // serialize in readable format for humas to change + b, err := json.MarshalIndent(translations, "", "\t") + if err != nil { + return err + } - return nil - }, + // avoid writing an empty file + if len(translations) == 0 { + fmt.Fprintln(os.Stderr, "No translations found") + return nil + } + + // stop without making any changes + if ctx.Bool("dry-run") { + return nil + } + + // support writing to stdout + if file == "-" { + fmt.Printf("%s\n", string(b)) + return nil } + + return writeTranslationsFile(b, file, f) } -// -- 1: key (default) -// L: Localize(in string, data ...any) string -// N: LocalizePlural(in string, count int, data ...any) string -// -- 2: key, default -// X: LocalizeKey(key, fallback string, data ...any) string -// XN: LocalizePluralKey(key, fallback string, count int, data ...any) string +// Write data to given file, optionally using same permissions as the original file +func writeTranslationsFile(b []byte, file string, f *os.File) error { + // default permissions + perm := fs.FileMode(0644) + + // use same permissions as original file when possible + if f != nil { + fi, err := f.Stat() + if err != nil { + return err + } + perm = fi.Mode().Perm() + } + + // use temporary file to do atomic change + nf, err := os.CreateTemp(filepath.Dir(file), filepath.Base(file)+"-*") + if err != nil { + return err + } -func updateTranslations(m map[string]interface{}, srcs []string) error { + n, err := nf.Write(b) + if err != nil { + return err + } + + if n < len(b) { + return io.ErrShortWrite + } + + if err := nf.Chmod(perm); err != nil { + return err + } + + if err := nf.Close(); err != nil { + return err + } + + // atomic switch to new file + return os.Rename(nf.Name(), file) +} + +// Update translations hash by scanning the given files +func updateTranslationsHash(ctx *cli.Context, m map[string]interface{}, srcs []string) error { for _, src := range srcs { + // get AST by parsing source fset := token.NewFileSet() af, err := parser.ParseFile(fset, src, nil, parser.AllErrors) if err != nil { return err } - ast.Walk(&visitor{m: m}, af) + // walk AST to find known translation calls + ast.Walk(&visitor{ctx: ctx, m: m}, af) } return nil } +// Visitor pattern with state machine to find translations fallback (and key) type visitor struct { + ctx *cli.Context state stateFn name string key string @@ -166,12 +211,13 @@ type visitor struct { m map[string]interface{} } -// Visitor pattern using a state machine while walking the tree +// Method to walk AST using interface for ast.Walk func (v *visitor) Visit(node ast.Node) ast.Visitor { if node == nil { return nil } + // start over any time there is no state if v.state == nil { v.state = translateNew v.name = "" @@ -179,6 +225,7 @@ func (v *visitor) Visit(node ast.Node) ast.Visitor { v.fallback = "" } + // run and get next state v.state = v.state(v, node) return v @@ -187,6 +234,7 @@ func (v *visitor) Visit(node ast.Node) ast.Visitor { // State machine to pick out translation key and fallback from AST type stateFn func(*visitor, ast.Node) stateFn +// All translation calls need to start with the literal "lang" func translateNew(v *visitor, node ast.Node) stateFn { ident, ok := node.(*ast.Ident) if !ok { @@ -200,6 +248,7 @@ func translateNew(v *visitor, node ast.Node) stateFn { return translateCall } +// A known translation method needs to be used func translateCall(v *visitor, node ast.Node) stateFn { ident, ok := node.(*ast.Ident) if !ok { @@ -209,10 +258,12 @@ func translateCall(v *visitor, node ast.Node) stateFn { v.name = ident.Name switch ident.Name { - case "L", "Localise": + // simple cases: only the first argument is relevant + case "L", "Localize": return translateLocalize case "N", "LocalizePlural": return translateLocalize + // more complex cases: first two arguments matter case "X", "LocalizeKey": return translateKey case "XN", "LocalizePluralKey": @@ -222,6 +273,7 @@ func translateCall(v *visitor, node ast.Node) stateFn { return nil } +// Parse first argument, use string as key and fallback, and finish func translateLocalize(v *visitor, node ast.Node) stateFn { basiclit, ok := node.(*ast.BasicLit) if !ok { @@ -239,6 +291,7 @@ func translateLocalize(v *visitor, node ast.Node) stateFn { return translateFinish(v, node) } +// Parse first argument and use as key func translateKey(v *visitor, node ast.Node) stateFn { basiclit, ok := node.(*ast.BasicLit) if !ok { @@ -255,6 +308,7 @@ func translateKey(v *visitor, node ast.Node) stateFn { return translateKeyFallback } +// Parse second argument and use as fallback, and finish func translateKeyFallback(v *visitor, node ast.Node) stateFn { basiclit, ok := node.(*ast.BasicLit) if !ok { @@ -271,16 +325,31 @@ func translateKeyFallback(v *visitor, node ast.Node) stateFn { return translateFinish(v, node) } +// Finish scan for translation and add to translation hash with the right type (singular or plural) func translateFinish(v *visitor, node ast.Node) stateFn { // only adding new keys, ignoring changed or removed (ha!) ones // removing is dangerous as there could be dynamic keys that get removed + // Ignore existing translations to prevent unintentional overwriting _, found := v.m[v.key] if found { - return nil + if !v.ctx.Bool("update") { + if v.ctx.Bool("verbose") { + fmt.Fprintf(os.Stderr, "ignoring: %s\n", v.key) + } + return nil + } + if v.ctx.Bool("verbose") { + fmt.Fprintf(os.Stderr, "updating: %s\n", v.key) + } + } else { + if v.ctx.Bool("verbose") { + fmt.Fprintf(os.Stderr, "adding: %s\n", v.key) + } } switch v.name { + // Plural translations use a nested map case "LocalizePlural", "LocalizePluralKey", "N", "XN": m := make(map[string]string) m["other"] = v.fallback