X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=binapigen%2Fgenerator.go;h=e5eed5a0def462f5e467c347692f842e3fdde8c8;hb=c94a962279858fb13eaacc689f47aed358373e44;hp=07c1b13466983d8c010f0e1eb9a9fcb87975df20;hpb=ceed73403bdb61387d04be8b47183e9c4a970749;p=govpp.git diff --git a/binapigen/generator.go b/binapigen/generator.go index 07c1b13..e5eed5a 100644 --- a/binapigen/generator.go +++ b/binapigen/generator.go @@ -15,130 +15,131 @@ package binapigen import ( + "bufio" "bytes" "fmt" - "go/format" + "go/ast" + "go/parser" + "go/printer" + "go/token" "io/ioutil" "os" "path" "path/filepath" - "regexp" + "sort" + "strconv" + "strings" "github.com/sirupsen/logrus" "git.fd.io/govpp.git/binapigen/vppapi" ) -type Options struct { - VPPVersion string // version of VPP that produced API files - - FilesToGenerate []string // list of API files to generate - - ImportPrefix string // defines import path prefix for importing types - ImportTypes bool // generate packages for import types - IncludeAPIVersion bool // include constant with API version string - IncludeComments bool // include parts of original source in comments - IncludeBinapiNames bool // include binary API names as struct tag - IncludeServices bool // include service interface with client implementation - IncludeVppVersion bool // include info about used VPP version -} - type Generator struct { - Options - Files []*File - FilesByPath map[string]*File FilesByName map[string]*File + FilesByPath map[string]*File + + opts Options + apifiles []*vppapi.File + vppVersion string - enumsByName map[string]*Enum - aliasesByName map[string]*Alias - structsByName map[string]*Struct - unionsByName map[string]*Union + filesToGen []string + genfiles []*GenFile - genfiles []*GenFile + enumsByName map[string]*Enum + aliasesByName map[string]*Alias + structsByName map[string]*Struct + unionsByName map[string]*Union + messagesByName map[string]*Message } -func New(opts Options, apifiles []*vppapi.File) (*Generator, error) { - g := &Generator{ - Options: opts, - FilesByPath: make(map[string]*File), - FilesByName: make(map[string]*File), - enumsByName: map[string]*Enum{}, - aliasesByName: map[string]*Alias{}, - structsByName: map[string]*Struct{}, - unionsByName: map[string]*Union{}, +func New(opts Options, apiFiles []*vppapi.File, filesToGen []string) (*Generator, error) { + gen := &Generator{ + FilesByName: make(map[string]*File), + FilesByPath: make(map[string]*File), + opts: opts, + apifiles: apiFiles, + filesToGen: filesToGen, + enumsByName: map[string]*Enum{}, + aliasesByName: map[string]*Alias{}, + structsByName: map[string]*Struct{}, + unionsByName: map[string]*Union{}, + messagesByName: map[string]*Message{}, } - logrus.Debugf("adding %d VPP API files to generator", len(apifiles)) - for _, apifile := range apifiles { - filename := apifile.Path - if filename == "" { - filename = apifile.Name - } - if _, ok := g.FilesByPath[filename]; ok { - return nil, fmt.Errorf("duplicate file name: %q", filename) - } - if _, ok := g.FilesByName[apifile.Name]; ok { + // Normalize API files + SortFilesByImports(gen.apifiles) + for _, apiFile := range apiFiles { + RemoveImportedTypes(gen.apifiles, apiFile) + SortFileObjectsByName(apiFile) + } + + // prepare package names and import paths + packageNames := make(map[string]GoPackageName) + importPaths := make(map[string]GoImportPath) + for _, apifile := range gen.apifiles { + filename := getFilename(apifile) + packageNames[filename] = cleanPackageName(apifile.Name) + importPaths[filename] = GoImportPath(path.Join(gen.opts.ImportPrefix, baseName(apifile.Name))) + } + + logrus.Debugf("adding %d VPP API files to generator", len(gen.apifiles)) + + for _, apifile := range gen.apifiles { + if _, ok := gen.FilesByName[apifile.Name]; ok { return nil, fmt.Errorf("duplicate file: %q", apifile.Name) } - file, err := newFile(g, apifile) + filename := getFilename(apifile) + file, err := newFile(gen, apifile, packageNames[filename], importPaths[filename]) if err != nil { - return nil, err + return nil, fmt.Errorf("loading file %s failed: %w", apifile.Name, err) } - g.Files = append(g.Files, file) - g.FilesByPath[filename] = file - g.FilesByName[apifile.Name] = file + gen.Files = append(gen.Files, file) + gen.FilesByName[apifile.Name] = file + gen.FilesByPath[apifile.Path] = file logrus.Debugf("added file %q (path: %v)", apifile.Name, apifile.Path) - if len(file.Imports) > 0 { - logrus.Debugf(" - %d imports: %v", len(file.Imports), file.Imports) - } } - if len(opts.FilesToGenerate) > 0 { - logrus.Debugf("Checking %d files to generate: %v", len(opts.FilesToGenerate), opts.FilesToGenerate) - for _, genfile := range opts.FilesToGenerate { - file, ok := g.FilesByPath[genfile] - if !ok { - file, ok = g.FilesByName[genfile] - if !ok { - return nil, fmt.Errorf("no API file found for: %v", genfile) - } - } - file.Generate = true - if opts.ImportTypes { + // mark files for generation + if len(gen.filesToGen) > 0 { + logrus.Debugf("Checking %d files to generate: %v", len(gen.filesToGen), gen.filesToGen) + for _, genFile := range gen.filesToGen { + markGen := func(file *File) { + file.Generate = true // generate all imported files - for _, impFile := range file.importedFiles(g) { + for _, impFile := range file.importedFiles(gen) { impFile.Generate = true } } + if file, ok := gen.FilesByName[genFile]; ok { + markGen(file) + continue + } + logrus.Debugf("File %s was not found by name", genFile) + if file, ok := gen.FilesByPath[genFile]; ok { + markGen(file) + continue + } + return nil, fmt.Errorf("no API file found for: %v", genFile) } } else { - logrus.Debugf("Files to generate not specified, marking all %d files to generate", len(g.Files)) - for _, file := range g.Files { + logrus.Debugf("Files to generate not specified, marking all %d files for generate", len(gen.Files)) + for _, file := range gen.Files { file.Generate = true } } - logrus.Debugf("Resolving imported types") - for _, file := range g.Files { - if !file.Generate { - // skip resolving for non-generated files - continue - } - var importedFiles []*File - for _, impFile := range file.importedFiles(g) { - if !impFile.Generate { - // exclude imports of non-generated files - continue - } - importedFiles = append(importedFiles, impFile) - } - file.loadTypeImports(g, importedFiles) - } + return gen, nil +} - return g, nil +func getFilename(file *vppapi.File) string { + if file.Path == "" { + return file.Name + } + return file.Path } func (g *Generator) Generate() error { @@ -147,127 +148,191 @@ func (g *Generator) Generate() error { } logrus.Infof("Generating %d files", len(g.genfiles)) + for _, genfile := range g.genfiles { - if err := writeSourceTo(genfile.filename, genfile.Content()); err != nil { - return fmt.Errorf("writing source for RPC package %s failed: %v", genfile.filename, err) + content, err := genfile.Content() + if err != nil { + return err + } + if err := writeSourceTo(genfile.filename, content); err != nil { + return fmt.Errorf("writing source package %s failed: %v", genfile.filename, err) } } return nil } type GenFile struct { - *Generator - filename string - file *File - outputDir string - buf bytes.Buffer + gen *Generator + file *File + filename string + goImportPath GoImportPath + buf bytes.Buffer + manualImports map[GoImportPath]bool + packageNames map[GoImportPath]GoPackageName } -func (g *Generator) NewGenFile(filename string) *GenFile { +// NewGenFile creates new generated file with +func (g *Generator) NewGenFile(filename string, importPath GoImportPath) *GenFile { f := &GenFile{ - Generator: g, - filename: filename, + gen: g, + filename: filename, + goImportPath: importPath, + manualImports: make(map[GoImportPath]bool), + packageNames: make(map[GoImportPath]GoPackageName), } g.genfiles = append(g.genfiles, f) return f } -func (f *GenFile) Content() []byte { - return f.buf.Bytes() +func (g *GenFile) Write(p []byte) (n int, err error) { + return g.buf.Write(p) } -func writeSourceTo(outputFile string, b []byte) error { - // create output directory - packageDir := filepath.Dir(outputFile) - if err := os.MkdirAll(packageDir, 0775); err != nil { - return fmt.Errorf("creating output dir %s failed: %v", packageDir, err) - } +func (g *GenFile) Import(importPath GoImportPath) { + g.manualImports[importPath] = true +} - // format generated source code - gosrc, err := format.Source(b) - if err != nil { - _ = ioutil.WriteFile(outputFile, b, 0666) - return fmt.Errorf("formatting source code failed: %v", err) +func (g *GenFile) GoIdent(ident GoIdent) string { + if ident.GoImportPath == g.goImportPath { + return ident.GoName } - - // write generated code to output file - if err := ioutil.WriteFile(outputFile, gosrc, 0666); err != nil { - return fmt.Errorf("writing to output file %s failed: %v", outputFile, err) + if packageName, ok := g.packageNames[ident.GoImportPath]; ok { + return string(packageName) + "." + ident.GoName } + packageName := cleanPackageName(baseName(string(ident.GoImportPath))) + g.packageNames[ident.GoImportPath] = packageName + return string(packageName) + "." + ident.GoName +} - lines := bytes.Count(gosrc, []byte("\n")) - logf("wrote %d lines (%d bytes) of code to: %q", lines, len(gosrc), outputFile) - - return nil +func (g *GenFile) P(v ...interface{}) { + for _, x := range v { + switch x := x.(type) { + case GoIdent: + fmt.Fprint(&g.buf, g.GoIdent(x)) + default: + fmt.Fprint(&g.buf, x) + } + } + fmt.Fprintln(&g.buf) } -func listImports(genfile *GenFile) map[string]string { - var importPath = genfile.ImportPrefix - if importPath == "" { - importPath = resolveImportPath(genfile.outputDir) - logrus.Debugf("resolved import path: %s", importPath) +func (g *GenFile) Content() ([]byte, error) { + if !strings.HasSuffix(g.filename, ".go") { + return g.buf.Bytes(), nil } - imports := map[string]string{} - for _, imp := range genfile.file.imports { - if _, ok := imports[imp]; !ok { - imports[imp] = path.Join(importPath, imp) - } + return g.injectImports(g.buf.Bytes()) +} + +func getImportClass(importPath string) int { + if !strings.Contains(importPath, ".") { + return 0 /* std */ } - return imports + return 1 /* External */ } -func resolveImportPath(outputDir string) string { - absPath, err := filepath.Abs(outputDir) +// injectImports parses source, injects import block declaration with all imports and return formatted +func (g *GenFile) injectImports(original []byte) ([]byte, error) { + // Parse source code + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "", original, parser.ParseComments) if err != nil { - panic(err) - } - modRoot := findModuleRoot(absPath) - if modRoot == "" { - logrus.Fatalf("module root not found at: %s", absPath) + var src bytes.Buffer + s := bufio.NewScanner(bytes.NewReader(original)) + for line := 1; s.Scan(); line++ { + fmt.Fprintf(&src, "%5d\t%s\n", line, s.Bytes()) + } + return nil, fmt.Errorf("%v: unparsable Go source: %v\n%v", g.filename, err, src.String()) } - modPath := findModulePath(path.Join(modRoot, "go.mod")) - if modPath == "" { - logrus.Fatalf("module path not found") + type Import struct { + Name string + Path string } - relDir, err := filepath.Rel(modRoot, absPath) - if err != nil { - panic(err) + // Prepare list of all imports + var importPaths []Import + for importPath := range g.packageNames { + importPaths = append(importPaths, Import{ + Name: string(g.packageNames[importPath]), + Path: string(importPath), + }) } - return filepath.Join(modPath, relDir) -} - -func findModuleRoot(dir string) (root string) { - if dir == "" { - panic("dir not set") + for importPath := range g.manualImports { + if _, ok := g.packageNames[importPath]; ok { + continue + } + importPaths = append(importPaths, Import{ + Name: "_", + Path: string(importPath), + }) } - dir = filepath.Clean(dir) - - // Look for enclosing go.mod. - for { - if fi, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() { - return dir + // Sort imports by import path + sort.Slice(importPaths, func(i, j int) bool { + ci := getImportClass(importPaths[i].Path) + cj := getImportClass(importPaths[j].Path) + if ci == cj { + return importPaths[i].Path < importPaths[j].Path } - d := filepath.Dir(dir) - if d == dir { - break + return ci < cj + }) + // Inject new import block into parsed AST + if len(importPaths) > 0 { + // Find import block position + pos := file.Package + tokFile := fset.File(file.Package) + pkgLine := tokFile.Line(file.Package) + for _, c := range file.Comments { + if tokFile.Line(c.Pos()) > pkgLine { + break + } + pos = c.End() } - dir = d + // Prepare the import block + impDecl := &ast.GenDecl{Tok: token.IMPORT, TokPos: pos, Lparen: pos, Rparen: pos} + for i, importPath := range importPaths { + var name *ast.Ident + if importPath.Name == "_" || strings.Contains(importPath.Path, ".") { + name = &ast.Ident{Name: importPath.Name, NamePos: pos} + } + value := strconv.Quote(importPath.Path) + if i < len(importPaths)-1 { + if getImportClass(importPath.Path) != getImportClass(importPaths[i+1].Path) { + value += "\n" + } + } + impDecl.Specs = append(impDecl.Specs, &ast.ImportSpec{ + Name: name, + Path: &ast.BasicLit{Kind: token.STRING, Value: value, ValuePos: pos}, + EndPos: pos, + }) + } + + file.Decls = append([]ast.Decl{impDecl}, file.Decls...) + } + // Reformat source code + var out bytes.Buffer + cfg := &printer.Config{ + Mode: printer.TabIndent | printer.UseSpaces, + Tabwidth: 8, } - return "" + if err = cfg.Fprint(&out, fset, file); err != nil { + return nil, fmt.Errorf("%v: can not reformat Go source: %v", g.filename, err) + } + return out.Bytes(), nil } -var ( - modulePathRE = regexp.MustCompile(`module[ \t]+([^ \t\r\n]+)`) -) - -func findModulePath(file string) string { - data, err := ioutil.ReadFile(file) - if err != nil { - return "" +func writeSourceTo(outputFile string, b []byte) error { + // create output directory + packageDir := filepath.Dir(outputFile) + if err := os.MkdirAll(packageDir, 0775); err != nil { + return fmt.Errorf("creating output dir %s failed: %v", packageDir, err) } - m := modulePathRE.FindSubmatch(data) - if m == nil { - return "" + + // write generated code to output file + if err := ioutil.WriteFile(outputFile, b, 0666); err != nil { + return fmt.Errorf("writing to output file %s failed: %v", outputFile, err) } - return string(m[1]) + + lines := bytes.Count(b, []byte("\n")) + logf("wrote %d lines (%d bytes) to: %q", lines, len(b), outputFile) + + return nil }