libgo/go/go/doc/doc.go

   1 // Copyright 2009 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // The doc package extracts source code documentation from a Go AST.
   6 package doc
   7
   8 import (
   9         "go/ast"
  10         "go/token"
  11         "regexp"
  12         "sort"
  13 )
  14
  15
  16 // ----------------------------------------------------------------------------
  17
  18 type typeDoc struct {
  19         // len(decl.Specs) == 1, and the element type is *ast.TypeSpec
  20         // if the type declaration hasn't been seen yet, decl is nil
  21         decl *ast.GenDecl
  22         // values, factory functions, and methods associated with the type
  23         values    []*ast.GenDecl // consts and vars
  24         factories map[string]*ast.FuncDecl
  25         methods   map[string]*ast.FuncDecl
  26 }
  27
  28
  29 // docReader accumulates documentation for a single package.
  30 // It modifies the AST: Comments (declaration documentation)
  31 // that have been collected by the DocReader are set to nil
  32 // in the respective AST nodes so that they are not printed
  33 // twice (once when printing the documentation and once when
  34 // printing the corresponding AST node).
  35 //
  36 type docReader struct {
  37         doc     *ast.CommentGroup // package documentation, if any
  38         pkgName string
  39         values  []*ast.GenDecl // consts and vars
  40         types   map[string]*typeDoc
  41         funcs   map[string]*ast.FuncDecl
  42         bugs    []*ast.CommentGroup
  43 }
  44
  45
  46 func (doc *docReader) init(pkgName string) {
  47         doc.pkgName = pkgName
  48         doc.types = make(map[string]*typeDoc)
  49         doc.funcs = make(map[string]*ast.FuncDecl)
  50 }
  51
  52
  53 func (doc *docReader) addDoc(comments *ast.CommentGroup) {
  54         if doc.doc == nil {
  55                 // common case: just one package comment
  56                 doc.doc = comments
  57                 return
  58         }
  59
  60         // More than one package comment: Usually there will be only
  61         // one file with a package comment, but it's better to collect
  62         // all comments than drop them on the floor.
  63         // (This code isn't particularly clever - no amortized doubling is
  64         // used - but this situation occurs rarely and is not time-critical.)
  65         n1 := len(doc.doc.List)
  66         n2 := len(comments.List)
  67         list := make([]*ast.Comment, n1+1+n2) // + 1 for separator line
  68         copy(list, doc.doc.List)
  69         list[n1] = &ast.Comment{token.Position{}, []byte("//")} // separator line
  70         copy(list[n1+1:], comments.List)
  71         doc.doc = &ast.CommentGroup{list}
  72 }
  73
  74
  75 func (doc *docReader) addType(decl *ast.GenDecl) {
  76         spec := decl.Specs[0].(*ast.TypeSpec)
  77         typ := doc.lookupTypeDoc(spec.Name.Name)
  78         // typ should always be != nil since declared types
  79         // are always named - be conservative and check
  80         if typ != nil {
  81                 // a type should be added at most once, so typ.decl
  82                 // should be nil - if it isn't, simply overwrite it
  83                 typ.decl = decl
  84         }
  85 }
  86
  87
  88 func (doc *docReader) lookupTypeDoc(name string) *typeDoc {
  89         if name == "" {
  90                 return nil // no type docs for anonymous types
  91         }
  92         if tdoc, found := doc.types[name]; found {
  93                 return tdoc
  94         }
  95         // type wasn't found - add one without declaration
  96         tdoc := &typeDoc{nil, nil, make(map[string]*ast.FuncDecl), make(map[string]*ast.FuncDecl)}
  97         doc.types[name] = tdoc
  98         return tdoc
  99 }
 100
 101
 102 func baseTypeName(typ ast.Expr) string {
 103         switch t := typ.(type) {
 104         case *ast.Ident:
 105                 // if the type is not exported, the effect to
 106                 // a client is as if there were no type name
 107                 if t.IsExported() {
 108                         return string(t.Name)
 109                 }
 110         case *ast.StarExpr:
 111                 return baseTypeName(t.X)
 112         }
 113         return ""
 114 }
 115
 116
 117 func (doc *docReader) addValue(decl *ast.GenDecl) {
 118         // determine if decl should be associated with a type
 119         // Heuristic: For each typed entry, determine the type name, if any.
 120         //            If there is exactly one type name that is sufficiently
 121         //            frequent, associate the decl with the respective type.
 122         domName := ""
 123         domFreq := 0
 124         prev := ""
 125         for _, s := range decl.Specs {
 126                 if v, ok := s.(*ast.ValueSpec); ok {
 127                         name := ""
 128                         switch {
 129                         case v.Type != nil:
 130                                 // a type is present; determine it's name
 131                                 name = baseTypeName(v.Type)
 132                         case decl.Tok == token.CONST:
 133                                 // no type is present but we have a constant declaration;
 134                                 // use the previous type name (w/o more type information
 135                                 // we cannot handle the case of unnamed variables with
 136                                 // initializer expressions except for some trivial cases)
 137                                 name = prev
 138                         }
 139                         if name != "" {
 140                                 // entry has a named type
 141                                 if domName != "" && domName != name {
 142                                         // more than one type name - do not associate
 143                                         // with any type
 144                                         domName = ""
 145                                         break
 146                                 }
 147                                 domName = name
 148                                 domFreq++
 149                         }
 150                         prev = name
 151                 }
 152         }
 153
 154         // determine values list
 155         const threshold = 0.75
 156         values := &doc.values
 157         if domName != "" && domFreq >= int(float(len(decl.Specs))*threshold) {
 158                 // typed entries are sufficiently frequent
 159                 typ := doc.lookupTypeDoc(domName)
 160                 if typ != nil {
 161                         values = &typ.values // associate with that type
 162                 }
 163         }
 164
 165         *values = append(*values, decl)
 166 }
 167
 168
 169 // Helper function to set the table entry for function f. Makes sure that
 170 // at least one f with associated documentation is stored in table, if there
 171 // are multiple f's with the same name.
 172 func setFunc(table map[string]*ast.FuncDecl, f *ast.FuncDecl) {
 173         name := f.Name.Name
 174         if g, exists := table[name]; exists && g.Doc != nil {
 175                 // a function with the same name has already been registered;
 176                 // since it has documentation, assume f is simply another
 177                 // implementation and ignore it
 178                 // TODO(gri) consider collecting all functions, or at least
 179                 //           all comments
 180                 return
 181         }
 182         // function doesn't exist or has no documentation; use f
 183         table[name] = f
 184 }
 185
 186
 187 func (doc *docReader) addFunc(fun *ast.FuncDecl) {
 188         name := fun.Name.Name
 189
 190         // determine if it should be associated with a type
 191         if fun.Recv != nil {
 192                 // method
 193                 typ := doc.lookupTypeDoc(baseTypeName(fun.Recv.List[0].Type))
 194                 if typ != nil {
 195                         // exported receiver type
 196                         setFunc(typ.methods, fun)
 197                 }
 198                 // otherwise don't show the method
 199                 // TODO(gri): There may be exported methods of non-exported types
 200                 // that can be called because of exported values (consts, vars, or
 201                 // function results) of that type. Could determine if that is the
 202                 // case and then show those methods in an appropriate section.
 203                 return
 204         }
 205
 206         // perhaps a factory function
 207         // determine result type, if any
 208         if fun.Type.Results.NumFields() >= 1 {
 209                 res := fun.Type.Results.List[0]
 210                 if len(res.Names) <= 1 {
 211                         // exactly one (named or anonymous) result associated
 212                         // with the first type in result signature (there may
 213                         // be more than one result)
 214                         tname := baseTypeName(res.Type)
 215                         typ := doc.lookupTypeDoc(tname)
 216                         if typ != nil {
 217                                 // named and exported result type
 218
 219                                 // Work-around for failure of heuristic: In package os
 220                                 // too many functions are considered factory functions
 221                                 // for the Error type. Eliminate manually for now as
 222                                 // this appears to be the only important case in the
 223                                 // current library where the heuristic fails.
 224                                 if doc.pkgName == "os" && tname == "Error" &&
 225                                         name != "NewError" && name != "NewSyscallError" {
 226                                         // not a factory function for os.Error
 227                                         setFunc(doc.funcs, fun) // treat as ordinary function
 228                                         return
 229                                 }
 230
 231                                 setFunc(typ.factories, fun)
 232                                 return
 233                         }
 234                 }
 235         }
 236
 237         // ordinary function
 238         setFunc(doc.funcs, fun)
 239 }
 240
 241
 242 func (doc *docReader) addDecl(decl ast.Decl) {
 243         switch d := decl.(type) {
 244         case *ast.GenDecl:
 245                 if len(d.Specs) > 0 {
 246                         switch d.Tok {
 247                         case token.CONST, token.VAR:
 248                                 // constants and variables are always handled as a group
 249                                 doc.addValue(d)
 250                         case token.TYPE:
 251                                 // types are handled individually
 252                                 var noPos token.Position
 253                                 for _, spec := range d.Specs {
 254                                         // make a (fake) GenDecl node for this TypeSpec
 255                                         // (we need to do this here - as opposed to just
 256                                         // for printing - so we don't lose the GenDecl
 257                                         // documentation)
 258                                         //
 259                                         // TODO(gri): Consider just collecting the TypeSpec
 260                                         // node (and copy in the GenDecl.doc if there is no
 261                                         // doc in the TypeSpec - this is currently done in
 262                                         // makeTypeDocs below). Simpler data structures, but
 263                                         // would lose GenDecl documentation if the TypeSpec
 264                                         // has documentation as well.
 265                                         doc.addType(&ast.GenDecl{d.Doc, d.Pos(), token.TYPE, noPos, []ast.Spec{spec}, noPos})
 266                                         // A new GenDecl node is created, no need to nil out d.Doc.
 267                                 }
 268                         }
 269                 }
 270         case *ast.FuncDecl:
 271                 doc.addFunc(d)
 272         }
 273 }
 274
 275
 276 func copyCommentList(list []*ast.Comment) []*ast.Comment {
 277         return append([]*ast.Comment(nil), list...)
 278 }
 279
 280 var (
 281         bug_markers = regexp.MustCompile("^/[/*][ \t]*BUG\\(.*\\):[ \t]*") // BUG(uid):
 282         bug_content = regexp.MustCompile("[^ \n\r\t]+")                    // at least one non-whitespace char
 283 )
 284
 285
 286 // addFile adds the AST for a source file to the docReader.
 287 // Adding the same AST multiple times is a no-op.
 288 //
 289 func (doc *docReader) addFile(src *ast.File) {
 290         // add package documentation
 291         if src.Doc != nil {
 292                 doc.addDoc(src.Doc)
 293                 src.Doc = nil // doc consumed - remove from ast.File node
 294         }
 295
 296         // add all declarations
 297         for _, decl := range src.Decls {
 298                 doc.addDecl(decl)
 299         }
 300
 301         // collect BUG(...) comments
 302         for _, c := range src.Comments {
 303                 text := c.List[0].Text
 304                 if m := bug_markers.FindIndex(text); m != nil {
 305                         // found a BUG comment; maybe empty
 306                         if btxt := text[m[1]:]; bug_content.Match(btxt) {
 307                                 // non-empty BUG comment; collect comment without BUG prefix
 308                                 list := copyCommentList(c.List)
 309                                 list[0].Text = text[m[1]:]
 310                                 doc.bugs = append(doc.bugs, &ast.CommentGroup{list})
 311                         }
 312                 }
 313         }
 314         src.Comments = nil // consumed unassociated comments - remove from ast.File node
 315 }
 316
 317
 318 func NewFileDoc(file *ast.File) *PackageDoc {
 319         var r docReader
 320         r.init(file.Name.Name)
 321         r.addFile(file)
 322         return r.newDoc("", nil)
 323 }
 324
 325
 326 func NewPackageDoc(pkg *ast.Package, importpath string) *PackageDoc {
 327         var r docReader
 328         r.init(pkg.Name)
 329         filenames := make([]string, len(pkg.Files))
 330         i := 0
 331         for filename, f := range pkg.Files {
 332                 r.addFile(f)
 333                 filenames[i] = filename
 334                 i++
 335         }
 336         return r.newDoc(importpath, filenames)
 337 }
 338
 339
 340 // ----------------------------------------------------------------------------
 341 // Conversion to external representation
 342
 343 // ValueDoc is the documentation for a group of declared
 344 // values, either vars or consts.
 345 //
 346 type ValueDoc struct {
 347         Doc   string
 348         Decl  *ast.GenDecl
 349         order int
 350 }
 351
 352 type sortValueDoc []*ValueDoc
 353
 354 func (p sortValueDoc) Len() int      { return len(p) }
 355 func (p sortValueDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
 356
 357
 358 func declName(d *ast.GenDecl) string {
 359         if len(d.Specs) != 1 {
 360                 return ""
 361         }
 362
 363         switch v := d.Specs[0].(type) {
 364         case *ast.ValueSpec:
 365                 return v.Names[0].Name
 366         case *ast.TypeSpec:
 367                 return v.Name.Name
 368         }
 369
 370         return ""
 371 }
 372
 373
 374 func (p sortValueDoc) Less(i, j int) bool {
 375         // sort by name
 376         // pull blocks (name = "") up to top
 377         // in original order
 378         if ni, nj := declName(p[i].Decl), declName(p[j].Decl); ni != nj {
 379                 return ni < nj
 380         }
 381         return p[i].order < p[j].order
 382 }
 383
 384
 385 func makeValueDocs(list []*ast.GenDecl, tok token.Token) []*ValueDoc {
 386         d := make([]*ValueDoc, len(list)) // big enough in any case
 387         n := 0
 388         for i, decl := range list {
 389                 if decl.Tok == tok {
 390                         d[n] = &ValueDoc{CommentText(decl.Doc), decl, i}
 391                         n++
 392                         decl.Doc = nil // doc consumed - removed from AST
 393                 }
 394         }
 395         d = d[0:n]
 396         sort.Sort(sortValueDoc(d))
 397         return d
 398 }
 399
 400
 401 // FuncDoc is the documentation for a func declaration,
 402 // either a top-level function or a method function.
 403 //
 404 type FuncDoc struct {
 405         Doc  string
 406         Recv ast.Expr // TODO(rsc): Would like string here
 407         Name string
 408         Decl *ast.FuncDecl
 409 }
 410
 411 type sortFuncDoc []*FuncDoc
 412
 413 func (p sortFuncDoc) Len() int           { return len(p) }
 414 func (p sortFuncDoc) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 415 func (p sortFuncDoc) Less(i, j int) bool { return p[i].Name < p[j].Name }
 416
 417
 418 func makeFuncDocs(m map[string]*ast.FuncDecl) []*FuncDoc {
 419         d := make([]*FuncDoc, len(m))
 420         i := 0
 421         for _, f := range m {
 422                 doc := new(FuncDoc)
 423                 doc.Doc = CommentText(f.Doc)
 424                 f.Doc = nil // doc consumed - remove from ast.FuncDecl node
 425                 if f.Recv != nil {
 426                         doc.Recv = f.Recv.List[0].Type
 427                 }
 428                 doc.Name = f.Name.Name
 429                 doc.Decl = f
 430                 d[i] = doc
 431                 i++
 432         }
 433         sort.Sort(sortFuncDoc(d))
 434         return d
 435 }
 436
 437
 438 // TypeDoc is the documentation for a declared type.
 439 // Consts and Vars are sorted lists of constants and variables of (mostly) that type.
 440 // Factories is a sorted list of factory functions that return that type.
 441 // Methods is a sorted list of method functions on that type.
 442 type TypeDoc struct {
 443         Doc       string
 444         Type      *ast.TypeSpec
 445         Consts    []*ValueDoc
 446         Vars      []*ValueDoc
 447         Factories []*FuncDoc
 448         Methods   []*FuncDoc
 449         Decl      *ast.GenDecl
 450         order     int
 451 }
 452
 453 type sortTypeDoc []*TypeDoc
 454
 455 func (p sortTypeDoc) Len() int      { return len(p) }
 456 func (p sortTypeDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
 457 func (p sortTypeDoc) Less(i, j int) bool {
 458         // sort by name
 459         // pull blocks (name = "") up to top
 460         // in original order
 461         if ni, nj := p[i].Type.Name.Name, p[j].Type.Name.Name; ni != nj {
 462                 return ni < nj
 463         }
 464         return p[i].order < p[j].order
 465 }
 466
 467
 468 // NOTE(rsc): This would appear not to be correct for type ( )
 469 // blocks, but the doc extractor above has split them into
 470 // individual declarations.
 471 func (doc *docReader) makeTypeDocs(m map[string]*typeDoc) []*TypeDoc {
 472         d := make([]*TypeDoc, len(m))
 473         i := 0
 474         for _, old := range m {
 475                 // all typeDocs should have a declaration associated with
 476                 // them after processing an entire package - be conservative
 477                 // and check
 478                 if decl := old.decl; decl != nil {
 479                         typespec := decl.Specs[0].(*ast.TypeSpec)
 480                         t := new(TypeDoc)
 481                         doc := typespec.Doc
 482                         typespec.Doc = nil // doc consumed - remove from ast.TypeSpec node
 483                         if doc == nil {
 484                                 // no doc associated with the spec, use the declaration doc, if any
 485                                 doc = decl.Doc
 486                         }
 487                         decl.Doc = nil // doc consumed - remove from ast.Decl node
 488                         t.Doc = CommentText(doc)
 489                         t.Type = typespec
 490                         t.Consts = makeValueDocs(old.values, token.CONST)
 491                         t.Vars = makeValueDocs(old.values, token.VAR)
 492                         t.Factories = makeFuncDocs(old.factories)
 493                         t.Methods = makeFuncDocs(old.methods)
 494                         t.Decl = old.decl
 495                         t.order = i
 496                         d[i] = t
 497                         i++
 498                 } else {
 499                         // no corresponding type declaration found - move any associated
 500                         // values, factory functions, and methods back to the top-level
 501                         // so that they are not lost (this should only happen if a package
 502                         // file containing the explicit type declaration is missing or if
 503                         // an unqualified type name was used after a "." import)
 504                         // 1) move values
 505                         doc.values = append(doc.values, old.values...)
 506                         // 2) move factory functions
 507                         for name, f := range old.factories {
 508                                 doc.funcs[name] = f
 509                         }
 510                         // 3) move methods
 511                         for name, f := range old.methods {
 512                                 // don't overwrite functions with the same name
 513                                 if _, found := doc.funcs[name]; !found {
 514                                         doc.funcs[name] = f
 515                                 }
 516                         }
 517                 }
 518         }
 519         d = d[0:i] // some types may have been ignored
 520         sort.Sort(sortTypeDoc(d))
 521         return d
 522 }
 523
 524
 525 func makeBugDocs(list []*ast.CommentGroup) []string {
 526         d := make([]string, len(list))
 527         for i, g := range list {
 528                 d[i] = CommentText(g)
 529         }
 530         return d
 531 }
 532
 533
 534 // PackageDoc is the documentation for an entire package.
 535 //
 536 type PackageDoc struct {
 537         PackageName string
 538         ImportPath  string
 539         Filenames   []string
 540         Doc         string
 541         Consts      []*ValueDoc
 542         Types       []*TypeDoc
 543         Vars        []*ValueDoc
 544         Funcs       []*FuncDoc
 545         Bugs        []string
 546 }
 547
 548
 549 // newDoc returns the accumulated documentation for the package.
 550 //
 551 func (doc *docReader) newDoc(importpath string, filenames []string) *PackageDoc {
 552         p := new(PackageDoc)
 553         p.PackageName = doc.pkgName
 554         p.ImportPath = importpath
 555         sort.SortStrings(filenames)
 556         p.Filenames = filenames
 557         p.Doc = CommentText(doc.doc)
 558         // makeTypeDocs may extend the list of doc.values and
 559         // doc.funcs and thus must be called before any other
 560         // function consuming those lists
 561         p.Types = doc.makeTypeDocs(doc.types)
 562         p.Consts = makeValueDocs(doc.values, token.CONST)
 563         p.Vars = makeValueDocs(doc.values, token.VAR)
 564         p.Funcs = makeFuncDocs(doc.funcs)
 565         p.Bugs = makeBugDocs(doc.bugs)
 566         return p
 567 }
 568
 569
 570 // ----------------------------------------------------------------------------
 571 // Filtering by name
 572
 573 type Filter func(string) bool
 574
 575
 576 func matchDecl(d *ast.GenDecl, f Filter) bool {
 577         for _, d := range d.Specs {
 578                 switch v := d.(type) {
 579                 case *ast.ValueSpec:
 580                         for _, name := range v.Names {
 581                                 if f(name.Name) {
 582                                         return true
 583                                 }
 584                         }
 585                 case *ast.TypeSpec:
 586                         if f(v.Name.Name) {
 587                                 return true
 588                         }
 589                 }
 590         }
 591         return false
 592 }
 593
 594
 595 func filterValueDocs(a []*ValueDoc, f Filter) []*ValueDoc {
 596         w := 0
 597         for _, vd := range a {
 598                 if matchDecl(vd.Decl, f) {
 599                         a[w] = vd
 600                         w++
 601                 }
 602         }
 603         return a[0:w]
 604 }
 605
 606
 607 func filterFuncDocs(a []*FuncDoc, f Filter) []*FuncDoc {
 608         w := 0
 609         for _, fd := range a {
 610                 if f(fd.Name) {
 611                         a[w] = fd
 612                         w++
 613                 }
 614         }
 615         return a[0:w]
 616 }
 617
 618
 619 func filterTypeDocs(a []*TypeDoc, f Filter) []*TypeDoc {
 620         w := 0
 621         for _, td := range a {
 622                 n := 0 // number of matches
 623                 if matchDecl(td.Decl, f) {
 624                         n = 1
 625                 } else {
 626                         // type name doesn't match, but we may have matching consts, vars, factories or methods
 627                         td.Consts = filterValueDocs(td.Consts, f)
 628                         td.Vars = filterValueDocs(td.Vars, f)
 629                         td.Factories = filterFuncDocs(td.Factories, f)
 630                         td.Methods = filterFuncDocs(td.Methods, f)
 631                         n += len(td.Consts) + len(td.Vars) + len(td.Factories) + len(td.Methods)
 632                 }
 633                 if n > 0 {
 634                         a[w] = td
 635                         w++
 636                 }
 637         }
 638         return a[0:w]
 639 }
 640
 641
 642 // Filter eliminates documentation for names that don't pass through the filter f.
 643 // TODO: Recognize "Type.Method" as a name.
 644 //
 645 func (p *PackageDoc) Filter(f Filter) {
 646         p.Consts = filterValueDocs(p.Consts, f)
 647         p.Vars = filterValueDocs(p.Vars, f)
 648         p.Types = filterTypeDocs(p.Types, f)
 649         p.Funcs = filterFuncDocs(p.Funcs, f)
 650         p.Doc = "" // don't show top-level package doc
 651 }