libgo/go/cmd/go/internal/modfetch/proxy.go

   1 // Copyright 2018 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 package modfetch
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/url"
  14         "os"
  15         "path"
  16         pathpkg "path"
  17         "path/filepath"
  18         "strings"
  19         "sync"
  20         "time"
  21
  22         "cmd/go/internal/base"
  23         "cmd/go/internal/cfg"
  24         "cmd/go/internal/modfetch/codehost"
  25         "cmd/go/internal/web"
  26
  27         "golang.org/x/mod/module"
  28         "golang.org/x/mod/semver"
  29 )
  30
  31 var HelpGoproxy = &base.Command{
  32         UsageLine: "goproxy",
  33         Short:     "module proxy protocol",
  34         Long: `
  35 A Go module proxy is any web server that can respond to GET requests for
  36 URLs of a specified form. The requests have no query parameters, so even
  37 a site serving from a fixed file system (including a file:/// URL)
  38 can be a module proxy.
  39
  40 The GET requests sent to a Go module proxy are:
  41
  42 GET $GOPROXY/<module>/@v/list returns a list of known versions of the given
  43 module, one per line.
  44
  45 GET $GOPROXY/<module>/@v/<version>.info returns JSON-formatted metadata
  46 about that version of the given module.
  47
  48 GET $GOPROXY/<module>/@v/<version>.mod returns the go.mod file
  49 for that version of the given module.
  50
  51 GET $GOPROXY/<module>/@v/<version>.zip returns the zip archive
  52 for that version of the given module.
  53
  54 GET $GOPROXY/<module>/@latest returns JSON-formatted metadata about the
  55 latest known version of the given module in the same format as
  56 <module>/@v/<version>.info. The latest version should be the version of
  57 the module the go command may use if <module>/@v/list is empty or no
  58 listed version is suitable. <module>/@latest is optional and may not
  59 be implemented by a module proxy.
  60
  61 When resolving the latest version of a module, the go command will request
  62 <module>/@v/list, then, if no suitable versions are found, <module>/@latest.
  63 The go command prefers, in order: the semantically highest release version,
  64 the semantically highest pre-release version, and the chronologically
  65 most recent pseudo-version. In Go 1.12 and earlier, the go command considered
  66 pseudo-versions in <module>/@v/list to be pre-release versions, but this is
  67 no longer true since Go 1.13.
  68
  69 To avoid problems when serving from case-sensitive file systems,
  70 the <module> and <version> elements are case-encoded, replacing every
  71 uppercase letter with an exclamation mark followed by the corresponding
  72 lower-case letter: github.com/Azure encodes as github.com/!azure.
  73
  74 The JSON-formatted metadata about a given module corresponds to
  75 this Go data structure, which may be expanded in the future:
  76
  77     type Info struct {
  78         Version string    // version string
  79         Time    time.Time // commit time
  80     }
  81
  82 The zip archive for a specific version of a given module is a
  83 standard zip file that contains the file tree corresponding
  84 to the module's source code and related files. The archive uses
  85 slash-separated paths, and every file path in the archive must
  86 begin with <module>@<version>/, where the module and version are
  87 substituted directly, not case-encoded. The root of the module
  88 file tree corresponds to the <module>@<version>/ prefix in the
  89 archive.
  90
  91 Even when downloading directly from version control systems,
  92 the go command synthesizes explicit info, mod, and zip files
  93 and stores them in its local cache, $GOPATH/pkg/mod/cache/download,
  94 the same as if it had downloaded them directly from a proxy.
  95 The cache layout is the same as the proxy URL space, so
  96 serving $GOPATH/pkg/mod/cache/download at (or copying it to)
  97 https://example.com/proxy would let other users access those
  98 cached module versions with GOPROXY=https://example.com/proxy.
  99 `,
 100 }
 101
 102 var proxyOnce struct {
 103         sync.Once
 104         list []string
 105         err  error
 106 }
 107
 108 func proxyURLs() ([]string, error) {
 109         proxyOnce.Do(func() {
 110                 if cfg.GONOPROXY != "" && cfg.GOPROXY != "direct" {
 111                         proxyOnce.list = append(proxyOnce.list, "noproxy")
 112                 }
 113                 for _, proxyURL := range strings.Split(cfg.GOPROXY, ",") {
 114                         proxyURL = strings.TrimSpace(proxyURL)
 115                         if proxyURL == "" {
 116                                 continue
 117                         }
 118                         if proxyURL == "off" {
 119                                 // "off" always fails hard, so can stop walking list.
 120                                 proxyOnce.list = append(proxyOnce.list, "off")
 121                                 break
 122                         }
 123                         if proxyURL == "direct" {
 124                                 proxyOnce.list = append(proxyOnce.list, "direct")
 125                                 // For now, "direct" is the end of the line. We may decide to add some
 126                                 // sort of fallback behavior for them in the future, so ignore
 127                                 // subsequent entries for forward-compatibility.
 128                                 break
 129                         }
 130
 131                         // Single-word tokens are reserved for built-in behaviors, and anything
 132                         // containing the string ":/" or matching an absolute file path must be a
 133                         // complete URL. For all other paths, implicitly add "https://".
 134                         if strings.ContainsAny(proxyURL, ".:/") && !strings.Contains(proxyURL, ":/") && !filepath.IsAbs(proxyURL) && !path.IsAbs(proxyURL) {
 135                                 proxyURL = "https://" + proxyURL
 136                         }
 137
 138                         // Check that newProxyRepo accepts the URL.
 139                         // It won't do anything with the path.
 140                         _, err := newProxyRepo(proxyURL, "golang.org/x/text")
 141                         if err != nil {
 142                                 proxyOnce.err = err
 143                                 return
 144                         }
 145                         proxyOnce.list = append(proxyOnce.list, proxyURL)
 146                 }
 147         })
 148
 149         return proxyOnce.list, proxyOnce.err
 150 }
 151
 152 // TryProxies iterates f over each configured proxy (including "noproxy" and
 153 // "direct" if applicable) until f returns an error that is not
 154 // equivalent to os.ErrNotExist.
 155 //
 156 // TryProxies then returns that final error.
 157 //
 158 // If GOPROXY is set to "off", TryProxies invokes f once with the argument
 159 // "off".
 160 func TryProxies(f func(proxy string) error) error {
 161         proxies, err := proxyURLs()
 162         if err != nil {
 163                 return err
 164         }
 165         if len(proxies) == 0 {
 166                 return f("off")
 167         }
 168
 169         var lastAttemptErr error
 170         for _, proxy := range proxies {
 171                 err = f(proxy)
 172                 if !errors.Is(err, os.ErrNotExist) {
 173                         lastAttemptErr = err
 174                         break
 175                 }
 176
 177                 // The error indicates that the module does not exist.
 178                 // In general we prefer to report the last such error,
 179                 // because it indicates the error that occurs after all other
 180                 // options have been exhausted.
 181                 //
 182                 // However, for modules in the NOPROXY list, the most useful error occurs
 183                 // first (with proxy set to "noproxy"), and the subsequent errors are all
 184                 // errNoProxy (which is not particularly helpful). Do not overwrite a more
 185                 // useful error with errNoproxy.
 186                 if lastAttemptErr == nil || !errors.Is(err, errNoproxy) {
 187                         lastAttemptErr = err
 188                 }
 189         }
 190         return lastAttemptErr
 191 }
 192
 193 type proxyRepo struct {
 194         url  *url.URL
 195         path string
 196 }
 197
 198 func newProxyRepo(baseURL, path string) (Repo, error) {
 199         base, err := url.Parse(baseURL)
 200         if err != nil {
 201                 return nil, err
 202         }
 203         switch base.Scheme {
 204         case "http", "https":
 205                 // ok
 206         case "file":
 207                 if *base != (url.URL{Scheme: base.Scheme, Path: base.Path, RawPath: base.RawPath}) {
 208                         return nil, fmt.Errorf("invalid file:// proxy URL with non-path elements: %s", web.Redacted(base))
 209                 }
 210         case "":
 211                 return nil, fmt.Errorf("invalid proxy URL missing scheme: %s", web.Redacted(base))
 212         default:
 213                 return nil, fmt.Errorf("invalid proxy URL scheme (must be https, http, file): %s", web.Redacted(base))
 214         }
 215
 216         enc, err := module.EscapePath(path)
 217         if err != nil {
 218                 return nil, err
 219         }
 220
 221         base.Path = strings.TrimSuffix(base.Path, "/") + "/" + enc
 222         base.RawPath = strings.TrimSuffix(base.RawPath, "/") + "/" + pathEscape(enc)
 223         return &proxyRepo{base, path}, nil
 224 }
 225
 226 func (p *proxyRepo) ModulePath() string {
 227         return p.path
 228 }
 229
 230 // versionError returns err wrapped in a ModuleError for p.path.
 231 func (p *proxyRepo) versionError(version string, err error) error {
 232         if version != "" && version != module.CanonicalVersion(version) {
 233                 return &module.ModuleError{
 234                         Path: p.path,
 235                         Err: &module.InvalidVersionError{
 236                                 Version: version,
 237                                 Pseudo:  IsPseudoVersion(version),
 238                                 Err:     err,
 239                         },
 240                 }
 241         }
 242
 243         return &module.ModuleError{
 244                 Path:    p.path,
 245                 Version: version,
 246                 Err:     err,
 247         }
 248 }
 249
 250 func (p *proxyRepo) getBytes(path string) ([]byte, error) {
 251         body, err := p.getBody(path)
 252         if err != nil {
 253                 return nil, err
 254         }
 255         defer body.Close()
 256         return ioutil.ReadAll(body)
 257 }
 258
 259 func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) {
 260         fullPath := pathpkg.Join(p.url.Path, path)
 261
 262         target := *p.url
 263         target.Path = fullPath
 264         target.RawPath = pathpkg.Join(target.RawPath, pathEscape(path))
 265
 266         resp, err := web.Get(web.DefaultSecurity, &target)
 267         if err != nil {
 268                 return nil, err
 269         }
 270         if err := resp.Err(); err != nil {
 271                 resp.Body.Close()
 272                 return nil, err
 273         }
 274         return resp.Body, nil
 275 }
 276
 277 func (p *proxyRepo) Versions(prefix string) ([]string, error) {
 278         data, err := p.getBytes("@v/list")
 279         if err != nil {
 280                 return nil, p.versionError("", err)
 281         }
 282         var list []string
 283         for _, line := range strings.Split(string(data), "\n") {
 284                 f := strings.Fields(line)
 285                 if len(f) >= 1 && semver.IsValid(f[0]) && strings.HasPrefix(f[0], prefix) && !IsPseudoVersion(f[0]) {
 286                         list = append(list, f[0])
 287                 }
 288         }
 289         SortVersions(list)
 290         return list, nil
 291 }
 292
 293 func (p *proxyRepo) latest() (*RevInfo, error) {
 294         data, err := p.getBytes("@v/list")
 295         if err != nil {
 296                 return nil, p.versionError("", err)
 297         }
 298
 299         var (
 300                 bestTime             time.Time
 301                 bestTimeIsFromPseudo bool
 302                 bestVersion          string
 303         )
 304
 305         for _, line := range strings.Split(string(data), "\n") {
 306                 f := strings.Fields(line)
 307                 if len(f) >= 1 && semver.IsValid(f[0]) {
 308                         // If the proxy includes timestamps, prefer the timestamp it reports.
 309                         // Otherwise, derive the timestamp from the pseudo-version.
 310                         var (
 311                                 ft             time.Time
 312                                 ftIsFromPseudo = false
 313                         )
 314                         if len(f) >= 2 {
 315                                 ft, _ = time.Parse(time.RFC3339, f[1])
 316                         } else if IsPseudoVersion(f[0]) {
 317                                 ft, _ = PseudoVersionTime(f[0])
 318                                 ftIsFromPseudo = true
 319                         } else {
 320                                 // Repo.Latest promises that this method is only called where there are
 321                                 // no tagged versions. Ignore any tagged versions that were added in the
 322                                 // meantime.
 323                                 continue
 324                         }
 325                         if bestTime.Before(ft) {
 326                                 bestTime = ft
 327                                 bestTimeIsFromPseudo = ftIsFromPseudo
 328                                 bestVersion = f[0]
 329                         }
 330                 }
 331         }
 332         if bestVersion == "" {
 333                 return nil, p.versionError("", codehost.ErrNoCommits)
 334         }
 335
 336         if bestTimeIsFromPseudo {
 337                 // We parsed bestTime from the pseudo-version, but that's in UTC and we're
 338                 // supposed to report the timestamp as reported by the VCS.
 339                 // Stat the selected version to canonicalize the timestamp.
 340                 //
 341                 // TODO(bcmills): Should we also stat other versions to ensure that we
 342                 // report the correct Name and Short for the revision?
 343                 return p.Stat(bestVersion)
 344         }
 345
 346         return &RevInfo{
 347                 Version: bestVersion,
 348                 Name:    bestVersion,
 349                 Short:   bestVersion,
 350                 Time:    bestTime,
 351         }, nil
 352 }
 353
 354 func (p *proxyRepo) Stat(rev string) (*RevInfo, error) {
 355         encRev, err := module.EscapeVersion(rev)
 356         if err != nil {
 357                 return nil, p.versionError(rev, err)
 358         }
 359         data, err := p.getBytes("@v/" + encRev + ".info")
 360         if err != nil {
 361                 return nil, p.versionError(rev, err)
 362         }
 363         info := new(RevInfo)
 364         if err := json.Unmarshal(data, info); err != nil {
 365                 return nil, p.versionError(rev, err)
 366         }
 367         if info.Version != rev && rev == module.CanonicalVersion(rev) && module.Check(p.path, rev) == nil {
 368                 // If we request a correct, appropriate version for the module path, the
 369                 // proxy must return either exactly that version or an error — not some
 370                 // arbitrary other version.
 371                 return nil, p.versionError(rev, fmt.Errorf("proxy returned info for version %s instead of requested version", info.Version))
 372         }
 373         return info, nil
 374 }
 375
 376 func (p *proxyRepo) Latest() (*RevInfo, error) {
 377         data, err := p.getBytes("@latest")
 378         if err != nil {
 379                 if !errors.Is(err, os.ErrNotExist) {
 380                         return nil, p.versionError("", err)
 381                 }
 382                 return p.latest()
 383         }
 384         info := new(RevInfo)
 385         if err := json.Unmarshal(data, info); err != nil {
 386                 return nil, p.versionError("", err)
 387         }
 388         return info, nil
 389 }
 390
 391 func (p *proxyRepo) GoMod(version string) ([]byte, error) {
 392         if version != module.CanonicalVersion(version) {
 393                 return nil, p.versionError(version, fmt.Errorf("internal error: version passed to GoMod is not canonical"))
 394         }
 395
 396         encVer, err := module.EscapeVersion(version)
 397         if err != nil {
 398                 return nil, p.versionError(version, err)
 399         }
 400         data, err := p.getBytes("@v/" + encVer + ".mod")
 401         if err != nil {
 402                 return nil, p.versionError(version, err)
 403         }
 404         return data, nil
 405 }
 406
 407 func (p *proxyRepo) Zip(dst io.Writer, version string) error {
 408         if version != module.CanonicalVersion(version) {
 409                 return p.versionError(version, fmt.Errorf("internal error: version passed to Zip is not canonical"))
 410         }
 411
 412         encVer, err := module.EscapeVersion(version)
 413         if err != nil {
 414                 return p.versionError(version, err)
 415         }
 416         body, err := p.getBody("@v/" + encVer + ".zip")
 417         if err != nil {
 418                 return p.versionError(version, err)
 419         }
 420         defer body.Close()
 421
 422         lr := &io.LimitedReader{R: body, N: codehost.MaxZipFile + 1}
 423         if _, err := io.Copy(dst, lr); err != nil {
 424                 return p.versionError(version, err)
 425         }
 426         if lr.N <= 0 {
 427                 return p.versionError(version, fmt.Errorf("downloaded zip file too large"))
 428         }
 429         return nil
 430 }
 431
 432 // pathEscape escapes s so it can be used in a path.
 433 // That is, it escapes things like ? and # (which really shouldn't appear anyway).
 434 // It does not escape / to %2F: our REST API is designed so that / can be left as is.
 435 func pathEscape(s string) string {
 436         return strings.ReplaceAll(url.PathEscape(s), "%2F", "/")
 437 }