1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
20 "cmd/go/internal/base"
22 "cmd/go/internal/lockedfile"
24 "cmd/go/internal/renameio"
26 "golang.org/x/mod/module"
27 "golang.org/x/mod/sumdb/dirhash"
28 modzip "golang.org/x/mod/zip"
31 var downloadCache par.Cache
33 // Download downloads the specific module version to the
34 // local download cache and returns the name of the directory
35 // corresponding to the root of the module's file tree.
36 func Download(mod module.Version) (dir string, err error) {
38 // Do not download to current directory.
39 return "", fmt.Errorf("missing modfetch.PkgMod")
42 // The par.Cache here avoids duplicate work.
47 c := downloadCache.Do(mod, func() interface{} {
48 dir, err := DownloadDir(mod)
50 return cached{"", err}
52 if err := download(mod, dir); err != nil {
53 return cached{"", err}
56 return cached{dir, nil}
61 func download(mod module.Version, dir string) (err error) {
62 // If the directory exists, the module has already been extracted.
63 fi, err := os.Stat(dir)
64 if err == nil && fi.IsDir() {
68 // To avoid cluttering the cache with extraneous files,
69 // DownloadZip uses the same lockfile as Download.
70 // Invoke DownloadZip before locking the file.
71 zipfile, err := DownloadZip(mod)
76 unlock, err := lockVersion(mod)
82 // Check whether the directory was populated while we were waiting on the lock.
83 fi, err = os.Stat(dir)
84 if err == nil && fi.IsDir() {
88 // Clean up any remaining temporary directories from previous runs.
89 // This is only safe to do because the lock file ensures that their writers
90 // are no longer active.
91 parentDir := filepath.Dir(dir)
92 tmpPrefix := filepath.Base(dir) + ".tmp-"
93 if old, err := filepath.Glob(filepath.Join(parentDir, tmpPrefix+"*")); err == nil {
94 for _, path := range old {
95 RemoveAll(path) // best effort
99 // Extract the zip file to a temporary directory, then rename it to the
100 // final path. That way, we can use the existence of the source directory to
101 // signal that it has been extracted successfully, and if someone deletes
102 // the entire directory (e.g. as an attempt to prune out file corruption)
103 // the module cache will still be left in a recoverable state.
104 if err := os.MkdirAll(parentDir, 0777); err != nil {
107 tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix)
117 if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil {
118 fmt.Fprintf(os.Stderr, "-> %s\n", err)
122 if err := os.Rename(tmpDir, dir); err != nil {
127 // Make dir read-only only *after* renaming it.
128 // os.Rename was observed to fail for read-only directories on macOS.
129 makeDirsReadOnly(dir)
134 var downloadZipCache par.Cache
136 // DownloadZip downloads the specific module version to the
137 // local zip cache and returns the name of the zip file.
138 func DownloadZip(mod module.Version) (zipfile string, err error) {
139 // The par.Cache here avoids duplicate work.
144 c := downloadZipCache.Do(mod, func() interface{} {
145 zipfile, err := CachePath(mod, "zip")
147 return cached{"", err}
150 // Skip locking if the zipfile already exists.
151 if _, err := os.Stat(zipfile); err == nil {
152 return cached{zipfile, nil}
155 // The zip file does not exist. Acquire the lock and create it.
156 if cfg.CmdName != "mod download" {
157 fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, mod.Version)
159 unlock, err := lockVersion(mod)
161 return cached{"", err}
165 // Double-check that the zipfile was not created while we were waiting for
167 if _, err := os.Stat(zipfile); err == nil {
168 return cached{zipfile, nil}
170 if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
171 return cached{"", err}
173 if err := downloadZip(mod, zipfile); err != nil {
174 return cached{"", err}
176 return cached{zipfile, nil}
178 return c.zipfile, c.err
181 func downloadZip(mod module.Version, zipfile string) (err error) {
182 // Clean up any remaining tempfiles from previous runs.
183 // This is only safe to do because the lock file ensures that their
184 // writers are no longer active.
185 for _, base := range []string{zipfile, zipfile + "hash"} {
186 if old, err := filepath.Glob(renameio.Pattern(base)); err == nil {
187 for _, path := range old {
188 os.Remove(path) // best effort
193 // From here to the os.Rename call below is functionally almost equivalent to
194 // renameio.WriteToFile, with one key difference: we want to validate the
195 // contents of the file (by hashing it) before we commit it. Because the file
196 // is zip-compressed, we need an actual file — or at least an io.ReaderAt — to
197 // validate it: we can't just tee the stream as we write it.
198 f, err := ioutil.TempFile(filepath.Dir(zipfile), filepath.Base(renameio.Pattern(zipfile)))
209 err = TryProxies(func(proxy string) error {
210 repo, err := Lookup(proxy, mod.Path)
214 return repo.Zip(f, mod.Version)
220 // Double-check that the paths within the zip file are well-formed.
222 // TODO(bcmills): There is a similar check within the Unzip function. Can we eliminate one?
227 z, err := zip.NewReader(f, fi.Size())
231 prefix := mod.Path + "@" + mod.Version + "/"
232 for _, f := range z.File {
233 if !strings.HasPrefix(f.Name, prefix) {
234 return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], f.Name)
238 // Sync the file before renaming it: otherwise, after a crash the reader may
239 // observe a 0-length file instead of the actual contents.
240 // See https://golang.org/issue/22397#issuecomment-380831736.
241 if err := f.Sync(); err != nil {
244 if err := f.Close(); err != nil {
248 // Hash the zip file and check the sum before renaming to the final location.
249 hash, err := dirhash.HashZip(f.Name(), dirhash.DefaultHash)
253 if err := checkModSum(mod, hash); err != nil {
257 if err := renameio.WriteFile(zipfile+"hash", []byte(hash), 0666); err != nil {
260 if err := os.Rename(f.Name(), zipfile); err != nil {
264 // TODO(bcmills): Should we make the .zip and .ziphash files read-only to discourage tampering?
269 // makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
270 // and its transitive contents.
271 func makeDirsReadOnly(dir string) {
272 type pathMode struct {
276 var dirs []pathMode // in lexical order
277 filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
278 if err == nil && info.Mode()&0222 != 0 {
280 dirs = append(dirs, pathMode{path, info.Mode()})
286 // Run over list backward to chmod children before parents.
287 for i := len(dirs) - 1; i >= 0; i-- {
288 os.Chmod(dirs[i].path, dirs[i].mode&^0222)
292 // RemoveAll removes a directory written by Download or Unzip, first applying
293 // any permission changes needed to do so.
294 func RemoveAll(dir string) error {
295 // Module cache has 0555 directories; make them writable in order to remove content.
296 filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
298 return nil // ignore errors walking in file system
305 return os.RemoveAll(dir)
308 var GoSumFile string // path to go.sum; set by package modload
317 m map[module.Version][]string // content of go.sum file (+ go.modverify if present)
318 checked map[modSum]bool // sums actually checked during execution
319 dirty bool // whether we added any new sums to m
320 overwrite bool // if true, overwrite go.sum without incorporating its contents
321 enabled bool // whether to use go.sum at all
322 modverify string // path to go.modverify, to be deleted
325 // initGoSum initializes the go.sum data.
326 // The boolean it returns reports whether the
327 // use of go.sum is now enabled.
328 // The goSum lock must be held.
329 func initGoSum() (bool, error) {
337 goSum.m = make(map[module.Version][]string)
338 goSum.checked = make(map[modSum]bool)
339 data, err := lockedfile.Read(GoSumFile)
340 if err != nil && !os.IsNotExist(err) {
344 readGoSum(goSum.m, GoSumFile, data)
346 // Add old go.modverify file.
347 // We'll delete go.modverify in WriteGoSum.
348 alt := strings.TrimSuffix(GoSumFile, ".sum") + ".modverify"
349 if data, err := renameio.ReadFile(alt); err == nil {
350 migrate := make(map[module.Version][]string)
351 readGoSum(migrate, alt, data)
352 for mod, sums := range migrate {
353 for _, sum := range sums {
354 addModSumLocked(mod, sum)
357 goSum.modverify = alt
362 // emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
363 // A bug caused us to write these into go.sum files for non-modules.
364 // We detect and remove them.
365 const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="
367 // readGoSum parses data, which is the content of file,
368 // and adds it to goSum.m. The goSum lock must be held.
369 func readGoSum(dst map[module.Version][]string, file string, data []byte) error {
374 i := bytes.IndexByte(data, '\n')
376 line, data = data, nil
378 line, data = data[:i], data[i+1:]
380 f := strings.Fields(string(line))
382 // blank line; skip it
386 return fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
388 if f[2] == emptyGoModHash {
392 mod := module.Version{Path: f[0], Version: f[1]}
393 dst[mod] = append(dst[mod], f[2])
398 // checkMod checks the given module's checksum.
399 func checkMod(mod module.Version) {
401 // Do not use current directory.
405 // Do the file I/O before acquiring the go.sum lock.
406 ziphash, err := CachePath(mod, "ziphash")
408 base.Fatalf("verifying %v", module.VersionError(mod, err))
410 data, err := renameio.ReadFile(ziphash)
412 if errors.Is(err, os.ErrNotExist) {
413 // This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes.
416 base.Fatalf("verifying %v", module.VersionError(mod, err))
418 h := strings.TrimSpace(string(data))
419 if !strings.HasPrefix(h, "h1:") {
420 base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h)))
423 if err := checkModSum(mod, h); err != nil {
424 base.Fatalf("%s", err)
428 // goModSum returns the checksum for the go.mod contents.
429 func goModSum(data []byte) (string, error) {
430 return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
431 return ioutil.NopCloser(bytes.NewReader(data)), nil
435 // checkGoMod checks the given module's go.mod checksum;
436 // data is the go.mod content.
437 func checkGoMod(path, version string, data []byte) error {
438 h, err := goModSum(data)
440 return &module.ModuleError{Path: path, Version: version, Err: fmt.Errorf("verifying go.mod: %v", err)}
443 return checkModSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
446 // checkModSum checks that the recorded checksum for mod is h.
447 func checkModSum(mod module.Version, h string) error {
448 // We lock goSum when manipulating it,
449 // but we arrange to release the lock when calling checkSumDB,
450 // so that parallel calls to checkModHash can execute parallel calls
453 // Check whether mod+h is listed in go.sum already. If so, we're done.
455 inited, err := initGoSum()
459 done := inited && haveModSumLocked(mod, h)
466 // Not listed, so we want to add them.
467 // Consult checksum database if appropriate.
469 // Calls base.Fatalf if mismatch detected.
470 if err := checkSumDB(mod, h); err != nil {
475 // Add mod+h to go.sum, if it hasn't appeared already.
478 addModSumLocked(mod, h)
484 // haveModSumLocked reports whether the pair mod,h is already listed in go.sum.
485 // If it finds a conflicting pair instead, it calls base.Fatalf.
486 // goSum.mu must be locked.
487 func haveModSumLocked(mod module.Version, h string) bool {
488 goSum.checked[modSum{mod, h}] = true
489 for _, vh := range goSum.m[mod] {
493 if strings.HasPrefix(vh, "h1:") {
494 base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tgo.sum: %v"+goSumMismatch, mod.Path, mod.Version, h, vh)
500 // addModSumLocked adds the pair mod,h to go.sum.
501 // goSum.mu must be locked.
502 func addModSumLocked(mod module.Version, h string) {
503 if haveModSumLocked(mod, h) {
506 if len(goSum.m[mod]) > 0 {
507 fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h)
509 goSum.m[mod] = append(goSum.m[mod], h)
513 // checkSumDB checks the mod, h pair against the Go checksum database.
514 // It calls base.Fatalf if the hash is to be rejected.
515 func checkSumDB(mod module.Version, h string) error {
516 db, lines, err := lookupSumDB(mod)
518 return module.VersionError(mod, fmt.Errorf("verifying module: %v", err))
521 have := mod.Path + " " + mod.Version + " " + h
522 prefix := mod.Path + " " + mod.Version + " h1:"
523 for _, line := range lines {
527 if strings.HasPrefix(line, prefix) {
528 return module.VersionError(mod, fmt.Errorf("verifying module: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, h, db, line[len(prefix)-len("h1:"):]))
534 // Sum returns the checksum for the downloaded copy of the given module,
535 // if present in the download cache.
536 func Sum(mod module.Version) string {
538 // Do not use current directory.
542 ziphash, err := CachePath(mod, "ziphash")
546 data, err := renameio.ReadFile(ziphash)
550 return strings.TrimSpace(string(data))
553 // WriteGoSum writes the go.sum file if it needs to be updated.
556 defer goSum.mu.Unlock()
559 // If we haven't read the go.sum file yet, don't bother writing it: at best,
560 // we could rename the go.modverify file if it isn't empty, but we haven't
561 // needed to touch it so far — how important could it be?
565 // Don't bother opening the go.sum file if we don't have anything to add.
568 if cfg.BuildMod == "readonly" {
569 base.Fatalf("go: updates to go.sum needed, disabled by -mod=readonly")
572 // Make a best-effort attempt to acquire the side lock, only to exclude
573 // previous versions of the 'go' command from making simultaneous edits.
574 if unlock, err := SideLock(); err == nil {
578 err := lockedfile.Transform(GoSumFile, func(data []byte) ([]byte, error) {
579 if !goSum.overwrite {
580 // Incorporate any sums added by other processes in the meantime.
581 // Add only the sums that we actually checked: the user may have edited or
582 // truncated the file to remove erroneous hashes, and we shouldn't restore
583 // them without good reason.
584 goSum.m = make(map[module.Version][]string, len(goSum.m))
585 readGoSum(goSum.m, GoSumFile, data)
586 for ms := range goSum.checked {
587 addModSumLocked(ms.mod, ms.sum)
592 var mods []module.Version
593 for m := range goSum.m {
594 mods = append(mods, m)
599 for _, m := range mods {
602 for _, h := range list {
603 fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
606 return buf.Bytes(), nil
610 base.Fatalf("go: updating go.sum: %v", err)
613 goSum.checked = make(map[modSum]bool)
615 goSum.overwrite = false
617 if goSum.modverify != "" {
618 os.Remove(goSum.modverify) // best effort
622 // TrimGoSum trims go.sum to contain only the modules for which keep[m] is true.
623 func TrimGoSum(keep map[module.Version]bool) {
625 defer goSum.mu.Unlock()
626 inited, err := initGoSum()
628 base.Fatalf("%s", err)
634 for m := range goSum.m {
635 // If we're keeping x@v we also keep x@v/go.mod.
636 // Map x@v/go.mod back to x@v for the keep lookup.
637 noGoMod := module.Version{Path: m.Path, Version: strings.TrimSuffix(m.Version, "/go.mod")}
638 if !keep[m] && !keep[noGoMod] {
641 goSum.overwrite = true
646 const goSumMismatch = `
649 This download does NOT match an earlier download recorded in go.sum.
650 The bits may have been replaced on the origin server, or an attacker may
651 have intercepted the download attempt.
653 For more information, see 'go help module-auth'.
656 const sumdbMismatch = `
659 This download does NOT match the one reported by the checksum server.
660 The bits may have been replaced on the origin server, or an attacker may
661 have intercepted the download attempt.
663 For more information, see 'go help module-auth'.
666 const hashVersionMismatch = `
669 This download is listed in go.sum, but using an unknown hash algorithm.
670 The download cannot be verified.
672 For more information, see 'go help module-auth'.
676 var HelpModuleAuth = &base.Command{
677 UsageLine: "module-auth",
678 Short: "module authentication using go.sum",
680 The go command tries to authenticate every downloaded module,
681 checking that the bits downloaded for a specific module version today
682 match bits downloaded yesterday. This ensures repeatable builds
683 and detects introduction of unexpected changes, malicious or not.
685 In each module's root, alongside go.mod, the go command maintains
686 a file named go.sum containing the cryptographic checksums of the
687 module's dependencies.
689 The form of each line in go.sum is three fields:
691 <module> <version>[/go.mod] <hash>
693 Each known module version results in two lines in the go.sum file.
694 The first line gives the hash of the module version's file tree.
695 The second line appends "/go.mod" to the version and gives the hash
696 of only the module version's (possibly synthesized) go.mod file.
697 The go.mod-only hash allows downloading and authenticating a
698 module version's go.mod file, which is needed to compute the
699 dependency graph, without also downloading all the module's source code.
701 The hash begins with an algorithm prefix of the form "h<N>:".
702 The only defined algorithm prefix is "h1:", which uses SHA-256.
704 Module authentication failures
706 The go command maintains a cache of downloaded packages and computes
707 and records the cryptographic checksum of each package at download time.
708 In normal operation, the go command checks the main module's go.sum file
709 against these precomputed checksums instead of recomputing them on
710 each command invocation. The 'go mod verify' command checks that
711 the cached copies of module downloads still match both their recorded
712 checksums and the entries in go.sum.
714 In day-to-day development, the checksum of a given module version
715 should never change. Each time a dependency is used by a given main
716 module, the go command checks its local cached copy, freshly
717 downloaded or not, against the main module's go.sum. If the checksums
718 don't match, the go command reports the mismatch as a security error
719 and refuses to run the build. When this happens, proceed with caution:
720 code changing unexpectedly means today's build will not match
721 yesterday's, and the unexpected change may not be beneficial.
723 If the go command reports a mismatch in go.sum, the downloaded code
724 for the reported module version does not match the one used in a
725 previous build of the main module. It is important at that point
726 to find out what the right checksum should be, to decide whether
727 go.sum is wrong or the downloaded code is wrong. Usually go.sum is right:
728 you want to use the same code you used yesterday.
730 If a downloaded module is not yet included in go.sum and it is a publicly
731 available module, the go command consults the Go checksum database to fetch
732 the expected go.sum lines. If the downloaded code does not match those
733 lines, the go command reports the mismatch and exits. Note that the
734 database is not consulted for module versions already listed in go.sum.
736 If a go.sum mismatch is reported, it is always worth investigating why
737 the code downloaded today differs from what was downloaded yesterday.
739 The GOSUMDB environment variable identifies the name of checksum database
740 to use and optionally its public key and URL, as in:
742 GOSUMDB="sum.golang.org"
743 GOSUMDB="sum.golang.org+<publickey>"
744 GOSUMDB="sum.golang.org+<publickey> https://sum.golang.org"
746 The go command knows the public key of sum.golang.org, and also that the name
747 sum.golang.google.cn (available inside mainland China) connects to the
748 sum.golang.org checksum database; use of any other database requires giving
749 the public key explicitly.
750 The URL defaults to "https://" followed by the database name.
752 GOSUMDB defaults to "sum.golang.org", the Go checksum database run by Google.
753 See https://sum.golang.org/privacy for the service's privacy policy.
755 If GOSUMDB is set to "off", or if "go get" is invoked with the -insecure flag,
756 the checksum database is not consulted, and all unrecognized modules are
757 accepted, at the cost of giving up the security guarantee of verified repeatable
758 downloads for all modules. A better way to bypass the checksum database
759 for specific modules is to use the GOPRIVATE or GONOSUMDB environment
760 variables. See 'go help module-private' for details.
762 The 'go env -w' command (see 'go help env') can be used to set these variables
763 for future go command invocations.
767 var HelpModulePrivate = &base.Command{
768 UsageLine: "module-private",
769 Short: "module configuration for non-public modules",
771 The go command defaults to downloading modules from the public Go module
772 mirror at proxy.golang.org. It also defaults to validating downloaded modules,
773 regardless of source, against the public Go checksum database at sum.golang.org.
774 These defaults work well for publicly available source code.
776 The GOPRIVATE environment variable controls which modules the go command
777 considers to be private (not available publicly) and should therefore not use the
778 proxy or checksum database. The variable is a comma-separated list of
779 glob patterns (in the syntax of Go's path.Match) of module path prefixes.
782 GOPRIVATE=*.corp.example.com,rsc.io/private
784 causes the go command to treat as private any module with a path prefix
785 matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
786 and rsc.io/private/quux.
788 The GOPRIVATE environment variable may be used by other tools as well to
789 identify non-public modules. For example, an editor could use GOPRIVATE
790 to decide whether to hyperlink a package import to a godoc.org page.
792 For fine-grained control over module download and validation, the GONOPROXY
793 and GONOSUMDB environment variables accept the same kind of glob list
794 and override GOPRIVATE for the specific decision of whether to use the proxy
795 and checksum database, respectively.
797 For example, if a company ran a module proxy serving private modules,
798 users would configure go using:
800 GOPRIVATE=*.corp.example.com
801 GOPROXY=proxy.example.com
804 This would tell the go command and other tools that modules beginning with
805 a corp.example.com subdomain are private but that the company proxy should
806 be used for downloading both public and private modules, because
807 GONOPROXY has been set to a pattern that won't match any modules,
808 overriding GOPRIVATE.
810 The 'go env -w' command (see 'go help env') can be used to set these variables
811 for future go command invocations.