Source file src/cmd/go/internal/work/buildid.go
1 // Copyright 2017 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package work 6 7 import ( 8 "bytes" 9 "fmt" 10 "os" 11 "os/exec" 12 "strings" 13 "sync" 14 15 "cmd/go/internal/base" 16 "cmd/go/internal/cache" 17 "cmd/go/internal/cfg" 18 "cmd/go/internal/fsys" 19 "cmd/go/internal/str" 20 "cmd/internal/buildid" 21 "cmd/internal/quoted" 22 "cmd/internal/telemetry/counter" 23 ) 24 25 // Build IDs 26 // 27 // Go packages and binaries are stamped with build IDs that record both 28 // the action ID, which is a hash of the inputs to the action that produced 29 // the packages or binary, and the content ID, which is a hash of the action 30 // output, namely the archive or binary itself. The hash is the same one 31 // used by the build artifact cache (see cmd/go/internal/cache), but 32 // truncated when stored in packages and binaries, as the full length is not 33 // needed and is a bit unwieldy. The precise form is 34 // 35 // actionID/[.../]contentID 36 // 37 // where the actionID and contentID are prepared by buildid.HashToString below. 38 // and are found by looking for the first or last slash. 39 // Usually the buildID is simply actionID/contentID, but see below for an 40 // exception. 41 // 42 // The build ID serves two primary purposes. 43 // 44 // 1. The action ID half allows installed packages and binaries to serve as 45 // one-element cache entries. If we intend to build math.a with a given 46 // set of inputs summarized in the action ID, and the installed math.a already 47 // has that action ID, we can reuse the installed math.a instead of rebuilding it. 48 // 49 // 2. The content ID half allows the easy preparation of action IDs for steps 50 // that consume a particular package or binary. The content hash of every 51 // input file for a given action must be included in the action ID hash. 52 // Storing the content ID in the build ID lets us read it from the file with 53 // minimal I/O, instead of reading and hashing the entire file. 54 // This is especially effective since packages and binaries are typically 55 // the largest inputs to an action. 56 // 57 // Separating action ID from content ID is important for reproducible builds. 58 // The compiler is compiled with itself. If an output were represented by its 59 // own action ID (instead of content ID) when computing the action ID of 60 // the next step in the build process, then the compiler could never have its 61 // own input action ID as its output action ID (short of a miraculous hash collision). 62 // Instead we use the content IDs to compute the next action ID, and because 63 // the content IDs converge, so too do the action IDs and therefore the 64 // build IDs and the overall compiler binary. See cmd/dist's cmdbootstrap 65 // for the actual convergence sequence. 66 // 67 // The “one-element cache” purpose is a bit more complex for installed 68 // binaries. For a binary, like cmd/gofmt, there are two steps: compile 69 // cmd/gofmt/*.go into main.a, and then link main.a into the gofmt binary. 70 // We do not install gofmt's main.a, only the gofmt binary. Being able to 71 // decide that the gofmt binary is up-to-date means computing the action ID 72 // for the final link of the gofmt binary and comparing it against the 73 // already-installed gofmt binary. But computing the action ID for the link 74 // means knowing the content ID of main.a, which we did not keep. 75 // To sidestep this problem, each binary actually stores an expanded build ID: 76 // 77 // actionID(binary)/actionID(main.a)/contentID(main.a)/contentID(binary) 78 // 79 // (Note that this can be viewed equivalently as: 80 // 81 // actionID(binary)/buildID(main.a)/contentID(binary) 82 // 83 // Storing the buildID(main.a) in the middle lets the computations that care 84 // about the prefix or suffix halves ignore the middle and preserves the 85 // original build ID as a contiguous string.) 86 // 87 // During the build, when it's time to build main.a, the gofmt binary has the 88 // information needed to decide whether the eventual link would produce 89 // the same binary: if the action ID for main.a's inputs matches and then 90 // the action ID for the link step matches when assuming the given main.a 91 // content ID, then the binary as a whole is up-to-date and need not be rebuilt. 92 // 93 // This is all a bit complex and may be simplified once we can rely on the 94 // main cache, but at least at the start we will be using the content-based 95 // staleness determination without a cache beyond the usual installed 96 // package and binary locations. 97 98 const buildIDSeparator = "/" 99 100 // actionID returns the action ID half of a build ID. 101 func actionID(buildID string) string { 102 i := strings.Index(buildID, buildIDSeparator) 103 if i < 0 { 104 return buildID 105 } 106 return buildID[:i] 107 } 108 109 // contentID returns the content ID half of a build ID. 110 func contentID(buildID string) string { 111 return buildID[strings.LastIndex(buildID, buildIDSeparator)+1:] 112 } 113 114 // toolID returns the unique ID to use for the current copy of the 115 // named tool (asm, compile, cover, link). 116 // 117 // It is important that if the tool changes (for example a compiler bug is fixed 118 // and the compiler reinstalled), toolID returns a different string, so that old 119 // package archives look stale and are rebuilt (with the fixed compiler). 120 // This suggests using a content hash of the tool binary, as stored in the build ID. 121 // 122 // Unfortunately, we can't just open the tool binary, because the tool might be 123 // invoked via a wrapper program specified by -toolexec and we don't know 124 // what the wrapper program does. In particular, we want "-toolexec toolstash" 125 // to continue working: it does no good if "-toolexec toolstash" is executing a 126 // stashed copy of the compiler but the go command is acting as if it will run 127 // the standard copy of the compiler. The solution is to ask the tool binary to tell 128 // us its own build ID using the "-V=full" flag now supported by all tools. 129 // Then we know we're getting the build ID of the compiler that will actually run 130 // during the build. (How does the compiler binary know its own content hash? 131 // We store it there using updateBuildID after the standard link step.) 132 // 133 // A final twist is that we'd prefer to have reproducible builds for release toolchains. 134 // It should be possible to cross-compile for Windows from either Linux or Mac 135 // or Windows itself and produce the same binaries, bit for bit. If the tool ID, 136 // which influences the action ID half of the build ID, is based on the content ID, 137 // then the Linux compiler binary and Mac compiler binary will have different tool IDs 138 // and therefore produce executables with different action IDs. 139 // To avoid this problem, for releases we use the release version string instead 140 // of the compiler binary's content hash. This assumes that all compilers built 141 // on all different systems are semantically equivalent, which is of course only true 142 // modulo bugs. (Producing the exact same executables also requires that the different 143 // build setups agree on details like $GOROOT and file name paths, but at least the 144 // tool IDs do not make it impossible.) 145 func (b *Builder) toolID(name string) string { 146 b.id.Lock() 147 id := b.toolIDCache[name] 148 b.id.Unlock() 149 150 if id != "" { 151 return id 152 } 153 154 path := base.Tool(name) 155 desc := "go tool " + name 156 157 // Special case: undocumented -vettool overrides usual vet, 158 // for testing vet or supplying an alternative analysis tool. 159 if name == "vet" && VetTool != "" { 160 path = VetTool 161 desc = VetTool 162 } 163 164 cmdline := str.StringList(cfg.BuildToolexec, path, "-V=full") 165 cmd := exec.Command(cmdline[0], cmdline[1:]...) 166 var stdout, stderr strings.Builder 167 cmd.Stdout = &stdout 168 cmd.Stderr = &stderr 169 if err := cmd.Run(); err != nil { 170 if stderr.Len() > 0 { 171 os.Stderr.WriteString(stderr.String()) 172 } 173 base.Fatalf("go: error obtaining buildID for %s: %v", desc, err) 174 } 175 176 line := stdout.String() 177 f := strings.Fields(line) 178 if len(f) < 3 || f[0] != name && path != VetTool || f[1] != "version" || f[2] == "devel" && !strings.HasPrefix(f[len(f)-1], "buildID=") { 179 base.Fatalf("go: parsing buildID from %s -V=full: unexpected output:\n\t%s", desc, line) 180 } 181 if f[2] == "devel" { 182 // On the development branch, use the content ID part of the build ID. 183 id = contentID(f[len(f)-1]) 184 } else { 185 // For a release, the output is like: "compile version go1.9.1 X:framepointer". 186 // Use the whole line. 187 id = strings.TrimSpace(line) 188 } 189 190 b.id.Lock() 191 b.toolIDCache[name] = id 192 b.id.Unlock() 193 194 return id 195 } 196 197 // gccToolID returns the unique ID to use for a tool that is invoked 198 // by the GCC driver. This is used particularly for gccgo, but this can also 199 // be used for gcc, g++, gfortran, etc.; those tools all use the GCC 200 // driver under different names. The approach used here should also 201 // work for sufficiently new versions of clang. Unlike toolID, the 202 // name argument is the program to run. The language argument is the 203 // type of input file as passed to the GCC driver's -x option. 204 // 205 // For these tools we have no -V=full option to dump the build ID, 206 // but we can run the tool with -v -### to reliably get the compiler proper 207 // and hash that. That will work in the presence of -toolexec. 208 // 209 // In order to get reproducible builds for released compilers, we 210 // detect a released compiler by the absence of "experimental" in the 211 // --version output, and in that case we just use the version string. 212 // 213 // gccToolID also returns the underlying executable for the compiler. 214 // The caller assumes that stat of the exe can be used, combined with the id, 215 // to detect changes in the underlying compiler. The returned exe can be empty, 216 // which means to rely only on the id. 217 func (b *Builder) gccToolID(name, language string) (id, exe string, err error) { 218 key := name + "." + language 219 b.id.Lock() 220 id = b.toolIDCache[key] 221 exe = b.toolIDCache[key+".exe"] 222 b.id.Unlock() 223 224 if id != "" { 225 return id, exe, nil 226 } 227 228 // Invoke the driver with -### to see the subcommands and the 229 // version strings. Use -x to set the language. Pretend to 230 // compile an empty file on standard input. 231 cmdline := str.StringList(cfg.BuildToolexec, name, "-###", "-x", language, "-c", "-") 232 cmd := exec.Command(cmdline[0], cmdline[1:]...) 233 // Force untranslated output so that we see the string "version". 234 cmd.Env = append(os.Environ(), "LC_ALL=C") 235 out, err := cmd.CombinedOutput() 236 if err != nil { 237 return "", "", fmt.Errorf("%s: %v; output: %q", name, err, out) 238 } 239 240 version := "" 241 lines := strings.Split(string(out), "\n") 242 for _, line := range lines { 243 fields := strings.Fields(line) 244 for i, field := range fields { 245 if strings.HasSuffix(field, ":") { 246 // Avoid parsing fields of lines like "Configured with: …", which may 247 // contain arbitrary substrings. 248 break 249 } 250 if field == "version" && i < len(fields)-1 { 251 // Check that the next field is plausibly a version number. 252 // We require only that it begins with an ASCII digit, 253 // since we don't know what version numbering schemes a given 254 // C compiler may use. (Clang and GCC mostly seem to follow the scheme X.Y.Z, 255 // but in https://go.dev/issue/64619 we saw "8.3 [DragonFly]", and who knows 256 // what other C compilers like "zig cc" might report?) 257 next := fields[i+1] 258 if len(next) > 0 && next[0] >= '0' && next[0] <= '9' { 259 version = line 260 break 261 } 262 } 263 } 264 if version != "" { 265 break 266 } 267 } 268 if version == "" { 269 return "", "", fmt.Errorf("%s: can not find version number in %q", name, out) 270 } 271 272 if !strings.Contains(version, "experimental") { 273 // This is a release. Use this line as the tool ID. 274 id = version 275 } else { 276 // This is a development version. The first line with 277 // a leading space is the compiler proper. 278 compiler := "" 279 for _, line := range lines { 280 if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " (in-process)") { 281 compiler = line 282 break 283 } 284 } 285 if compiler == "" { 286 return "", "", fmt.Errorf("%s: can not find compilation command in %q", name, out) 287 } 288 289 fields, _ := quoted.Split(compiler) 290 if len(fields) == 0 { 291 return "", "", fmt.Errorf("%s: compilation command confusion %q", name, out) 292 } 293 exe = fields[0] 294 if !strings.ContainsAny(exe, `/\`) { 295 if lp, err := cfg.LookPath(exe); err == nil { 296 exe = lp 297 } 298 } 299 id, err = buildid.ReadFile(exe) 300 if err != nil { 301 return "", "", err 302 } 303 304 // If we can't find a build ID, use a hash. 305 if id == "" { 306 id = b.fileHash(exe) 307 } 308 } 309 310 b.id.Lock() 311 b.toolIDCache[key] = id 312 b.toolIDCache[key+".exe"] = exe 313 b.id.Unlock() 314 315 return id, exe, nil 316 } 317 318 // Check if assembler used by gccgo is GNU as. 319 func assemblerIsGas() bool { 320 cmd := exec.Command(BuildToolchain.compiler(), "-print-prog-name=as") 321 assembler, err := cmd.Output() 322 if err == nil { 323 cmd := exec.Command(strings.TrimSpace(string(assembler)), "--version") 324 out, err := cmd.Output() 325 return err == nil && strings.Contains(string(out), "GNU") 326 } else { 327 return false 328 } 329 } 330 331 // gccgoBuildIDFile creates an assembler file that records the 332 // action's build ID in an SHF_EXCLUDE section for ELF files or 333 // in a CSECT in XCOFF files. 334 func (b *Builder) gccgoBuildIDFile(a *Action) (string, error) { 335 sfile := a.Objdir + "_buildid.s" 336 337 var buf bytes.Buffer 338 if cfg.Goos == "aix" { 339 fmt.Fprintf(&buf, "\t.csect .go.buildid[XO]\n") 340 } else if (cfg.Goos != "solaris" && cfg.Goos != "illumos") || assemblerIsGas() { 341 fmt.Fprintf(&buf, "\t"+`.section .go.buildid,"e"`+"\n") 342 } else if cfg.Goarch == "sparc" || cfg.Goarch == "sparc64" { 343 fmt.Fprintf(&buf, "\t"+`.section ".go.buildid",#exclude`+"\n") 344 } else { // cfg.Goarch == "386" || cfg.Goarch == "amd64" 345 fmt.Fprintf(&buf, "\t"+`.section .go.buildid,#exclude`+"\n") 346 } 347 fmt.Fprintf(&buf, "\t.byte ") 348 for i := 0; i < len(a.buildID); i++ { 349 if i > 0 { 350 if i%8 == 0 { 351 fmt.Fprintf(&buf, "\n\t.byte ") 352 } else { 353 fmt.Fprintf(&buf, ",") 354 } 355 } 356 fmt.Fprintf(&buf, "%#02x", a.buildID[i]) 357 } 358 fmt.Fprintf(&buf, "\n") 359 if cfg.Goos != "solaris" && cfg.Goos != "illumos" && cfg.Goos != "aix" { 360 secType := "@progbits" 361 if cfg.Goarch == "arm" { 362 secType = "%progbits" 363 } 364 fmt.Fprintf(&buf, "\t"+`.section .note.GNU-stack,"",%s`+"\n", secType) 365 fmt.Fprintf(&buf, "\t"+`.section .note.GNU-split-stack,"",%s`+"\n", secType) 366 } 367 368 if err := b.Shell(a).writeFile(sfile, buf.Bytes()); err != nil { 369 return "", err 370 } 371 372 return sfile, nil 373 } 374 375 // buildID returns the build ID found in the given file. 376 // If no build ID is found, buildID returns the content hash of the file. 377 func (b *Builder) buildID(file string) string { 378 b.id.Lock() 379 id := b.buildIDCache[file] 380 b.id.Unlock() 381 382 if id != "" { 383 return id 384 } 385 386 id, err := buildid.ReadFile(file) 387 if err != nil { 388 id = b.fileHash(file) 389 } 390 391 b.id.Lock() 392 b.buildIDCache[file] = id 393 b.id.Unlock() 394 395 return id 396 } 397 398 // fileHash returns the content hash of the named file. 399 func (b *Builder) fileHash(file string) string { 400 file, _ = fsys.OverlayPath(file) 401 sum, err := cache.FileHash(file) 402 if err != nil { 403 return "" 404 } 405 return buildid.HashToString(sum) 406 } 407 408 var ( 409 counterCacheHit = counter.New("go/buildcache/hit") 410 counterCacheMiss = counter.New("go/buildcache/miss") 411 412 onceIncStdlibRecompiled sync.Once 413 stdlibRecompiled = counter.New("go/buildcache/stdlib-recompiled") 414 ) 415 416 // useCache tries to satisfy the action a, which has action ID actionHash, 417 // by using a cached result from an earlier build. At the moment, the only 418 // cached result is the installed package or binary at target. 419 // If useCache decides that the cache can be used, it sets a.buildID 420 // and a.built for use by parent actions and then returns true. 421 // Otherwise it sets a.buildID to a temporary build ID for use in the build 422 // and returns false. When useCache returns false the expectation is that 423 // the caller will build the target and then call updateBuildID to finish the 424 // build ID computation. 425 // When useCache returns false, it may have initiated buffering of output 426 // during a's work. The caller should defer b.flushOutput(a), to make sure 427 // that flushOutput is eventually called regardless of whether the action 428 // succeeds. The flushOutput call must happen after updateBuildID. 429 func (b *Builder) useCache(a *Action, actionHash cache.ActionID, target string, printOutput bool) (ok bool) { 430 // The second half of the build ID here is a placeholder for the content hash. 431 // It's important that the overall buildID be unlikely verging on impossible 432 // to appear in the output by chance, but that should be taken care of by 433 // the actionID half; if it also appeared in the input that would be like an 434 // engineered 120-bit partial SHA256 collision. 435 a.actionID = actionHash 436 actionID := buildid.HashToString(actionHash) 437 if a.json != nil { 438 a.json.ActionID = actionID 439 } 440 contentID := actionID // temporary placeholder, likely unique 441 a.buildID = actionID + buildIDSeparator + contentID 442 443 // Executable binaries also record the main build ID in the middle. 444 // See "Build IDs" comment above. 445 if a.Mode == "link" { 446 mainpkg := a.Deps[0] 447 a.buildID = actionID + buildIDSeparator + mainpkg.buildID + buildIDSeparator + contentID 448 } 449 450 // If user requested -a, we force a rebuild, so don't use the cache. 451 if cfg.BuildA { 452 if p := a.Package; p != nil && !p.Stale { 453 p.Stale = true 454 p.StaleReason = "build -a flag in use" 455 } 456 // Begin saving output for later writing to cache. 457 a.output = []byte{} 458 return false 459 } 460 461 defer func() { 462 // Increment counters for cache hits and misses based on the return value 463 // of this function. Don't increment counters if we return early because of 464 // cfg.BuildA above because we don't even look at the cache in that case. 465 if ok { 466 counterCacheHit.Inc() 467 } else { 468 if a.Package != nil && a.Package.Standard { 469 onceIncStdlibRecompiled.Do(stdlibRecompiled.Inc) 470 } 471 counterCacheMiss.Inc() 472 } 473 }() 474 475 c := cache.Default() 476 477 if target != "" { 478 buildID, _ := buildid.ReadFile(target) 479 if strings.HasPrefix(buildID, actionID+buildIDSeparator) { 480 a.buildID = buildID 481 if a.json != nil { 482 a.json.BuildID = a.buildID 483 } 484 a.built = target 485 // Poison a.Target to catch uses later in the build. 486 a.Target = "DO NOT USE - " + a.Mode 487 return true 488 } 489 // Special case for building a main package: if the only thing we 490 // want the package for is to link a binary, and the binary is 491 // already up-to-date, then to avoid a rebuild, report the package 492 // as up-to-date as well. See "Build IDs" comment above. 493 // TODO(rsc): Rewrite this code to use a TryCache func on the link action. 494 if !b.NeedExport && a.Mode == "build" && len(a.triggers) == 1 && a.triggers[0].Mode == "link" { 495 if id := strings.Split(buildID, buildIDSeparator); len(id) == 4 && id[1] == actionID { 496 // Temporarily assume a.buildID is the package build ID 497 // stored in the installed binary, and see if that makes 498 // the upcoming link action ID a match. If so, report that 499 // we built the package, safe in the knowledge that the 500 // link step will not ask us for the actual package file. 501 // Note that (*Builder).LinkAction arranged that all of 502 // a.triggers[0]'s dependencies other than a are also 503 // dependencies of a, so that we can be sure that, 504 // other than a.buildID, b.linkActionID is only accessing 505 // build IDs of completed actions. 506 oldBuildID := a.buildID 507 a.buildID = id[1] + buildIDSeparator + id[2] 508 linkID := buildid.HashToString(b.linkActionID(a.triggers[0])) 509 if id[0] == linkID { 510 // Best effort attempt to display output from the compile and link steps. 511 // If it doesn't work, it doesn't work: reusing the cached binary is more 512 // important than reprinting diagnostic information. 513 if printOutput { 514 showStdout(b, c, a, "stdout") // compile output 515 showStdout(b, c, a, "link-stdout") // link output 516 } 517 518 // Poison a.Target to catch uses later in the build. 519 a.Target = "DO NOT USE - main build pseudo-cache Target" 520 a.built = "DO NOT USE - main build pseudo-cache built" 521 if a.json != nil { 522 a.json.BuildID = a.buildID 523 } 524 return true 525 } 526 // Otherwise restore old build ID for main build. 527 a.buildID = oldBuildID 528 } 529 } 530 } 531 532 // Special case for linking a test binary: if the only thing we 533 // want the binary for is to run the test, and the test result is cached, 534 // then to avoid the link step, report the link as up-to-date. 535 // We avoid the nested build ID problem in the previous special case 536 // by recording the test results in the cache under the action ID half. 537 if len(a.triggers) == 1 && a.triggers[0].TryCache != nil && a.triggers[0].TryCache(b, a.triggers[0]) { 538 // Best effort attempt to display output from the compile and link steps. 539 // If it doesn't work, it doesn't work: reusing the test result is more 540 // important than reprinting diagnostic information. 541 if printOutput { 542 showStdout(b, c, a.Deps[0], "stdout") // compile output 543 showStdout(b, c, a.Deps[0], "link-stdout") // link output 544 } 545 546 // Poison a.Target to catch uses later in the build. 547 a.Target = "DO NOT USE - pseudo-cache Target" 548 a.built = "DO NOT USE - pseudo-cache built" 549 return true 550 } 551 552 // Check to see if the action output is cached. 553 if file, _, err := cache.GetFile(c, actionHash); err == nil { 554 if a.Mode == "preprocess PGO profile" { 555 // Preprocessed PGO profiles don't embed a build ID, so 556 // skip the build ID lookup. 557 // TODO(prattmic): better would be to add a build ID to the format. 558 a.built = file 559 a.Target = "DO NOT USE - using cache" 560 return true 561 } 562 if buildID, err := buildid.ReadFile(file); err == nil { 563 if printOutput { 564 showStdout(b, c, a, "stdout") 565 } 566 a.built = file 567 a.Target = "DO NOT USE - using cache" 568 a.buildID = buildID 569 if a.json != nil { 570 a.json.BuildID = a.buildID 571 } 572 if p := a.Package; p != nil && target != "" { 573 p.Stale = true 574 // Clearer than explaining that something else is stale. 575 p.StaleReason = "not installed but available in build cache" 576 } 577 return true 578 } 579 } 580 581 // If we've reached this point, we can't use the cache for the action. 582 if p := a.Package; p != nil && !p.Stale { 583 p.Stale = true 584 p.StaleReason = "build ID mismatch" 585 if b.IsCmdList { 586 // Since we may end up printing StaleReason, include more detail. 587 for _, p1 := range p.Internal.Imports { 588 if p1.Stale && p1.StaleReason != "" { 589 if strings.HasPrefix(p1.StaleReason, "stale dependency: ") { 590 p.StaleReason = p1.StaleReason 591 break 592 } 593 if strings.HasPrefix(p.StaleReason, "build ID mismatch") { 594 p.StaleReason = "stale dependency: " + p1.ImportPath 595 } 596 } 597 } 598 } 599 } 600 601 // Begin saving output for later writing to cache. 602 a.output = []byte{} 603 return false 604 } 605 606 func showStdout(b *Builder, c cache.Cache, a *Action, key string) error { 607 actionID := a.actionID 608 609 stdout, stdoutEntry, err := cache.GetBytes(c, cache.Subkey(actionID, key)) 610 if err != nil { 611 return err 612 } 613 614 if len(stdout) > 0 { 615 sh := b.Shell(a) 616 if cfg.BuildX || cfg.BuildN { 617 sh.ShowCmd("", "%s # internal", joinUnambiguously(str.StringList("cat", c.OutputFile(stdoutEntry.OutputID)))) 618 } 619 if !cfg.BuildN { 620 sh.Print(string(stdout)) 621 } 622 } 623 return nil 624 } 625 626 // flushOutput flushes the output being queued in a. 627 func (b *Builder) flushOutput(a *Action) { 628 b.Shell(a).Print(string(a.output)) 629 a.output = nil 630 } 631 632 // updateBuildID updates the build ID in the target written by action a. 633 // It requires that useCache was called for action a and returned false, 634 // and that the build was then carried out and given the temporary 635 // a.buildID to record as the build ID in the resulting package or binary. 636 // updateBuildID computes the final content ID and updates the build IDs 637 // in the binary. 638 // 639 // Keep in sync with src/cmd/buildid/buildid.go 640 func (b *Builder) updateBuildID(a *Action, target string, rewrite bool) error { 641 sh := b.Shell(a) 642 643 if cfg.BuildX || cfg.BuildN { 644 if rewrite { 645 sh.ShowCmd("", "%s # internal", joinUnambiguously(str.StringList(base.Tool("buildid"), "-w", target))) 646 } 647 if cfg.BuildN { 648 return nil 649 } 650 } 651 652 c := cache.Default() 653 654 // Cache output from compile/link, even if we don't do the rest. 655 switch a.Mode { 656 case "build": 657 cache.PutBytes(c, cache.Subkey(a.actionID, "stdout"), a.output) 658 case "link": 659 // Even though we don't cache the binary, cache the linker text output. 660 // We might notice that an installed binary is up-to-date but still 661 // want to pretend to have run the linker. 662 // Store it under the main package's action ID 663 // to make it easier to find when that's all we have. 664 for _, a1 := range a.Deps { 665 if p1 := a1.Package; p1 != nil && p1.Name == "main" { 666 cache.PutBytes(c, cache.Subkey(a1.actionID, "link-stdout"), a.output) 667 break 668 } 669 } 670 } 671 672 // Find occurrences of old ID and compute new content-based ID. 673 r, err := os.Open(target) 674 if err != nil { 675 return err 676 } 677 matches, hash, err := buildid.FindAndHash(r, a.buildID, 0) 678 r.Close() 679 if err != nil { 680 return err 681 } 682 newID := a.buildID[:strings.LastIndex(a.buildID, buildIDSeparator)] + buildIDSeparator + buildid.HashToString(hash) 683 if len(newID) != len(a.buildID) { 684 return fmt.Errorf("internal error: build ID length mismatch %q vs %q", a.buildID, newID) 685 } 686 687 // Replace with new content-based ID. 688 a.buildID = newID 689 if a.json != nil { 690 a.json.BuildID = a.buildID 691 } 692 if len(matches) == 0 { 693 // Assume the user specified -buildid= to override what we were going to choose. 694 return nil 695 } 696 697 if rewrite { 698 w, err := os.OpenFile(target, os.O_RDWR, 0) 699 if err != nil { 700 return err 701 } 702 err = buildid.Rewrite(w, matches, newID) 703 if err != nil { 704 w.Close() 705 return err 706 } 707 if err := w.Close(); err != nil { 708 return err 709 } 710 } 711 712 // Cache package builds, but not binaries (link steps). 713 // The expectation is that binaries are not reused 714 // nearly as often as individual packages, and they're 715 // much larger, so the cache-footprint-to-utility ratio 716 // of binaries is much lower for binaries. 717 // Not caching the link step also makes sure that repeated "go run" at least 718 // always rerun the linker, so that they don't get too fast. 719 // (We don't want people thinking go is a scripting language.) 720 // Note also that if we start caching binaries, then we will 721 // copy the binaries out of the cache to run them, and then 722 // that will mean the go process is itself writing a binary 723 // and then executing it, so we will need to defend against 724 // ETXTBSY problems as discussed in exec.go and golang.org/issue/22220. 725 if a.Mode == "build" { 726 r, err := os.Open(target) 727 if err == nil { 728 if a.output == nil { 729 panic("internal error: a.output not set") 730 } 731 outputID, _, err := c.Put(a.actionID, r) 732 r.Close() 733 if err == nil && cfg.BuildX { 734 sh.ShowCmd("", "%s # internal", joinUnambiguously(str.StringList("cp", target, c.OutputFile(outputID)))) 735 } 736 if b.NeedExport { 737 if err != nil { 738 return err 739 } 740 a.Package.Export = c.OutputFile(outputID) 741 a.Package.BuildID = a.buildID 742 } 743 } 744 } 745 746 return nil 747 } 748