diff --git a/src/BUILD.plz b/src/BUILD.plz index c0eaec7e7..87bc4a0e8 100644 --- a/src/BUILD.plz +++ b/src/BUILD.plz @@ -10,8 +10,8 @@ go_binary( deps = [ "///third_party/go/github.com_thought-machine_go-flags//:go-flags", "///third_party/go/go.uber.org_automaxprocs//maxprocs", - "//src/audit", "//src/assets", + "//src/audit", "//src/build", "//src/cache", "//src/clean", diff --git a/src/audit/BUILD b/src/audit/BUILD index c08aef8f9..7511020ba 100644 --- a/src/audit/BUILD +++ b/src/audit/BUILD @@ -3,8 +3,8 @@ go_library( srcs = [ "audit.go", ], - visibility = ["PUBLIC"], pgo_file = "//:pgo", + visibility = ["PUBLIC"], deps = [ "//src/cli/logging", "//src/fs", diff --git a/src/build/build_step.go b/src/build/build_step.go index 303c75097..7b3200791 100644 --- a/src/build/build_step.go +++ b/src/build/build_step.go @@ -3,6 +3,7 @@ package build import ( "bytes" + "context" "encoding/hex" "errors" "fmt" @@ -59,11 +60,11 @@ var successfulLocalTargetBuildDuration = metrics.NewHistogramVec( ) // Build implements the core logic for building a single target. -func Build(state *core.BuildState, target *core.BuildTarget, remote bool) { +func Build(ctx context.Context, state *core.BuildState, target *core.BuildTarget, remote bool) { state = state.ForTarget(target) target.SetState(core.Building) start := time.Now() - if err := buildTarget(state, target, remote); err != nil { + if err := buildTarget(ctx, state, target, remote); err != nil { if errors.Is(err, errStop) { target.SetState(core.Stopped) state.LogBuildResult(target, core.TargetBuildStopped, "Build stopped") @@ -161,7 +162,7 @@ func prepareOnly(state *core.BuildState, target *core.BuildTarget) error { // b) attempt to fetch the outputs from the cache based on the output hash // 3. Actually build the rule // 4. Store result in the cache -func buildTarget(state *core.BuildState, target *core.BuildTarget, runRemotely bool) (err error) { +func buildTarget(ctx context.Context, state *core.BuildState, target *core.BuildTarget, runRemotely bool) (err error) { defer func() { if r := recover(); r != nil { if e, ok := r.(error); ok { @@ -285,7 +286,7 @@ func buildTarget(state *core.BuildState, target *core.BuildTarget, runRemotely b // If we fail to hash our outputs, we get a nil hash so we'll attempt to pull the outputs from the cache // // N.B. Important we do not go through state.TargetHasher here since it memoises and - // this calculation might be incorrect. + // this calculation might be incorrect. oldOutputHash := outputHashOrNil(target, target.FullOutputs(), state.PathHasher, state.PathHasher.NewHash) cacheKey = mustShortTargetHash(state, target) @@ -320,7 +321,7 @@ func buildTarget(state *core.BuildState, target *core.BuildTarget, runRemotely b } state.LogBuildResult(target, core.TargetBuilding, target.BuildingDescription) - metadata, err = build(state, target, cacheKey) + metadata, err = build(ctx, state, target, cacheKey) if err != nil { return err } @@ -509,7 +510,7 @@ func retrieveArtifacts(state *core.BuildState, target *core.BuildTarget, oldOutp // runBuildCommand runs the actual command to build a target. // On success it returns the stdout of the target, otherwise an error. -func runBuildCommand(state *core.BuildState, target *core.BuildTarget, command string, inputHash []byte) ([]byte, error) { +func runBuildCommand(ctx context.Context, state *core.BuildState, target *core.BuildTarget, command string, inputHash []byte) ([]byte, error) { if target.IsRemoteFile { return nil, fetchRemoteFile(state, target) } @@ -519,7 +520,7 @@ func runBuildCommand(state *core.BuildState, target *core.BuildTarget, command s env := core.StampedBuildEnvironment(state, target, inputHash, filepath.Join(core.RepoRoot, target.TmpDir()), target.Stamp).ToSlice() log.Debug("Building target %s\nENVIRONMENT:\n%s\n%s", target.Label, env, command) audit.WriteBuildCommand(target.Label.String(), env, command) - out, combined, err := state.ProcessExecutor.ExecWithTimeoutShell(target, target.TmpDir(), env, target.BuildTimeout, state.ShowAllOutput, false, process.NewSandboxConfig(target.Sandbox, target.Sandbox), command) + out, combined, err := state.ProcessExecutor.ExecWithTimeoutShell(ctx, target, target.TmpDir(), env, target.BuildTimeout, state.ShowAllOutput, false, process.NewSandboxConfig(target.Sandbox, target.Sandbox), command) if err != nil { return nil, fmt.Errorf("Error building target %s: %s\n%s", target.Label, err, combined) } @@ -1219,14 +1220,14 @@ func (r *progressReader) Read(b []byte) (int, error) { } // build builds a target locally, it errors if a remote worker is needed since this has beeen removed. -func build(state *core.BuildState, target *core.BuildTarget, inputHash []byte) (*core.BuildMetadata, error) { +func build(ctx context.Context, state *core.BuildState, target *core.BuildTarget, inputHash []byte) (*core.BuildMetadata, error) { metadata := new(core.BuildMetadata) workerCmd, _, localCmd, err := core.WorkerCommandAndArgs(state, target) if err != nil { return nil, err } else if workerCmd == "" { - metadata.Stdout, err = runBuildCommand(state, target, localCmd, inputHash) + metadata.Stdout, err = runBuildCommand(ctx, state, target, localCmd, inputHash) return metadata, err } return nil, fmt.Errorf("Persistent workers are no longer supported, found worker command: %s", workerCmd) diff --git a/src/build/build_step_stress_test.go b/src/build/build_step_stress_test.go index 6a4830e80..c6c712f37 100644 --- a/src/build/build_step_stress_test.go +++ b/src/build/build_step_stress_test.go @@ -26,7 +26,7 @@ var state *core.BuildState func TestBuildLotsOfTargets(t *testing.T) { config, _ := core.ReadConfigFiles(fs.HostFS, nil, nil) config.Please.NumThreads = 10 - state = core.NewBuildState(config) + state = core.NewBuildState(t.Context(), config) state.Parser = &fakeParser{ PostBuildFunctions: buildFunctionMap{}, } diff --git a/src/build/build_step_test.go b/src/build/build_step_test.go index 0e4642ce8..2cbeeb83b 100644 --- a/src/build/build_step_test.go +++ b/src/build/build_step_test.go @@ -8,6 +8,7 @@ package build import ( + "context" "encoding/hex" "fmt" "io" @@ -29,37 +30,37 @@ import ( var cache core.Cache func TestBuildTargetWithNoDeps(t *testing.T) { - state, target := newState("//package1:target1") + state, target := newState(t, "//package1:target1") target.AddOutput("file1") - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) assert.NoError(t, err) assert.Equal(t, core.Built, target.State()) } func TestFailedBuildTarget(t *testing.T) { - state, target := newState("//package1:target1a") + state, target := newState(t, "//package1:target1a") target.Command = "false" - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) assert.Error(t, err) } func TestBuildTargetWhichNeedsRebuilding(t *testing.T) { // The output file for this target already exists, but it should still get rebuilt // because there's no rule hash file. - state, target := newState("//package1:target2") + state, target := newState(t, "//package1:target2") target.AddOutput("file2") - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) assert.NoError(t, err) assert.Equal(t, core.Built, target.State()) } func TestBuildTargetWhichDoesntNeedRebuilding(t *testing.T) { // We write a rule hash for this target before building it, so we don't need to build again. - state, target := newState("//package1:target3") + state, target := newState(t, "//package1:target3") target.AddOutput("file3") StoreTargetMetadata(target, new(core.BuildMetadata)) assert.NoError(t, writeRuleHash(state, target)) - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) assert.NoError(t, err) assert.Equal(t, core.Reused, target.State()) } @@ -67,51 +68,51 @@ func TestBuildTargetWhichDoesntNeedRebuilding(t *testing.T) { func TestModifiedBuildTargetStillNeedsRebuilding(t *testing.T) { // Similar to above, but if we change the target such that the rule hash no longer matches, // it should get rebuilt. - state, target := newState("//package1:target4") + state, target := newState(t, "//package1:target4") target.AddOutput("file4") assert.NoError(t, writeRuleHash(state, target)) target.Command = "echo -n 'wibble wibble wibble' > $OUT" target.RuleHash = nil // Have to force a reset of this - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) assert.NoError(t, err) assert.Equal(t, core.Built, target.State()) } func TestSymlinkedOutputs(t *testing.T) { // Test behaviour when the output is a symlink. - state, target := newState("//package1:target5") + state, target := newState(t, "//package1:target5") target.AddOutput("file5") target.AddSource(core.FileLabel{File: "src5", Package: "package1"}) target.Command = "ln -s $SRC $OUT" - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) assert.NoError(t, err) assert.Equal(t, core.Built, target.State()) } func TestPreBuildFunction(t *testing.T) { // Test modifying a command in the pre-build function. - state, target := newState("//package1:target6") + state, target := newState(t, "//package1:target6") target.AddOutput("file6") target.Command = "" // Target now won't produce the needed output target.PreBuildFunction = preBuildFunction(func(target *core.BuildTarget) error { target.Command = "echo 'wibble wibble wibble' > $OUT" return nil }) - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) assert.NoError(t, err) assert.Equal(t, core.Built, target.State()) } func TestPostBuildFunction(t *testing.T) { // Test modifying a command in the post-build function. - state, target := newState("//package1:target7") + state, target := newState(t, "//package1:target7") target.Command = "echo -n 'wibble wibble wibble' | tee file7" target.PostBuildFunction = postBuildFunction(func(target *core.BuildTarget, output string) error { target.AddOutput("file7") assert.Equal(t, "wibble wibble wibble", output) return nil }) - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) assert.NoError(t, err) assert.Equal(t, core.Built, target.State()) assert.Equal(t, []string{"file7"}, target.Outputs()) @@ -120,7 +121,7 @@ func TestPostBuildFunction(t *testing.T) { func TestOutputDir(t *testing.T) { newTarget := func() (*core.BuildState, *core.BuildTarget) { // Test modifying a command in the post-build function. - state, target := newState("//package1:target8") + state, target := newState(t, "//package1:target8") target.Command = "mkdir OUT_DIR && touch OUT_DIR/file7" target.OutputDirectories = append(target.OutputDirectories, "OUT_DIR") @@ -129,7 +130,7 @@ func TestOutputDir(t *testing.T) { state, target := newTarget() - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) require.NoError(t, err) assert.Equal(t, []string{"file7"}, target.Outputs()) @@ -141,7 +142,7 @@ func TestOutputDir(t *testing.T) { // Run again to load the outputs from the metadata state, target = newTarget() - err = buildTarget(state, target, false) + err = buildTarget(t.Context(), state, target, false) require.NoError(t, err) assert.Equal(t, []string{"file7"}, target.Outputs()) assert.Equal(t, core.Reused, target.State()) @@ -150,7 +151,7 @@ func TestOutputDir(t *testing.T) { func TestOutputDirDoubleStar(t *testing.T) { newTarget := func(withDoubleStar bool) (*core.BuildState, *core.BuildTarget) { // Test modifying a command in the post-build function. - state, target := newState("//package1:target8") + state, target := newState(t, "//package1:target8") target.Command = "mkdir -p OUT_DIR/foo && touch OUT_DIR/foo/file7 && chmod 777 OUT_DIR/foo/file7" if withDoubleStar { @@ -164,7 +165,7 @@ func TestOutputDirDoubleStar(t *testing.T) { state, target := newTarget(false) - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) require.NoError(t, err) assert.Equal(t, []string{"foo"}, target.Outputs()) @@ -180,7 +181,7 @@ func TestOutputDirDoubleStar(t *testing.T) { state, target = newTarget(true) - err = buildTarget(state, target, false) + err = buildTarget(t.Context(), state, target, false) require.NoError(t, err) assert.Equal(t, []string{"foo/file7"}, target.Outputs()) @@ -191,11 +192,11 @@ func TestOutputDirDoubleStar(t *testing.T) { func TestCacheRetrieval(t *testing.T) { // Test retrieving stuff from the cache - state, target := newState("//package1:target8") + state, target := newState(t, "//package1:target8") target.AddOutput("file8") target.Command = "false" // Will fail if we try to build it. state.Cache = cache - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) assert.NoError(t, err) assert.Equal(t, core.Cached, target.State()) } @@ -203,7 +204,7 @@ func TestCacheRetrieval(t *testing.T) { func TestPostBuildFunctionAndCache(t *testing.T) { // Test the often subtle and quick to anger interaction of post-build function and cache. // In this case when it fails to retrieve the post-build output it should still call the function after building. - state, target := newState("//package1:target9") + state, target := newState(t, "//package1:target9") target.AddOutput("file9") target.Command = "echo -n 'wibble wibble wibble' | tee $OUT" called := false @@ -213,7 +214,7 @@ func TestPostBuildFunctionAndCache(t *testing.T) { return nil }) state.Cache = cache - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) assert.NoError(t, err) assert.Equal(t, core.Built, target.State()) assert.True(t, called) @@ -222,7 +223,7 @@ func TestPostBuildFunctionAndCache(t *testing.T) { func TestPostBuildFunctionAndCache2(t *testing.T) { // Test the often subtle and quick to anger interaction of post-build function and cache. // In this case it succeeds in retrieving the post-build output but must still call the function. - state, target := newState("//package1:target10") + state, target := newState(t, "//package1:target10") target.AddOutput("file10") target.Command = "echo 'wibble wibble wibble' | tee $OUT" called := false @@ -233,14 +234,14 @@ func TestPostBuildFunctionAndCache2(t *testing.T) { return nil }) state.Cache = cache - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) assert.NoError(t, err) assert.Equal(t, core.Cached, target.State()) assert.True(t, called) } func TestInitPyCreation(t *testing.T) { - state, _ := newState("//pypkg:wevs") + state, _ := newState(t, "//pypkg:wevs") target1 := newPyFilegroup(state, "//pypkg:target1", "file1.py") target2 := newPyFilegroup(state, "//pypkg:target2", "__init__.py") _, err := buildFilegroup(state, target1) @@ -254,7 +255,7 @@ func TestInitPyCreation(t *testing.T) { } func TestRecursiveInitPyCreation(t *testing.T) { - state, _ := newState("//package1/package2:wevs") + state, _ := newState(t, "//package1/package2:wevs") target1 := newPyFilegroup(state, "//package1/package2:target1", "file1.py") _, err := buildFilegroup(state, target1) assert.NoError(t, err) @@ -263,7 +264,7 @@ func TestRecursiveInitPyCreation(t *testing.T) { } func TestGoModCreation(t *testing.T) { - state, _ := newState("//package_go/subpackage:wevs") + state, _ := newState(t, "//package_go/subpackage:wevs") target := newPyFilegroup(state, "//package1/package2:target1", "file1.py") target.AddLabel("go") _, err := buildFilegroup(state, target) @@ -272,16 +273,16 @@ func TestGoModCreation(t *testing.T) { } func TestCreatePlzOutGo(t *testing.T) { - state, target := newState("//package1:target") + state, target := newState(t, "//package1:target") target.AddLabel("link:plz-out/go/${PKG}/src") target.AddOutput("file1.go") assert.False(t, fs.PathExists("plz-out/go")) - assert.NoError(t, buildTarget(state, target, false)) + assert.NoError(t, buildTarget(t.Context(), state, target, false)) assert.True(t, fs.PathExists("plz-out/go/package1/src/file1.go")) } func TestLicenceEnforcement(t *testing.T) { - state, target := newState("//pkg:good") + state, target := newState(t, "//pkg:good") state.Config.Licences.Reject = append(state.Config.Licences.Reject, "gpl") state.Config.Licences.Accept = append(state.Config.Licences.Accept, "mit") @@ -299,7 +300,7 @@ func TestLicenceEnforcement(t *testing.T) { checkLicences(state, target) // Now construct a new "bad" target. - state, target = newState("//pkg:bad") + state, target = newState(t, "//pkg:bad") state.Config.Licences.Reject = append(state.Config.Licences.Reject, "gpl") state.Config.Licences.Accept = append(state.Config.Licences.Accept, "mit") @@ -311,7 +312,7 @@ func TestLicenceEnforcement(t *testing.T) { } func TestFileGroupBinDir(t *testing.T) { - state, target := newState("//package1:bindir") + state, target := newState(t, "//package1:bindir") target.AddSource(core.FileLabel{File: "package2", Package: target.Label.PackageName}) target.IsBinary = true target.IsFilegroup = true @@ -335,7 +336,7 @@ func TestFileGroupBinDir(t *testing.T) { } func TestOutputHash(t *testing.T) { - state, target := newState("//package3:target1") + state, target := newState(t, "//package3:target1") target.AddOutput("file1") target.Hashes = []string{"634b027b1b69e1242d40d53e312b3b4ac7710f55be81f289b549446ef6778bee"} b, err := state.TargetHasher.OutputHash(target) @@ -344,7 +345,7 @@ func TestOutputHash(t *testing.T) { } func TestCheckRuleHashes(t *testing.T) { - state, target := newState("//package3:target1") + state, target := newState(t, "//package3:target1") target.AddOutput("file1") // This is the normal sha1 hash calculation with no combining. @@ -375,7 +376,7 @@ func TestCheckRuleHashes(t *testing.T) { } func TestHashCheckers(t *testing.T) { - state, target := newStateWithHashCheckers("//package3:target1", "sha256", "xxhash") + state, target := newStateWithHashCheckers(t, "//package3:target1", "sha256", "xxhash") target.AddOutput("file1") b, err := state.TargetHasher.OutputHash(target) @@ -398,7 +399,7 @@ func TestHashCheckers(t *testing.T) { } func TestFetchLocalRemoteFile(t *testing.T) { - state, target := newState("//package4:target1") + state, target := newState(t, "//package4:target1") target.AddSource(core.URLLabel("file://" + os.Getenv("TMP_DIR") + "/src/build/test_data/local_remote_file.txt")) target.AddOutput("local_remote_file.txt") @@ -415,7 +416,7 @@ func TestFetchLocalRemoteFile(t *testing.T) { } func TestFetchLocalRemoteFileCannotBeRelative(t *testing.T) { - state, target := newState("//package4:target2") + state, target := newState(t, "//package4:target2") target.AddSource(core.URLLabel("src/build/test_data/local_remote_file.txt")) target.AddOutput("local_remote_file.txt") err := fetchRemoteFile(state, target) @@ -423,7 +424,7 @@ func TestFetchLocalRemoteFileCannotBeRelative(t *testing.T) { } func TestFetchLocalRemoteFileCannotBeWithinRepo(t *testing.T) { - state, target := newState("//package4:target2") + state, target := newState(t, "//package4:target2") target.AddSource(core.URLLabel("file://" + os.Getenv("TMP_DIR") + "/src/build/test_data/local_remote_file.txt")) target.AddOutput("local_remote_file.txt") err := fetchRemoteFile(state, target) @@ -433,21 +434,21 @@ func TestFetchLocalRemoteFileCannotBeWithinRepo(t *testing.T) { func TestBuildMetadatafileIsCreated(t *testing.T) { stdOut := "wibble wibble wibble" - state, target := newState("//package1:mdtest") + state, target := newState(t, "//package1:mdtest") target.AddOutput("file1") - err := buildTarget(state, target, false) + err := buildTarget(t.Context(), state, target, false) require.NoError(t, err) assert.False(t, target.BuildCouldModifyTarget()) assert.True(t, fs.FileExists(filepath.Join(target.OutDir(), target.TargetBuildMetadataFileName()))) - state, target = newState("//package1:mdtest_post_build") + state, target = newState(t, "//package1:mdtest_post_build") target.Command = fmt.Sprintf("echo -n '%s' | tee $OUT", stdOut) target.AddOutput("file1") target.PostBuildFunction = postBuildFunction(func(target *core.BuildTarget, output string) error { assert.Equal(t, stdOut, output) return nil }) - err = buildTarget(state, target, false) + err = buildTarget(t.Context(), state, target, false) require.NoError(t, err) assert.True(t, target.BuildCouldModifyTarget()) assert.True(t, fs.FileExists(filepath.Join(target.OutDir(), target.TargetBuildMetadataFileName()))) @@ -498,7 +499,7 @@ func TestSha1SingleHash(t *testing.T) { for _, test := range testCases { t.Run(test.name+" foo", func(t *testing.T) { - state, target := newStateWithHashFunc("//hash_test:hash_test", test.algorithm) + state, target := newStateWithHashFunc(t, "//hash_test:hash_test", test.algorithm) target.AddOutput("foo.txt") @@ -507,7 +508,7 @@ func TestSha1SingleHash(t *testing.T) { assert.Equal(t, test.fooHash, hex.EncodeToString(h)) }) t.Run(test.name+" foo and bar", func(t *testing.T) { - state, target := newStateWithHashFunc("//hash_test:hash_test", test.algorithm) + state, target := newStateWithHashFunc(t, "//hash_test:hash_test", test.algorithm) target.AddOutput("foo.txt") target.AddOutput("bar.txt") @@ -519,7 +520,8 @@ func TestSha1SingleHash(t *testing.T) { } } -func newStateWithHashCheckers(label, hashFunction string, hashCheckers ...string) (*core.BuildState, *core.BuildTarget) { +func newStateWithHashCheckers(t *testing.T, label, hashFunction string, hashCheckers ...string) (*core.BuildState, *core.BuildTarget) { + t.Helper() config, _ := core.ReadConfigFiles(fs.HostFS, nil, nil) if hashFunction != "" { config.Build.HashFunction = hashFunction @@ -527,7 +529,7 @@ func newStateWithHashCheckers(label, hashFunction string, hashCheckers ...string if len(hashCheckers) > 0 { config.Build.HashCheckers = hashCheckers } - state := core.NewBuildState(config) + state := core.NewBuildState(t.Context(), config) state.Config.Parse.BuildFileName = []string{"BUILD_FILE"} target := core.NewBuildTarget(core.ParseBuildLabel(label, "")) target.Command = fmt.Sprintf("echo 'output of %s' > $OUT", target.Label) @@ -538,10 +540,11 @@ func newStateWithHashCheckers(label, hashFunction string, hashCheckers ...string return state, target } -func newStateWithHashFunc(label, hashFunc string) (*core.BuildState, *core.BuildTarget) { +func newStateWithHashFunc(t *testing.T, label, hashFunc string) (*core.BuildState, *core.BuildTarget) { + t.Helper() config, _ := core.ReadConfigFiles(fs.HostFS, nil, nil) config.Build.HashFunction = hashFunc - state := core.NewBuildState(config) + state := core.NewBuildState(t.Context(), config) state.Config.Parse.BuildFileName = []string{"BUILD_FILE"} target := core.NewBuildTarget(core.ParseBuildLabel(label, "")) target.Command = fmt.Sprintf("echo 'output of %s' > $OUT", target.Label) @@ -552,9 +555,10 @@ func newStateWithHashFunc(label, hashFunc string) (*core.BuildState, *core.Build return state, target } -func newState(label string) (*core.BuildState, *core.BuildTarget) { +func newState(t *testing.T, label string) (*core.BuildState, *core.BuildTarget) { + t.Helper() config, _ := core.ReadConfigFiles(fs.HostFS, nil, nil) - state := core.NewBuildState(config) + state := core.NewBuildState(t.Context(), config) state.Config.Parse.BuildFileName = []string{"BUILD_FILE"} target := core.NewBuildTarget(core.ParseBuildLabel(label, "")) target.Command = fmt.Sprintf("echo 'output of %s' > $OUT", target.Label) @@ -665,7 +669,7 @@ func TestMain(m *testing.M) { // Move ourselves to the root of the test data tree wd, _ := os.Getwd() core.RepoRoot = filepath.Join(wd, "src/build/test_data") - Init(core.NewDefaultBuildState()) + Init(core.NewDefaultBuildState(context.Background())) if err := os.Chdir(core.RepoRoot); err != nil { panic(err) } diff --git a/src/build/remote_file_test.go b/src/build/remote_file_test.go index 6fe86a670..5075e8bb7 100644 --- a/src/build/remote_file_test.go +++ b/src/build/remote_file_test.go @@ -26,7 +26,7 @@ func listen(s *http.Server) net.Listener { } func TestHeader(t *testing.T) { - state, target := newState("//pkg:header_test") + state, target := newState(t, "//pkg:header_test") target.IsRemoteFile = true target.Sources = []core.BuildInput{core.URLLabel("http://localhost:8080/header")} target.AddOutput("header") @@ -46,7 +46,7 @@ func TestHeader(t *testing.T) { } func TestSecretHeader(t *testing.T) { - state, target := newState("//pkg:header_test") + state, target := newState(t, "//pkg:header_test") target.IsRemoteFile = true target.Sources = []core.BuildInput{core.URLLabel("http://localhost:8080/header")} target.AddOutput("header") @@ -73,7 +73,7 @@ func TestSecretHeader(t *testing.T) { } func TestBasicAuth(t *testing.T) { - state, target := newState("//pkg:header_test") + state, target := newState(t, "//pkg:header_test") target.IsRemoteFile = true target.Sources = []core.BuildInput{core.URLLabel("http://localhost:8080/header")} target.AddOutput("header") diff --git a/src/core/build_env_test.go b/src/core/build_env_test.go index ca2b22210..6cc806988 100644 --- a/src/core/build_env_test.go +++ b/src/core/build_env_test.go @@ -63,7 +63,7 @@ func TestExecEnvironment(t *testing.T) { target.AddOutput("out_file1") target.AddDatum(FileLabel{File: "data_file1", Package: "pkg"}) - env := ExecEnvironment(NewDefaultBuildState(), target, "/path/to/runtime/dir") + env := ExecEnvironment(NewDefaultBuildState(t.Context()), target, "/path/to/runtime/dir") assert.Equal(t, env["PLZ_ENV"], "1") assert.Equal(t, env["DATA"], "pkg/data_file1") @@ -79,7 +79,7 @@ func TestExecEnvironment(t *testing.T) { func TestExecEnvironmentTestTarget(t *testing.T) { t.Setenv("TERM", "my-term") - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) rootPkg := NewPackage("") // Set up tool 1. @@ -122,7 +122,7 @@ func TestExecEnvironmentTestTarget(t *testing.T) { func TestExecEnvironmentDebugTarget(t *testing.T) { t.Setenv("TERM", "my-term") - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) // Set up tool 1. rootPkg := NewPackage("") @@ -157,7 +157,7 @@ func TestExecEnvironmentDebugTarget(t *testing.T) { func TestExecEnvironmentDebugTestTarget(t *testing.T) { t.Setenv("TERM", "my-term") - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) // Set up tool 1. rootPkg := NewPackage("") @@ -191,7 +191,7 @@ func TestExecEnvironmentDebugTestTarget(t *testing.T) { } func TestDeduplicateEnvVars(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.NeedCoverage = true target := NewBuildTarget(NewBuildLabel("pkg", "t")) diff --git a/src/core/build_label_test.go b/src/core/build_label_test.go index e24e7fdd3..f77e74c00 100644 --- a/src/core/build_label_test.go +++ b/src/core/build_label_test.go @@ -109,7 +109,7 @@ func TestCompleteError(t *testing.T) { } func TestSubrepoLabel(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) assert.EqualValues(t, BuildLabel{PackageName: "", Name: "test"}, subrepoLabel("test", "")) assert.EqualValues(t, BuildLabel{PackageName: "package", Name: "test"}, subrepoLabel("package/test", "")) // This isn't really valid (the caller shouldn't need to call it in such a case) @@ -128,7 +128,7 @@ func TestSubrepoLabel(t *testing.T) { func TestPluginSubrepoLabel(t *testing.T) { subrepoLabel := BuildLabel{PackageName: "foo/bar", Name: "plugin"} - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Plugin = map[string]*Plugin{} state.Config.Plugin["plugin"] = &Plugin{Target: subrepoLabel} state.Graph.AddSubrepo(&Subrepo{Name: "foowin_amd64", Arch: cli.NewArch("foowin", "amd64")}) diff --git a/src/core/build_target_test.go b/src/core/build_target_test.go index 4360be131..216032302 100644 --- a/src/core/build_target_test.go +++ b/src/core/build_target_test.go @@ -55,7 +55,7 @@ func TestTestDirSubrepo(t *testing.T) { } func TestCanSee(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) target1 := makeTarget1("//src/build/python:lib1", "") target2 := makeTarget1("//src/build/python:lib2", "PUBLIC") target3 := makeTarget1("//src/test/python:lib3", "//src/test/...") @@ -89,7 +89,7 @@ func TestCanSee(t *testing.T) { func TestCanSeeExperimental(t *testing.T) { config := DefaultConfiguration() config.Parse.ExperimentalDir = []string{"experimental"} - state := NewBuildState(config) + state := NewBuildState(t.Context(), config) target1 := makeTarget1("//src/core:target1", "") target2 := makeTarget1("//experimental/user:target2", "PUBLIC") @@ -112,7 +112,7 @@ func TestCheckDependencyVisibility(t *testing.T) { target7 := makeTarget1("//src/test/python:test1", "", target5, target4) target7.Test = new(TestFields) - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Graph.AddTarget(target1) state.Graph.AddTarget(target2) state.Graph.AddTarget(target3) @@ -312,7 +312,7 @@ func TestGetCommandConfig(t *testing.T) { } func TestGetCommand(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Build.Config = "dbg" state.Config.Build.FallbackConfig = "opt" target := makeTarget1("//src/core:target1", "PUBLIC") @@ -331,7 +331,7 @@ func TestGetCommand(t *testing.T) { } func TestGetTestCommand(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Build.Config = "dbg" state.Config.Build.FallbackConfig = "opt" target := makeTarget1("//src/core:target1", "PUBLIC") @@ -612,7 +612,7 @@ func TestAllLocalSourcePaths(t *testing.T) { } func TestAllURLs(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) target := makeTarget1("//src/core:remote1", "") target.IsRemoteFile = true target.AddSource(URLLabel("https://github.com/thought-machine/please")) @@ -750,7 +750,7 @@ func TestBuildTargetOwnBuildInputs(t *testing.T) { buildFiles := []string{"BUILD_FILE"} t.Run("file as source is in package", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -766,7 +766,7 @@ func TestBuildTargetOwnBuildInputs(t *testing.T) { }) t.Run("file as named source is in package", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -782,7 +782,7 @@ func TestBuildTargetOwnBuildInputs(t *testing.T) { }) t.Run("file as source is subpackage", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -798,7 +798,7 @@ func TestBuildTargetOwnBuildInputs(t *testing.T) { }) t.Run("file as named source is subpackage", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -814,7 +814,7 @@ func TestBuildTargetOwnBuildInputs(t *testing.T) { }) t.Run("file as source is in subpackage", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -830,7 +830,7 @@ func TestBuildTargetOwnBuildInputs(t *testing.T) { }) t.Run("file as named source is in subpackage", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -846,7 +846,7 @@ func TestBuildTargetOwnBuildInputs(t *testing.T) { }) t.Run("file as data is in package", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -862,7 +862,7 @@ func TestBuildTargetOwnBuildInputs(t *testing.T) { }) t.Run("file as named data is in package", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -878,7 +878,7 @@ func TestBuildTargetOwnBuildInputs(t *testing.T) { }) t.Run("file as data is subpackage", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -894,7 +894,7 @@ func TestBuildTargetOwnBuildInputs(t *testing.T) { }) t.Run("file as named data is subpackage", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -910,7 +910,7 @@ func TestBuildTargetOwnBuildInputs(t *testing.T) { }) t.Run("file as data is in subpackage", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -926,7 +926,7 @@ func TestBuildTargetOwnBuildInputs(t *testing.T) { }) t.Run("file as named data is in subpackage", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -945,7 +945,7 @@ func TestBuildTargetOwnBuildInputs(t *testing.T) { func TestBuildTargetOwnBuildOutput(t *testing.T) { buildFiles := []string{"BUILD_FILE"} t.Run("file is in package", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -958,7 +958,7 @@ func TestBuildTargetOwnBuildOutput(t *testing.T) { }) t.Run("file is subpackage", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") @@ -971,7 +971,7 @@ func TestBuildTargetOwnBuildOutput(t *testing.T) { }) t.Run("file is in subpackage", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = buildFiles target := makeTarget1("//src/core/test_data/project", "PUBLIC") diff --git a/src/core/command_replacements_test.go b/src/core/command_replacements_test.go index 510aee77e..623f74ae3 100644 --- a/src/core/command_replacements_test.go +++ b/src/core/command_replacements_test.go @@ -3,6 +3,7 @@ package core import ( + "context" "encoding/base64" "os" "path/filepath" @@ -18,7 +19,7 @@ var state *BuildState const testHash = "gB4sUwsLkB1ODYKUxYrKGlpdYUI" func init() { - state = NewDefaultBuildState() + state = NewDefaultBuildState(context.Background()) state.TargetHasher = &testHasher{} wd, _ = os.Getwd() } @@ -161,7 +162,7 @@ func TestBazelCompatReplacements(t *testing.T) { target := makeTarget2("//path/to:target", "cp $< $@", nil) assert.Equal(t, "cp $< $@", replaceSequences(state, target)) // In Bazel compat mode we do though. - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Bazel.Compatibility = true assert.Equal(t, "cp $SRCS $OUTS", replaceSequences(state, target)) // @D is the output dir, for us it's the tmp dir. diff --git a/src/core/cycle_detector_test.go b/src/core/cycle_detector_test.go index 70506f59b..5fb67eed9 100644 --- a/src/core/cycle_detector_test.go +++ b/src/core/cycle_detector_test.go @@ -39,7 +39,7 @@ func TestCycleDetector(t *testing.T) { } t.Run("NoCycle", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) newTarget(state, "//src:a", "//src:b", "//src:c") newTarget(state, "//src:b", "//src:d", "//src:e") newTarget(state, "//src:c", "//src:b", "//src:f") @@ -54,7 +54,7 @@ func TestCycleDetector(t *testing.T) { }) t.Run("Cycle", func(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) newTarget(state, "//src:a", "//src:b", "//src:c") newTarget(state, "//src:b", "//src:d", "//src:e") newTarget(state, "//src:c", "//src:b", "//src:f") diff --git a/src/core/package_test.go b/src/core/package_test.go index 4b100fd12..8f116deb7 100644 --- a/src/core/package_test.go +++ b/src/core/package_test.go @@ -17,7 +17,7 @@ func TestRegisterSubinclude(t *testing.T) { } func TestRegisterOutput(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) target1 := NewBuildTarget(ParseBuildLabel("//src/core:target1", "")) target2 := NewBuildTarget(ParseBuildLabel("//src/core:target2", "")) pkg := NewPackage("src/core") @@ -34,7 +34,7 @@ func TestRegisterOutput(t *testing.T) { } func TestRegisterOutputNonFilegroupTargets(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) target1 := NewBuildTarget(ParseBuildLabel("//src/core:target1", "")) target2 := NewBuildTarget(ParseBuildLabel("//src/core:target2", "")) @@ -45,7 +45,7 @@ func TestRegisterOutputNonFilegroupTargets(t *testing.T) { } func TestRegisterOutputFilegroupAndNonFilegroupTargets(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) target1 := NewBuildTarget(ParseBuildLabel("//src/core:target1", "")) target2 := NewBuildTarget(ParseBuildLabel("//src/core:target2", "")) @@ -60,7 +60,7 @@ func TestRegisterOutputFilegroupAndNonFilegroupTargets(t *testing.T) { } func TestRegisterOutputFilegroupTargets(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) target1 := NewBuildTarget(ParseBuildLabel("//src/core:target1", "")) target1.IsFilegroup = true @@ -89,7 +89,7 @@ func TestAllChildren(t *testing.T) { } func TestFindOwningPackages(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.Config.Parse.BuildFileName = []string{"BUILD_FILE"} pkgs := FindOwningPackages(state, []string{"src/core/test_data/test_subfolder1/whatever.txt"}) assert.Equal(t, []BuildLabel{ParseBuildLabel("//src/core/test_data:all", "")}, pkgs) @@ -103,7 +103,7 @@ func TestIsIncludedIn(t *testing.T) { } func TestVerifyOutputs(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) target1 := NewBuildTarget(ParseBuildLabel("//src/core:target1", "")) target2 := NewBuildTarget(ParseBuildLabel("//src/core:target2", "")) pkg := NewPackage("src/core") diff --git a/src/core/state.go b/src/core/state.go index 18a03d70a..f3630e59b 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -2,6 +2,7 @@ package core import ( "bytes" + "context" "crypto/sha1" "crypto/sha256" "fmt" @@ -287,7 +288,6 @@ type stateProgress struct { numParses atomic.Int64 mutex sync.Mutex closeOnce sync.Once - resultOnce sync.Once // Used to track subinclude() calls that block until targets are built. Keyed by their label. pendingTargets *cmap.Map[BuildLabel, chan struct{}] // Used to track general package parsing requests. Keyed by a packageKey struct. @@ -308,6 +308,17 @@ type stateProgress struct { results chan *BuildResult // Internal result stream, used to intermediate them for the cycle checker. internalResults chan *BuildResult + // workerCtx is cancelled when workers should stop producing results. + workerCtx context.Context //nolint:containedctx + workerCancel context.CancelFunc + // dispatcherCtx is cancelled after all workers have finished, signalling + // the result dispatcher (forwardResults) to drain remaining results and + // shut down. + dispatcherCtx context.Context //nolint:containedctx + dispatcherCancel context.CancelFunc + // dispatcherWg tracks the result dispatcher goroutine so CloseResults + // can wait for it to finish draining. + dispatcherWg sync.WaitGroup // The cycle checker itself. cycleDetector cycleDetector } @@ -413,24 +424,31 @@ func (state *BuildState) taskDone(wasSynthetic bool) { } } +// WorkerContext returns the context that workers should use to detect shutdown. +func (state *BuildState) WorkerContext() context.Context { + return state.progress.workerCtx +} + // Stop stops the worker queues after any current tasks are done. func (state *BuildState) Stop() { state.progress.closeOnce.Do(func() { + state.progress.workerCancel() close(state.pendingParses) close(state.pendingActions) }) } -// CloseResults closes the result channels. +// CloseResults signals the result-forwarding goroutine to drain any remaining +// results and close the external results channel. It blocks until forwarding +// is complete. +// +// This must only be called after all workers have finished (i.e. after the +// worker WaitGroup is done), so that no new results are produced while +// draining. func (state *BuildState) CloseResults() { state.progress.cycleDetector.Stop() - state.progress.mutex.Lock() - defer state.progress.mutex.Unlock() - if state.progress.results != nil { - state.progress.resultOnce.Do(func() { - close(state.progress.results) - }) - } + state.progress.dispatcherCancel() + state.progress.dispatcherWg.Wait() } // IsOriginalTarget returns true if a target is an original target, ie. one specified on the command line. @@ -624,52 +642,94 @@ func (state *BuildState) logResult(result *BuildResult) { } } -// forwardResults runs indefinitely, forwarding results from the internal -// channel to the external one. On the way it checks if we need to do -// cycle detection. -func (state *BuildState) forwardResults() { - defer func() { - if r := recover(); r != nil { - // Ensure we don't get a "send on closed channel" when the - // outward results channel is closed. - log.Debug("%s", r) - } - }() +// forwardResults forwards results from the internal channel to the external +// one. On the way it checks if we need to do cycle detection. +// +// It runs until its context is cancelled, at which point it drains any +// remaining results from internalResults and closes the external results +// channel. +func (state *BuildState) forwardResults(ctx context.Context) { + defer state.progress.dispatcherWg.Done() + defer state.closeResults() activeTargets := map[*BuildTarget]struct{}{} // Persist this one timer throughout so we don't generate bazillions of them. t := time.NewTimer(cycleCheckDuration) t.Stop() - var result *BuildResult + stopTimer := func() { + if !t.Stop() { + select { + case <-t.C: + default: + } + } + } + for { + // When no targets are active, arm the cycle detection timer. if len(activeTargets) == 0 { t.Reset(cycleCheckDuration) select { - case result = <-state.progress.internalResults: - // This has to be properly managed to prevent hangs. - if !t.Stop() { - <-t.C - } + case <-ctx.Done(): + state.drainResults(activeTargets) + return case <-t.C: go state.checkForCycles() go dumpGoroutineInfo() - // Still need to get a result! - result = <-state.progress.internalResults + case result := <-state.progress.internalResults: + stopTimer() + state.forwardResult(result, activeTargets) + continue } - } else { - result = <-state.progress.internalResults } - if target := result.target; target != nil { - if result.Status.IsActive() { - activeTargets[target] = struct{}{} - } else { - delete(activeTargets, target) - } + + // Wait for a result or shutdown. + select { + case <-ctx.Done(): + state.drainResults(activeTargets) + return + case result := <-state.progress.internalResults: + state.forwardResult(result, activeTargets) + } + } +} + +// forwardResult sends a single result to the external results channel and +// updates active target tracking. +func (state *BuildState) forwardResult(result *BuildResult, activeTargets map[*BuildTarget]struct{}) { + if target := result.target; target != nil { + if result.Status.IsActive() { + activeTargets[target] = struct{}{} + } else { + delete(activeTargets, target) } - state.progress.mutex.Lock() - if state.progress.results != nil { - state.progress.results <- result + } + state.progress.mutex.Lock() + if state.progress.results != nil { + state.progress.results <- result + } + state.progress.mutex.Unlock() +} + +// drainResults forwards any remaining results from the internal channel +// to the external results channel without blocking. +func (state *BuildState) drainResults(activeTargets map[*BuildTarget]struct{}) { + for { + select { + case result := <-state.progress.internalResults: + state.forwardResult(result, activeTargets) + default: + return } - state.progress.mutex.Unlock() + } +} + +// closeResults closes the external results channel under the mutex. +func (state *BuildState) closeResults() { + state.progress.mutex.Lock() + defer state.progress.mutex.Unlock() + if state.progress.results != nil { + close(state.progress.results) + state.progress.results = nil } } @@ -1451,7 +1511,7 @@ func executorFromConfig(config *Configuration) *process.Executor { // NewBuildState constructs and returns a new BuildState. // Everyone should use this rather than attempting to construct it themselves; // callers can't initialise all the required private fields. -func NewBuildState(config *Configuration) *BuildState { +func NewBuildState(ctx context.Context, config *Configuration) *BuildState { graph := NewGraph() state := &BuildState{ Graph: graph, @@ -1497,14 +1557,17 @@ func NewBuildState(config *Configuration) *BuildState { for _, exp := range config.Parse.ExperimentalDir { state.experimentalLabels = append(state.experimentalLabels, BuildLabel{PackageName: exp, Name: "..."}) } - go state.forwardResults() + state.progress.workerCtx, state.progress.workerCancel = context.WithCancel(ctx) + state.progress.dispatcherCtx, state.progress.dispatcherCancel = context.WithCancel(ctx) + state.progress.dispatcherWg.Add(1) + go state.forwardResults(state.progress.dispatcherCtx) return state } // NewDefaultBuildState creates a BuildState for the default configuration. // This is useful for tests etc that don't need to customise anything about it. -func NewDefaultBuildState() *BuildState { - return NewBuildState(DefaultConfiguration()) +func NewDefaultBuildState(ctx context.Context) *BuildState { + return NewBuildState(ctx, DefaultConfiguration()) } // A BuildResult represents a single event in the build process, i.e. a target starting or finishing diff --git a/src/core/state_test.go b/src/core/state_test.go index 010988a24..33614fd23 100644 --- a/src/core/state_test.go +++ b/src/core/state_test.go @@ -8,7 +8,7 @@ import ( ) func TestExpandOriginalLabels(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}, true) state.AddOriginalTarget(BuildLabel{PackageName: "src/parse", Name: "parse"}, true) state.Include = []string{"go"} @@ -33,7 +33,7 @@ func TestExpandOriginalLabels(t *testing.T) { } func TestExpandOriginalTestLabels(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}, true) state.NeedTests = true state.Include = []string{"go"} @@ -49,7 +49,7 @@ func TestExpandOriginalTestLabels(t *testing.T) { } func TestExpandVisibleOriginalTargets(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}, true) addTarget(state, "//src/core:target1", "py") @@ -58,7 +58,7 @@ func TestExpandVisibleOriginalTargets(t *testing.T) { } func TestExpandOriginalSubLabels(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}, true) state.AddOriginalTarget(BuildLabel{PackageName: "src/core/tests", Name: "all"}, true) state.Include = []string{"go"} @@ -75,7 +75,7 @@ func TestExpandOriginalSubLabels(t *testing.T) { } func TestExpandOriginalLabelsOrdering(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) state.AddOriginalTarget(BuildLabel{PackageName: "src/parse", Name: "parse"}, true) state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}, true) state.AddOriginalTarget(BuildLabel{PackageName: "src/core/tests", Name: "all"}, true) @@ -94,7 +94,7 @@ func TestExpandOriginalLabelsOrdering(t *testing.T) { } func TestAddTargetFilegroupPackageOutputs(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) pkg := NewPackage("src/core") target := NewBuildTarget(ParseBuildLabel("//src/core:test", "")) @@ -110,7 +110,7 @@ func TestAddTargetFilegroupPackageOutputs(t *testing.T) { } func TestAddDepsToTarget(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) _, builds := state.TaskQueues() pkg := NewPackage("src/core") target1 := addTargetDeps(state, pkg, "//src/core:target1", "//src/core:target2") diff --git a/src/core/utils_benchmark_test.go b/src/core/utils_benchmark_test.go index cec960cf1..1db03ed24 100644 --- a/src/core/utils_benchmark_test.go +++ b/src/core/utils_benchmark_test.go @@ -6,7 +6,7 @@ import ( ) func BenchmarkIterInputsControl(b *testing.B) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(b.Context()) target := NewBuildTarget(NewBuildLabel("src/foo", "foo_lib")) state.Graph.AddTarget(target) @@ -20,7 +20,7 @@ func BenchmarkIterInputsControl(b *testing.B) { } func BenchmarkIterInputsSimple(b *testing.B) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(b.Context()) target := NewBuildTarget(NewBuildLabel("src/foo", "foo_lib")) target.NeedsTransitiveDependencies = true @@ -52,7 +52,7 @@ func BenchmarkIterInputsSimple(b *testing.B) { } func BenchmarkIterInputsNamedSources(b *testing.B) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(b.Context()) target := NewBuildTarget(NewBuildLabel("src/foo", "foo_lib")) target.NeedsTransitiveDependencies = true diff --git a/src/core/utils_test.go b/src/core/utils_test.go index adfa07dd6..ad21c7281 100644 --- a/src/core/utils_test.go +++ b/src/core/utils_test.go @@ -34,7 +34,7 @@ func TestCollapseHash2(t *testing.T) { } func TestIterSources(t *testing.T) { - state := NewDefaultBuildState() + state := NewDefaultBuildState(t.Context()) graph := buildGraph() type SourcePair struct{ Src, Tmp string } diff --git a/src/exec/exec.go b/src/exec/exec.go index 93ca0deab..eda370671 100644 --- a/src/exec/exec.go +++ b/src/exec/exec.go @@ -111,7 +111,8 @@ func exec(state *core.BuildState, outputMode process.OutputMode, target *core.Bu } env = append(core.ExecEnvironment(state, target, filepath.Join(core.RepoRoot, runtimeDir)).ToSlice(), env...) - out, _, err := state.ProcessExecutor.ExecWithTimeoutShellStdStreams(target, runtimeDir, env, time.Duration(math.MaxInt64), false, foreground, sandbox, cmd, outputMode == process.Default) + ctx := context.TODO() // TODO: wire up to signal handling or a timeout + out, _, err := state.ProcessExecutor.ExecWithTimeoutShellStdStreams(ctx, target, runtimeDir, env, time.Duration(math.MaxInt64), false, foreground, sandbox, cmd, outputMode == process.Default) return out, err }); err != nil { log.Error("Failed to execute %s: %s", target, err) diff --git a/src/exec/exec_test.go b/src/exec/exec_test.go index 07c371290..caad46338 100644 --- a/src/exec/exec_test.go +++ b/src/exec/exec_test.go @@ -16,13 +16,13 @@ import ( func TestNoBinaryTargetNoOverrideCommand(t *testing.T) { target := core.NewBuildTarget(core.NewBuildLabel("pkg", "t")) - err := exec(core.NewDefaultBuildState(), process.Default, target, ".", nil, nil, nil, "", false, process.NoSandbox) + err := exec(core.NewDefaultBuildState(t.Context()), process.Default, target, ".", nil, nil, nil, "", false, process.NoSandbox) assert.Error(t, err) assert.Contains(t, err.Error(), "target needs to be a binary") } func TestPrepareRuntimeDir(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) target := core.NewBuildTarget(core.NewBuildLabel("pkg", "t")) target.BuildTimeout = 10 * time.Second @@ -34,7 +34,7 @@ func TestPrepareRuntimeDir(t *testing.T) { if err := build.StoreTargetMetadata(target, &core.BuildMetadata{}); err != nil { panic(err) } - build.Build(state, target, false) + build.Build(t.Context(), state, target, false) err := core.PrepareRuntimeDir(state, target, "plz-out/exec/pkg") assert.Nil(t, err) @@ -44,7 +44,7 @@ func TestPrepareRuntimeDir(t *testing.T) { func TestSimpleOverrideCommand(t *testing.T) { target := core.NewBuildTarget(core.NewBuildLabel("pkg", "t")) - cmd, err := resolveCmd(core.NewDefaultBuildState(), target, []string{"ls", "-l"}, "", "runtime/dir", process.NoSandbox) + cmd, err := resolveCmd(core.NewDefaultBuildState(t.Context()), target, []string{"ls", "-l"}, "", "runtime/dir", process.NoSandbox) assert.Nil(t, err) assert.Equal(t, "ls -l", cmd) } @@ -54,7 +54,7 @@ func TestOverrideCommandWithSequence(t *testing.T) { target.AddOutput("my-binary") target.IsBinary = true - cmd, err := resolveCmd(core.NewDefaultBuildState(), target, []string{"$(out_exe", "//pkg:t)"}, "", "runtime/dir", process.NoSandbox) + cmd, err := resolveCmd(core.NewDefaultBuildState(t.Context()), target, []string{"$(out_exe", "//pkg:t)"}, "", "runtime/dir", process.NoSandbox) assert.Nil(t, err) assert.Equal(t, "plz-out/bin/pkg/my-binary", cmd) } @@ -64,7 +64,7 @@ func TestCommandWithMultipleOutputs(t *testing.T) { target.AddOutput("my-out-1") target.AddOutput("my-out-2") - cmd, err := resolveCmd(core.NewDefaultBuildState(), target, nil, "", "runtime/dir", process.NoSandbox) + cmd, err := resolveCmd(core.NewDefaultBuildState(t.Context()), target, nil, "", "runtime/dir", process.NoSandbox) assert.Empty(t, cmd) assert.Error(t, err) assert.Contains(t, err.Error(), "it has 2 outputs") @@ -76,7 +76,7 @@ func TestCommandMountNotSandboxed(t *testing.T) { target := core.NewBuildTarget(core.NewBuildLabel("pkg", "t")) target.AddOutput("my-out") - cmd, err := resolveCmd(core.NewDefaultBuildState(), target, nil, "", "runtime/dir", process.NoSandbox) + cmd, err := resolveCmd(core.NewDefaultBuildState(t.Context()), target, nil, "", "runtime/dir", process.NoSandbox) assert.Nil(t, err) assert.Equal(t, filepath.Join(core.RepoRoot, "runtime/dir/my-out"), cmd) } @@ -85,13 +85,13 @@ func TestCommandMountSandboxed(t *testing.T) { target := core.NewBuildTarget(core.NewBuildLabel("pkg", "t")) target.AddOutput("my-out") - cmd, err := resolveCmd(core.NewDefaultBuildState(), target, nil, "", "runtime/dir", process.NewSandboxConfig(false, true)) + cmd, err := resolveCmd(core.NewDefaultBuildState(t.Context()), target, nil, "", "runtime/dir", process.NewSandboxConfig(false, true)) assert.Nil(t, err) assert.Equal(t, filepath.Join(core.SandboxDir, "my-out"), cmd) } func TestExec(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) target := core.NewBuildTarget(core.NewBuildLabel("pkg", "t")) state.Graph.AddTarget(target) @@ -100,7 +100,7 @@ func TestExec(t *testing.T) { } func TestCommandExitCode(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) target := core.NewBuildTarget(core.NewBuildLabel("pkg", "t")) state.Graph.AddTarget(target) diff --git a/src/gc/rewrite_test.go b/src/gc/rewrite_test.go index 8911b0378..285e4d088 100644 --- a/src/gc/rewrite_test.go +++ b/src/gc/rewrite_test.go @@ -12,7 +12,7 @@ import ( ) func TestRewriteFile(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) // Copy file to avoid any issues with links etc. wd, _ := os.Getwd() err := fs.CopyFile("src/gc/test_data/before.build", filepath.Join(wd, "test.build"), 0644) diff --git a/src/hashes/hash_rewriter_test.go b/src/hashes/hash_rewriter_test.go index a63aee1ae..9333e88fd 100644 --- a/src/hashes/hash_rewriter_test.go +++ b/src/hashes/hash_rewriter_test.go @@ -12,7 +12,7 @@ import ( ) func TestRewriteHashes(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) // Copy file to avoid any issues with links etc. wd, _ := os.Getwd() err := fs.CopyFile("src/hashes/test_data/before.build", filepath.Join(wd, "test.build"), 0644) diff --git a/src/help/rules.go b/src/help/rules.go index 2054ac61c..58a47a348 100644 --- a/src/help/rules.go +++ b/src/help/rules.go @@ -1,6 +1,7 @@ package help import ( + "context" "encoding/json" iofs "io/fs" "os" @@ -41,13 +42,14 @@ func PrintRuleArgs(files cli.StdinStrings) { } func newState() *core.BuildState { + ctx := context.TODO() // If we're in a repo, we might be able to read some stuff from there. if core.FindRepoRoot() { if config, err := core.ReadDefaultConfigFiles(fs.HostFS, nil); err == nil { - return core.NewBuildState(config) + return core.NewBuildState(ctx, config) } } - return core.NewDefaultBuildState() + return core.NewDefaultBuildState(ctx) } // AllBuiltinFunctions returns all the builtin functions, including any defined diff --git a/src/help/rules_test.go b/src/help/rules_test.go index 027f4b354..4f5a023e6 100644 --- a/src/help/rules_test.go +++ b/src/help/rules_test.go @@ -10,7 +10,7 @@ import ( ) func TestRuleArgs(t *testing.T) { - funcMap := getFunctionsFromState(core.NewDefaultBuildState()) + funcMap := getFunctionsFromState(core.NewDefaultBuildState(t.Context())) env := getRuleArgs(funcMap) assert.True(t, len(env.Functions) > 20) // Don't care exactly how many there are, but it should have a fair few. rule := env.Functions["http_archive"] @@ -34,7 +34,7 @@ func TestRuleArgs(t *testing.T) { } func TestMultilineComment(t *testing.T) { - funcMap := getFunctionsFromState(core.NewDefaultBuildState()) + funcMap := getFunctionsFromState(core.NewDefaultBuildState(t.Context())) env := getRuleArgs(funcMap) rule := env.Functions["new_http_archive"] assert.True(t, strings.Count(rule.Comment, "\n") > 1) diff --git a/src/parse/asp/builtins_test.go b/src/parse/asp/builtins_test.go index 02e622b0c..47bf10242 100644 --- a/src/parse/asp/builtins_test.go +++ b/src/parse/asp/builtins_test.go @@ -12,7 +12,7 @@ import ( func TestPackageName(t *testing.T) { s := &scope{ pkg: &core.Package{Name: "test/package"}, - state: core.NewBuildState(core.DefaultConfiguration()), + state: core.NewBuildState(t.Context(), core.DefaultConfiguration()), } assert.Equal(t, "test/package", packageName(s, []pyObject{pyNone{}, pyNone{}}).String()) assert.Equal(t, "test/package", packageName(s, []pyObject{pyString(":test"), pyNone{}}).String()) @@ -20,7 +20,7 @@ func TestPackageName(t *testing.T) { } func TestGetLabels(t *testing.T) { - state := core.NewBuildState(core.DefaultConfiguration()) + state := core.NewBuildState(t.Context(), core.DefaultConfiguration()) bottom := core.NewBuildTarget(core.NewBuildLabel("pkg", "bottom")) bottom.AddLabel("target:bottom") diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go index 511f00b05..de96a2e86 100644 --- a/src/parse/asp/interpreter_test.go +++ b/src/parse/asp/interpreter_test.go @@ -4,6 +4,7 @@ package asp import ( + "context" "fmt" "testing" @@ -19,7 +20,7 @@ func parseFileToStatements(filename string) (*scope, []*Statement, error) { } func parseFileToStatementsInPkg(filename string, pkg *core.Package) (*scope, []*Statement, error) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(context.Background()) state.Config.BuildConfig = map[string]string{"parser-engine": "python27"} parser := NewParser(state) @@ -592,7 +593,7 @@ func TestIsSemver(t *testing.T) { } func TestJSON(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) parser := NewParser(state) src, err := rules.ReadAsset("builtins.build_defs") @@ -657,7 +658,7 @@ func TestSemverCheck(t *testing.T) { } func TestLogConfigVariable(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) parser := NewParser(state) src, err := rules.ReadAsset("builtins.build_defs") diff --git a/src/parse/asp/label_context_test.go b/src/parse/asp/label_context_test.go index 9b99625bf..42d7fe6b9 100644 --- a/src/parse/asp/label_context_test.go +++ b/src/parse/asp/label_context_test.go @@ -10,10 +10,11 @@ import ( "github.com/thought-machine/please/src/core" ) -func newScope(pkgName, subrepo, plugin string) *scope { +func newScope(t *testing.T, pkgName, subrepo, plugin string) *scope { + t.Helper() s := &scope{ pkg: core.NewPackageSubrepo(pkgName, subrepo), - state: core.NewBuildState(core.DefaultConfiguration()), + state: core.NewBuildState(t.Context(), core.DefaultConfiguration()), } if plugin != "" { s.state.Config.PluginDefinition.Name = plugin @@ -33,7 +34,7 @@ func TestParseLabelContext(t *testing.T) { { testName: "Test parse absolute label with subrepo using @", label: "@other_subrepo//test:target", - scope: newScope("subrepo_package", "subrepo", ""), + scope: newScope(t, "subrepo_package", "subrepo", ""), subrepo: "other_subrepo", pkg: "test", name: "target", @@ -41,7 +42,7 @@ func TestParseLabelContext(t *testing.T) { { testName: "Test parse absolute label with subrepo using ///", label: "///other_subrepo//test:target", - scope: newScope("subrepo_package", "subrepo", ""), + scope: newScope(t, "subrepo_package", "subrepo", ""), subrepo: "other_subrepo", pkg: "test", name: "target", @@ -49,7 +50,7 @@ func TestParseLabelContext(t *testing.T) { { testName: "Test host reference using @", label: "@//test:target", - scope: newScope("subrepo_package", "subrepo", ""), + scope: newScope(t, "subrepo_package", "subrepo", ""), subrepo: "", pkg: "test", name: "target", @@ -57,7 +58,7 @@ func TestParseLabelContext(t *testing.T) { { testName: "Test host reference using ///", label: "/////test:target", - scope: newScope("subrepo_package", "subrepo", ""), + scope: newScope(t, "subrepo_package", "subrepo", ""), subrepo: "", pkg: "test", name: "target", @@ -65,7 +66,7 @@ func TestParseLabelContext(t *testing.T) { { testName: "Test label relative to subrepo", label: "//test:target", - scope: newScope("subrepo_package", "subrepo", ""), + scope: newScope(t, "subrepo_package", "subrepo", ""), subrepo: "subrepo", pkg: "test", name: "target", @@ -73,7 +74,7 @@ func TestParseLabelContext(t *testing.T) { { testName: "Test label relative to package in subrepo", label: ":target", - scope: newScope("subrepo_package", "subrepo", ""), + scope: newScope(t, "subrepo_package", "subrepo", ""), subrepo: "subrepo", pkg: "subrepo_package", name: "target", @@ -81,7 +82,7 @@ func TestParseLabelContext(t *testing.T) { { testName: "Test host arch is stripped", label: fmt.Sprintf("///%s//test:target", (&arch).String()), - scope: newScope("pkg", "", ""), + scope: newScope(t, "pkg", "", ""), subrepo: "", pkg: "test", name: "target", @@ -89,7 +90,7 @@ func TestParseLabelContext(t *testing.T) { { testName: "Test host arch is stripped from subrepo", label: fmt.Sprintf("///subrepo@%s//test:target", (&arch).String()), - scope: newScope("pkg", "", ""), + scope: newScope(t, "pkg", "", ""), subrepo: "subrepo", pkg: "test", name: "target", @@ -97,7 +98,7 @@ func TestParseLabelContext(t *testing.T) { { testName: "Test host arch is stripped from nested subrepo", label: "///foowin_amd64//test:target", - scope: newScope("pkg", "subrepo2@foowin_amd64", ""), + scope: newScope(t, "pkg", "subrepo2@foowin_amd64", ""), subrepo: "foowin_amd64", pkg: "test", name: "target", @@ -105,7 +106,7 @@ func TestParseLabelContext(t *testing.T) { { testName: "Test host plugin is stripped", label: "///foo//test:target", - scope: newScope("subrepo_package", "", "foo"), + scope: newScope(t, "subrepo_package", "", "foo"), subrepo: "", pkg: "test", name: "target", diff --git a/src/parse/asp/logging_test.go b/src/parse/asp/logging_test.go index fb8ac0bf1..791a9843a 100644 --- a/src/parse/asp/logging_test.go +++ b/src/parse/asp/logging_test.go @@ -5,6 +5,7 @@ package asp import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -21,7 +22,7 @@ type record struct { //nolint:unused } func parseFile2(filename string) (*scope, error) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(context.Background()) pkg := core.NewPackage("test/package") pkg.Filename = "test/package/BUILD" parser := NewParser(state) diff --git a/src/parse/asp/main/main.go b/src/parse/asp/main/main.go index 285feb63d..067d4ff24 100644 --- a/src/parse/asp/main/main.go +++ b/src/parse/asp/main/main.go @@ -4,6 +4,7 @@ package main import ( + "context" "fmt" iofs "io/fs" "os" @@ -223,7 +224,8 @@ func main() { } config.Please.NumThreads = opts.NumThreads - state := core.NewBuildState(config) + ctx := context.TODO() + state := core.NewBuildState(ctx, config) if opts.BuildDefsDir != "" { mustLoadBuildDefsDir(state, opts.BuildDefsDir) } diff --git a/src/parse/asp/targets_test.go b/src/parse/asp/targets_test.go index c70c85279..87e0c4124 100644 --- a/src/parse/asp/targets_test.go +++ b/src/parse/asp/targets_test.go @@ -9,7 +9,7 @@ import ( ) func TestValidateTargetNoSandbox(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) foo := core.NewBuildTarget(core.NewBuildLabel("pkg", "foo")) foo.Sandbox = false @@ -27,7 +27,7 @@ func TestValidateTargetNoSandbox(t *testing.T) { } func TestValidateTargetSandbox(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) state.Config.Sandbox.ExcludeableTargets = []core.BuildLabel{core.NewBuildLabel("third_party", "all")} foo := core.NewBuildTarget(core.NewBuildLabel("pkg", "foo")) diff --git a/src/parse/parse_step.go b/src/parse/parse_step.go index 2c9689fea..b9cbc0c5c 100644 --- a/src/parse/parse_step.go +++ b/src/parse/parse_step.go @@ -6,6 +6,7 @@ package parse import ( + "context" "errors" "fmt" iofs "io/fs" @@ -31,18 +32,18 @@ var ErrMissingBuildFile = errors.New("build file not found") // targets with at least one matching label are added. Any targets with a label in 'exclude' are not added. // 'forSubinclude' is set when the parse is required for a subinclude target so should proceed // even when we're not otherwise building targets. -func Parse(state *core.BuildState, label, dependent core.BuildLabel, mode core.ParseMode) { - if err := parse(state, label, dependent, mode); err != nil { +func Parse(ctx context.Context, state *core.BuildState, label, dependent core.BuildLabel, mode core.ParseMode) { + if err := parse(ctx, state, label, dependent, mode); err != nil { state.LogBuildError(label, core.ParseFailed, err, "Failed to parse package") } } -func parse(state *core.BuildState, label, dependent core.BuildLabel, mode core.ParseMode) error { +func parse(ctx context.Context, state *core.BuildState, label, dependent core.BuildLabel, mode core.ParseMode) error { if t := state.Graph.Target(label); t != nil && t.State() < core.Active { return state.ActivateTarget(nil, label, dependent, mode) } - subrepo, err := checkSubrepo(state, label, dependent, mode) + subrepo, err := checkSubrepo(ctx, state, label, dependent, mode) if err != nil { return err } @@ -104,7 +105,7 @@ func inSamePackage(label, dependent core.BuildLabel) bool { // The subrepo target can be inferred from the subrepo name using convention i.e. ///foo/bar//:baz has a subrepo label // //foo:bar. checkSubrepo parses package foo, expecting a call to `subrepo()` that registers a subrepo named foo/bar, // so it can return it. -func checkSubrepo(state *core.BuildState, label, dependent core.BuildLabel, mode core.ParseMode) (*core.Subrepo, error) { +func checkSubrepo(ctx context.Context, state *core.BuildState, label, dependent core.BuildLabel, mode core.ParseMode) (*core.Subrepo, error) { if label.Subrepo == "" { return nil, nil } @@ -126,14 +127,14 @@ func checkSubrepo(state *core.BuildState, label, dependent core.BuildLabel, mode } // Try parsing the package in the host repo first. - s, err := maybeParseSubrepoPackage(state, sl.PackageName, sl.Subrepo, label, mode) + s, err := maybeParseSubrepoPackage(ctx, state, sl.PackageName, sl.Subrepo, label, mode) if err != nil || s != nil { return s, err } if sl.Subrepo != dependent.Subrepo { // They may have meant a subrepo that was defined in the dependent label's subrepo rather than the host repo - s, err = maybeParseSubrepoPackage(state, sl.PackageName, dependent.Subrepo, label, mode) + s, err = maybeParseSubrepoPackage(ctx, state, sl.PackageName, dependent.Subrepo, label, mode) if err != nil || s != nil { return s, err } @@ -144,7 +145,7 @@ func checkSubrepo(state *core.BuildState, label, dependent core.BuildLabel, mode // maybeParseSubrepoPackage parses a package to make sure subrepos are available, returning the subrepo if it exists. // Returns nothing if the package doesn't exist, or the package doesn't define the subrepo. -func maybeParseSubrepoPackage(state *core.BuildState, subrepoPkg, subrepoSubrepo string, dependent core.BuildLabel, mode core.ParseMode) (*core.Subrepo, error) { +func maybeParseSubrepoPackage(ctx context.Context, state *core.BuildState, subrepoPkg, subrepoSubrepo string, dependent core.BuildLabel, mode core.ParseMode) (*core.Subrepo, error) { // First, check whether this is an architecture subrepo. The built-in architecture subrepos - which are implicitly // defined at the top level - should be registered regardless of whether a top-level BUILD file exists, so we need to // perform this check before we check whether the subrepo package exists. @@ -157,7 +158,7 @@ func maybeParseSubrepoPackage(state *core.BuildState, subrepoPkg, subrepoSubrepo if state.Graph.Package(subrepoPkg, subrepoSubrepo) == nil { // Don't have it already, must parse. label := core.BuildLabel{Subrepo: subrepoSubrepo, PackageName: subrepoPkg, Name: "all"} - if err := parse(state, label, dependent, mode|core.ParseModeForSubinclude); err != nil { + if err := parse(ctx, state, label, dependent, mode|core.ParseModeForSubinclude); err != nil { // When we try and parse a subrepo package, but the BUILD file or directory doesn't exist, return nil so // this gets handled later on, in the same way as when the package does exist but doesn't define the subrepo if errors.Is(err, ErrMissingBuildFile) { diff --git a/src/parse/parse_step_test.go b/src/parse/parse_step_test.go index d21034f54..042f45337 100644 --- a/src/parse/parse_step_test.go +++ b/src/parse/parse_step_test.go @@ -3,6 +3,7 @@ package parse import ( + "context" "testing" "time" @@ -94,7 +95,7 @@ func makeTarget(label string, deps ...string) *core.BuildTarget { // makeState creates a new build state with optionally one or two packages in it. // Used in various tests above. func makeState(withPackage1, withPackage2 bool) *core.BuildState { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(context.Background()) if withPackage1 { pkg := core.NewPackage("package1") state.Graph.AddPackage(pkg) diff --git a/src/please.go b/src/please.go index 26e16a02e..cb365235b 100644 --- a/src/please.go +++ b/src/please.go @@ -676,11 +676,12 @@ var buildFunctions = map[string]func() int{ return 1 }, "clean": func() int { + ctx := context.TODO() config.Cache.DirClean = false // don't run the normal cleaner if len(opts.Clean.Args.Targets) == 0 && core.InitialPackage()[0].PackageName == "" { if len(opts.BuildFlags.Include) == 0 && len(opts.BuildFlags.Exclude) == 0 { // Clean everything, doesn't require parsing at all. - state := core.NewBuildState(config) + state := core.NewBuildState(ctx, config) clean.Clean(config, cache.NewCache(state), !opts.Clean.NoBackground) return 0 } @@ -885,6 +886,7 @@ var buildFunctions = map[string]func() int{ }) }, "query.whatinputs": func() int { + ctx := context.TODO() files := opts.Query.WhatInputs.Args.Files.Get() // Make all these relative to the repo root; many things do not work if they're absolute. for i, file := range files { @@ -899,7 +901,7 @@ var buildFunctions = map[string]func() int{ } } // We only need this to retrieve the BuildFileName - state := core.NewBuildState(config) + state := core.NewBuildState(ctx, config) labels := make([]core.BuildLabel, 0, len(files)) for _, file := range files { labels = append(labels, core.FindOwningPackage(state, file)) @@ -1128,6 +1130,7 @@ func prettyOutput(interactiveOutput bool, plainOutput bool, verbosity cli.Verbos // Please starts & runs the main build process through to its completion. func Please(targets []core.BuildLabel, config *core.Configuration, shouldBuild, shouldTest bool) (bool, *core.BuildState) { + ctx := context.TODO() if opts.BuildFlags.NumThreads > 0 { config.Please.NumThreads = opts.BuildFlags.NumThreads config.Parse.NumThreads = opts.BuildFlags.NumThreads @@ -1139,7 +1142,7 @@ func Please(targets []core.BuildLabel, config *core.Configuration, shouldBuild, } else if debug || debugFailingTests { config.Build.Config = "dbg" } - state := core.NewBuildState(config) + state := core.NewBuildState(ctx, config) state.KeepGoing = opts.BehaviorFlags.KeepGoing state.VerifyHashes = !opts.BehaviorFlags.NoHashVerification // Only one of these two can be passed diff --git a/src/plz/plz.go b/src/plz/plz.go index 7c893a816..861467ed1 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -77,13 +77,14 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, config * } // Start up all the build workers + ctx := state.WorkerContext() var wg sync.WaitGroup wg.Add(2) go func() { for task := range parses { go func(task core.ParseTask) { state.Parses().Add(1) - parse.Parse(state, task.Label, task.Dependent, task.Mode) + parse.Parse(ctx, state, task.Label, task.Dependent, task.Mode) state.Parses().Add(-1) state.TaskDone() }(task) @@ -102,9 +103,9 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, config * switch task.Type { case core.TestTask: - test.Test(state, task.Target, isRemote, int(task.Run)) + test.Test(ctx, state, task.Target, isRemote, int(task.Run)) case core.BuildTask: - build.Build(state, task.Target, isRemote) + build.Build(ctx, state, task.Target, isRemote) } }(task) } @@ -131,10 +132,11 @@ func RunHost(targets []core.BuildLabel, state *core.BuildState) { // findOriginalTasks finds the original parse tasks for the original set of targets. func findOriginalTasks(state *core.BuildState, preTargets, targets []core.BuildLabel, arch cli.Arch) { + ctx := state.WorkerContext() if state.Config.Bazel.Compatibility && fs.FileExists("WORKSPACE") { // We have to parse the WORKSPACE file before anything else to understand subrepos. // This is a bit crap really since it inhibits parallelism for the first step. - parse.Parse(state, core.NewBuildLabel("workspace", "all"), core.OriginalTarget, core.ParseModeNormal) + parse.Parse(ctx, state, core.NewBuildLabel("workspace", "all"), core.OriginalTarget, core.ParseModeNormal) } if arch.Arch != "" && arch != cli.HostArch() { // Set up a new subrepo for this architecture. @@ -173,7 +175,8 @@ func findOriginalTaskSet(state *core.BuildState, targets []core.BuildLabel, addT } func queueTargetsForDebug(state *core.BuildState, target core.BuildLabel) { - parse.Parse(state, target, core.OriginalTarget, core.ParseModeNormal) + ctx := state.WorkerContext() + parse.Parse(ctx, state, target, core.OriginalTarget, core.ParseModeNormal) t := state.Graph.TargetOrDie(target) for _, tool := range t.AllDebugTools() { if l, ok := tool.Label(); ok { diff --git a/src/process/exec_other.go b/src/process/exec_other.go index 1f3902adc..d6846b14f 100644 --- a/src/process/exec_other.go +++ b/src/process/exec_other.go @@ -9,8 +9,8 @@ import ( ) // ExecCommand executes an external command. -// N.B. This does not start the command - the caller must handle that (or use one -// of the other functions which are higher-level interfaces). +// N.B. This does not start the command - the caller must handle that (or +// use one of the other functions which are higher-level interfaces). func (e *Executor) ExecCommand(sandbox SandboxConfig, foreground bool, command string, args ...string) *exec.Cmd { cmd := exec.Command(command, args...) cmd.SysProcAttr = &syscall.SysProcAttr{ diff --git a/src/process/process.go b/src/process/process.go index f76e15fce..8cfe19434 100644 --- a/src/process/process.go +++ b/src/process/process.go @@ -149,14 +149,14 @@ func runCommand(cmd *exec.Cmd, ch chan<- error) { // ExecWithTimeoutShell runs an external command within a Bash shell. // Other arguments are as ExecWithTimeout. // Note that the command is deliberately a single string. -func (e *Executor) ExecWithTimeoutShell(target Target, dir string, env []string, timeout time.Duration, showOutput, foreground bool, sandbox SandboxConfig, cmd string) ([]byte, []byte, error) { - return e.ExecWithTimeoutShellStdStreams(target, dir, env, timeout, showOutput, foreground, sandbox, cmd, false) +func (e *Executor) ExecWithTimeoutShell(ctx context.Context, target Target, dir string, env []string, timeout time.Duration, showOutput, foreground bool, sandbox SandboxConfig, cmd string) ([]byte, []byte, error) { + return e.ExecWithTimeoutShellStdStreams(ctx, target, dir, env, timeout, showOutput, foreground, sandbox, cmd, false) } // ExecWithTimeoutShellStdStreams is as ExecWithTimeoutShell but optionally attaches stdin to the subprocess. -func (e *Executor) ExecWithTimeoutShellStdStreams(target Target, dir string, env []string, timeout time.Duration, showOutput, foreground bool, sandbox SandboxConfig, cmd string, attachStdStreams bool) ([]byte, []byte, error) { +func (e *Executor) ExecWithTimeoutShellStdStreams(ctx context.Context, target Target, dir string, env []string, timeout time.Duration, showOutput, foreground bool, sandbox SandboxConfig, cmd string, attachStdStreams bool) ([]byte, []byte, error) { c := BashCommand("bash", cmd, target.ShouldExitOnError()) - return e.ExecWithTimeout(context.Background(), target, dir, env, timeout, showOutput, attachStdStreams, attachStdStreams, foreground, sandbox, c) + return e.ExecWithTimeout(ctx, target, dir, env, timeout, showOutput, attachStdStreams, attachStdStreams, foreground, sandbox, c) } // KillProcess kills a process, attempting to send it a SIGTERM first followed by a SIGKILL diff --git a/src/process/process_test.go b/src/process/process_test.go index 48519ab6c..4e8926ae2 100644 --- a/src/process/process_test.go +++ b/src/process/process_test.go @@ -29,7 +29,7 @@ func TestExecWithTimeoutDeadline(t *testing.T) { func TestExecWithTimeoutOutput(t *testing.T) { targ := &target{} - out, stderr, err := New().ExecWithTimeoutShell(targ, "", nil, 10*time.Second, false, false, NoSandbox, "echo hello") + out, stderr, err := New().ExecWithTimeoutShell(t.Context(), targ, "", nil, 10*time.Second, false, false, NoSandbox, "echo hello") assert.NoError(t, err) assert.Equal(t, "hello\n", string(out)) assert.Equal(t, "hello\n", string(stderr)) @@ -37,7 +37,7 @@ func TestExecWithTimeoutOutput(t *testing.T) { func TestExecWithTimeoutStderr(t *testing.T) { targ := &target{} - out, stderr, err := New().ExecWithTimeoutShell(targ, "", nil, 10*time.Second, false, false, NoSandbox, "echo hello 1>&2") + out, stderr, err := New().ExecWithTimeoutShell(t.Context(), targ, "", nil, 10*time.Second, false, false, NoSandbox, "echo hello 1>&2") assert.NoError(t, err) assert.Equal(t, "", string(out)) assert.Equal(t, "hello\n", string(stderr)) diff --git a/src/query/changes_test.go b/src/query/changes_test.go index ed4124b68..c762d35e1 100644 --- a/src/query/changes_test.go +++ b/src/query/changes_test.go @@ -11,8 +11,8 @@ import ( ) func TestDiffGraphs(t *testing.T) { - s1 := core.NewDefaultBuildState() - s2 := core.NewDefaultBuildState() + s1 := core.NewDefaultBuildState(t.Context()) + s2 := core.NewDefaultBuildState(t.Context()) t1 := addTarget(s1, "//src/query:changes", nil, "src/query/changes.go") t2 := addTarget(s2, "//src/query:changes", nil, "src/query/changes.go") addTarget(s1, "//src/query:changes_test", t1, "src/query/changes_test.go") @@ -29,8 +29,8 @@ func TestDiffGraphs(t *testing.T) { } func TestDiffGraphsIncludeNothing(t *testing.T) { - s1 := core.NewDefaultBuildState() - s2 := core.NewDefaultBuildState() + s1 := core.NewDefaultBuildState(t.Context()) + s2 := core.NewDefaultBuildState(t.Context()) t1 := addTarget(s1, "//src/core:core", nil, "src/core/core.go") t2 := addTarget(s1, "//src/query:changes", t1, "src/query/changes.go") addTarget(s1, "//src/query:changes_test", t2, "src/query/changes_test.go") @@ -41,8 +41,8 @@ func TestDiffGraphsIncludeNothing(t *testing.T) { } func TestDiffGraphsIncludeDirect(t *testing.T) { - s1 := core.NewDefaultBuildState() - s2 := core.NewDefaultBuildState() + s1 := core.NewDefaultBuildState(t.Context()) + s2 := core.NewDefaultBuildState(t.Context()) t1 := addTarget(s1, "//src/core:core", nil, "src/core/core.go") t2 := addTarget(s1, "//src/query:changes", t1, "src/query/changes.go") addTarget(s1, "//src/query:changes_test", t2, "src/query/changes_test.go") @@ -53,8 +53,8 @@ func TestDiffGraphsIncludeDirect(t *testing.T) { } func TestDiffGraphsLevel(t *testing.T) { - s1 := core.NewDefaultBuildState() - s2 := core.NewDefaultBuildState() + s1 := core.NewDefaultBuildState(t.Context()) + s2 := core.NewDefaultBuildState(t.Context()) t1 := addTarget(s1, "//src/core:core", nil, "src/core/core.go") t2 := addTarget(s1, "//src/query:changes", t1, "src/query/changes.go") t3 := addTarget(s1, "//src/query:changes_test", t2, "src/query/changes_test.go") @@ -67,8 +67,8 @@ func TestDiffGraphsLevel(t *testing.T) { } func TestDiffGraphsIncludeTransitive(t *testing.T) { - s1 := core.NewDefaultBuildState() - s2 := core.NewDefaultBuildState() + s1 := core.NewDefaultBuildState(t.Context()) + s2 := core.NewDefaultBuildState(t.Context()) t1 := addTarget(s1, "//src/core:core", nil, "src/core/core.go") t2 := addTarget(s1, "//src/query:changes", t1, "src/query/changes.go") addTarget(s1, "//src/query:changes_test", t2, "src/query/changes_test.go") @@ -79,14 +79,14 @@ func TestDiffGraphsIncludeTransitive(t *testing.T) { } func TestDiffGraphsStopsAtSubrepos(t *testing.T) { - s1 := core.NewDefaultBuildState() + s1 := core.NewDefaultBuildState(t.Context()) t1 := addTarget(s1, "//:modfile", nil, "go.mod") t2 := addTarget(s1, "//third_party/go:mod", t1) t3 := addTarget(s1, "///third_party/go/mod//:mod", nil) t3.Subrepo = core.NewSubrepo(s1, "go_mod", "third_party/go", t2, cli.Arch{}, false) addTarget(s1, "//src/core:core", t3) - s2 := core.NewDefaultBuildState() + s2 := core.NewDefaultBuildState(t.Context()) t1 = addTarget(s2, "//:modfile", nil, "go.mod") t2 = addTarget(s2, "//third_party/go:mod", t1) t3 = addTarget(s2, "///third_party/go/mod//:mod", nil) @@ -98,14 +98,14 @@ func TestDiffGraphsStopsAtSubrepos(t *testing.T) { } func TestDiffGraphsStillChecksTargetsInSubrepos(t *testing.T) { - s1 := core.NewDefaultBuildState() + s1 := core.NewDefaultBuildState(t.Context()) t1 := addTarget(s1, "//:modfile", nil, "go.mod") t2 := addTarget(s1, "//third_party/go:mod", t1) t3 := addTarget(s1, "///third_party/go/mod//:mod", nil) t3.Subrepo = core.NewSubrepo(s1, "go_mod", "third_party/go", t2, cli.Arch{}, false) addTarget(s1, "//src/core:core", t3) - s2 := core.NewDefaultBuildState() + s2 := core.NewDefaultBuildState(t.Context()) t1 = addTarget(s2, "//:modfile", nil, "go.mod") t2 = addTarget(s2, "//third_party/go:mod", t1) t3 = addTarget(s2, "///third_party/go/mod//:mod", nil, "test.go") @@ -119,7 +119,7 @@ func TestDiffGraphsStillChecksTargetsInSubrepos(t *testing.T) { } func TestChangesIncludesDataDirs(t *testing.T) { - s := core.NewDefaultBuildState() + s := core.NewDefaultBuildState(t.Context()) t1 := addTarget(s, "//src/core:core", nil, "src/core/core.go") t2 := addTarget(s, "//src/query:changes", t1, "src/query/changes.go") t3 := addTarget(s, "//src/query:changes_test", t2, "src/query/changes_test.go") @@ -128,8 +128,8 @@ func TestChangesIncludesDataDirs(t *testing.T) { } func TestSameToolHashNoChange(t *testing.T) { - s1 := core.NewDefaultBuildState() - s2 := core.NewDefaultBuildState() + s1 := core.NewDefaultBuildState(t.Context()) + s2 := core.NewDefaultBuildState(t.Context()) target := addTarget(s1, "//src/core:core", nil, "src/core/core.go") target.AddTool(core.SystemPathLabel{Name: "non-existent", Path: s1.Config.Path()}) target = addTarget(s2, "//src/core:core", nil, "src/core/core.go") @@ -138,7 +138,7 @@ func TestSameToolHashNoChange(t *testing.T) { } func TestChangesIncludesRootTarget(t *testing.T) { - s := core.NewDefaultBuildState() + s := core.NewDefaultBuildState(t.Context()) t1 := addTarget(s, "//:file", nil, "file.go") assert.EqualValues(t, []core.BuildLabel{t1.Label}, Changes(s, []string{"file.go"}, 0, false)) } diff --git a/src/query/deps_test.go b/src/query/deps_test.go index 8ff749484..719ccc089 100644 --- a/src/query/deps_test.go +++ b/src/query/deps_test.go @@ -10,7 +10,7 @@ import ( ) func TestQueryDeps(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) pkg1 := core.NewPackage("tools/performance") pkg2 := core.NewPackage("third_party/python") diff --git a/src/query/graph_test.go b/src/query/graph_test.go index 1b755ebca..c9d63c6a5 100644 --- a/src/query/graph_test.go +++ b/src/query/graph_test.go @@ -41,7 +41,7 @@ func TestQueryPackage(t *testing.T) { func makeGraph(t *testing.T) *core.BuildState { t.Helper() - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) graph := state.Graph pkg1 := core.NewPackage("package1") pkg1.AddTarget(makeTarget("//package1:target1")) diff --git a/src/query/print_test.go b/src/query/print_test.go index 9e68bf1a5..a8ce6ad35 100644 --- a/src/query/print_test.go +++ b/src/query/print_test.go @@ -2,6 +2,7 @@ package query import ( "bytes" + "context" "encoding/json" "testing" "time" @@ -13,7 +14,7 @@ import ( "github.com/thought-machine/please/src/parse" ) -var order = parse.BuildRuleArgOrder(core.NewDefaultBuildState()) +var order = parse.BuildRuleArgOrder(core.NewDefaultBuildState(context.Background())) func TestAllFieldsArePresentAndAccountedFor(t *testing.T) { target := &core.BuildTarget{} diff --git a/src/query/reverse_deps_test.go b/src/query/reverse_deps_test.go index 1f1f4cbbb..5c56bb333 100644 --- a/src/query/reverse_deps_test.go +++ b/src/query/reverse_deps_test.go @@ -9,7 +9,7 @@ import ( ) func TestReverseDeps(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) graph := state.Graph root := core.NewBuildTarget(core.ParseBuildLabel("//package:root", "")) @@ -35,7 +35,7 @@ func TestReverseDeps(t *testing.T) { } func TestReverseDepsWithHidden(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) graph := state.Graph foo := core.NewBuildTarget(core.ParseBuildLabel("//package:foo", "")) diff --git a/src/query/runtime_deps_test.go b/src/query/runtime_deps_test.go index 9a0ca1154..2401e40ea 100644 --- a/src/query/runtime_deps_test.go +++ b/src/query/runtime_deps_test.go @@ -86,7 +86,7 @@ func TestRuntimeDeps(t *testing.T) { }, } { t.Run(test.Description, func(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) testPkg := core.NewPackage("test") addTarget = func(name string, fromSrcs, fromDeps bool, srcs, deps, runtimeDeps []*core.BuildTarget) *core.BuildTarget { diff --git a/src/query/whatoutputs_test.go b/src/query/whatoutputs_test.go index 4dcc9dd83..fad77e6dd 100644 --- a/src/query/whatoutputs_test.go +++ b/src/query/whatoutputs_test.go @@ -1,6 +1,7 @@ package query import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -9,7 +10,7 @@ import ( ) func makeTarget2(g *core.BuildGraph, label string, filegroup bool, outputs ...string) *core.BuildTarget { - s := core.NewDefaultBuildState() + s := core.NewDefaultBuildState(context.Background()) l := core.ParseBuildLabel(label, "") t := core.NewBuildTarget(l) diff --git a/src/remote/impl_test.go b/src/remote/impl_test.go index 338871a67..8f342e995 100644 --- a/src/remote/impl_test.go +++ b/src/remote/impl_test.go @@ -28,11 +28,13 @@ import ( "github.com/thought-machine/please/src/core" ) -func newClient() *Client { - return newClientInstance("wibble") +func newClient(t *testing.T) *Client { + t.Helper() + return newClientInstance(t, "wibble") } -func newClientInstance(name string) *Client { +func newClientInstance(t *testing.T, name string) *Client { + t.Helper() config := core.DefaultConfiguration() config.Build.Path = []string{"/usr/local/bin", "/usr/bin", "/bin"} config.Build.HashFunction = "sha256" @@ -40,7 +42,7 @@ func newClientInstance(name string) *Client { config.Remote.Instance = name config.Remote.Secure = false config.Remote.Platform = []string{"OSFamily=linux"} - state := core.NewBuildState(config) + state := core.NewBuildState(t.Context(), config) state.Config.Remote.URL = "127.0.0.1:9987" state.Config.Remote.AssetURL = state.Config.Remote.URL state.Cache = cache.NewCache(state) diff --git a/src/remote/remote_test.go b/src/remote/remote_test.go index 05af2cd9b..833fc47ff 100644 --- a/src/remote/remote_test.go +++ b/src/remote/remote_test.go @@ -18,7 +18,7 @@ import ( ) func TestInit(t *testing.T) { - c := newClient() + c := newClient(t) assert.NoError(t, c.CheckInitialised()) } @@ -28,7 +28,7 @@ func TestBadAPIVersion(t *testing.T) { defer server.Reset() server.HighAPIVersion.Major = 1 server.LowAPIVersion.Major = 1 - c := newClient() + c := newClient(t) assert.Error(t, c.CheckInitialised()) assert.Contains(t, c.CheckInitialised().Error(), "1.0.0 - 1.1.0") } @@ -40,12 +40,12 @@ func TestUnsupportedDigest(t *testing.T) { pb.DigestFunction_SHA384, pb.DigestFunction_SHA512, } - c := newClient() + c := newClient(t) assert.Error(t, c.CheckInitialised()) } func TestExecuteBuild(t *testing.T) { - c := newClient() + c := newClient(t) target := core.NewBuildTarget(core.BuildLabel{PackageName: "package", Name: "target2"}) target.AddSource(core.FileLabel{File: "src1.txt", Package: "package"}) target.AddSource(core.FileLabel{File: "src2.txt", Package: "package"}) @@ -72,7 +72,7 @@ func (f postBuildFunction) String() string { return "" } func TestExecutePostBuildFunction(t *testing.T) { t.Skip("Post-build function currently triggered at a higher level") - c := newClient() + c := newClient(t) target := core.NewBuildTarget(core.BuildLabel{PackageName: "package", Name: "target5"}) target.BuildTimeout = time.Minute target.Command = "echo 'wibble wibble wibble' | tee file7" @@ -87,7 +87,7 @@ func TestExecutePostBuildFunction(t *testing.T) { } func TestExecuteFetch(t *testing.T) { - c := newClient() + c := newClient(t) target := core.NewBuildTarget(core.BuildLabel{PackageName: "package", Name: "remote1"}) target.IsRemoteFile = true target.AddSource(core.URLLabel("https://get.please.build/linux_amd64/14.2.0/please_14.2.0.tar.gz")) @@ -99,7 +99,7 @@ func TestExecuteFetch(t *testing.T) { } func TestExecuteTest(t *testing.T) { - c := newClientInstance("test") + c := newClientInstance(t, "test") target := core.NewBuildTarget(core.BuildLabel{PackageName: "package", Name: "target3"}) target.AddOutput("remote_test") target.Test = new(core.TestFields) @@ -120,7 +120,7 @@ func TestExecuteTest(t *testing.T) { } func TestExecuteTestWithCoverage(t *testing.T) { - c := newClientInstance("test") + c := newClientInstance(t, "test") c.state.NeedCoverage = true // bit of a hack but we need to turn this on somehow target := core.NewBuildTarget(core.BuildLabel{PackageName: "package", Name: "target4"}) target.AddOutput("remote_test") @@ -158,7 +158,7 @@ src/core/build_target.go:177.44,179.2 1 0 `) func TestNoAbsolutePaths(t *testing.T) { - c := newClientInstance("test") + c := newClientInstance(t, "test") tool := core.NewBuildTarget(core.BuildLabel{PackageName: "package", Name: "tool"}) tool.AddOutput("bin") c.state.Graph.AddTarget(tool) @@ -178,7 +178,7 @@ func TestNoAbsolutePaths(t *testing.T) { } func TestNoAbsolutePaths2(t *testing.T) { - c := newClientInstance("test") + c := newClientInstance(t, "test") tool := core.NewBuildTarget(core.BuildLabel{PackageName: "package", Name: "tool"}) tool.AddOutput("bin") c.state.Graph.AddTarget(tool) @@ -195,7 +195,7 @@ func TestNoAbsolutePaths2(t *testing.T) { } func TestRemoteFilesHashConsistently(t *testing.T) { - c := newClientInstance("test") + c := newClientInstance(t, "test") target := core.NewBuildTarget(core.BuildLabel{PackageName: "package", Name: "download"}) target.IsRemoteFile = true target.AddSource(core.URLLabel("https://localhost/file")) @@ -211,7 +211,7 @@ func TestRemoteFilesHashConsistently(t *testing.T) { } func TestOutDirsSetOutsOnTarget(t *testing.T) { - c := newClientInstance("mock") + c := newClientInstance(t, "mock") foo := []byte("this is the content of foo") fooDigest := digest.NewFromBlob(foo) @@ -315,7 +315,7 @@ func TestDirectoryMetadataStore(t *testing.T) { } func TestTargetPlatform(t *testing.T) { - c := newClientInstance("platform_test") + c := newClientInstance(t, "platform_test") c.platform = convertPlatform(c.state.Config.Remote.Platform) // Bit of a hack but we can't go through the normal path. target := core.NewBuildTarget(core.BuildLabel{PackageName: "package", Name: "target"}) cmd, err := c.buildCommand(target, &pb.Directory{}, false, false, false, 0) diff --git a/src/run/run_test.go b/src/run/run_test.go index 4cdcccfe1..7f22a6925 100644 --- a/src/run/run_test.go +++ b/src/run/run_test.go @@ -18,7 +18,7 @@ func init() { } func TestSequential(t *testing.T) { - state, labels1, labels2 := makeState(core.DefaultConfiguration()) + state, labels1, labels2 := makeState(t, core.DefaultConfiguration()) code := Sequential(state, labels1, nil, process.Quiet, false, false, false, "") assert.Equal(t, 0, code) code = Sequential(state, labels2, nil, process.Default, false, false, false, "") @@ -26,7 +26,7 @@ func TestSequential(t *testing.T) { } func TestParallel(t *testing.T) { - state, labels1, labels2 := makeState(core.DefaultConfiguration()) + state, labels1, labels2 := makeState(t, core.DefaultConfiguration()) code := Parallel(context.Background(), state, labels1, nil, 5, process.Default, false, false, false, false, "") assert.Equal(t, 0, code) code = Parallel(context.Background(), state, labels2, nil, 5, process.Quiet, false, false, false, false, "") @@ -36,7 +36,7 @@ func TestParallel(t *testing.T) { func TestEnvVars(t *testing.T) { config := core.DefaultConfiguration() config.Build.Path = []string{"/wibble"} - state, lab1, _ := makeState(config) + state, lab1, _ := makeState(t, config) t.Setenv("PATH", "/usr/local/bin:/usr/bin:/bin") env := environ(state, state.Graph.TargetOrDie(lab1[0].BuildLabel), false, false) @@ -47,8 +47,9 @@ func TestEnvVars(t *testing.T) { assert.Contains(t, env, "PATH=:/wibble", env) } -func makeState(config *core.Configuration) (*core.BuildState, []core.AnnotatedOutputLabel, []core.AnnotatedOutputLabel) { - state := core.NewBuildState(config) +func makeState(t *testing.T, config *core.Configuration) (*core.BuildState, []core.AnnotatedOutputLabel, []core.AnnotatedOutputLabel) { + t.Helper() + state := core.NewBuildState(t.Context(), config) target1 := core.NewBuildTarget(core.ParseBuildLabel("//:true", "")) target1.IsBinary = true target1.AddOutput("true") diff --git a/src/test/coverage_test.go b/src/test/coverage_test.go index ad64fddf5..8d33dfbde 100644 --- a/src/test/coverage_test.go +++ b/src/test/coverage_test.go @@ -221,7 +221,7 @@ func TestIstanbulCoverage2(t *testing.T) { } func TestIncrementalStats(t *testing.T) { - state := core.NewDefaultBuildState() + state := core.NewDefaultBuildState(t.Context()) state.Config.Cover.FileExtension = []string{".go"} cov := core.TestCoverage{ Files: map[string][]core.LineCoverage{ diff --git a/src/test/test_step.go b/src/test/test_step.go index 02788964a..8d5d03663 100644 --- a/src/test/test_step.go +++ b/src/test/test_step.go @@ -33,7 +33,7 @@ var numUploadFailures int64 const maxUploadFailures int64 = 10 // Test runs the tests for a single target. -func Test(state *core.BuildState, target *core.BuildTarget, remote bool, run int) { +func Test(ctx context.Context, state *core.BuildState, target *core.BuildTarget, remote bool, run int) { // Defer this so that no matter what happens in this test run, we always call target.CompleteRun defer func() { runsAllCompleted := target.CompleteRun(state) @@ -51,10 +51,10 @@ func Test(state *core.BuildState, target *core.BuildTarget, remote bool, run int }() state.LogTestRunning(target, run, core.TargetTesting, "Testing...") - test(state.ForTarget(target), target.Label, target, remote, run) + test(ctx, state.ForTarget(target), target.Label, target, remote, run) } -func test(state *core.BuildState, label core.BuildLabel, target *core.BuildTarget, runRemotely bool, run int) { +func test(ctx context.Context, state *core.BuildState, label core.BuildLabel, target *core.BuildTarget, runRemotely bool, run int) { target.StartTestSuite() hash, err := runtimeHash(state, target, runRemotely, run) @@ -186,7 +186,7 @@ func test(state *core.BuildState, label core.BuildLabel, target *core.BuildTarge coverage := &core.TestCoverage{} if state.NumTestRuns == 1 { var results core.TestSuite - results, coverage = doFlakeRun(state, target, run, runRemotely) + results, coverage = doFlakeRun(ctx, state, target, run, runRemotely) target.AddTestResults(results) if target.Test.Results.TestCases.AllSucceeded() { @@ -197,13 +197,13 @@ func test(state *core.BuildState, label core.BuildLabel, target *core.BuildTarge for run := 1; run <= int(state.NumTestRuns); run++ { state.LogTestRunning(target, run, core.TargetTesting, "Testing...") var results core.TestSuite - results, coverage = doTest(state, target, runRemotely, 1) // Sequential tests re-use run 1's test dir + results, coverage = doTest(ctx, state, target, runRemotely, 1) // Sequential tests re-use run 1's test dir target.AddTestResults(results) } } else { state.LogTestRunning(target, run, core.TargetTesting, "Testing...") var results core.TestSuite - results, coverage = doTest(state, target, runRemotely, run) + results, coverage = doTest(ctx, state, target, runRemotely, run) target.AddTestResults(results) } @@ -231,7 +231,7 @@ func retrieveFromCache(state *core.BuildState, target *core.BuildTarget, hash [] } // doFlakeRun runs a test repeatably until it succeeds or exceeds the max number of flakes for the test -func doFlakeRun(state *core.BuildState, target *core.BuildTarget, run int, runRemotely bool) (core.TestSuite, *core.TestCoverage) { +func doFlakeRun(ctx context.Context, state *core.BuildState, target *core.BuildTarget, run int, runRemotely bool) (core.TestSuite, *core.TestCoverage) { coverage := &core.TestCoverage{} results := core.TestSuite{} @@ -239,7 +239,7 @@ func doFlakeRun(state *core.BuildState, target *core.BuildTarget, run int, runRe for flakes := 1; flakes <= int(target.Test.Flakiness); flakes++ { state.LogTestRunning(target, run, core.TargetTesting, getFlakeStatus(flakes, int(target.Test.Flakiness))) - testSuite, cov := doTest(state, target, runRemotely, 1) // If we're running flakes, numRuns must be 1 + testSuite, cov := doTest(ctx, state, target, runRemotely, 1) // If we're running flakes, numRuns must be 1 results.TimedOut = results.TimedOut || testSuite.TimedOut results.Properties = testSuite.Properties @@ -349,19 +349,19 @@ func testCommandAndEnv(state *core.BuildState, target *core.BuildTarget, run int return replacedCmd, env, err } -func runTest(state *core.BuildState, target *core.BuildTarget, run int) ([]byte, error) { +func runTest(ctx context.Context, state *core.BuildState, target *core.BuildTarget, run int) ([]byte, error) { replacedCmd, env, err := testCommandAndEnv(state, target, run) if err != nil { return nil, err } log.Debugf("Running test %s#%d\nENVIRONMENT:\n%s\n%s", target.Label, run, env, replacedCmd) - _, stderr, err := state.ProcessExecutor.ExecWithTimeoutShellStdStreams(target, target.TestDir(run), env.ToSlice(), target.Test.Timeout, state.ShowAllOutput, false, process.NewSandboxConfig(target.Test.Sandbox, target.Test.Sandbox), replacedCmd, state.DebugFailingTests) + _, stderr, err := state.ProcessExecutor.ExecWithTimeoutShellStdStreams(ctx, target, target.TestDir(run), env.ToSlice(), target.Test.Timeout, state.ShowAllOutput, false, process.NewSandboxConfig(target.Test.Sandbox, target.Test.Sandbox), replacedCmd, state.DebugFailingTests) return stderr, err } -func doTest(state *core.BuildState, target *core.BuildTarget, runRemotely bool, run int) (core.TestSuite, *core.TestCoverage) { +func doTest(ctx context.Context, state *core.BuildState, target *core.BuildTarget, runRemotely bool, run int) (core.TestSuite, *core.TestCoverage) { startTime := time.Now() - metadata, resultsData, coverage, err := doTestResults(state, target, runRemotely, run) + metadata, resultsData, coverage, err := doTestResults(ctx, state, target, runRemotely, run) duration := time.Since(startTime) parsedSuite := parseTestOutput(string(metadata.Stdout), string(metadata.Stderr), err, duration, target, resultsData) return core.TestSuite{ @@ -375,7 +375,7 @@ func doTest(state *core.BuildState, target *core.BuildTarget, runRemotely bool, }, coverage } -func doTestResults(state *core.BuildState, target *core.BuildTarget, runRemotely bool, run int) (*core.BuildMetadata, [][]byte, *core.TestCoverage, error) { +func doTestResults(ctx context.Context, state *core.BuildState, target *core.BuildTarget, runRemotely bool, run int) (*core.BuildMetadata, [][]byte, *core.TestCoverage, error) { var err error var metadata *core.BuildMetadata @@ -386,7 +386,7 @@ func doTestResults(state *core.BuildState, target *core.BuildTarget, runRemotely } } else { var stdout []byte - stdout, err = prepareAndRunTest(state, target, run) + stdout, err = prepareAndRunTest(ctx, state, target, run) metadata = &core.BuildMetadata{Stdout: stdout} } @@ -409,12 +409,12 @@ func doTestResults(state *core.BuildState, target *core.BuildTarget, runRemotely } // prepareAndRunTest sets up a test directory and runs the test. -func prepareAndRunTest(state *core.BuildState, target *core.BuildTarget, run int) (stdout []byte, err error) { +func prepareAndRunTest(ctx context.Context, state *core.BuildState, target *core.BuildTarget, run int) (stdout []byte, err error) { if err = core.PrepareRuntimeDir(state, target, target.TestDir(run)); err != nil { state.LogBuildError(target.Label, core.TargetTestFailed, err, "Failed to prepare test directory for %s: %s", target.Label, err) return []byte{}, err } - return runTest(state, target, run) + return runTest(ctx, state, target, run) } func parseTestOutput(stdout string, stderr string, runError error, duration time.Duration, target *core.BuildTarget, resultsData [][]byte) core.TestSuite { diff --git a/src/watch/watch.go b/src/watch/watch.go index 3d7112f59..9a5ebef27 100644 --- a/src/watch/watch.go +++ b/src/watch/watch.go @@ -163,7 +163,7 @@ func anyTests(state *core.BuildState, labels []core.BuildLabel) bool { // build invokes a single build while watching. func build(ctx context.Context, state *core.BuildState, labels []core.BuildLabel, args []string, callback CallbackFunc) { // Set up a new state & copy relevant parts off the existing one. - ns := core.NewBuildState(state.Config) + ns := core.NewBuildState(ctx, state.Config) ns.Cache = state.Cache ns.VerifyHashes = state.VerifyHashes ns.NumTestRuns = state.NumTestRuns diff --git a/test/audit/BUILD b/test/audit/BUILD index b47d6e00c..e60abd6fb 100644 --- a/test/audit/BUILD +++ b/test/audit/BUILD @@ -2,6 +2,6 @@ subinclude("//test/build_defs") please_repo_e2e_test( name = "audit_test", - repo = "test_repo", plz_command = "./test_audit.sh", + repo = "test_repo", ) diff --git a/test/get_labels/BUILD b/test/get_labels/BUILD index e7c03693b..3490bb0dc 100644 --- a/test/get_labels/BUILD +++ b/test/get_labels/BUILD @@ -4,10 +4,10 @@ def maxdepth_test(name:str, deps:list=None, maxdepth:int, expected:list): test_case = genrule( name = name, outs = [f"{name}.sh"], + binary = True, cmd = { "opt": "echo '#!/bin/sh' > $OUTS", }, - binary = True, labels = [ f"name:{name}", "manual", @@ -41,63 +41,72 @@ def dep(name:str, deps:list=None): def echo_name_labels_up_to(maxdepth:int): def echo(name:str): - labels = get_labels(name, "name:", maxdepth=maxdepth) + labels = get_labels(name, "name:", maxdepth = maxdepth) set_command(name, "opt", " && ".join([get_command(name, "opt")] + [f"echo 'echo {l}' >> $OUTS" for l in labels])) + return echo -dep(name = "dep1", deps = [":dep3"]) +dep( + name = "dep1", + deps = [":dep3"], +) + dep(name = "dep2") -dep(name = "dep3", deps = [":dep4", ":dep5"]) + +dep( + name = "dep3", + deps = [ + ":dep4", + ":dep5", + ], +) + dep(name = "dep4") + dep(name = "dep5") maxdepth_test( name = "target_only", + expected = ["target_only"], + maxdepth = 0, deps = [ ":dep1", ":dep2", ], - maxdepth = 0, - expected = ["target_only"], ) maxdepth_test( name = "direct_deps", - deps = [ - ":dep1", - ":dep2", - ], - maxdepth = 1, expected = [ "dep1", "dep2", "direct_deps", ], -) - -maxdepth_test( - name = "second_level_deps", + maxdepth = 1, deps = [ ":dep1", ":dep2", - ":dep3", ], - maxdepth = 2, +) + +maxdepth_test( + name = "second_level_deps", expected = [ "dep1", "dep2", "dep3", "second_level_deps", ], -) - -maxdepth_test( - name = "all_deps", + maxdepth = 2, deps = [ ":dep1", ":dep2", + ":dep3", ], - maxdepth = -1, +) + +maxdepth_test( + name = "all_deps", expected = [ "all_deps", "dep1", @@ -106,4 +115,9 @@ maxdepth_test( "dep4", "dep5", ], + maxdepth = -1, + deps = [ + ":dep1", + ":dep2", + ], ) diff --git a/test/runtime_deps/BUILD b/test/runtime_deps/BUILD index 8f07efff6..6fbf840ea 100644 --- a/test/runtime_deps/BUILD +++ b/test/runtime_deps/BUILD @@ -13,14 +13,14 @@ please_repo_e2e_test( # print the target's direct run-time dependencies. please_repo_e2e_test( name = "query_print_test", - plz_command = " && ".join([ - "plz query print -f runtime_deps //test:runtime_deps_test_case > runtime_deps_test_case", - "plz query print -f runtime_deps //test:target_with_no_runtime_deps > target_with_no_runtime_deps", - ]), expected_output = { "runtime_deps_test_case": "//test:target_with_runtime_deps", "target_with_no_runtime_deps": "", }, + plz_command = " && ".join([ + "plz query print -f runtime_deps //test:runtime_deps_test_case > runtime_deps_test_case", + "plz query print -f runtime_deps //test:target_with_no_runtime_deps > target_with_no_runtime_deps", + ]), repo = "repo", ) @@ -50,20 +50,20 @@ _expected_deps = """\ please_repo_e2e_test( name = "query_deps_test", - plz_command = "plz query deps //test:runtime_deps_test_case > deps", expected_output = { "deps": _expected_deps, }, + plz_command = "plz query deps //test:runtime_deps_test_case > deps", repo = "repo", ) # Ensure that run-time dependencies are in fact considered dependencies by `plz query revdeps`. please_repo_e2e_test( name = "query_revdeps_test", - plz_command = "plz query revdeps //test:another_runtime_dep > revdeps", expected_output = { "revdeps": "//test:target_with_another_runtime_dep", }, + plz_command = "plz query revdeps //test:another_runtime_dep > revdeps", repo = "repo", ) diff --git a/test/source_list_files/BUILD b/test/source_list_files/BUILD index b6d95ab7e..95b835bf5 100644 --- a/test/source_list_files/BUILD +++ b/test/source_list_files/BUILD @@ -2,34 +2,34 @@ subinclude("//test/build_defs") please_repo_e2e_test( name = "srcs_test", - plz_command = "plz build //:with_srcs", expected_output = { - "plz-out/gen/with_srcs.txt": "a.txt\nb.txt", + "plz-out/gen/with_srcs.txt": "a.txt\nb.txt", }, + plz_command = "plz build //:with_srcs", repo = "test_repo", ) please_repo_e2e_test( name = "src_test", - plz_command = "plz build //:with_one_src", expected_output = { - "plz-out/gen/with_one_src.txt": "a.txt", + "plz-out/gen/with_one_src.txt": "a.txt", }, + plz_command = "plz build //:with_one_src", repo = "test_repo", ) please_repo_e2e_test( name = "named_srcs_test", - plz_command = "plz build //:with_named_srcs", expected_output = { - "plz-out/gen/with_named_srcs.txt": "a.txt\nb.txt", + "plz-out/gen/with_named_srcs.txt": "a.txt\nb.txt", }, + plz_command = "plz build //:with_named_srcs", repo = "test_repo", ) please_repo_e2e_test( name = "flag_not_set_test", + expected_failure = True, plz_command = "plz build //:with_flag_not_set", repo = "test_repo", - expected_failure = True, ) diff --git a/test/stamp/lib/lib.go b/test/stamp/lib/lib.go index be5578593..5512cb871 100644 --- a/test/stamp/lib/lib.go +++ b/test/stamp/lib/lib.go @@ -2,7 +2,7 @@ package lib // vars that will be overridden at build time with actual git data. // N.B. Must be a variable not a constant - constants aren't linker symbols and -// hence can't be replaced in the same way. +// hence can't be replaced in the same way. var ( GitRevision = "12345-revision" GitDescribe = "12345-describe" diff --git a/tools/build_langserver/lsp/lsp.go b/tools/build_langserver/lsp/lsp.go index b97936017..f567b91de 100644 --- a/tools/build_langserver/lsp/lsp.go +++ b/tools/build_langserver/lsp/lsp.go @@ -193,7 +193,8 @@ func (h *Handler) initialize(params *lsp.InitializeParams) (*lsp.InitializeResul log.Error("Error reading configuration: %s", err) config = core.DefaultConfiguration() } - h.state = core.NewBuildState(config) + ctx := context.TODO() + h.state = core.NewBuildState(ctx, config) h.state.NeedBuild = false // We need an unwrapped parser instance as well for raw access. h.parser = asp.NewParser(h.state)