Examples with Executor com.google.devtools.build.lib.actions.Executor used on opensource projects

Example 1 with Executor

use of com.google.devtools.build.lib.actions.Executor in project bazel by bazelbuild.

the class StandaloneTestStrategy method exec.

@Override
public void exec(TestRunnerAction action, ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException {
    Path execRoot = actionExecutionContext.getExecutor().getExecRoot();
    Path coverageDir = execRoot.getRelative(action.getCoverageDirectory());
    Path runfilesDir = getLocalRunfilesDirectory(action, actionExecutionContext, binTools, action.getLocalShellEnvironment(), action.isEnableRunfiles());
    Path tmpDir = tmpDirRoot.getChild(getTmpDirName(action.getExecutionSettings().getExecutable().getExecPath()));
    Map<String, String> env = setupEnvironment(action, execRoot, runfilesDir, tmpDir);
    Path workingDirectory = runfilesDir.getRelative(action.getRunfilesPrefix());
    ResolvedPaths resolvedPaths = action.resolve(execRoot);
    Map<String, String> info = new HashMap<>();
    // This key is only understood by StandaloneSpawnStrategy.
    info.put("timeout", "" + getTimeout(action));
    info.putAll(action.getTestProperties().getExecutionInfo());
    Spawn spawn = new SimpleSpawn(action, getArgs(COLLECT_COVERAGE, action), ImmutableMap.copyOf(env), ImmutableMap.copyOf(info), new RunfilesSupplierImpl(runfilesDir.asFragment(), action.getExecutionSettings().getRunfiles()), /*inputs=*/
    ImmutableList.copyOf(action.getInputs()), /*tools=*/
    ImmutableList.<Artifact>of(), /*filesetManifests=*/
    ImmutableList.<Artifact>of(), ImmutableList.copyOf(action.getSpawnOutputs()), action.getTestProperties().getLocalResourceUsage(executionOptions.usingLocalTestJobs()));
    Executor executor = actionExecutionContext.getExecutor();
    TestResultData.Builder dataBuilder = TestResultData.newBuilder();
    try {
        int maxAttempts = getTestAttempts(action);
        TestResultData data = executeTestAttempt(action, spawn, actionExecutionContext, execRoot, coverageDir, tmpDir, workingDirectory);
        int attempt;
        for (attempt = 1; data.getStatus() != BlazeTestStatus.PASSED && attempt < maxAttempts; attempt++) {
            processFailedTestAttempt(attempt, executor, action, dataBuilder, data, actionExecutionContext.getFileOutErr());
            data = executeTestAttempt(action, spawn, actionExecutionContext, execRoot, coverageDir, tmpDir, workingDirectory);
        }
        processLastTestAttempt(attempt, dataBuilder, data);
        ImmutableList.Builder<Pair<String, Path>> testOutputsBuilder = new ImmutableList.Builder<>();
        if (action.getTestLog().getPath().exists()) {
            testOutputsBuilder.add(Pair.of("test.log", action.getTestLog().getPath()));
        }
        if (resolvedPaths.getXmlOutputPath().exists()) {
            testOutputsBuilder.add(Pair.of("test.xml", resolvedPaths.getXmlOutputPath()));
        }
        executor.getEventBus().post(new TestAttempt(action, attempt, data.getTestPassed(), data.getRunDurationMillis(), testOutputsBuilder.build(), true));
        finalizeTest(actionExecutionContext, action, dataBuilder.build());
    } catch (IOException e) {
        executor.getEventHandler().handle(Event.error("Caught I/O exception: " + e));
        throw new EnvironmentalExecException("unexpected I/O exception", e);
    }
}

Example 2 with Executor

use of com.google.devtools.build.lib.actions.Executor in project bazel by bazelbuild.

the class WorkerSpawnStrategy method exec.

@Override
public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext, AtomicReference<Class<? extends SpawnActionContext>> writeOutputFiles) throws ExecException, InterruptedException {
    Executor executor = actionExecutionContext.getExecutor();
    if (!spawn.getExecutionInfo().containsKey("supports-workers") || !spawn.getExecutionInfo().get("supports-workers").equals("1")) {
        StandaloneSpawnStrategy standaloneStrategy = Preconditions.checkNotNull(executor.getContext(StandaloneSpawnStrategy.class));
        executor.getEventHandler().handle(Event.warn(String.format(ERROR_MESSAGE_PREFIX + REASON_NO_EXECUTION_INFO, spawn.getMnemonic())));
        standaloneStrategy.exec(spawn, actionExecutionContext);
        return;
    }
    EventBus eventBus = actionExecutionContext.getExecutor().getEventBus();
    ActionExecutionMetadata owner = spawn.getResourceOwner();
    eventBus.post(ActionStatusMessage.schedulingStrategy(owner));
    try (ResourceHandle handle = ResourceManager.instance().acquireResources(owner, spawn.getLocalResources())) {
        eventBus.post(ActionStatusMessage.runningStrategy(spawn.getResourceOwner(), "worker"));
        actuallyExec(spawn, actionExecutionContext, writeOutputFiles);
    }
}

Example 3 with Executor

use of com.google.devtools.build.lib.actions.Executor in project bazel by bazelbuild.

the class WorkerTestStrategy method execInWorker.

private TestResultData execInWorker(TestRunnerAction action, ActionExecutionContext actionExecutionContext, Map<String, String> environment, List<String> startupArgs, Path execRoot, int retriesLeft) throws ExecException, InterruptedException, IOException {
    Executor executor = actionExecutionContext.getExecutor();
    // TODO(kush): Remove once we're out of the experimental phase.
    executor.getEventHandler().handle(Event.warn("RUNNING TEST IN AN EXPERIMENTAL PERSISTENT WORKER. RESULTS MAY BE INACCURATE"));
    TestResultData.Builder builder = TestResultData.newBuilder();
    Path testLogPath = action.getTestLog().getPath();
    Worker worker = null;
    WorkerKey key = null;
    long startTime = executor.getClock().currentTimeMillis();
    try {
        HashCode workerFilesHash = WorkerFilesHash.getWorkerFilesHash(action.getTools(), actionExecutionContext);
        key = new WorkerKey(startupArgs, environment, execRoot, action.getMnemonic(), workerFilesHash, ImmutableMap.<PathFragment, Path>of(), ImmutableSet.<PathFragment>of(), /*mustBeSandboxed=*/
        false);
        worker = workerPool.borrowObject(key);
        WorkRequest request = WorkRequest.getDefaultInstance();
        request.writeDelimitedTo(worker.getOutputStream());
        worker.getOutputStream().flush();
        WorkResponse response = WorkResponse.parseDelimitedFrom(worker.getInputStream());
        actionExecutionContext.getFileOutErr().getErrorStream().write(response.getOutputBytes().toByteArray());
        long duration = executor.getClock().currentTimeMillis() - startTime;
        builder.addTestTimes(duration);
        builder.setRunDurationMillis(duration);
        if (response.getExitCode() == 0) {
            builder.setTestPassed(true).setStatus(BlazeTestStatus.PASSED).setCachable(true).setPassedLog(testLogPath.getPathString());
        } else {
            builder.setTestPassed(false).setStatus(BlazeTestStatus.FAILED).addFailedLogs(testLogPath.getPathString());
        }
        TestCase details = parseTestResult(action.resolve(actionExecutionContext.getExecutor().getExecRoot()).getXmlOutputPath());
        if (details != null) {
            builder.setTestCase(details);
        }
        return builder.build();
    } catch (IOException | InterruptedException e) {
        if (e instanceof InterruptedException) {
            // The user pressed Ctrl-C. Get out here quick.
            retriesLeft = 0;
        }
        if (worker != null) {
            workerPool.invalidateObject(key, worker);
            worker = null;
        }
        if (retriesLeft > 0) {
            // The worker process failed, but we still have some retries left. Let's retry with a fresh
            // worker.
            executor.getEventHandler().handle(Event.warn(key.getMnemonic() + " worker failed (" + e + "), invalidating and retrying with new worker..."));
            return execInWorker(action, actionExecutionContext, environment, startupArgs, execRoot, retriesLeft - 1);
        } else {
            throw new TestExecException(e.getMessage());
        }
    } finally {
        if (worker != null) {
            workerPool.returnObject(key, worker);
        }
    }
}

Example 4 with Executor

use of com.google.devtools.build.lib.actions.Executor in project bazel by bazelbuild.

the class CppLinkAction method execute.

@Override
@ThreadCompatible
public void execute(ActionExecutionContext actionExecutionContext) throws ActionExecutionException, InterruptedException {
    if (fake) {
        executeFake();
    } else {
        Executor executor = actionExecutionContext.getExecutor();
        try {
            // Collect input files
            List<ActionInput> allInputs = new ArrayList<>();
            Artifact.addExpandedArtifacts(getMandatoryInputs(), allInputs, actionExecutionContext.getArtifactExpander());
            ImmutableMap<String, String> executionInfo = ImmutableMap.of();
            if (needsToRunOnMac()) {
                executionInfo = ImmutableMap.of(ExecutionRequirements.REQUIRES_DARWIN, "");
            }
            Spawn spawn = new SimpleSpawn(this, ImmutableList.copyOf(getCommandLine()), getEnvironment(), executionInfo, ImmutableList.copyOf(allInputs), getOutputs().asList(), estimateResourceConsumptionLocal());
            executor.getSpawnActionContext(getMnemonic()).exec(spawn, actionExecutionContext);
        } catch (ExecException e) {
            throw e.toActionExecutionException("Linking of rule '" + getOwner().getLabel() + "'", executor.getVerboseFailures(), this);
        }
    }
}

Example 5 with Executor

use of com.google.devtools.build.lib.actions.Executor in project bazel by bazelbuild.

the class FakeCppCompileAction method execute.

@Override
@ThreadCompatible
public void execute(ActionExecutionContext actionExecutionContext) throws ActionExecutionException, InterruptedException {
    setModuleFileFlags();
    Executor executor = actionExecutionContext.getExecutor();
    // First, do a normal compilation, to generate the ".d" file. The generated object file is built
    // to a temporary location (tempOutputFile) and ignored afterwards.
    LOG.info("Generating " + getDotdFile());
    CppCompileActionContext context = executor.getContext(actionContext);
    CppCompileActionContext.Reply reply = null;
    try {
        // We delegate stdout/stderr to nowhere, i.e. same as redirecting to /dev/null.
        reply = context.execWithReply(this, actionExecutionContext.withFileOutErr(new FileOutErr()));
    } catch (ExecException e) {
        // We ignore failures here (other than capturing the Distributor reply).
        // The compilation may well fail (that's the whole point of negative compilation tests).
        // We execute it here just for the side effect of generating the ".d" file.
        reply = context.getReplyFromException(e, this);
        if (reply == null) {
            // This can only happen if the ExecException does not come from remote execution.
            throw e.toActionExecutionException("Fake C++ Compilation of rule '" + getOwner().getLabel() + "'", executor.getVerboseFailures(), this);
        }
    }
    IncludeScanningContext scanningContext = executor.getContext(IncludeScanningContext.class);
    Path execRoot = executor.getExecRoot();
    NestedSet<Artifact> discoveredInputs;
    if (getDotdFile() == null) {
        discoveredInputs = NestedSetBuilder.<Artifact>stableOrder().build();
    } else {
        HeaderDiscovery.Builder discoveryBuilder = new HeaderDiscovery.Builder().setAction(this).setDotdFile(getDotdFile()).setSourceFile(getSourceFile()).setSpecialInputsHandler(specialInputsHandler).setDependencySet(processDepset(execRoot, reply)).setPermittedSystemIncludePrefixes(getPermittedSystemIncludePrefixes(execRoot)).setAllowedDerivedinputsMap(getAllowedDerivedInputsMap());
        if (cppSemantics.needsIncludeValidation()) {
            discoveryBuilder.shouldValidateInclusions();
        }
        discoveredInputs = discoveryBuilder.build().discoverInputsFromDotdFiles(execRoot, scanningContext.getArtifactResolver());
    }
    // Clear in-memory .d files early.
    reply = null;
    // depends on.
    try {
        validateInclusions(discoveredInputs, actionExecutionContext.getArtifactExpander(), executor.getEventHandler());
    } catch (ActionExecutionException e) {
        // TODO(bazel-team): (2009) make this into an error, once most of the current warnings
        // are fixed.
        executor.getEventHandler().handle(Event.warn(getOwner().getLocation(), e.getMessage() + ";\n  this warning may eventually become an error"));
    }
    updateActionInputs(discoveredInputs);
    // Generate a fake ".o" file containing the command line needed to generate
    // the real object file.
    LOG.info("Generating " + outputFile);
    // A cc_fake_binary rule generates fake .o files and a fake target file,
    // which merely contain instructions on building the real target. We need to
    // be careful to use a new set of output file names in the instructions, as
    // to not overwrite the fake output files when someone tries to follow the
    // instructions. As the real compilation is executed by the test from its
    // runfiles directory (where writing is forbidden), we patch the command
    // line to write to $TEST_TMPDIR instead.
    final String outputPrefix = "$TEST_TMPDIR/";
    String argv = Joiner.on(' ').join(Iterables.transform(getArgv(outputFile.getExecPath()), new Function<String, String>() {

        @Override
        public String apply(String input) {
            String result = ShellEscaper.escapeString(input);
            // -c <tempOutputFile>, but here it has to be outputFile, so we replace it.
            if (input.equals(tempOutputFile.getPathString())) {
                result = outputPrefix + ShellEscaper.escapeString(outputFile.getExecPathString());
            }
            if (input.equals(outputFile.getExecPathString()) || input.equals(getDotdFile().getSafeExecPath().getPathString())) {
                result = outputPrefix + ShellEscaper.escapeString(input);
            }
            return result;
        }
    }));
    // the compilation would fail.
    try {
        // Ensure that the .d file and .o file are siblings, so that the "mkdir" below works for
        // both.
        Preconditions.checkState(outputFile.getExecPath().getParentDirectory().equals(getDotdFile().getSafeExecPath().getParentDirectory()));
        FileSystemUtils.writeContent(outputFile.getPath(), ISO_8859_1, outputFile.getPath().getBaseName() + ": " + "mkdir -p " + outputPrefix + "$(dirname " + outputFile.getExecPath() + ")" + " && " + argv + "\n");
    } catch (IOException e) {
        throw new ActionExecutionException("failed to create fake compile command for rule '" + getOwner().getLabel() + ": " + e.getMessage(), this, false);
    }
}