Examples with WorkRequest com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest used on opensource projects

Example 1 with WorkRequest

use of com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest in project bazel by bazelbuild.

the class WorkerTestStrategy method execInWorker.

private TestResultData execInWorker(TestRunnerAction action, ActionExecutionContext actionExecutionContext, Map<String, String> environment, List<String> startupArgs, Path execRoot, int retriesLeft) throws ExecException, InterruptedException, IOException {
    Executor executor = actionExecutionContext.getExecutor();
    // TODO(kush): Remove once we're out of the experimental phase.
    executor.getEventHandler().handle(Event.warn("RUNNING TEST IN AN EXPERIMENTAL PERSISTENT WORKER. RESULTS MAY BE INACCURATE"));
    TestResultData.Builder builder = TestResultData.newBuilder();
    Path testLogPath = action.getTestLog().getPath();
    Worker worker = null;
    WorkerKey key = null;
    long startTime = executor.getClock().currentTimeMillis();
    try {
        HashCode workerFilesHash = WorkerFilesHash.getWorkerFilesHash(action.getTools(), actionExecutionContext);
        key = new WorkerKey(startupArgs, environment, execRoot, action.getMnemonic(), workerFilesHash, ImmutableMap.<PathFragment, Path>of(), ImmutableSet.<PathFragment>of(), /*mustBeSandboxed=*/
        false);
        worker = workerPool.borrowObject(key);
        WorkRequest request = WorkRequest.getDefaultInstance();
        request.writeDelimitedTo(worker.getOutputStream());
        worker.getOutputStream().flush();
        WorkResponse response = WorkResponse.parseDelimitedFrom(worker.getInputStream());
        actionExecutionContext.getFileOutErr().getErrorStream().write(response.getOutputBytes().toByteArray());
        long duration = executor.getClock().currentTimeMillis() - startTime;
        builder.addTestTimes(duration);
        builder.setRunDurationMillis(duration);
        if (response.getExitCode() == 0) {
            builder.setTestPassed(true).setStatus(BlazeTestStatus.PASSED).setCachable(true).setPassedLog(testLogPath.getPathString());
        } else {
            builder.setTestPassed(false).setStatus(BlazeTestStatus.FAILED).addFailedLogs(testLogPath.getPathString());
        }
        TestCase details = parseTestResult(action.resolve(actionExecutionContext.getExecutor().getExecRoot()).getXmlOutputPath());
        if (details != null) {
            builder.setTestCase(details);
        }
        return builder.build();
    } catch (IOException | InterruptedException e) {
        if (e instanceof InterruptedException) {
            // The user pressed Ctrl-C. Get out here quick.
            retriesLeft = 0;
        }
        if (worker != null) {
            workerPool.invalidateObject(key, worker);
            worker = null;
        }
        if (retriesLeft > 0) {
            // The worker process failed, but we still have some retries left. Let's retry with a fresh
            // worker.
            executor.getEventHandler().handle(Event.warn(key.getMnemonic() + " worker failed (" + e + "), invalidating and retrying with new worker..."));
            return execInWorker(action, actionExecutionContext, environment, startupArgs, execRoot, retriesLeft - 1);
        } else {
            throw new TestExecException(e.getMessage());
        }
    } finally {
        if (worker != null) {
            workerPool.returnObject(key, worker);
        }
    }
}

Example 2 with WorkRequest

use of com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest in project bazel by bazelbuild.

the class BazelJavaBuilder method runPersistentWorker.

private static int runPersistentWorker() {
    while (true) {
        try {
            WorkRequest request = WorkRequest.parseDelimitedFrom(System.in);
            if (request == null) {
                break;
            }
            try (StringWriter sw = new StringWriter();
                PrintWriter pw = new PrintWriter(sw)) {
                int exitCode = processRequest(request.getArgumentsList(), pw);
                WorkResponse.newBuilder().setOutput(sw.toString()).setExitCode(exitCode).build().writeDelimitedTo(System.out);
                System.out.flush();
            }
        } catch (IOException e) {
            e.printStackTrace();
            return 1;
        }
    }
    return 0;
}

Example 3 with WorkRequest

use of com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest in project bazel by bazelbuild.

the class VanillaJavaBuilder method runPersistentWorker.

private static int runPersistentWorker() {
    while (true) {
        try {
            WorkRequest request = WorkRequest.parseDelimitedFrom(System.in);
            if (request == null) {
                break;
            }
            try (VanillaJavaBuilder builder = new VanillaJavaBuilder()) {
                VanillaJavaBuilderResult result = builder.run(request.getArgumentsList());
                WorkResponse response = WorkResponse.newBuilder().setOutput(result.output()).setExitCode(result.ok() ? 0 : 1).build();
                response.writeDelimitedTo(System.out);
            }
            System.out.flush();
        } catch (IOException e) {
            e.printStackTrace();
            return 1;
        }
    }
    return 0;
}

Example 4 with WorkRequest

use of com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest in project bazel by bazelbuild.

the class SingleJarWorker method dispatchWorkRequestsForever.

private void dispatchWorkRequestsForever() throws IOException {
    while (true) {
        WorkRequest workRequest = WorkRequest.parseDelimitedFrom(System.in);
        String[] args = workRequest.getArgumentsList().toArray(new String[0]);
        int returnCode = runSingleJar(args);
        outputResult(returnCode);
    }
}

Example 5 with WorkRequest

use of com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest in project bazel by bazelbuild.

the class WorkerSpawnStrategy method actuallyExec.

private void actuallyExec(Spawn spawn, ActionExecutionContext actionExecutionContext, AtomicReference<Class<? extends SpawnActionContext>> writeOutputFiles) throws ExecException, InterruptedException {
    Executor executor = actionExecutionContext.getExecutor();
    EventHandler eventHandler = executor.getEventHandler();
    if (executor.reportsSubcommands()) {
        executor.reportSubcommand(spawn);
    }
    // We assume that the spawn to be executed always gets at least one @flagfile.txt or
    // --flagfile=flagfile.txt argument, which contains the flags related to the work itself (as
    // opposed to start-up options for the executed tool). Thus, we can extract those elements from
    // its args and put them into the WorkRequest instead.
    List<String> flagfiles = new ArrayList<>();
    List<String> startupArgs = new ArrayList<>();
    for (String arg : spawn.getArguments()) {
        if (FLAG_FILE_PATTERN.matcher(arg).matches()) {
            flagfiles.add(arg);
        } else {
            startupArgs.add(arg);
        }
    }
    if (flagfiles.isEmpty()) {
        throw new UserExecException(String.format(ERROR_MESSAGE_PREFIX + REASON_NO_FLAGFILE, spawn.getMnemonic()));
    }
    if (Iterables.isEmpty(spawn.getToolFiles())) {
        throw new UserExecException(String.format(ERROR_MESSAGE_PREFIX + REASON_NO_TOOLS, spawn.getMnemonic()));
    }
    FileOutErr outErr = actionExecutionContext.getFileOutErr();
    ImmutableList<String> args = ImmutableList.<String>builder().addAll(startupArgs).add("--persistent_worker").addAll(MoreObjects.firstNonNull(extraFlags.get(spawn.getMnemonic()), ImmutableList.<String>of())).build();
    ImmutableMap<String, String> env = spawn.getEnvironment();
    try {
        ActionInputFileCache inputFileCache = actionExecutionContext.getActionInputFileCache();
        HashCode workerFilesHash = WorkerFilesHash.getWorkerFilesHash(spawn.getToolFiles(), actionExecutionContext);
        Map<PathFragment, Path> inputFiles = new SpawnHelpers(execRoot).getMounts(spawn, actionExecutionContext);
        Set<PathFragment> outputFiles = SandboxHelpers.getOutputFiles(spawn);
        WorkerKey key = new WorkerKey(args, env, execRoot, spawn.getMnemonic(), workerFilesHash, inputFiles, outputFiles, writeOutputFiles != null);
        WorkRequest.Builder requestBuilder = WorkRequest.newBuilder();
        for (String flagfile : flagfiles) {
            expandArgument(requestBuilder, flagfile);
        }
        List<ActionInput> inputs = ActionInputHelper.expandArtifacts(spawn.getInputFiles(), actionExecutionContext.getArtifactExpander());
        for (ActionInput input : inputs) {
            byte[] digestBytes = inputFileCache.getDigest(input);
            ByteString digest;
            if (digestBytes == null) {
                digest = ByteString.EMPTY;
            } else {
                digest = ByteString.copyFromUtf8(HashCode.fromBytes(digestBytes).toString());
            }
            requestBuilder.addInputsBuilder().setPath(input.getExecPathString()).setDigest(digest).build();
        }
        WorkResponse response = execInWorker(eventHandler, key, requestBuilder.build(), maxRetries, writeOutputFiles);
        outErr.getErrorStream().write(response.getOutputBytes().toByteArray());
        if (response.getExitCode() != 0) {
            throw new UserExecException(String.format("Worker process sent response with exit code: %d.", response.getExitCode()));
        }
    } catch (IOException e) {
        String message = CommandFailureUtils.describeCommandFailure(verboseFailures, spawn.getArguments(), env, execRoot.getPathString());
        throw new UserExecException(message, e);
    }
}