Examples with WorkResponse com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse used on opensource projects

Example 1 with WorkResponse

use of com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse in project bazel by bazelbuild.

the class WorkerTestStrategy method execInWorker.

private TestResultData execInWorker(TestRunnerAction action, ActionExecutionContext actionExecutionContext, Map<String, String> environment, List<String> startupArgs, Path execRoot, int retriesLeft) throws ExecException, InterruptedException, IOException {
    Executor executor = actionExecutionContext.getExecutor();
    // TODO(kush): Remove once we're out of the experimental phase.
    executor.getEventHandler().handle(Event.warn("RUNNING TEST IN AN EXPERIMENTAL PERSISTENT WORKER. RESULTS MAY BE INACCURATE"));
    TestResultData.Builder builder = TestResultData.newBuilder();
    Path testLogPath = action.getTestLog().getPath();
    Worker worker = null;
    WorkerKey key = null;
    long startTime = executor.getClock().currentTimeMillis();
    try {
        HashCode workerFilesHash = WorkerFilesHash.getWorkerFilesHash(action.getTools(), actionExecutionContext);
        key = new WorkerKey(startupArgs, environment, execRoot, action.getMnemonic(), workerFilesHash, ImmutableMap.<PathFragment, Path>of(), ImmutableSet.<PathFragment>of(), /*mustBeSandboxed=*/
        false);
        worker = workerPool.borrowObject(key);
        WorkRequest request = WorkRequest.getDefaultInstance();
        request.writeDelimitedTo(worker.getOutputStream());
        worker.getOutputStream().flush();
        WorkResponse response = WorkResponse.parseDelimitedFrom(worker.getInputStream());
        actionExecutionContext.getFileOutErr().getErrorStream().write(response.getOutputBytes().toByteArray());
        long duration = executor.getClock().currentTimeMillis() - startTime;
        builder.addTestTimes(duration);
        builder.setRunDurationMillis(duration);
        if (response.getExitCode() == 0) {
            builder.setTestPassed(true).setStatus(BlazeTestStatus.PASSED).setCachable(true).setPassedLog(testLogPath.getPathString());
        } else {
            builder.setTestPassed(false).setStatus(BlazeTestStatus.FAILED).addFailedLogs(testLogPath.getPathString());
        }
        TestCase details = parseTestResult(action.resolve(actionExecutionContext.getExecutor().getExecRoot()).getXmlOutputPath());
        if (details != null) {
            builder.setTestCase(details);
        }
        return builder.build();
    } catch (IOException | InterruptedException e) {
        if (e instanceof InterruptedException) {
            // The user pressed Ctrl-C. Get out here quick.
            retriesLeft = 0;
        }
        if (worker != null) {
            workerPool.invalidateObject(key, worker);
            worker = null;
        }
        if (retriesLeft > 0) {
            // The worker process failed, but we still have some retries left. Let's retry with a fresh
            // worker.
            executor.getEventHandler().handle(Event.warn(key.getMnemonic() + " worker failed (" + e + "), invalidating and retrying with new worker..."));
            return execInWorker(action, actionExecutionContext, environment, startupArgs, execRoot, retriesLeft - 1);
        } else {
            throw new TestExecException(e.getMessage());
        }
    } finally {
        if (worker != null) {
            workerPool.returnObject(key, worker);
        }
    }
}

Example 2 with WorkResponse

use of com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse in project bazel by bazelbuild.

the class VanillaJavaBuilder method runPersistentWorker.

private static int runPersistentWorker() {
    while (true) {
        try {
            WorkRequest request = WorkRequest.parseDelimitedFrom(System.in);
            if (request == null) {
                break;
            }
            try (VanillaJavaBuilder builder = new VanillaJavaBuilder()) {
                VanillaJavaBuilderResult result = builder.run(request.getArgumentsList());
                WorkResponse response = WorkResponse.newBuilder().setOutput(result.output()).setExitCode(result.ok() ? 0 : 1).build();
                response.writeDelimitedTo(System.out);
            }
            System.out.flush();
        } catch (IOException e) {
            e.printStackTrace();
            return 1;
        }
    }
    return 0;
}

Example 3 with WorkResponse

use of com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse in project bazel by bazelbuild.

the class ExperimentalTestRunner method runPersistentTestRunner.

private static int runPersistentTestRunner(String suiteClassName) {
    PrintStream originalStdOut = System.out;
    PrintStream originalStdErr = System.err;
    while (true) {
        try {
            WorkRequest request = WorkRequest.parseDelimitedFrom(System.in);
            if (request == null) {
                break;
            }
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            PrintStream ps = new PrintStream(baos, true);
            System.setOut(ps);
            System.setErr(ps);
            String[] arguments = request.getArgumentsList().toArray(new String[0]);
            int exitCode = -1;
            try {
                exitCode = runTestsInSuite(suiteClassName, arguments);
            } finally {
                System.setOut(originalStdOut);
                System.setErr(originalStdErr);
            }
            WorkResponse response = WorkResponse.newBuilder().setOutput(baos.toString()).setExitCode(exitCode).build();
            response.writeDelimitedTo(System.out);
            System.out.flush();
        } catch (IOException e) {
            e.printStackTrace();
            return 1;
        }
    }
    return 0;
}

Example 4 with WorkResponse

use of com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse in project bazel by bazelbuild.

the class WorkerSpawnStrategy method execInWorker.

private WorkResponse execInWorker(EventHandler eventHandler, WorkerKey key, WorkRequest request, int retriesLeft, AtomicReference<Class<? extends SpawnActionContext>> writeOutputFiles) throws IOException, InterruptedException, UserExecException {
    Worker worker = null;
    WorkResponse response = null;
    try {
        worker = workers.borrowObject(key);
        worker.prepareExecution(key);
        request.writeDelimitedTo(worker.getOutputStream());
        worker.getOutputStream().flush();
        RecordingInputStream recordingStream = new RecordingInputStream(worker.getInputStream());
        recordingStream.startRecording(4096);
        try {
            response = WorkResponse.parseDelimitedFrom(recordingStream);
        } catch (IOException e2) {
            // If protobuf couldn't parse the response, try to print whatever the failing worker wrote
            // to stdout - it's probably a stack trace or some kind of error message that will help the
            // user figure out why the compiler is failing.
            recordingStream.readRemaining();
            String data = recordingStream.getRecordedDataAsString(Charsets.UTF_8);
            eventHandler.handle(Event.warn("Worker process returned an unparseable WorkResponse:\n" + data));
            throw e2;
        }
        if (writeOutputFiles != null && !writeOutputFiles.compareAndSet(null, WorkerSpawnStrategy.class)) {
            throw new InterruptedException();
        }
        worker.finishExecution(key);
        if (response == null) {
            throw new UserExecException("Worker process did not return a WorkResponse. This is probably caused by a " + "bug in the worker, writing unexpected other data to stdout.");
        }
    } catch (IOException e) {
        if (worker != null) {
            workers.invalidateObject(key, worker);
            worker = null;
        }
        if (retriesLeft > 0) {
            // worker.
            if (workerVerbose) {
                eventHandler.handle(Event.warn(key.getMnemonic() + " worker failed (" + Throwables.getStackTraceAsString(e) + "), invalidating and retrying with new worker..."));
            } else {
                eventHandler.handle(Event.warn(key.getMnemonic() + " worker failed, invalidating and retrying with new worker..."));
            }
            return execInWorker(eventHandler, key, request, retriesLeft - 1, writeOutputFiles);
        } else {
            throw e;
        }
    } finally {
        if (worker != null) {
            workers.returnObject(key, worker);
        }
    }
    return response;
}

Example 5 with WorkResponse

use of com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse in project bazel by bazelbuild.

the class WorkerSpawnStrategy method actuallyExec.

private void actuallyExec(Spawn spawn, ActionExecutionContext actionExecutionContext, AtomicReference<Class<? extends SpawnActionContext>> writeOutputFiles) throws ExecException, InterruptedException {
    Executor executor = actionExecutionContext.getExecutor();
    EventHandler eventHandler = executor.getEventHandler();
    if (executor.reportsSubcommands()) {
        executor.reportSubcommand(spawn);
    }
    // We assume that the spawn to be executed always gets at least one @flagfile.txt or
    // --flagfile=flagfile.txt argument, which contains the flags related to the work itself (as
    // opposed to start-up options for the executed tool). Thus, we can extract those elements from
    // its args and put them into the WorkRequest instead.
    List<String> flagfiles = new ArrayList<>();
    List<String> startupArgs = new ArrayList<>();
    for (String arg : spawn.getArguments()) {
        if (FLAG_FILE_PATTERN.matcher(arg).matches()) {
            flagfiles.add(arg);
        } else {
            startupArgs.add(arg);
        }
    }
    if (flagfiles.isEmpty()) {
        throw new UserExecException(String.format(ERROR_MESSAGE_PREFIX + REASON_NO_FLAGFILE, spawn.getMnemonic()));
    }
    if (Iterables.isEmpty(spawn.getToolFiles())) {
        throw new UserExecException(String.format(ERROR_MESSAGE_PREFIX + REASON_NO_TOOLS, spawn.getMnemonic()));
    }
    FileOutErr outErr = actionExecutionContext.getFileOutErr();
    ImmutableList<String> args = ImmutableList.<String>builder().addAll(startupArgs).add("--persistent_worker").addAll(MoreObjects.firstNonNull(extraFlags.get(spawn.getMnemonic()), ImmutableList.<String>of())).build();
    ImmutableMap<String, String> env = spawn.getEnvironment();
    try {
        ActionInputFileCache inputFileCache = actionExecutionContext.getActionInputFileCache();
        HashCode workerFilesHash = WorkerFilesHash.getWorkerFilesHash(spawn.getToolFiles(), actionExecutionContext);
        Map<PathFragment, Path> inputFiles = new SpawnHelpers(execRoot).getMounts(spawn, actionExecutionContext);
        Set<PathFragment> outputFiles = SandboxHelpers.getOutputFiles(spawn);
        WorkerKey key = new WorkerKey(args, env, execRoot, spawn.getMnemonic(), workerFilesHash, inputFiles, outputFiles, writeOutputFiles != null);
        WorkRequest.Builder requestBuilder = WorkRequest.newBuilder();
        for (String flagfile : flagfiles) {
            expandArgument(requestBuilder, flagfile);
        }
        List<ActionInput> inputs = ActionInputHelper.expandArtifacts(spawn.getInputFiles(), actionExecutionContext.getArtifactExpander());
        for (ActionInput input : inputs) {
            byte[] digestBytes = inputFileCache.getDigest(input);
            ByteString digest;
            if (digestBytes == null) {
                digest = ByteString.EMPTY;
            } else {
                digest = ByteString.copyFromUtf8(HashCode.fromBytes(digestBytes).toString());
            }
            requestBuilder.addInputsBuilder().setPath(input.getExecPathString()).setDigest(digest).build();
        }
        WorkResponse response = execInWorker(eventHandler, key, requestBuilder.build(), maxRetries, writeOutputFiles);
        outErr.getErrorStream().write(response.getOutputBytes().toByteArray());
        if (response.getExitCode() != 0) {
            throw new UserExecException(String.format("Worker process sent response with exit code: %d.", response.getExitCode()));
        }
    } catch (IOException e) {
        String message = CommandFailureUtils.describeCommandFailure(verboseFailures, spawn.getArguments(), env, execRoot.getPathString());
        throw new UserExecException(message, e);
    }
}