Examples with WorkRequest com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest used on opensource projects

Example 6 with WorkRequest

use of com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest in project bazel by bazelbuild.

the class WorkerSpawnStrategy method actuallyExec.

private void actuallyExec(Spawn spawn, ActionExecutionContext actionExecutionContext, AtomicReference<Class<? extends SpawnActionContext>> writeOutputFiles) throws ExecException, InterruptedException {
    Executor executor = actionExecutionContext.getExecutor();
    EventHandler eventHandler = executor.getEventHandler();
    if (executor.reportsSubcommands()) {
        executor.reportSubcommand(spawn);
    }
    // We assume that the spawn to be executed always gets at least one @flagfile.txt or
    // --flagfile=flagfile.txt argument, which contains the flags related to the work itself (as
    // opposed to start-up options for the executed tool). Thus, we can extract those elements from
    // its args and put them into the WorkRequest instead.
    List<String> flagfiles = new ArrayList<>();
    List<String> startupArgs = new ArrayList<>();
    for (String arg : spawn.getArguments()) {
        if (FLAG_FILE_PATTERN.matcher(arg).matches()) {
            flagfiles.add(arg);
        } else {
            startupArgs.add(arg);
        }
    }
    if (flagfiles.isEmpty()) {
        throw new UserExecException(String.format(ERROR_MESSAGE_PREFIX + REASON_NO_FLAGFILE, spawn.getMnemonic()));
    }
    if (Iterables.isEmpty(spawn.getToolFiles())) {
        throw new UserExecException(String.format(ERROR_MESSAGE_PREFIX + REASON_NO_TOOLS, spawn.getMnemonic()));
    }
    FileOutErr outErr = actionExecutionContext.getFileOutErr();
    ImmutableList<String> args = ImmutableList.<String>builder().addAll(startupArgs).add("--persistent_worker").addAll(MoreObjects.firstNonNull(extraFlags.get(spawn.getMnemonic()), ImmutableList.<String>of())).build();
    ImmutableMap<String, String> env = spawn.getEnvironment();
    try {
        ActionInputFileCache inputFileCache = actionExecutionContext.getActionInputFileCache();
        HashCode workerFilesHash = WorkerFilesHash.getWorkerFilesHash(spawn.getToolFiles(), actionExecutionContext);
        Map<PathFragment, Path> inputFiles = new SpawnHelpers(execRoot).getMounts(spawn, actionExecutionContext);
        Set<PathFragment> outputFiles = SandboxHelpers.getOutputFiles(spawn);
        WorkerKey key = new WorkerKey(args, env, execRoot, spawn.getMnemonic(), workerFilesHash, inputFiles, outputFiles, writeOutputFiles != null);
        WorkRequest.Builder requestBuilder = WorkRequest.newBuilder();
        for (String flagfile : flagfiles) {
            expandArgument(requestBuilder, flagfile);
        }
        List<ActionInput> inputs = ActionInputHelper.expandArtifacts(spawn.getInputFiles(), actionExecutionContext.getArtifactExpander());
        for (ActionInput input : inputs) {
            byte[] digestBytes = inputFileCache.getDigest(input);
            ByteString digest;
            if (digestBytes == null) {
                digest = ByteString.EMPTY;
            } else {
                digest = ByteString.copyFromUtf8(HashCode.fromBytes(digestBytes).toString());
            }
            requestBuilder.addInputsBuilder().setPath(input.getExecPathString()).setDigest(digest).build();
        }
        WorkResponse response = execInWorker(eventHandler, key, requestBuilder.build(), maxRetries, writeOutputFiles);
        outErr.getErrorStream().write(response.getOutputBytes().toByteArray());
        if (response.getExitCode() != 0) {
            throw new UserExecException(String.format("Worker process sent response with exit code: %d.", response.getExitCode()));
        }
    } catch (IOException e) {
        String message = CommandFailureUtils.describeCommandFailure(verboseFailures, spawn.getArguments(), env, execRoot.getPathString());
        throw new UserExecException(message, e);
    }
}

Example 7 with WorkRequest

use of com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest in project bazel by bazelbuild.

the class ExampleWorker method runPersistentWorker.

private static void runPersistentWorker(ExampleWorkerOptions workerOptions) throws IOException {
    PrintStream originalStdOut = System.out;
    PrintStream originalStdErr = System.err;
    while (true) {
        try {
            WorkRequest request = WorkRequest.parseDelimitedFrom(System.in);
            if (request == null) {
                break;
            }
            inputs.clear();
            for (Input input : request.getInputsList()) {
                inputs.put(input.getPath(), input.getDigest().toStringUtf8());
            }
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            int exitCode = 0;
            try (PrintStream ps = new PrintStream(baos)) {
                System.setOut(ps);
                System.setErr(ps);
                if (poisoned) {
                    System.out.println("I'm a poisoned worker and this is not a protobuf.");
                    System.out.println("Here's a fake stack trace for you:");
                    System.out.println("    at com.example.Something(Something.java:83)");
                    System.out.println("    at java.lang.Thread.run(Thread.java:745)");
                    System.out.print("And now, 8k of random bytes: ");
                    byte[] b = new byte[8192];
                    new Random().nextBytes(b);
                    System.out.write(b);
                } else {
                    try {
                        processRequest(request.getArgumentsList());
                    } catch (Exception e) {
                        e.printStackTrace();
                        exitCode = 1;
                    }
                }
            } finally {
                System.setOut(originalStdOut);
                System.setErr(originalStdErr);
            }
            if (poisoned) {
                baos.writeTo(System.out);
            } else {
                WorkResponse.newBuilder().setOutput(baos.toString()).setExitCode(exitCode).build().writeDelimitedTo(System.out);
            }
            System.out.flush();
            if (workerOptions.exitAfter > 0 && workUnitCounter > workerOptions.exitAfter) {
                return;
            }
            if (workerOptions.poisonAfter > 0 && workUnitCounter > workerOptions.poisonAfter) {
                poisoned = true;
            }
        } finally {
            // Be a good worker process and consume less memory when idle.
            System.gc();
        }
    }
}

Example 8 with WorkRequest

use of com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest in project bazel by bazelbuild.

the class DexBuilder method runPersistentWorker.

/**
   * Implements a persistent worker process for use with Bazel (see {@code WorkerSpawnStrategy}).
   */
private static void runPersistentWorker() throws IOException {
    ExecutorService executor = newFixedThreadPool(Runtime.getRuntime().availableProcessors());
    Cache<DexingKey, byte[]> dexCache = CacheBuilder.newBuilder().maximumWeight(Math.min(Runtime.getRuntime().maxMemory() - 25 * ONE_MEG, 200 * ONE_MEG)).weigher(new Weigher<DexingKey, byte[]>() {

        @Override
        public int weigh(DexingKey key, byte[] value) {
            return key.classfileContent().length + value.length;
        }
    }).build();
    try {
        while (true) {
            WorkRequest request = WorkRequest.parseDelimitedFrom(System.in);
            if (request == null) {
                return;
            }
            // Redirect dx's output so we can return it in response
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            PrintStream ps = new PrintStream(baos, /*autoFlush*/
            true);
            DxConsole.out = DxConsole.err = ps;
            // Make sure that we exit nonzero in case uncaught errors occur during processRequest.
            int exitCode = 1;
            try {
                processRequest(executor, dexCache, request.getArgumentsList());
                // success!
                exitCode = 0;
            } catch (Exception e) {
                // Deliberate catch-all so we can capture a stack trace.
                // TODO(bazel-team): Consider canceling any outstanding futures created for this request
                e.printStackTrace(ps);
            } catch (Error e) {
                e.printStackTrace();
                // try capturing the error, may fail if out of memory
                e.printStackTrace(ps);
                // rethrow to kill the worker
                throw e;
            } finally {
                // Try sending a response no matter what
                String output;
                try {
                    output = baos.toString();
                } catch (Throwable t) {
                    // most likely out of memory, so log with minimal memory needs
                    t.printStackTrace();
                    output = "check worker log for exceptions";
                }
                WorkResponse.newBuilder().setOutput(output).setExitCode(exitCode).build().writeDelimitedTo(System.out);
                System.out.flush();
            }
        }
    } finally {
        executor.shutdown();
    }
}