Examples with EventHandler com.google.devtools.build.lib.events.EventHandler used on opensource projects

Example 1 with EventHandler

use of com.google.devtools.build.lib.events.EventHandler in project bazel by bazelbuild.

the class ParallelBuilderTest method testProgressReporting.

@Test
public void testProgressReporting() throws Exception {
    // Build three artifacts in 3 separate actions (baz depends on bar and bar
    // depends on foo.  Make sure progress is reported at the beginning of all
    // three actions.
    List<Artifact> sourceFiles = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        sourceFiles.add(createInputFile("file" + i));
    }
    Artifact foo = createDerivedArtifact("foo");
    Artifact bar = createDerivedArtifact("bar");
    Artifact baz = createDerivedArtifact("baz");
    bar.getPath().delete();
    baz.getPath().delete();
    final List<String> messages = new ArrayList<>();
    EventHandler handler = new EventHandler() {

        @Override
        public void handle(Event event) {
            EventKind k = event.getKind();
            if (k == EventKind.START || k == EventKind.FINISH) {
                // Remove the tmpDir as this is user specific and the assert would
                // fail below.
                messages.add(event.getMessage().replaceFirst(TestUtils.tmpDir(), "") + " " + event.getKind());
            }
        }
    };
    reporter.addHandler(handler);
    reporter.addHandler(new PrintingEventHandler(EventKind.ALL_EVENTS));
    registerAction(new TestAction(TestAction.NO_EFFECT, sourceFiles, asSet(foo)));
    registerAction(new TestAction(TestAction.NO_EFFECT, asSet(foo), asSet(bar)));
    registerAction(new TestAction(TestAction.NO_EFFECT, asSet(bar), asSet(baz)));
    buildArtifacts(baz);
    // Check that the percentages increase non-linearly, because foo has 10 input files
    List<String> expectedMessages = Lists.newArrayList("Test foo START", "Test foo FINISH", "Test bar START", "Test bar FINISH", "Test baz START", "Test baz FINISH");
    assertThat(messages).containsAllIn(expectedMessages);
    // Now do an incremental rebuild of bar and baz,
    // and check the incremental progress percentages.
    messages.clear();
    bar.getPath().delete();
    baz.getPath().delete();
    // This uses a new builder instance so that we refetch timestamps from
    // (in-memory) file system, rather than using cached entries.
    buildArtifacts(baz);
    expectedMessages = Lists.newArrayList("Test bar START", "Test bar FINISH", "Test baz START", "Test baz FINISH");
    assertThat(messages).containsAllIn(expectedMessages);
}

Example 2 with EventHandler

use of com.google.devtools.build.lib.events.EventHandler in project bazel by bazelbuild.

the class SkyframeLabelVisitorTest method testCrashInLoadPackageIsReportedEffectively.

// Regression test for "Bazel hangs on input of illegal rule".
@Test
public void testCrashInLoadPackageIsReportedEffectively() throws Exception {
    reporter.removeHandler(failFastHandler);
    // Inject a NullPointerException into loadPackage().  This is triggered by
    // any ERROR event.
    reporter.addHandler(new EventHandler() {

        @Override
        public void handle(Event event) {
            if (EventKind.ERRORS.contains(event.getKind())) {
                throw new NullPointerException("oops");
            }
        }
    });
    // Visitation of //x reaches package "bad" by many paths.  The first time,
    // loadPackage() crashes (because of the injected NPE).  Previously,
    // on a subsequent visitation, the visitor would get livelocked due the
    // stale PendingEntry stuck in the PackageCache.  With the fix, the NPE is
    // thrown.
    scratch.file("bad/BUILD", "this is a bad build file");
    scratch.file("x/BUILD", "sh_library(name='x', ", "           deps=['//bad:a', '//bad:b', '//bad:c',", "                 '//bad:d', '//bad:e', '//bad:f'])");
    try {
        // Used to get stuck.
        assertLabelsVisitedWithErrors(ImmutableSet.of("//x:x"), ImmutableSet.of("//x"));
        // unreachable
        fail();
    } catch (NullPointerException npe) {
    // This is expected for legacy blaze.
    } catch (RuntimeException re) {
        // This is expected for Skyframe blaze.
        assertThat(re.getCause()).isInstanceOf(NullPointerException.class);
    }
}

Example 3 with EventHandler

use of com.google.devtools.build.lib.events.EventHandler in project bazel by bazelbuild.

the class GenRuleAction method internalExecute.

@Override
protected void internalExecute(ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException {
    EventHandler reporter = actionExecutionContext.getExecutor().getEventHandler();
    checkInputsForDirectories(reporter, actionExecutionContext.getMetadataHandler());
    super.internalExecute(actionExecutionContext);
    checkOutputsForDirectories(reporter);
}

Example 4 with EventHandler

use of com.google.devtools.build.lib.events.EventHandler in project bazel by bazelbuild.

the class WorkerSpawnStrategy method actuallyExec.

private void actuallyExec(Spawn spawn, ActionExecutionContext actionExecutionContext, AtomicReference<Class<? extends SpawnActionContext>> writeOutputFiles) throws ExecException, InterruptedException {
    Executor executor = actionExecutionContext.getExecutor();
    EventHandler eventHandler = executor.getEventHandler();
    if (executor.reportsSubcommands()) {
        executor.reportSubcommand(spawn);
    }
    // We assume that the spawn to be executed always gets at least one @flagfile.txt or
    // --flagfile=flagfile.txt argument, which contains the flags related to the work itself (as
    // opposed to start-up options for the executed tool). Thus, we can extract those elements from
    // its args and put them into the WorkRequest instead.
    List<String> flagfiles = new ArrayList<>();
    List<String> startupArgs = new ArrayList<>();
    for (String arg : spawn.getArguments()) {
        if (FLAG_FILE_PATTERN.matcher(arg).matches()) {
            flagfiles.add(arg);
        } else {
            startupArgs.add(arg);
        }
    }
    if (flagfiles.isEmpty()) {
        throw new UserExecException(String.format(ERROR_MESSAGE_PREFIX + REASON_NO_FLAGFILE, spawn.getMnemonic()));
    }
    if (Iterables.isEmpty(spawn.getToolFiles())) {
        throw new UserExecException(String.format(ERROR_MESSAGE_PREFIX + REASON_NO_TOOLS, spawn.getMnemonic()));
    }
    FileOutErr outErr = actionExecutionContext.getFileOutErr();
    ImmutableList<String> args = ImmutableList.<String>builder().addAll(startupArgs).add("--persistent_worker").addAll(MoreObjects.firstNonNull(extraFlags.get(spawn.getMnemonic()), ImmutableList.<String>of())).build();
    ImmutableMap<String, String> env = spawn.getEnvironment();
    try {
        ActionInputFileCache inputFileCache = actionExecutionContext.getActionInputFileCache();
        HashCode workerFilesHash = WorkerFilesHash.getWorkerFilesHash(spawn.getToolFiles(), actionExecutionContext);
        Map<PathFragment, Path> inputFiles = new SpawnHelpers(execRoot).getMounts(spawn, actionExecutionContext);
        Set<PathFragment> outputFiles = SandboxHelpers.getOutputFiles(spawn);
        WorkerKey key = new WorkerKey(args, env, execRoot, spawn.getMnemonic(), workerFilesHash, inputFiles, outputFiles, writeOutputFiles != null);
        WorkRequest.Builder requestBuilder = WorkRequest.newBuilder();
        for (String flagfile : flagfiles) {
            expandArgument(requestBuilder, flagfile);
        }
        List<ActionInput> inputs = ActionInputHelper.expandArtifacts(spawn.getInputFiles(), actionExecutionContext.getArtifactExpander());
        for (ActionInput input : inputs) {
            byte[] digestBytes = inputFileCache.getDigest(input);
            ByteString digest;
            if (digestBytes == null) {
                digest = ByteString.EMPTY;
            } else {
                digest = ByteString.copyFromUtf8(HashCode.fromBytes(digestBytes).toString());
            }
            requestBuilder.addInputsBuilder().setPath(input.getExecPathString()).setDigest(digest).build();
        }
        WorkResponse response = execInWorker(eventHandler, key, requestBuilder.build(), maxRetries, writeOutputFiles);
        outErr.getErrorStream().write(response.getOutputBytes().toByteArray());
        if (response.getExitCode() != 0) {
            throw new UserExecException(String.format("Worker process sent response with exit code: %d.", response.getExitCode()));
        }
    } catch (IOException e) {
        String message = CommandFailureUtils.describeCommandFailure(verboseFailures, spawn.getArguments(), env, execRoot.getPathString());
        throw new UserExecException(message, e);
    }
}

Example 5 with EventHandler

use of com.google.devtools.build.lib.events.EventHandler in project bazel by bazelbuild.

the class RemoteSpawnStrategy method exec.

/** Executes the given {@code spawn}. */
@Override
public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException {
    ActionKey actionKey = null;
    String mnemonic = spawn.getMnemonic();
    Executor executor = actionExecutionContext.getExecutor();
    EventHandler eventHandler = executor.getEventHandler();
    RemoteActionCache actionCache = null;
    RemoteWorkExecutor workExecutor = null;
    if (spawn.isRemotable()) {
        // action to enable server-side parallelism (need a different gRPC channel per action).
        try {
            if (ConcurrentMapFactory.isRemoteCacheOptions(options)) {
                actionCache = new ConcurrentMapActionCache(ConcurrentMapFactory.create(options));
            } else if (GrpcActionCache.isRemoteCacheOptions(options)) {
                actionCache = new GrpcActionCache(options);
            }
            if (actionCache != null && RemoteWorkExecutor.isRemoteExecutionOptions(options)) {
                workExecutor = new RemoteWorkExecutor(options);
            }
        } catch (InvalidConfigurationException e) {
            eventHandler.handle(Event.warn(e.toString()));
        }
    }
    if (!spawn.isRemotable() || actionCache == null) {
        standaloneStrategy.exec(spawn, actionExecutionContext);
        return;
    }
    if (workExecutor == null) {
        execLocally(spawn, actionExecutionContext, actionCache, actionKey);
        return;
    }
    if (executor.reportsSubcommands()) {
        executor.reportSubcommand(spawn);
    }
    executor.getEventBus().post(ActionStatusMessage.runningStrategy(spawn.getResourceOwner(), "remote"));
    try {
        // Temporary hack: the TreeNodeRepository should be created and maintained upstream!
        TreeNodeRepository repository = new TreeNodeRepository(execRoot);
        List<ActionInput> inputs = ActionInputHelper.expandArtifacts(spawn.getInputFiles(), actionExecutionContext.getArtifactExpander());
        TreeNode inputRoot = repository.buildFromActionInputs(inputs);
        repository.computeMerkleDigests(inputRoot);
        Command command = buildCommand(spawn.getArguments(), spawn.getEnvironment());
        Action action = buildAction(spawn.getOutputFiles(), ContentDigests.computeDigest(command), repository.getMerkleDigest(inputRoot));
        // Look up action cache, and reuse the action output if it is found.
        actionKey = ContentDigests.computeActionKey(action);
        ActionResult result = this.options.remoteAcceptCached ? actionCache.getCachedActionResult(actionKey) : null;
        boolean acceptCachedResult = this.options.remoteAcceptCached;
        if (result != null) {
            // just update the TreeNodeRepository and continue the build.
            try {
                actionCache.downloadAllResults(result, execRoot);
                return;
            } catch (CacheNotFoundException e) {
                // Retry the action remotely and invalidate the results.
                acceptCachedResult = false;
            }
        }
        // Upload the command and all the inputs into the remote cache.
        actionCache.uploadBlob(command.toByteArray());
        // TODO(olaola): this should use the ActionInputFileCache for SHA1 digests!
        actionCache.uploadTree(repository, execRoot, inputRoot);
        // TODO(olaola): set BuildInfo and input total bytes as well.
        ExecuteRequest.Builder request = ExecuteRequest.newBuilder().setAction(action).setAcceptCached(acceptCachedResult).setTotalInputFileCount(inputs.size()).setTimeoutMillis(1000 * Spawns.getTimeoutSeconds(spawn, 120));
        // TODO(olaola): set sensible local and remote timouts.
        ExecuteReply reply = workExecutor.executeRemotely(request.build());
        ExecutionStatus status = reply.getStatus();
        result = reply.getResult();
        // action.
        if (status.getSucceeded()) {
            passRemoteOutErr(actionCache, result, actionExecutionContext.getFileOutErr());
            actionCache.downloadAllResults(result, execRoot);
            return;
        }
        if (status.getError() == ExecutionStatus.ErrorCode.EXEC_FAILED || !options.remoteAllowLocalFallback) {
            passRemoteOutErr(actionCache, result, actionExecutionContext.getFileOutErr());
            throw new UserExecException(status.getErrorDetail());
        }
        // For now, we retry locally on all other remote errors.
        // TODO(olaola): add remote retries on cache miss errors.
        execLocally(spawn, actionExecutionContext, actionCache, actionKey);
    } catch (IOException e) {
        throw new UserExecException("Unexpected IO error.", e);
    } catch (InterruptedException e) {
        eventHandler.handle(Event.warn(mnemonic + " remote work interrupted (" + e + ")"));
        Thread.currentThread().interrupt();
        throw e;
    } catch (StatusRuntimeException e) {
        String stackTrace = "";
        if (verboseFailures) {
            stackTrace = "\n" + Throwables.getStackTraceAsString(e);
        }
        eventHandler.handle(Event.warn(mnemonic + " remote work failed (" + e + ")" + stackTrace));
        if (options.remoteAllowLocalFallback) {
            execLocally(spawn, actionExecutionContext, actionCache, actionKey);
        } else {
            throw new UserExecException(e);
        }
    } catch (CacheNotFoundException e) {
        eventHandler.handle(Event.warn(mnemonic + " remote work results cache miss (" + e + ")"));
        if (options.remoteAllowLocalFallback) {
            execLocally(spawn, actionExecutionContext, actionCache, actionKey);
        } else {
            throw new UserExecException(e);
        }
    } catch (UnsupportedOperationException e) {
        eventHandler.handle(Event.warn(mnemonic + " unsupported operation for action cache (" + e + ")"));
    }
}