Examples with Executor com.google.devtools.build.lib.actions.Executor used on opensource projects

Example 16 with Executor

use of com.google.devtools.build.lib.actions.Executor in project bazel by bazelbuild.

the class RemoteSpawnStrategy method exec.

/** Executes the given {@code spawn}. */
@Override
public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException {
    ActionKey actionKey = null;
    String mnemonic = spawn.getMnemonic();
    Executor executor = actionExecutionContext.getExecutor();
    EventHandler eventHandler = executor.getEventHandler();
    RemoteActionCache actionCache = null;
    RemoteWorkExecutor workExecutor = null;
    if (spawn.isRemotable()) {
        // action to enable server-side parallelism (need a different gRPC channel per action).
        try {
            if (ConcurrentMapFactory.isRemoteCacheOptions(options)) {
                actionCache = new ConcurrentMapActionCache(ConcurrentMapFactory.create(options));
            } else if (GrpcActionCache.isRemoteCacheOptions(options)) {
                actionCache = new GrpcActionCache(options);
            }
            if (actionCache != null && RemoteWorkExecutor.isRemoteExecutionOptions(options)) {
                workExecutor = new RemoteWorkExecutor(options);
            }
        } catch (InvalidConfigurationException e) {
            eventHandler.handle(Event.warn(e.toString()));
        }
    }
    if (!spawn.isRemotable() || actionCache == null) {
        standaloneStrategy.exec(spawn, actionExecutionContext);
        return;
    }
    if (workExecutor == null) {
        execLocally(spawn, actionExecutionContext, actionCache, actionKey);
        return;
    }
    if (executor.reportsSubcommands()) {
        executor.reportSubcommand(spawn);
    }
    executor.getEventBus().post(ActionStatusMessage.runningStrategy(spawn.getResourceOwner(), "remote"));
    try {
        // Temporary hack: the TreeNodeRepository should be created and maintained upstream!
        TreeNodeRepository repository = new TreeNodeRepository(execRoot);
        List<ActionInput> inputs = ActionInputHelper.expandArtifacts(spawn.getInputFiles(), actionExecutionContext.getArtifactExpander());
        TreeNode inputRoot = repository.buildFromActionInputs(inputs);
        repository.computeMerkleDigests(inputRoot);
        Command command = buildCommand(spawn.getArguments(), spawn.getEnvironment());
        Action action = buildAction(spawn.getOutputFiles(), ContentDigests.computeDigest(command), repository.getMerkleDigest(inputRoot));
        // Look up action cache, and reuse the action output if it is found.
        actionKey = ContentDigests.computeActionKey(action);
        ActionResult result = this.options.remoteAcceptCached ? actionCache.getCachedActionResult(actionKey) : null;
        boolean acceptCachedResult = this.options.remoteAcceptCached;
        if (result != null) {
            // just update the TreeNodeRepository and continue the build.
            try {
                actionCache.downloadAllResults(result, execRoot);
                return;
            } catch (CacheNotFoundException e) {
                // Retry the action remotely and invalidate the results.
                acceptCachedResult = false;
            }
        }
        // Upload the command and all the inputs into the remote cache.
        actionCache.uploadBlob(command.toByteArray());
        // TODO(olaola): this should use the ActionInputFileCache for SHA1 digests!
        actionCache.uploadTree(repository, execRoot, inputRoot);
        // TODO(olaola): set BuildInfo and input total bytes as well.
        ExecuteRequest.Builder request = ExecuteRequest.newBuilder().setAction(action).setAcceptCached(acceptCachedResult).setTotalInputFileCount(inputs.size()).setTimeoutMillis(1000 * Spawns.getTimeoutSeconds(spawn, 120));
        // TODO(olaola): set sensible local and remote timouts.
        ExecuteReply reply = workExecutor.executeRemotely(request.build());
        ExecutionStatus status = reply.getStatus();
        result = reply.getResult();
        // action.
        if (status.getSucceeded()) {
            passRemoteOutErr(actionCache, result, actionExecutionContext.getFileOutErr());
            actionCache.downloadAllResults(result, execRoot);
            return;
        }
        if (status.getError() == ExecutionStatus.ErrorCode.EXEC_FAILED || !options.remoteAllowLocalFallback) {
            passRemoteOutErr(actionCache, result, actionExecutionContext.getFileOutErr());
            throw new UserExecException(status.getErrorDetail());
        }
        // For now, we retry locally on all other remote errors.
        // TODO(olaola): add remote retries on cache miss errors.
        execLocally(spawn, actionExecutionContext, actionCache, actionKey);
    } catch (IOException e) {
        throw new UserExecException("Unexpected IO error.", e);
    } catch (InterruptedException e) {
        eventHandler.handle(Event.warn(mnemonic + " remote work interrupted (" + e + ")"));
        Thread.currentThread().interrupt();
        throw e;
    } catch (StatusRuntimeException e) {
        String stackTrace = "";
        if (verboseFailures) {
            stackTrace = "\n" + Throwables.getStackTraceAsString(e);
        }
        eventHandler.handle(Event.warn(mnemonic + " remote work failed (" + e + ")" + stackTrace));
        if (options.remoteAllowLocalFallback) {
            execLocally(spawn, actionExecutionContext, actionCache, actionKey);
        } else {
            throw new UserExecException(e);
        }
    } catch (CacheNotFoundException e) {
        eventHandler.handle(Event.warn(mnemonic + " remote work results cache miss (" + e + ")"));
        if (options.remoteAllowLocalFallback) {
            execLocally(spawn, actionExecutionContext, actionCache, actionKey);
        } else {
            throw new UserExecException(e);
        }
    } catch (UnsupportedOperationException e) {
        eventHandler.handle(Event.warn(mnemonic + " unsupported operation for action cache (" + e + ")"));
    }
}

Example 17 with Executor

use of com.google.devtools.build.lib.actions.Executor in project bazel by bazelbuild.

the class CppCompileAction method execute.

@Override
@ThreadCompatible
public void execute(ActionExecutionContext actionExecutionContext) throws ActionExecutionException, InterruptedException {
    setModuleFileFlags();
    Executor executor = actionExecutionContext.getExecutor();
    CppCompileActionContext.Reply reply;
    try {
        reply = executor.getContext(actionContext).execWithReply(this, actionExecutionContext);
    } catch (ExecException e) {
        throw e.toActionExecutionException("C++ compilation of rule '" + getOwner().getLabel() + "'", executor.getVerboseFailures(), this);
    }
    ensureCoverageNotesFilesExist();
    // This is the .d file scanning part.
    IncludeScanningContext scanningContext = executor.getContext(IncludeScanningContext.class);
    Path execRoot = executor.getExecRoot();
    NestedSet<Artifact> discoveredInputs = discoverInputsFromDotdFiles(execRoot, scanningContext.getArtifactResolver(), reply);
    // Clear in-memory .d files early.
    reply = null;
    // Post-execute "include scanning", which modifies the action inputs to match what the compile
    // action actually used by incorporating the results of .d file parsing.
    updateActionInputs(discoveredInputs);
    // HeadersCheckingMode.NONE should only be used for ObjC build actions.
    if (cppSemantics.needsIncludeValidation()) {
        validateInclusions(discoveredInputs, actionExecutionContext.getArtifactExpander(), executor.getEventHandler());
    }
}

Example 18 with Executor

use of com.google.devtools.build.lib.actions.Executor in project bazel by bazelbuild.

the class CppCompileAction method discoverInputs.

@Nullable
@Override
public Iterable<Artifact> discoverInputs(ActionExecutionContext actionExecutionContext) throws ActionExecutionException, InterruptedException {
    Executor executor = actionExecutionContext.getExecutor();
    Iterable<Artifact> initialResult;
    actionExecutionContext.getExecutor().getEventBus().post(ActionStatusMessage.analysisStrategy(this));
    try {
        initialResult = executor.getContext(actionContext).findAdditionalInputs(this, actionExecutionContext, cppSemantics.getIncludeProcessing());
    } catch (ExecException e) {
        throw e.toActionExecutionException("Include scanning of rule '" + getOwner().getLabel() + "'", executor.getVerboseFailures(), this);
    }
    if (initialResult == null) {
        NestedSetBuilder<Artifact> result = NestedSetBuilder.stableOrder();
        if (useHeaderModules) {
            // Here, we cannot really know what the top-level modules are, so we just mark all
            // transitive modules as "top level".
            topLevelModules = Sets.newLinkedHashSet(context.getTransitiveModules(usePic).toCollection());
            result.addTransitive(context.getTransitiveModules(usePic));
        }
        result.addTransitive(prunableInputs);
        additionalInputs = result.build();
        return result.build();
    }
    Set<Artifact> initialResultSet = Sets.newLinkedHashSet(initialResult);
    if (shouldPruneModules) {
        if (CppFileTypes.CPP_MODULE.matches(sourceFile.getFilename())) {
            usedModules = ImmutableSet.of(sourceFile);
            initialResultSet.add(sourceFile);
        } else {
            usedModules = Sets.newLinkedHashSet();
            topLevelModules = null;
            for (CppCompilationContext.TransitiveModuleHeaders usedModule : context.getUsedModules(usePic, initialResultSet)) {
                usedModules.add(usedModule.getModule());
            }
            initialResultSet.addAll(usedModules);
        }
    }
    initialResult = initialResultSet;
    this.additionalInputs = initialResult;
    // In some cases, execution backends need extra files for each included file. Add them
    // to the set of inputs the caller may need to be aware of.
    Collection<Artifact> result = new HashSet<>();
    ArtifactResolver artifactResolver = executor.getContext(IncludeScanningContext.class).getArtifactResolver();
    for (Artifact artifact : initialResult) {
        result.addAll(specialInputsHandler.getInputsForIncludedFile(artifact, artifactResolver));
    }
    for (Artifact artifact : getInputs()) {
        result.addAll(specialInputsHandler.getInputsForIncludedFile(artifact, artifactResolver));
    }
    // TODO(ulfjack): This only works if include scanning is enabled; the cleanup is in progress,
    // and this needs to be fixed before we can even consider disabling it.
    resolvedInputs = ImmutableList.copyOf(result);
    Iterables.addAll(result, initialResult);
    return Preconditions.checkNotNull(result);
}

Example 19 with Executor

use of com.google.devtools.build.lib.actions.Executor in project bazel by bazelbuild.

the class ExtractInclusionAction method execute.

@Override
public void execute(ActionExecutionContext actionExecutionContext) throws ActionExecutionException, InterruptedException {
    Executor executor = actionExecutionContext.getExecutor();
    IncludeScanningContext context = executor.getContext(IncludeScanningContext.class);
    try {
        context.extractIncludes(actionExecutionContext, this, getPrimaryInput(), getPrimaryOutput());
    } catch (IOException e) {
        throw new ActionExecutionException(e, this, false);
    } catch (ExecException e) {
        throw e.toActionExecutionException(this);
    }
}

Example 20 with Executor

use of com.google.devtools.build.lib.actions.Executor in project bazel by bazelbuild.

the class JavaHeaderCompileAction method execute.

@Override
public void execute(ActionExecutionContext actionExecutionContext) throws ActionExecutionException, InterruptedException {
    if (!useDirectClasspath()) {
        super.execute(actionExecutionContext);
        return;
    }
    Executor executor = actionExecutionContext.getExecutor();
    SpawnActionContext context = getContext(executor);
    try {
        context.exec(getDirectSpawn(), actionExecutionContext);
    } catch (ExecException e) {
        // if the direct input spawn failed, try again with transitive inputs to produce better
        // better messages
        super.execute(actionExecutionContext);
        // the compilation should never fail with direct deps but succeed with transitive inputs
        if (fallbackError) {
            throw e.toActionExecutionException("header compilation failed unexpectedly", executor.getVerboseFailures(), this);
        }
        Event event = Event.warn(getOwner().getLocation(), "header compilation failed unexpectedly");
        executor.getEventHandler().handle(event);
    }
}