Examples with Entry com.google.devtools.build.lib.actions.cache.ActionCache.Entry used on opensource projects

Example 1 with Entry

use of com.google.devtools.build.lib.actions.cache.ActionCache.Entry in project bazel by bazelbuild.

the class ActionCacheChecker method checkMiddlemanAction.

/**
   * Special handling for the MiddlemanAction. Since MiddlemanAction output
   * artifacts are purely fictional and used only to stay within dependency
   * graph model limitations (action has to depend on artifacts, not on other
   * actions), we do not need to validate metadata for the outputs - only for
   * inputs. We also do not need to validate MiddlemanAction key, since action
   * cache entry key already incorporates that information for the middlemen
   * and we will experience a cache miss when it is different. Whenever it
   * encounters middleman artifacts as input artifacts for other actions, it
   * consults with the aggregated middleman digest computed here.
   */
protected void checkMiddlemanAction(Action action, EventHandler handler, MetadataHandler metadataHandler) {
    if (!cacheConfig.enabled()) {
        // Action cache is disabled, don't generate digests.
        return;
    }
    Artifact middleman = action.getPrimaryOutput();
    String cacheKey = middleman.getExecPathString();
    ActionCache.Entry entry = actionCache.get(cacheKey);
    boolean changed = false;
    if (entry != null) {
        if (entry.isCorrupted()) {
            reportCorruptedCacheEntry(handler, action);
            changed = true;
        } else if (validateArtifacts(entry, action, action.getInputs(), metadataHandler, false)) {
            reportChanged(handler, action);
            changed = true;
        }
    } else {
        reportChangedDeps(handler, action);
        changed = true;
    }
    if (changed) {
        // Compute the aggregated middleman digest.
        // Since we never validate action key for middlemen, we should not store
        // it in the cache entry and just use empty string instead.
        entry = new ActionCache.Entry("", ImmutableMap.<String, String>of(), false);
        for (Artifact input : action.getInputs()) {
            entry.addFile(input.getExecPath(), metadataHandler.getMetadataMaybe(input));
        }
    }
    metadataHandler.setDigestForVirtualArtifact(middleman, entry.getFileDigest());
    if (changed) {
        actionCache.put(cacheKey, entry);
    }
}

Example 2 with Entry

use of com.google.devtools.build.lib.actions.cache.ActionCache.Entry in project bazel by bazelbuild.

the class ActionCacheChecker method getCachedInputs.

@Nullable
public Iterable<Artifact> getCachedInputs(Action action, PackageRootResolver resolver) throws PackageRootResolutionException, InterruptedException {
    ActionCache.Entry entry = getCacheEntry(action);
    if (entry == null || entry.isCorrupted()) {
        return ImmutableList.of();
    }
    List<PathFragment> outputs = new ArrayList<>();
    for (Artifact output : action.getOutputs()) {
        outputs.add(output.getExecPath());
    }
    List<PathFragment> inputExecPaths = new ArrayList<>();
    for (String path : entry.getPaths()) {
        PathFragment execPath = new PathFragment(path);
        // most efficient.
        if (!outputs.contains(execPath)) {
            inputExecPaths.add(execPath);
        }
    }
    // Note that this method may trigger a violation of the desirable invariant that getInputs()
    // is a superset of getMandatoryInputs(). See bug about an "action not in canonical form"
    // error message and the integration test test_crosstool_change_and_failure().
    Map<PathFragment, Artifact> allowedDerivedInputsMap = new HashMap<>();
    for (Artifact derivedInput : action.getAllowedDerivedInputs()) {
        if (!derivedInput.isSourceArtifact()) {
            allowedDerivedInputsMap.put(derivedInput.getExecPath(), derivedInput);
        }
    }
    List<Artifact> inputArtifacts = new ArrayList<>();
    List<PathFragment> unresolvedPaths = new ArrayList<>();
    for (PathFragment execPath : inputExecPaths) {
        Artifact artifact = allowedDerivedInputsMap.get(execPath);
        if (artifact != null) {
            inputArtifacts.add(artifact);
        } else {
            // Remember this execPath, we will try to resolve it as a source artifact.
            unresolvedPaths.add(execPath);
        }
    }
    Map<PathFragment, Artifact> resolvedArtifacts = artifactResolver.resolveSourceArtifacts(unresolvedPaths, resolver);
    if (resolvedArtifacts == null) {
        // We are missing some dependencies. We need to rerun this update later.
        return null;
    }
    for (PathFragment execPath : unresolvedPaths) {
        Artifact artifact = resolvedArtifacts.get(execPath);
        // was used before) and will force action execution.
        if (artifact != null) {
            inputArtifacts.add(artifact);
        }
    }
    return inputArtifacts;
}

Example 3 with Entry

use of com.google.devtools.build.lib.actions.cache.ActionCache.Entry in project bazel by bazelbuild.

the class CompactPersistentActionCache method decode.

/**
   * Creates new action cache entry using given compressed entry data. Data
   * will stay in the compressed format until entry is actually used by the
   * dependency checker.
   */
private static ActionCache.Entry decode(StringIndexer indexer, byte[] data) throws IOException {
    try {
        ByteBuffer source = ByteBuffer.wrap(data);
        byte[] actionKeyBytes = new byte[VarInt.getVarInt(source)];
        source.get(actionKeyBytes);
        String actionKey = new String(actionKeyBytes, ISO_8859_1);
        Md5Digest md5Digest = DigestUtils.read(source);
        int count = VarInt.getVarInt(source);
        ImmutableList.Builder<String> builder = new ImmutableList.Builder<>();
        for (int i = 0; i < count; i++) {
            int id = VarInt.getVarInt(source);
            String filename = (id >= 0 ? indexer.getStringForIndex(id) : null);
            if (filename == null) {
                throw new IOException("Corrupted file index");
            }
            builder.add(filename);
        }
        Md5Digest usedClientEnvDigest = DigestUtils.read(source);
        if (source.remaining() > 0) {
            throw new IOException("serialized entry data has not been fully decoded");
        }
        return new Entry(actionKey, usedClientEnvDigest, count == NO_INPUT_DISCOVERY_COUNT ? null : builder.build(), md5Digest);
    } catch (BufferUnderflowException e) {
        throw new IOException("encoded entry data is incomplete", e);
    }
}

Example 4 with Entry

use of com.google.devtools.build.lib.actions.cache.ActionCache.Entry in project bazel by bazelbuild.

the class ActionCacheChecker method afterExecution.

public void afterExecution(Action action, Token token, MetadataHandler metadataHandler, Map<String, String> clientEnv) throws IOException {
    if (!cacheConfig.enabled()) {
        // Action cache is disabled, don't generate digests.
        return;
    }
    Preconditions.checkArgument(token != null);
    String key = token.cacheKey;
    if (actionCache.get(key) != null) {
        // This cache entry has already been updated by a shared action. We don't need to do it again.
        return;
    }
    Map<String, String> usedClientEnv = computeUsedClientEnv(action, clientEnv);
    ActionCache.Entry entry = new ActionCache.Entry(action.getKey(), usedClientEnv, action.discoversInputs());
    for (Artifact output : action.getOutputs()) {
        // Remove old records from the cache if they used different key.
        String execPath = output.getExecPathString();
        if (!key.equals(execPath)) {
            actionCache.remove(execPath);
        }
        if (!metadataHandler.artifactOmitted(output)) {
            // Output files *must* exist and be accessible after successful action execution.
            Metadata metadata = metadataHandler.getMetadata(output);
            Preconditions.checkState(metadata != null);
            entry.addFile(output.getExecPath(), metadata);
        }
    }
    for (Artifact input : action.getInputs()) {
        entry.addFile(input.getExecPath(), metadataHandler.getMetadataMaybe(input));
    }
    entry.getFileDigest();
    actionCache.put(key, entry);
}

Example 5 with Entry

use of com.google.devtools.build.lib.actions.cache.ActionCache.Entry in project bazel by bazelbuild.

the class ActionCacheChecker method getTokenIfNeedToExecute.

/**
   * Checks whether {@code action} needs to be executed and returns a non-null Token if so.
   *
   * <p>The method checks if any of the action's inputs or outputs have changed. Returns a non-null
   * {@link Token} if the action needs to be executed, and null otherwise.
   *
   * <p>If this method returns non-null, indicating that the action will be executed, the
   * metadataHandler's {@link MetadataHandler#discardOutputMetadata} method must be called, so that
   * it does not serve stale metadata for the action's outputs after the action is executed.
   */
// Note: the handler should only be used for DEPCHECKER events; there's no
// guarantee it will be available for other events.
public Token getTokenIfNeedToExecute(Action action, Iterable<Artifact> resolvedCacheArtifacts, Map<String, String> clientEnv, EventHandler handler, MetadataHandler metadataHandler) {
    // TODO(bazel-team): (2010) For RunfilesAction/SymlinkAction and similar actions that
    // produce only symlinks we should not check whether inputs are valid at all - all that matters
    // that inputs and outputs are still exist (and new inputs have not appeared). All other checks
    // are unnecessary. In other words, the only metadata we should check for them is file existence
    // itself.
    MiddlemanType middlemanType = action.getActionType();
    if (middlemanType.isMiddleman()) {
        // propagate invalidation of their inputs.
        if (middlemanType != MiddlemanType.ERROR_PROPAGATING_MIDDLEMAN) {
            checkMiddlemanAction(action, handler, metadataHandler);
        }
        return null;
    }
    if (!cacheConfig.enabled()) {
        return new Token(getKeyString(action));
    }
    Iterable<Artifact> actionInputs = action.getInputs();
    // Resolve action inputs from cache, if necessary.
    boolean inputsDiscovered = action.inputsDiscovered();
    if (!inputsDiscovered && resolvedCacheArtifacts != null) {
        // The action doesn't know its inputs, but the caller has a good idea of what they are.
        Preconditions.checkState(action.discoversInputs(), "Actions that don't know their inputs must discover them: %s", action);
        actionInputs = resolvedCacheArtifacts;
    }
    ActionCache.Entry entry = getCacheEntry(action);
    if (mustExecute(action, entry, handler, metadataHandler, actionInputs, clientEnv)) {
        if (entry != null) {
            removeCacheEntry(action);
        }
        return new Token(getKeyString(action));
    }
    if (!inputsDiscovered) {
        action.updateInputs(actionInputs);
    }
    return null;
}