Examples with RemoteOutputArtifact com.google.idea.blaze.base.command.buildresult.RemoteOutputArtifact used on opensource projects

Example 6 with RemoteOutputArtifact

use of com.google.idea.blaze.base.command.buildresult.RemoteOutputArtifact in project intellij by bazelbuild.

the class RemoteOutputsCache method updateCache.

private void updateCache(BlazeContext context, Set<RemoteOutputArtifact> toCache, RemoteOutputArtifacts previousOutputs) {
    Map<String, RemoteOutputArtifact> newState = toCache.stream().collect(toImmutableMap(RemoteOutputsCache::getCacheKey, Functions.identity()));
    Map<String, File> cachedFiles = readCachedFiles();
    try {
        Map<String, RemoteOutputArtifact> updatedOutputs = FileCacheDiffer.findUpdatedOutputs(newState, cachedFiles, previousOutputs);
        List<File> removed = cachedFiles.entrySet().stream().filter(e -> !newState.containsKey(e.getKey())).map(Map.Entry::getValue).collect(toImmutableList());
        // Ensure the cache dir exists
        if (!cacheDir.exists()) {
            if (!cacheDir.mkdirs()) {
                IssueOutput.error("Could not create remote outputs cache directory").submit(context);
                context.setHasError();
                return;
            }
        }
        ListenableFuture<?> downloadArtifactsFuture = RemoteArtifactPrefetcher.getInstance().downloadArtifacts(/* projectName= */
        project.getName(), /* outputArtifacts= */
        BlazeArtifact.getRemoteArtifacts(updatedOutputs.values()));
        FutureUtil.waitForFuture(context, downloadArtifactsFuture).timed("PrefetchRemoteOutput", EventType.Prefetching).withProgressMessage("Prefetching output artifacts...").run();
        List<ListenableFuture<?>> futures = new ArrayList<>(copyLocally(updatedOutputs));
        futures.addAll(deleteCacheFiles(removed));
        Futures.allAsList(futures).get();
        this.cachedFiles = newState.keySet().stream().collect(toImmutableMap(Functions.identity(), k -> new File(cacheDir, k)));
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        context.setCancelled();
    } catch (ExecutionException e) {
        IssueOutput.warn("Remote outputs synchronization didn't complete: " + e.getMessage()).submit(context);
    }
}

Example 7 with RemoteOutputArtifact

use of com.google.idea.blaze.base.command.buildresult.RemoteOutputArtifact in project intellij by bazelbuild.

the class PackageManifestReader method readPackageManifestFiles.

/**
 * @return A map from java source absolute file path to declared package string.
 */
public Map<TargetKey, Map<ArtifactLocation, String>> readPackageManifestFiles(Project project, BlazeContext context, ArtifactLocationDecoder decoder, Map<TargetKey, ArtifactLocation> javaPackageManifests, ListeningExecutorService executorService) {
    Map<OutputArtifact, TargetKey> fileToLabelMap = Maps.newHashMap();
    for (Map.Entry<TargetKey, ArtifactLocation> entry : javaPackageManifests.entrySet()) {
        TargetKey key = entry.getKey();
        BlazeArtifact artifact = decoder.resolveOutput(entry.getValue());
        if (artifact instanceof OutputArtifact) {
            fileToLabelMap.put((OutputArtifact) artifact, key);
        }
    }
    ArtifactsDiff diff;
    try {
        diff = ArtifactsDiff.diffArtifacts(artifactState, fileToLabelMap.keySet());
        artifactState = diff.getNewState();
    } catch (InterruptedException e) {
        throw new ProcessCanceledException(e);
    } catch (ExecutionException e) {
        context.setHasError();
        IssueOutput.error("Updating package manifest files failed: " + e).submit(context);
        throw new AssertionError("Unhandled exception", e);
    }
    // Find all not cached {@link RemoteOutputArtifact} and download them before parsing manifest
    // file
    ImmutableList<RemoteOutputArtifact> toDownload = BlazeArtifact.getRemoteArtifacts(diff.getUpdatedOutputs()).stream().filter(a -> findArtifactInCache(project, a) == null).collect(toImmutableList());
    ListenableFuture<?> fetchRemoteArtifactFuture = RemoteArtifactPrefetcher.getInstance().downloadArtifacts(project.getName(), toDownload);
    ListenableFuture<PrefetchStats> fetchLocalFilesFuture = PrefetchService.getInstance().prefetchFiles(BlazeArtifact.getLocalFiles(diff.getUpdatedOutputs()), true, false);
    if (!FutureUtil.waitForFuture(context, Futures.allAsList(fetchRemoteArtifactFuture, fetchLocalFilesFuture)).timed("FetchPackageManifests", EventType.Prefetching).withProgressMessage("Reading package manifests...").run().success()) {
        return null;
    }
    try {
        long bytesConsumed = toDownload.stream().mapToLong(RemoteOutputArtifact::getLength).sum() + fetchLocalFilesFuture.get().bytesPrefetched();
        if (bytesConsumed > 0) {
            context.output(new NetworkTrafficUsedOutput(bytesConsumed, "packagemanifest"));
        }
    } catch (InterruptedException | ExecutionException e) {
        // Should never happen - the future has already completed.
        logger.error(e);
    // carry on - failing to log the stats should not affect anything else.
    }
    List<ListenableFuture<Void>> futures = Lists.newArrayList();
    for (OutputArtifact file : diff.getUpdatedOutputs()) {
        futures.add(executorService.submit(() -> {
            Map<ArtifactLocation, String> manifest = parseManifestFile(project, file);
            manifestMap.put(fileToLabelMap.get(file), manifest);
            return null;
        }));
    }
    for (ArtifactState file : diff.getRemovedOutputs()) {
        TargetKey key = this.fileToLabelMap.get(file);
        if (key != null) {
            manifestMap.remove(key);
        }
    }
    this.fileToLabelMap = fileToLabelMap.entrySet().stream().filter(e -> diff.getNewState().containsKey(e.getKey().getKey())).collect(toImmutableMap(e -> e.getKey().toArtifactState(), Map.Entry::getValue));
    try {
        Futures.allAsList(futures).get();
    } catch (ExecutionException | InterruptedException e) {
        logger.error(e);
        throw new IllegalStateException("Could not read sources");
    }
    return manifestMap;
}

Example 8 with RemoteOutputArtifact

use of com.google.idea.blaze.base.command.buildresult.RemoteOutputArtifact in project intellij by bazelbuild.

the class FileCacheDiffer method shouldUpdateRemote.

private static boolean shouldUpdateRemote(RemoteOutputArtifact newOutput, RemoteOutputArtifacts previousOutputs) {
    RemoteOutputArtifact previous = previousOutputs.findRemoteOutput(newOutput.getRelativePath());
    if (previous == null) {
        return true;
    }
    ArtifactState previousState = previous.toArtifactState();
    ArtifactState newState = newOutput.toArtifactState();
    return previousState == null || (newState != null && previousState.isMoreRecent(newState));
}

Example 9 with RemoteOutputArtifact

use of com.google.idea.blaze.base.command.buildresult.RemoteOutputArtifact in project intellij by bazelbuild.

the class RemoteOutputArtifacts method appendNewOutputs.

/**
 * Merges this set of outputs with another set, returning a new {@link RemoteOutputArtifacts}
 * instance.
 */
public RemoteOutputArtifacts appendNewOutputs(Set<OutputArtifact> outputs) {
    HashMap<String, RemoteOutputArtifact> map = new HashMap<>(remoteOutputArtifacts);
    // more recently built artifacts replace existing ones with the same path
    outputs.forEach(a -> {
        String key = a.getKey();
        if (!(a instanceof RemoteOutputArtifact)) {
            map.remove(key);
        } else {
            RemoteOutputArtifact newOutput = (RemoteOutputArtifact) a;
            RemoteOutputArtifact other = map.get(key);
            if (other == null || other.getSyncTimeMillis() < newOutput.getSyncTimeMillis()) {
                map.put(key, newOutput);
            }
        }
    });
    return new RemoteOutputArtifacts(ImmutableMap.copyOf(map));
}

Example 10 with RemoteOutputArtifact

use of com.google.idea.blaze.base.command.buildresult.RemoteOutputArtifact in project intellij by bazelbuild.

the class RemoteOutputsCacheTest method testDotInDirectory.

@Test
public void testDotInDirectory() {
    RemoteOutputArtifact artifact = mock(RemoteOutputArtifact.class);
    when(artifact.getKey()).thenReturn("k8-opt/foo.bar/foo.bar");
    assertThat(RemoteOutputsCache.getCacheKey(artifact)).isEqualTo("foo_df1c67cb.bar");
}