Examples with PrefetchStats com.google.idea.blaze.base.prefetch.PrefetchStats used on opensource projects

Example 1 with PrefetchStats

use of com.google.idea.blaze.base.prefetch.PrefetchStats in project intellij by bazelbuild.

the class PackageManifestReader method readPackageManifestFiles.

/**
 * @return A map from java source absolute file path to declared package string.
 */
public Map<TargetKey, Map<ArtifactLocation, String>> readPackageManifestFiles(Project project, BlazeContext context, ArtifactLocationDecoder decoder, Map<TargetKey, ArtifactLocation> javaPackageManifests, ListeningExecutorService executorService) {
    Map<OutputArtifact, TargetKey> fileToLabelMap = Maps.newHashMap();
    for (Map.Entry<TargetKey, ArtifactLocation> entry : javaPackageManifests.entrySet()) {
        TargetKey key = entry.getKey();
        BlazeArtifact artifact = decoder.resolveOutput(entry.getValue());
        if (artifact instanceof OutputArtifact) {
            fileToLabelMap.put((OutputArtifact) artifact, key);
        }
    }
    ArtifactsDiff diff;
    try {
        diff = ArtifactsDiff.diffArtifacts(artifactState, fileToLabelMap.keySet());
        artifactState = diff.getNewState();
    } catch (InterruptedException e) {
        throw new ProcessCanceledException(e);
    } catch (ExecutionException e) {
        context.setHasError();
        IssueOutput.error("Updating package manifest files failed: " + e).submit(context);
        throw new AssertionError("Unhandled exception", e);
    }
    // Find all not cached {@link RemoteOutputArtifact} and download them before parsing manifest
    // file
    ImmutableList<RemoteOutputArtifact> toDownload = BlazeArtifact.getRemoteArtifacts(diff.getUpdatedOutputs()).stream().filter(a -> findArtifactInCache(project, a) == null).collect(toImmutableList());
    ListenableFuture<?> fetchRemoteArtifactFuture = RemoteArtifactPrefetcher.getInstance().downloadArtifacts(project.getName(), toDownload);
    ListenableFuture<PrefetchStats> fetchLocalFilesFuture = PrefetchService.getInstance().prefetchFiles(BlazeArtifact.getLocalFiles(diff.getUpdatedOutputs()), true, false);
    if (!FutureUtil.waitForFuture(context, Futures.allAsList(fetchRemoteArtifactFuture, fetchLocalFilesFuture)).timed("FetchPackageManifests", EventType.Prefetching).withProgressMessage("Reading package manifests...").run().success()) {
        return null;
    }
    try {
        long bytesConsumed = toDownload.stream().mapToLong(RemoteOutputArtifact::getLength).sum() + fetchLocalFilesFuture.get().bytesPrefetched();
        if (bytesConsumed > 0) {
            context.output(new NetworkTrafficUsedOutput(bytesConsumed, "packagemanifest"));
        }
    } catch (InterruptedException | ExecutionException e) {
        // Should never happen - the future has already completed.
        logger.error(e);
    // carry on - failing to log the stats should not affect anything else.
    }
    List<ListenableFuture<Void>> futures = Lists.newArrayList();
    for (OutputArtifact file : diff.getUpdatedOutputs()) {
        futures.add(executorService.submit(() -> {
            Map<ArtifactLocation, String> manifest = parseManifestFile(project, file);
            manifestMap.put(fileToLabelMap.get(file), manifest);
            return null;
        }));
    }
    for (ArtifactState file : diff.getRemovedOutputs()) {
        TargetKey key = this.fileToLabelMap.get(file);
        if (key != null) {
            manifestMap.remove(key);
        }
    }
    this.fileToLabelMap = fileToLabelMap.entrySet().stream().filter(e -> diff.getNewState().containsKey(e.getKey().getKey())).collect(toImmutableMap(e -> e.getKey().toArtifactState(), Map.Entry::getValue));
    try {
        Futures.allAsList(futures).get();
    } catch (ExecutionException | InterruptedException e) {
        logger.error(e);
        throw new IllegalStateException("Could not read sources");
    }
    return manifestMap;
}

Example 2 with PrefetchStats

use of com.google.idea.blaze.base.prefetch.PrefetchStats in project intellij by bazelbuild.

the class ProjectUpdateSyncTask method run.

private void run(BlazeContext context) throws SyncCanceledException, SyncFailedException {
    TargetMap targetMap = targetData.targetMap;
    RemoteOutputArtifacts oldRemoteState = RemoteOutputArtifacts.fromProjectData(oldProjectData);
    RemoteOutputArtifacts newRemoteState = targetData.remoteOutputs;
    ArtifactLocationDecoder artifactLocationDecoder = new ArtifactLocationDecoderImpl(blazeInfo, projectState.getWorkspacePathResolver(), newRemoteState);
    Scope.push(context, childContext -> {
        childContext.push(new TimingScope("UpdateRemoteOutputsCache", EventType.Prefetching));
        RemoteOutputsCache.getInstance(project).updateCache(context, targetMap, projectState.getLanguageSettings(), newRemoteState, oldRemoteState, /* clearCache= */
        syncMode == SyncMode.FULL);
    });
    SyncState.Builder syncStateBuilder = new SyncState.Builder();
    Scope.push(context, childContext -> {
        childContext.push(new TimingScope("UpdateSyncState", EventType.Other));
        for (BlazeSyncPlugin syncPlugin : BlazeSyncPlugin.EP_NAME.getExtensions()) {
            syncPlugin.updateSyncState(project, childContext, workspaceRoot, projectState.getProjectViewSet(), projectState.getLanguageSettings(), projectState.getBlazeVersionData(), projectState.getWorkingSet(), artifactLocationDecoder, targetMap, syncStateBuilder, oldProjectData != null ? oldProjectData.getSyncState() : null, syncMode);
        }
    });
    if (context.isCancelled()) {
        throw new SyncCanceledException();
    }
    if (context.hasErrors()) {
        throw new SyncFailedException();
    }
    BlazeProjectData newProjectData = new BlazeProjectData(targetData, blazeInfo, projectState.getBlazeVersionData(), projectState.getWorkspacePathResolver(), artifactLocationDecoder, projectState.getLanguageSettings(), syncStateBuilder.build());
    FileCaches.onSync(project, context, projectState.getProjectViewSet(), newProjectData, oldProjectData, syncMode);
    ListenableFuture<PrefetchStats> prefetch = PrefetchService.getInstance().prefetchProjectFiles(project, projectState.getProjectViewSet(), newProjectData);
    FutureResult<PrefetchStats> result = FutureUtil.waitForFuture(context, prefetch).withProgressMessage("Prefetching files...").timed("PrefetchFiles", EventType.Prefetching).onError("Prefetch failed").run();
    if (result.success()) {
        long prefetched = result.result().bytesPrefetched();
        if (prefetched > 0) {
            context.output(new NetworkTrafficUsedOutput(prefetched, "prefetch"));
        }
    }
    ListenableFuture<DirectoryStructure> directoryStructureFuture = DirectoryStructure.getRootDirectoryStructure(project, workspaceRoot, projectState.getProjectViewSet());
    refreshVirtualFileSystem(context, project, newProjectData);
    DirectoryStructure directoryStructure = FutureUtil.waitForFuture(context, directoryStructureFuture).withProgressMessage("Computing directory structure...").timed("DirectoryStructure", EventType.Other).onError("Directory structure computation failed").run().result();
    if (directoryStructure == null) {
        throw new SyncFailedException();
    }
    boolean success = updateProject(context, projectState.getProjectViewSet(), projectState.getBlazeVersionData(), directoryStructure, oldProjectData, newProjectData);
    if (!success) {
        throw new SyncFailedException();
    }
}