use of com.google.idea.blaze.base.filecache.ArtifactsDiff in project intellij by bazelbuild.
the class BlazeIdeInterfaceAspectsImpl method updateTargetMap.
@Nullable
private static TargetMapAndInterfaceState updateTargetMap(Project project, BlazeContext context, WorkspaceRoot workspaceRoot, SyncProjectState projectState, BlazeBuildOutputs buildResult, boolean mergeWithOldState, @Nullable BlazeProjectData oldProjectData) {
// any old state that we have in an attempt not to lose too much code.
if (buildResult.buildResult.status == BuildResult.Status.BUILD_ERROR) {
mergeWithOldState = true;
}
TargetMap oldTargetMap = oldProjectData != null ? oldProjectData.getTargetMap() : null;
BlazeIdeInterfaceState prevState = oldProjectData != null ? oldProjectData.getTargetData().ideInterfaceState : null;
Predicate<String> ideInfoPredicate = AspectStrategy.ASPECT_OUTPUT_FILE_PREDICATE;
Collection<OutputArtifact> files = buildResult.getOutputGroupArtifacts(group -> group.startsWith(OutputGroup.INFO.prefix)).stream().filter(f -> ideInfoPredicate.test(f.getKey())).distinct().collect(toImmutableList());
ArtifactsDiff diff;
try {
diff = ArtifactsDiff.diffArtifacts(prevState != null ? prevState.ideInfoFileState : null, files);
} catch (InterruptedException e) {
throw new ProcessCanceledException(e);
} catch (ExecutionException e) {
IssueOutput.error("Failed to diff aspect output files: " + e).submit(context);
return null;
}
// if we're merging with the old state, no files are removed
int targetCount = files.size() + (mergeWithOldState ? diff.getRemovedOutputs().size() : 0);
int removedCount = mergeWithOldState ? 0 : diff.getRemovedOutputs().size();
context.output(PrintOutput.log(String.format("Total rules: %d, new/changed: %d, removed: %d", targetCount, diff.getUpdatedOutputs().size(), removedCount)));
ListenableFuture<?> downloadArtifactsFuture = RemoteArtifactPrefetcher.getInstance().downloadArtifacts(/* projectName= */
project.getName(), /* outputArtifacts= */
BlazeArtifact.getRemoteArtifacts(diff.getUpdatedOutputs()));
ListenableFuture<?> loadFilesInJvmFuture = RemoteArtifactPrefetcher.getInstance().loadFilesInJvm(/* outputArtifacts= */
BlazeArtifact.getRemoteArtifacts(diff.getUpdatedOutputs()));
if (!FutureUtil.waitForFuture(context, Futures.allAsList(downloadArtifactsFuture, loadFilesInJvmFuture)).timed("PrefetchRemoteAspectOutput", EventType.Prefetching).withProgressMessage("Reading IDE info result...").run().success()) {
return null;
}
ListenableFuture<?> fetchLocalFilesFuture = PrefetchService.getInstance().prefetchFiles(/* files= */
BlazeArtifact.getLocalFiles(diff.getUpdatedOutputs()), /* refetchCachedFiles= */
true, /* fetchFileTypes= */
false);
if (!FutureUtil.waitForFuture(context, fetchLocalFilesFuture).timed("FetchAspectOutput", EventType.Prefetching).withProgressMessage("Reading IDE info result...").run().success()) {
return null;
}
ImportRoots importRoots = ImportRoots.builder(workspaceRoot, Blaze.getBuildSystem(project)).add(projectState.getProjectViewSet()).build();
BlazeConfigurationHandler configHandler = new BlazeConfigurationHandler(projectState.getBlazeInfo());
TargetMapAndInterfaceState state = updateState(project, context, prevState, diff, configHandler, projectState.getBlazeVersionData(), projectState.getLanguageSettings(), importRoots, mergeWithOldState, oldTargetMap);
if (state == null) {
return null;
}
// prefetch ide-resolve genfiles
Scope.push(context, childContext -> {
childContext.push(new TimingScope("GenfilesPrefetchBuildArtifacts", EventType.Other));
ImmutableList<OutputArtifact> resolveOutputs = ImmutableList.copyOf(buildResult.getOutputGroupArtifacts(group -> group.startsWith(OutputGroup.RESOLVE.prefix)));
prefetchGenfiles(context, resolveOutputs);
});
return state;
}
use of com.google.idea.blaze.base.filecache.ArtifactsDiff in project intellij by bazelbuild.
the class PackageManifestReader method readPackageManifestFiles.
/**
* @return A map from java source absolute file path to declared package string.
*/
public Map<TargetKey, Map<ArtifactLocation, String>> readPackageManifestFiles(Project project, BlazeContext context, ArtifactLocationDecoder decoder, Map<TargetKey, ArtifactLocation> javaPackageManifests, ListeningExecutorService executorService) {
Map<OutputArtifact, TargetKey> fileToLabelMap = Maps.newHashMap();
for (Map.Entry<TargetKey, ArtifactLocation> entry : javaPackageManifests.entrySet()) {
TargetKey key = entry.getKey();
BlazeArtifact artifact = decoder.resolveOutput(entry.getValue());
if (artifact instanceof OutputArtifact) {
fileToLabelMap.put((OutputArtifact) artifact, key);
}
}
ArtifactsDiff diff;
try {
diff = ArtifactsDiff.diffArtifacts(artifactState, fileToLabelMap.keySet());
artifactState = diff.getNewState();
} catch (InterruptedException e) {
throw new ProcessCanceledException(e);
} catch (ExecutionException e) {
context.setHasError();
IssueOutput.error("Updating package manifest files failed: " + e).submit(context);
throw new AssertionError("Unhandled exception", e);
}
// Find all not cached {@link RemoteOutputArtifact} and download them before parsing manifest
// file
ImmutableList.Builder<RemoteOutputArtifact> toDownload = ImmutableList.builder();
for (OutputArtifact outputArtifact : diff.getUpdatedOutputs()) {
if (!(outputArtifact instanceof RemoteOutputArtifact)) {
continue;
}
if (findArtifactInCache(project, outputArtifact) != null) {
continue;
}
toDownload.add((RemoteOutputArtifact) outputArtifact);
}
ListenableFuture<?> fetchRemoteArtifactFuture = RemoteArtifactPrefetcher.getInstance().downloadArtifacts(project.getName(), toDownload.build());
ListenableFuture<?> fetchFuture = PrefetchService.getInstance().prefetchFiles(BlazeArtifact.getLocalFiles(diff.getUpdatedOutputs()), true, false);
if (!FutureUtil.waitForFuture(context, Futures.allAsList(fetchRemoteArtifactFuture, fetchFuture)).timed("FetchPackageManifests", EventType.Prefetching).withProgressMessage("Reading package manifests...").run().success()) {
return null;
}
List<ListenableFuture<Void>> futures = Lists.newArrayList();
for (OutputArtifact file : diff.getUpdatedOutputs()) {
futures.add(executorService.submit(() -> {
Map<ArtifactLocation, String> manifest = parseManifestFile(project, file);
manifestMap.put(fileToLabelMap.get(file), manifest);
return null;
}));
}
for (ArtifactState file : diff.getRemovedOutputs()) {
TargetKey key = this.fileToLabelMap.get(file);
if (key != null) {
manifestMap.remove(key);
}
}
this.fileToLabelMap = fileToLabelMap.entrySet().stream().filter(e -> diff.getNewState().containsKey(e.getKey().getKey())).collect(toImmutableMap(e -> e.getKey().toArtifactState(), Map.Entry::getValue));
try {
Futures.allAsList(futures).get();
} catch (ExecutionException | InterruptedException e) {
logger.error(e);
throw new IllegalStateException("Could not read sources");
}
return manifestMap;
}
use of com.google.idea.blaze.base.filecache.ArtifactsDiff in project intellij by bazelbuild.
the class EmptyLibrary method getUpdatedEmptyJarTracker.
/**
* Takes the current set of artifacts and artifacts known from previous sync to calculate which of
* new artifacts are empty. Stores the status of each artifact in {@link EmptyJarTracker}.
*
* <p>Artifacts that have changed are fetched and passed to {@link EmptyLibraryFilter} to check if
* they are empty or not.
*
* <p>NOTE: This method does Network IO and File IO, keep an eye on performance
*/
private static EmptyJarTracker getUpdatedEmptyJarTracker(Project project, BlazeContext context, Collection<OutputArtifact> newArtifacts, EmptyJarTracker oldTracker) {
ImmutableMap<String, ArtifactState> oldState = oldTracker.getState();
try {
// Calculate artifacts that have changed or been removed since last sync
ArtifactsDiff diff = ArtifactsDiff.diffArtifacts(oldState, newArtifacts);
ImmutableList<OutputArtifact> updated = diff.getUpdatedOutputs();
ImmutableSet<ArtifactState> removed = diff.getRemovedOutputs();
// Copy over tracking data from previous sync, and remove entries which are no longer valid.
EmptyJarTracker.Builder builder = EmptyJarTracker.builder();
builder.addAllEntries(oldTracker);
builder.removeEntries(removed);
// Prefetch updated artifacts
ListenableFuture<?> future = RemoteArtifactPrefetcher.getInstance().downloadArtifacts(project.getName(), BlazeArtifact.getRemoteArtifacts(updated));
FutureUtil.waitForFuture(context, future).timed("FetchJarsForEmptyStatusTracking", EventType.Prefetching).withProgressMessage("Fetching JARs to track empty status..").run();
// Evaluate if the updated artifacts are empty or not
Map<ArtifactState, Boolean> updatedStatuses = getEmptyStatusInParallel(updated, new EmptyLibraryFilter(), FetchExecutor.EXECUTOR);
builder.addAllEntries(updatedStatuses);
if (!updated.isEmpty()) {
context.output(PrintOutput.log(String.format("[Empty JAR Filter] Calculated empty status of %d JARs", updated.size())));
}
if (!removed.isEmpty()) {
context.output(PrintOutput.log(String.format("[Empty JAR Filter] Removed %d JARs", removed.size())));
}
return builder.build();
} catch (InterruptedException e) {
String message = "Updating EmptyJarTracker failed.";
logger.warn(message, e);
IssueOutput.warn(message).submit(context);
} catch (ExecutionException e) {
Thread.currentThread().interrupt();
context.setCancelled();
}
// Something went wrong, return old tracking data
return oldTracker;
}
use of com.google.idea.blaze.base.filecache.ArtifactsDiff in project intellij by bazelbuild.
the class JdepsFileReader method doLoadJdepsFiles.
@Nullable
private JdepsState doLoadJdepsFiles(Project project, BlazeContext context, ArtifactLocationDecoder decoder, @Nullable JdepsState oldState, Collection<TargetIdeInfo> targetsToLoad, SyncMode syncMode) throws InterruptedException, ExecutionException {
Map<OutputArtifact, TargetKey> fileToTargetMap = Maps.newHashMap();
for (TargetIdeInfo target : targetsToLoad) {
BlazeArtifact output = resolveJdepsOutput(decoder, target);
if (output instanceof OutputArtifact) {
fileToTargetMap.put((OutputArtifact) output, target.getKey());
}
}
ArtifactsDiff diff = ArtifactsDiff.diffArtifacts(oldState != null ? oldState.getArtifactState() : null, fileToTargetMap.keySet());
// TODO: handle prefetching for arbitrary OutputArtifacts
List<OutputArtifact> outputArtifacts = diff.getUpdatedOutputs();
// already. Additional logging to identify what is going wrong.
if (!outputArtifacts.isEmpty() && !SyncMode.involvesBlazeBuild(syncMode)) {
logger.warn("ArtifactDiff: " + outputArtifacts.size() + " outputs need to be updated during SyncMode.NO_BUILD ");
if (oldState == null) {
logger.warn("ArtifactDiff: oldState == null, we failed to load prior JdepsState.");
} else {
// Do not list all artifacts since it may be pretty long.
if (oldState.getArtifactState().size() != fileToTargetMap.size()) {
logger.warn("Existing artifact state does not match with target map." + " [oldState.getArtifactState().size() = " + oldState.getArtifactState().size() + ", fileToTargetMap.size() = " + fileToTargetMap.size() + "]");
}
}
}
ListenableFuture<?> downloadArtifactsFuture = RemoteArtifactPrefetcher.getInstance().downloadArtifacts(/* projectName= */
project.getName(), /* outputArtifacts= */
BlazeArtifact.getRemoteArtifacts(outputArtifacts));
ListenableFuture<?> fetchLocalFilesFuture = PrefetchService.getInstance().prefetchFiles(BlazeArtifact.getLocalFiles(outputArtifacts), true, false);
if (!FutureUtil.waitForFuture(context, Futures.allAsList(downloadArtifactsFuture, fetchLocalFilesFuture)).timed("FetchJdeps", EventType.Prefetching).withProgressMessage("Reading jdeps files...").run().success()) {
return null;
}
AtomicLong totalSizeLoaded = new AtomicLong(0);
List<ListenableFuture<Result>> futures = Lists.newArrayList();
for (OutputArtifact updatedFile : outputArtifacts) {
futures.add(FetchExecutor.EXECUTOR.submit(() -> {
totalSizeLoaded.addAndGet(updatedFile.getLength());
try (InputStream inputStream = updatedFile.getInputStream()) {
Deps.Dependencies dependencies = Deps.Dependencies.parseFrom(inputStream);
if (dependencies == null) {
return null;
}
List<String> deps = dependencies.getDependencyList().stream().filter(dep -> relevantDep(dep)).map(Dependency::getPath).collect(toImmutableList());
TargetKey targetKey = fileToTargetMap.get(updatedFile);
return new Result(updatedFile, targetKey, deps);
} catch (IOException e) {
logger.info("Could not read jdeps file: " + updatedFile);
return null;
}
}));
}
JdepsState.Builder state = JdepsState.builder();
if (oldState != null) {
state.list.addAll(oldState.data);
}
state.removeArtifacts(diff.getUpdatedOutputs().stream().map(OutputArtifact::toArtifactState).collect(toImmutableList()));
state.removeArtifacts(diff.getRemovedOutputs());
for (Result result : Futures.allAsList(futures).get()) {
if (result != null) {
state.list.add(JdepsData.create(result.targetKey, result.dependencies, result.output.toArtifactState()));
}
}
context.output(PrintOutput.log(String.format("Loaded %d jdeps files, total size %dkB", diff.getUpdatedOutputs().size(), totalSizeLoaded.get() / 1024)));
return state.build();
}
Aggregations