Examples with SourceToOutputMapping org.jetbrains.jps.builders.storage.SourceToOutputMapping used on opensource projects

Example 6 with SourceToOutputMapping

use of org.jetbrains.jps.builders.storage.SourceToOutputMapping in project intellij-community by JetBrains.

the class BuildOutputConsumerImpl method registerOutput.

private void registerOutput(final File output, boolean isDirectory, Collection<String> sourcePaths) throws IOException {
    final String outputPath = FileUtil.toSystemIndependentName(output.getPath());
    for (File outputRoot : myOutputs) {
        final String outputRootPath = FileUtil.toSystemIndependentName(outputRoot.getPath());
        final String relativePath = FileUtil.getRelativePath(outputRootPath, outputPath, '/');
        if (relativePath != null && !relativePath.startsWith("../")) {
            // the relative path must be under the root or equal to it
            if (isDirectory) {
                addEventsRecursively(output, outputRootPath, relativePath);
            } else {
                myFileGeneratedEvent.add(outputRootPath, relativePath);
            }
        }
    }
    final SourceToOutputMapping mapping = myContext.getProjectDescriptor().dataManager.getSourceToOutputMap(myTarget);
    for (String sourcePath : sourcePaths) {
        if (myRegisteredSources.add(FileUtil.toSystemIndependentName(sourcePath))) {
            mapping.setOutput(sourcePath, outputPath);
        } else {
            mapping.appendOutput(sourcePath, outputPath);
        }
    }
}

Example 7 with SourceToOutputMapping

use of org.jetbrains.jps.builders.storage.SourceToOutputMapping in project intellij-community by JetBrains.

the class IncProjectBuilder method processDeletedPaths.

//private static void createClasspathIndex(final BuildTargetChunk chunk) {
//  final Set<File> outputDirs = new THashSet<File>(FileUtil.FILE_HASHING_STRATEGY);
//  for (BuildTarget<?> target : chunk.getTargets()) {
//    if (target instanceof ModuleBuildTarget) {
//      File outputDir = ((ModuleBuildTarget)target).getOutputDir();
//      if (outputDir != null && outputDirs.add(outputDir)) {
//        try {
//          BufferedWriter writer = new BufferedWriter(new FileWriter(new File(outputDir, CLASSPATH_INDEX_FILE_NAME)));
//          try {
//            writeIndex(writer, outputDir, "");
//          }
//          finally {
//            writer.close();
//          }
//        }
//        catch (IOException e) {
//          // Ignore. Failed to create optional classpath index
//        }
//      }
//    }
//  }
//}
//private static void writeIndex(final BufferedWriter writer, final File file, final String path) throws IOException {
//  writer.write(path);
//  writer.write('\n');
//  final File[] files = file.listFiles();
//  if (files != null) {
//    for (File child : files) {
//      final String _path = path.isEmpty() ? child.getName() : path + "/" + child.getName();
//      writeIndex(writer, child, _path);
//    }
//  }
//}
private boolean processDeletedPaths(CompileContext context, final Set<? extends BuildTarget<?>> targets) throws ProjectBuildException {
    boolean doneSomething = false;
    try {
        // cleanup outputs
        final Map<BuildTarget<?>, Collection<String>> targetToRemovedSources = new HashMap<>();
        final THashSet<File> dirsToDelete = new THashSet<>(FileUtil.FILE_HASHING_STRATEGY);
        for (BuildTarget<?> target : targets) {
            final Collection<String> deletedPaths = myProjectDescriptor.fsState.getAndClearDeletedPaths(target);
            if (deletedPaths.isEmpty()) {
                continue;
            }
            targetToRemovedSources.put(target, deletedPaths);
            if (isTargetOutputCleared(context, target)) {
                continue;
            }
            final int buildTargetId = context.getProjectDescriptor().getTargetsState().getBuildTargetId(target);
            final boolean shouldPruneEmptyDirs = target instanceof ModuleBasedTarget;
            final SourceToOutputMapping sourceToOutputStorage = context.getProjectDescriptor().dataManager.getSourceToOutputMap(target);
            final ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger();
            // actually delete outputs associated with removed paths
            final Collection<String> pathsForIteration;
            if (myIsTestMode) {
                // ensure predictable order in test logs
                pathsForIteration = new ArrayList<>(deletedPaths);
                Collections.sort((List<String>) pathsForIteration);
            } else {
                pathsForIteration = deletedPaths;
            }
            for (String deletedSource : pathsForIteration) {
                // deleting outputs corresponding to non-existing source
                final Collection<String> outputs = sourceToOutputStorage.getOutputs(deletedSource);
                if (outputs != null && !outputs.isEmpty()) {
                    List<String> deletedOutputPaths = new ArrayList<>();
                    final OutputToTargetRegistry outputToSourceRegistry = context.getProjectDescriptor().dataManager.getOutputToTargetRegistry();
                    for (String output : outputToSourceRegistry.getSafeToDeleteOutputs(outputs, buildTargetId)) {
                        final boolean deleted = BuildOperations.deleteRecursively(output, deletedOutputPaths, shouldPruneEmptyDirs ? dirsToDelete : null);
                        if (deleted) {
                            doneSomething = true;
                        }
                    }
                    for (String outputPath : outputs) {
                        outputToSourceRegistry.removeMapping(outputPath, buildTargetId);
                    }
                    if (!deletedOutputPaths.isEmpty()) {
                        if (logger.isEnabled()) {
                            logger.logDeletedFiles(deletedOutputPaths);
                        }
                        context.processMessage(new FileDeletedEvent(deletedOutputPaths));
                    }
                }
                if (target instanceof ModuleBuildTarget) {
                    // check if deleted source was associated with a form
                    final OneToManyPathsMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap();
                    final Collection<String> boundForms = sourceToFormMap.getState(deletedSource);
                    if (boundForms != null) {
                        for (String formPath : boundForms) {
                            final File formFile = new File(formPath);
                            if (formFile.exists()) {
                                FSOperations.markDirty(context, CompilationRound.CURRENT, formFile);
                            }
                        }
                        sourceToFormMap.remove(deletedSource);
                    }
                }
            }
        }
        if (!targetToRemovedSources.isEmpty()) {
            final Map<BuildTarget<?>, Collection<String>> existing = Utils.REMOVED_SOURCES_KEY.get(context);
            if (existing != null) {
                for (Map.Entry<BuildTarget<?>, Collection<String>> entry : existing.entrySet()) {
                    final Collection<String> paths = targetToRemovedSources.get(entry.getKey());
                    if (paths != null) {
                        paths.addAll(entry.getValue());
                    } else {
                        targetToRemovedSources.put(entry.getKey(), entry.getValue());
                    }
                }
            }
            Utils.REMOVED_SOURCES_KEY.set(context, targetToRemovedSources);
        }
        FSOperations.pruneEmptyDirs(context, dirsToDelete);
    } catch (IOException e) {
        throw new ProjectBuildException(e);
    }
    return doneSomething;
}

Example 8 with SourceToOutputMapping

use of org.jetbrains.jps.builders.storage.SourceToOutputMapping in project intellij-community by JetBrains.

the class IncProjectBuilder method runModuleLevelBuilders.

// return true if changed something, false otherwise
private boolean runModuleLevelBuilders(final CompileContext context, final ModuleChunk chunk) throws ProjectBuildException, IOException {
    for (BuilderCategory category : BuilderCategory.values()) {
        for (ModuleLevelBuilder builder : myBuilderRegistry.getBuilders(category)) {
            builder.chunkBuildStarted(context, chunk);
        }
    }
    boolean doneSomething = false;
    boolean rebuildFromScratchRequested = false;
    float stageCount = myTotalModuleLevelBuilderCount;
    final int modulesInChunk = chunk.getModules().size();
    int buildersPassed = 0;
    boolean nextPassRequired;
    ChunkBuildOutputConsumerImpl outputConsumer = new ChunkBuildOutputConsumerImpl(context);
    try {
        do {
            nextPassRequired = false;
            myProjectDescriptor.fsState.beforeNextRoundStart(context, chunk);
            DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget> dirtyFilesHolder = new DirtyFilesHolderBase<JavaSourceRootDescriptor, ModuleBuildTarget>(context) {

                @Override
                public void processDirtyFiles(@NotNull FileProcessor<JavaSourceRootDescriptor, ModuleBuildTarget> processor) throws IOException {
                    FSOperations.processFilesToRecompile(context, chunk, processor);
                }
            };
            if (!JavaBuilderUtil.isForcedRecompilationAllJavaModules(context)) {
                final Map<ModuleBuildTarget, Set<File>> cleanedSources = BuildOperations.cleanOutputsCorrespondingToChangedFiles(context, dirtyFilesHolder);
                for (Map.Entry<ModuleBuildTarget, Set<File>> entry : cleanedSources.entrySet()) {
                    final ModuleBuildTarget target = entry.getKey();
                    final Set<File> files = entry.getValue();
                    if (!files.isEmpty()) {
                        final SourceToOutputMapping mapping = context.getProjectDescriptor().dataManager.getSourceToOutputMap(target);
                        for (File srcFile : files) {
                            mapping.setOutputs(srcFile.getPath(), Collections.<String>emptyList());
                        }
                    }
                }
            }
            BUILDER_CATEGORY_LOOP: for (BuilderCategory category : BuilderCategory.values()) {
                final List<ModuleLevelBuilder> builders = myBuilderRegistry.getBuilders(category);
                if (category == BuilderCategory.CLASS_POST_PROCESSOR) {
                    // ensure changes from instrumenters are visible to class post-processors
                    saveInstrumentedClasses(outputConsumer);
                }
                if (builders.isEmpty()) {
                    continue;
                }
                try {
                    for (ModuleLevelBuilder builder : builders) {
                        processDeletedPaths(context, chunk.getTargets());
                        long start = System.nanoTime();
                        int processedSourcesBefore = outputConsumer.getNumberOfProcessedSources();
                        final ModuleLevelBuilder.ExitCode buildResult = builder.build(context, chunk, dirtyFilesHolder, outputConsumer);
                        storeBuilderStatistics(builder, System.nanoTime() - start, outputConsumer.getNumberOfProcessedSources() - processedSourcesBefore);
                        doneSomething |= (buildResult != ModuleLevelBuilder.ExitCode.NOTHING_DONE);
                        if (buildResult == ModuleLevelBuilder.ExitCode.ABORT) {
                            throw new StopBuildException("Builder " + builder.getPresentableName() + " requested build stop");
                        }
                        context.checkCanceled();
                        if (buildResult == ModuleLevelBuilder.ExitCode.ADDITIONAL_PASS_REQUIRED) {
                            nextPassRequired = true;
                        } else if (buildResult == ModuleLevelBuilder.ExitCode.CHUNK_REBUILD_REQUIRED) {
                            if (!rebuildFromScratchRequested && !JavaBuilderUtil.isForcedRecompilationAllJavaModules(context)) {
                                LOG.info("Builder " + builder.getPresentableName() + " requested rebuild of module chunk " + chunk.getName());
                                // allow rebuild from scratch only once per chunk
                                rebuildFromScratchRequested = true;
                                try {
                                    // forcibly mark all files in the chunk dirty
                                    context.getProjectDescriptor().fsState.clearContextRoundData(context);
                                    FSOperations.markDirty(context, CompilationRound.NEXT, chunk, null);
                                    // reverting to the beginning
                                    myTargetsProcessed -= (buildersPassed * modulesInChunk) / stageCount;
                                    stageCount = myTotalModuleLevelBuilderCount;
                                    buildersPassed = 0;
                                    nextPassRequired = true;
                                    outputConsumer.clear();
                                    break BUILDER_CATEGORY_LOOP;
                                } catch (Exception e) {
                                    throw new ProjectBuildException(e);
                                }
                            } else {
                                LOG.debug("Builder " + builder.getPresentableName() + " requested second chunk rebuild");
                            }
                        }
                        buildersPassed++;
                        updateDoneFraction(context, modulesInChunk / (stageCount));
                    }
                } finally {
                    final boolean moreToCompile = JavaBuilderUtil.updateMappingsOnRoundCompletion(context, dirtyFilesHolder, chunk);
                    if (moreToCompile) {
                        nextPassRequired = true;
                    }
                    if (nextPassRequired && !rebuildFromScratchRequested) {
                        // recalculate basis
                        myTargetsProcessed -= (buildersPassed * modulesInChunk) / stageCount;
                        stageCount += myTotalModuleLevelBuilderCount;
                        myTargetsProcessed += (buildersPassed * modulesInChunk) / stageCount;
                    }
                }
            }
        } while (nextPassRequired);
    } finally {
        saveInstrumentedClasses(outputConsumer);
        outputConsumer.fireFileGeneratedEvents();
        outputConsumer.clear();
        for (BuilderCategory category : BuilderCategory.values()) {
            for (ModuleLevelBuilder builder : myBuilderRegistry.getBuilders(category)) {
                builder.chunkBuildFinished(context, chunk);
            }
        }
    }
    return doneSomething;
}

Example 9 with SourceToOutputMapping

use of org.jetbrains.jps.builders.storage.SourceToOutputMapping in project intellij-community by JetBrains.

the class IncArtifactBuilder method build.

@Override
public void build(@NotNull ArtifactBuildTarget target, @NotNull DirtyFilesHolder<ArtifactRootDescriptor, ArtifactBuildTarget> holder, @NotNull BuildOutputConsumer outputConsumer, @NotNull final CompileContext context) throws ProjectBuildException {
    JpsArtifact artifact = target.getArtifact();
    String outputFilePath = artifact.getOutputFilePath();
    if (StringUtil.isEmpty(outputFilePath)) {
        context.processMessage(new CompilerMessage(BUILDER_NAME, BuildMessage.Kind.ERROR, "Cannot build '" + artifact.getName() + "' artifact: output path is not specified"));
        return;
    }
    final ProjectDescriptor pd = context.getProjectDescriptor();
    final ArtifactSorter sorter = new ArtifactSorter(pd.getModel());
    final Map<JpsArtifact, JpsArtifact> selfIncludingNameMap = sorter.getArtifactToSelfIncludingNameMap();
    final JpsArtifact selfIncluding = selfIncludingNameMap.get(artifact);
    if (selfIncluding != null) {
        String name = selfIncluding.equals(artifact) ? "it" : "'" + selfIncluding.getName() + "' artifact";
        context.processMessage(new CompilerMessage(BUILDER_NAME, BuildMessage.Kind.ERROR, "Cannot build '" + artifact.getName() + "' artifact: " + name + " includes itself in the output layout"));
        return;
    }
    try {
        final Collection<String> deletedFiles = holder.getRemovedFiles(target);
        String messageText = "Building artifact '" + artifact.getName() + "'...";
        context.processMessage(new ProgressMessage(messageText));
        LOG.debug(messageText);
        runArtifactTasks(context, target.getArtifact(), ArtifactBuildTaskProvider.ArtifactBuildPhase.PRE_PROCESSING);
        final SourceToOutputMapping srcOutMapping = pd.dataManager.getSourceToOutputMap(target);
        final ArtifactOutputToSourceMapping outSrcMapping = pd.dataManager.getStorage(target, ArtifactOutToSourceStorageProvider.INSTANCE);
        final TIntObjectHashMap<Set<String>> filesToProcess = new TIntObjectHashMap<>();
        final MultiMap<String, String> filesToDelete = new MultiMap<>();
        final Set<String> deletedOutputPaths = new THashSet<>(FileUtil.PATH_HASHING_STRATEGY);
        for (String sourcePath : deletedFiles) {
            final Collection<String> outputPaths = srcOutMapping.getOutputs(sourcePath);
            if (outputPaths != null) {
                for (String outputPath : outputPaths) {
                    if (deletedOutputPaths.add(outputPath)) {
                        collectSourcesCorrespondingToOutput(outputPath, sourcePath, deletedFiles, outSrcMapping, filesToProcess, filesToDelete);
                    }
                }
            }
        }
        final Set<String> changedOutputPaths = new THashSet<>(FileUtil.PATH_HASHING_STRATEGY);
        holder.processDirtyFiles(new FileProcessor<ArtifactRootDescriptor, ArtifactBuildTarget>() {

            @Override
            public boolean apply(ArtifactBuildTarget target, File file, ArtifactRootDescriptor root) throws IOException {
                int rootIndex = root.getRootIndex();
                String sourcePath = FileUtil.toSystemIndependentName(file.getPath());
                addFileToProcess(filesToProcess, rootIndex, sourcePath, deletedFiles);
                final Collection<String> outputPaths = srcOutMapping.getOutputs(sourcePath);
                if (outputPaths != null) {
                    for (String outputPath : outputPaths) {
                        if (changedOutputPaths.add(outputPath)) {
                            collectSourcesCorrespondingToOutput(outputPath, sourcePath, deletedFiles, outSrcMapping, filesToProcess, filesToDelete);
                        }
                    }
                }
                return true;
            }
        });
        BuildOperations.cleanOutputsCorrespondingToChangedFiles(context, holder);
        for (String outputPath : changedOutputPaths) {
            outSrcMapping.remove(outputPath);
        }
        if (filesToDelete.isEmpty() && filesToProcess.isEmpty()) {
            return;
        }
        deleteOutdatedFiles(filesToDelete, context, srcOutMapping, outSrcMapping);
        context.checkCanceled();
        context.processMessage(new ProgressMessage("Building artifact '" + artifact.getName() + "': copying files..."));
        final Set<JarInfo> changedJars = new THashSet<>();
        for (ArtifactRootDescriptor descriptor : pd.getBuildRootIndex().getTargetRoots(target, context)) {
            context.checkCanceled();
            final Set<String> sourcePaths = filesToProcess.get(descriptor.getRootIndex());
            if (sourcePaths == null)
                continue;
            for (String sourcePath : sourcePaths) {
                if (!descriptor.getFilter().shouldBeCopied(sourcePath, pd)) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("File " + sourcePath + " will be skipped because it isn't accepted by filter");
                    }
                    continue;
                }
                DestinationInfo destination = descriptor.getDestinationInfo();
                if (destination instanceof ExplodedDestinationInfo) {
                    descriptor.copyFromRoot(sourcePath, descriptor.getRootIndex(), destination.getOutputPath(), context, outputConsumer, outSrcMapping);
                } else {
                    List<ArtifactOutputToSourceMapping.SourcePathAndRootIndex> sources = outSrcMapping.getState(destination.getOutputFilePath());
                    if (sources == null || sources.size() > 0 && sources.get(0).getRootIndex() == descriptor.getRootIndex()) {
                        outSrcMapping.update(destination.getOutputFilePath(), Collections.<ArtifactOutputToSourceMapping.SourcePathAndRootIndex>emptyList());
                        changedJars.add(((JarDestinationInfo) destination).getJarInfo());
                    }
                }
            }
        }
        context.checkCanceled();
        JarsBuilder builder = new JarsBuilder(changedJars, context, outputConsumer, outSrcMapping);
        builder.buildJars();
        runArtifactTasks(context, artifact, ArtifactBuildTaskProvider.ArtifactBuildPhase.FINISHING_BUILD);
        runArtifactTasks(context, artifact, ArtifactBuildTaskProvider.ArtifactBuildPhase.POST_PROCESSING);
    } catch (IOException e) {
        throw new ProjectBuildException(e);
    }
}

Example 10 with SourceToOutputMapping

use of org.jetbrains.jps.builders.storage.SourceToOutputMapping in project intellij-community by JetBrains.

the class BuildOperations method dropRemovedPaths.

private static boolean dropRemovedPaths(CompileContext context, BuildTargetChunk chunk) throws IOException {
    final Map<BuildTarget<?>, Collection<String>> map = Utils.REMOVED_SOURCES_KEY.get(context);
    boolean dropped = false;
    if (map != null) {
        for (BuildTarget<?> target : chunk.getTargets()) {
            final Collection<String> paths = map.remove(target);
            if (paths != null) {
                final SourceToOutputMapping storage = context.getProjectDescriptor().dataManager.getSourceToOutputMap(target);
                for (String path : paths) {
                    storage.remove(path);
                    dropped = true;
                }
            }
        }
    }
    return dropped;
}