Examples with RepositoryComparator org.finos.legend.pure.m3.serialization.runtime.RepositoryComparator used on opensource projects

Example 1 with RepositoryComparator

use of org.finos.legend.pure.m3.serialization.runtime.RepositoryComparator in project legend-pure by finos.

the class IncrementalCompiler_New method compile.

// ----------
// Compile
// ----------
@Override
SourceMutation compile(RichIterable<? extends Source> sources, Iterable<? extends CompilerEventHandler> compilerEventHandlers) throws PureCompilationException, PureParserException {
    MutableSet<CoreInstance> potentialToProcess = this.walkTheGraphForUnload(this.toUnload).withAll(this.toProcess).withAll(this.toUnbind);
    this.unload();
    this.toProcess = this.removeNodesFromRemovedSources(this.toProcess);
    IncrementalCompilerTransaction threadLocalTransaction = this.transactionManager.getThreadLocalTransaction();
    SourceMutation result;
    MutableListMultimap<String, Source> compiledSourcesByRepo = Multimaps.mutable.list.empty();
    if (sources.isEmpty()) {
        // We must compile even if the set of sources is empty, as post-processing or validation may be required for nodes from already compiled sources.
        boolean shouldCreateNewRepoTransaction = this.isTransactionalByDefault && (threadLocalTransaction == null);
        IncrementalCompilerTransaction repoTransaction = shouldCreateNewRepoTransaction ? this.newTransaction(true) : threadLocalTransaction;
        MutableSet<CoreInstance> repoTransactionInstances = Sets.mutable.empty();
        try {
            result = this.compileRepoSources(repoTransaction, "Pure", 1, 1, sources, this.toProcess.toImmutable(), this.toUnbind.toImmutable(), Lists.mutable.<SourceState>with(), repoTransactionInstances);
            if (shouldCreateNewRepoTransaction) {
                repoTransaction.commit();
            }
        } catch (Exception e) {
            if (shouldCreateNewRepoTransaction) {
                this.rollBack(repoTransaction, e, repoTransactionInstances);
            }
            throw e;
        }
    } else {
        result = new SourceMutation();
        Multimap<String, ? extends Source> sourcesByRepo = sources.groupBy(s -> PureCodeStorage.getSourceRepoName(s.getId()));
        Multimap<String, ? extends Source> sourcesByRepoNew = sources.groupBy(PureCodeStorage.GET_SOURCE_REPO);
        Multimap<String, SourceState> sourceStatesByRepo = this.oldSourceStates.groupBy(GET_SOURCE_STATE_REPO);
        Multimap<String, CoreInstance> potentialRepos = potentialToProcess.groupBy(GET_COREINSTANCE_REPO_NAME);
        MutableSet<String> allReposToCompile = Sets.mutable.withAll(sourcesByRepo.keysView()).withAll(potentialRepos.keysView());
        ListIterable<String> repoCompileOrder = allReposToCompile.toSortedList(new RepositoryComparator(this.codeStorage.getAllRepositories()));
        MutableList<String> newCompileOrder = Lists.mutable.empty();
        repoCompileOrder.forEach(repo -> {
            if ((repo != null) && repo.startsWith("model")) {
                if (!newCompileOrder.contains("model-all")) {
                    newCompileOrder.add("model-all");
                }
            } else {
                newCompileOrder.add(repo);
            }
        });
        int repoCount = newCompileOrder.size();
        boolean shouldCreateNewRepoTransactions = this.isTransactionalByDefault && (threadLocalTransaction == null);
        newCompileOrder.forEachWithIndex((repo, i) -> {
            IncrementalCompilerTransaction repoTransaction = shouldCreateNewRepoTransactions ? this.newTransaction(true) : threadLocalTransaction;
            RichIterable<? extends Source> repoSources = sourcesByRepoNew.get(repo);
            RichIterable<CoreInstance> toProcessThisRepo = this.toProcess.selectWith(CORE_INSTANCE_IS_FROM_REPO, repo).toImmutable();
            RichIterable<CoreInstance> toUnbindThisRepo = this.toUnbind.selectWith(CORE_INSTANCE_IS_FROM_REPO, repo).toImmutable();
            MutableSet<CoreInstance> repoTransactionInstances = Sets.mutable.empty();
            SourceMutation repoResult;
            try {
                repoResult = this.compileRepoSources(repoTransaction, repo, i + 1, repoCount, repoSources, toProcessThisRepo, toUnbindThisRepo, sourceStatesByRepo.get(repo), repoTransactionInstances);
                if (shouldCreateNewRepoTransactions) {
                    repoTransaction.commit();
                }
            } catch (Exception e) {
                if (shouldCreateNewRepoTransactions) {
                    this.rollBack(repoTransaction, e, repoTransactionInstances);
                }
                throw e;
            }
            result.merge(repoResult);
            if (repo == null) {
                compiledSourcesByRepo.putAll(repo, repoSources);
            } else if ("model-all".equals(repo)) {
                compiledSourcesByRepo.putAll(repoSources.groupBy(source -> PureCodeStorage.getSourceRepoName(source.getId())));
            } else {
                compiledSourcesByRepo.putAll(repo, repoSources);
            }
        });
    }
    this.runEventHandlers(compilerEventHandlers, this.processed, compiledSourcesByRepo);
    this.processed.clear();
    return result;
}

Example 2 with RepositoryComparator

use of org.finos.legend.pure.m3.serialization.runtime.RepositoryComparator in project legend-pure by finos.

the class ClassLoaderPureGraphCache method buildRepoAndSources.

@Override
public boolean buildRepoAndSources(ModelRepository modelRepository, SourceRegistry sourceRegistry, ParserLibrary parserLibrary, Context context, ProcessorSupport processorSupport, Message message) {
    if (this.runtime == null) {
        return false;
    }
    try {
        CodeStorage codeStorage = this.runtime.getCodeStorage();
        MutableList<String> repoNames = codeStorage.getAllRepoNames().toSortedList(new RepositoryComparator(codeStorage.getAllRepositories()));
        PureRepositoryJarLibrary jarLibrary = SimplePureRepositoryJarLibrary.newLibrary(GraphLoader.findJars(repoNames, this.classLoader, message));
        GraphLoader loader = new GraphLoader(modelRepository, context, parserLibrary, this.runtime.getIncrementalCompiler().getDslLibrary(), sourceRegistry, null, jarLibrary, this.forkJoinPool);
        for (String repoName : repoNames) {
            loader.loadRepository(repoName, message);
        }
        this.state.update(true, -1L, true, null);
        return true;
    } catch (Exception e) {
        modelRepository.clear();
        this.state.update(false, -1L, false, e);
        return false;
    } catch (Error e) {
        this.state.update(false, -1L, false, e);
        throw e;
    }
}

Example 3 with RepositoryComparator

use of org.finos.legend.pure.m3.serialization.runtime.RepositoryComparator in project legend-pure by finos.

the class FSGraphLoaderPureGraphCache method buildFromCaches.

@Override
public boolean buildFromCaches(ModelRepository modelRepository, SourceRegistry sources, ParserLibrary library, Context context, ProcessorSupport processorSupport, Message message) {
    PureRepositoryJarLibrary jarLibrary = SimplePureRepositoryJarLibrary.newLibraryFromDirectory(getCacheLocation());
    final GraphLoader loader = new GraphLoader(modelRepository, context, library, this.pureRuntime.getIncrementalCompiler().getDslLibrary(), sources, null, jarLibrary, this.forkJoinPool);
    CodeStorage codeStorage = this.pureRuntime.getCodeStorage();
    MutableList<String> repoNames = codeStorage.getAllRepoNames().toSortedList(new RepositoryComparator(codeStorage.getAllRepositories()));
    if (shouldAddRootRepo()) {
        repoNames.add(ROOT_REPOSITORY_NAME);
    }
    if (this.allowBuildingFromRepoSubset) {
        repoNames.removeIf(new Predicate<String>() {

            @Override
            public boolean accept(String repoName) {
                return !loader.isKnownRepository(repoName);
            }
        });
    }
    for (String repoName : repoNames) {
        loader.loadRepository(repoName, message);
    }
    updateCacheState();
    return true;
}

Example 4 with RepositoryComparator

use of org.finos.legend.pure.m3.serialization.runtime.RepositoryComparator in project legend-pure by finos.

the class JavaStandaloneLibraryGenerator method compile.

private static GenerateAndCompile compile(PureRuntime runtime, boolean writeJavaSourcesToDisk, Path pathToWriteTo, Iterable<? extends CompiledExtension> extensions, boolean addExternalAPI, String externalAPIPackage) throws PureJavaCompileException {
    // Group sources by repo and separate platform sources
    MutableListMultimap<String, Source> sourcesByRepo = runtime.getSourceRegistry().getSources().groupBy((Source source) -> PureCodeStorage.getSourceRepoName(source.getId()), Multimaps.mutable.list.empty());
    // Generate and compile Java code
    GenerateAndCompile generateAndCompile = new GenerateAndCompile(new Message(""), VoidJavaCompilerEventObserver.VOID_JAVA_COMPILER_EVENT_OBSERVER);
    JavaSourceCodeGenerator javaSourceCodeGenerator = new JavaSourceCodeGenerator(runtime.getProcessorSupport(), runtime.getCodeStorage(), writeJavaSourcesToDisk, pathToWriteTo, false, extensions, "UserCode", externalAPIPackage);
    RichIterable<CodeRepository> repositories = runtime.getCodeStorage().getAllRepositories();
    javaSourceCodeGenerator.collectClassesToSerialize();
    generateAndCompile.generateAndCompileJavaCodeForSources(TreeSortedMap.newMap(new RepositoryComparator(repositories), sourcesByRepo.toMap()), javaSourceCodeGenerator);
    if (addExternalAPI) {
        generateAndCompile.generateAndCompileExternalizableAPI(javaSourceCodeGenerator, externalAPIPackage);
    }
    return generateAndCompile;
}

Example 5 with RepositoryComparator

use of org.finos.legend.pure.m3.serialization.runtime.RepositoryComparator in project legend-pure by finos.

the class IncrementalCompiler_Old method compile.

// ----------
// Compile
// ----------
@Override
SourceMutation compile(RichIterable<? extends Source> sources, Iterable<? extends CompilerEventHandler> compilerEventHandlers) throws PureCompilationException, PureParserException {
    this.unload();
    this.toProcess = this.removeNodesFromSourcesInScope(sources, this.toProcess);
    MutableSet<CoreInstance> copyToProcess = Sets.mutable.withAll(this.toProcess);
    IncrementalCompilerTransaction threadLocalTransaction = this.transactionManager.getThreadLocalTransaction();
    SourceMutation result;
    MutableListMultimap<String, Source> compiledSourcesByRepo = Multimaps.mutable.list.empty();
    if (sources.isEmpty()) {
        // We must compile even if the set of sources is empty, as post-processing or validation may be required for nodes from already compiled sources.
        IncrementalCompilerTransaction repoTransaction = this.isTransactionalByDefault && threadLocalTransaction == null ? this.newTransaction(true) : threadLocalTransaction;
        try {
            result = this.compileRepoSources(repoTransaction, "Pure", 1, 1, sources, this.toProcess);
            if (this.isTransactionalByDefault && threadLocalTransaction == null) {
                repoTransaction.commit();
            }
        } catch (RuntimeException e) {
            if (this.isTransactionalByDefault && threadLocalTransaction == null) {
                this.rollBack(repoTransaction, e);
            }
            throw e;
        }
    } else {
        result = new SourceMutation();
        Multimap<String, ? extends Source> sourcesByRepo = sources.groupBy((Source source) -> PureCodeStorage.getSourceRepoName(source.getId()));
        Multimap<String, CoreInstance> toProcessByRepo = this.toProcess.groupBy(GET_COREINSTANCE_REPO_NAME);
        MutableSet<String> allReposToCompile = Sets.mutable.withAll(sourcesByRepo.keysView()).withAll(toProcessByRepo.keysView());
        int repoCount = allReposToCompile.size();
        allReposToCompile.toSortedList(new RepositoryComparator(this.codeStorage.getAllRepositories())).forEachWithIndex((repo, i) -> {
            IncrementalCompilerTransaction repoTransaction = this.isTransactionalByDefault && threadLocalTransaction == null ? this.newTransaction(true) : threadLocalTransaction;
            RichIterable<? extends Source> repoSources = sourcesByRepo.get(repo);
            RichIterable<CoreInstance> toProcessThisRepo = toProcessByRepo.get(repo);
            SourceMutation repoResult;
            try {
                repoResult = this.compileRepoSources(repoTransaction, repo, i + 1, repoCount, repoSources, toProcessThisRepo);
                if (this.isTransactionalByDefault && threadLocalTransaction == null) {
                    repoTransaction.commit();
                }
            } catch (RuntimeException e) {
                if (this.isTransactionalByDefault && threadLocalTransaction == null) {
                    this.rollBack(repoTransaction, e);
                }
                throw e;
            }
            result.merge(repoResult);
            compiledSourcesByRepo.putAll(repo, repoSources);
        });
    }
    this.runEventHandlers(compilerEventHandlers, copyToProcess, compiledSourcesByRepo);
    return result;
}