Examples with SourceState org.finos.legend.pure.m3.serialization.runtime.SourceState used on opensource projects

Example 1 with SourceState

use of org.finos.legend.pure.m3.serialization.runtime.SourceState in project legend-pure by finos.

the class IncrementalCompiler_New method compile.

// ----------
// Compile
// ----------
@Override
SourceMutation compile(RichIterable<? extends Source> sources, Iterable<? extends CompilerEventHandler> compilerEventHandlers) throws PureCompilationException, PureParserException {
    MutableSet<CoreInstance> potentialToProcess = this.walkTheGraphForUnload(this.toUnload).withAll(this.toProcess).withAll(this.toUnbind);
    this.unload();
    this.toProcess = this.removeNodesFromRemovedSources(this.toProcess);
    IncrementalCompilerTransaction threadLocalTransaction = this.transactionManager.getThreadLocalTransaction();
    SourceMutation result;
    MutableListMultimap<String, Source> compiledSourcesByRepo = Multimaps.mutable.list.empty();
    if (sources.isEmpty()) {
        // We must compile even if the set of sources is empty, as post-processing or validation may be required for nodes from already compiled sources.
        boolean shouldCreateNewRepoTransaction = this.isTransactionalByDefault && (threadLocalTransaction == null);
        IncrementalCompilerTransaction repoTransaction = shouldCreateNewRepoTransaction ? this.newTransaction(true) : threadLocalTransaction;
        MutableSet<CoreInstance> repoTransactionInstances = Sets.mutable.empty();
        try {
            result = this.compileRepoSources(repoTransaction, "Pure", 1, 1, sources, this.toProcess.toImmutable(), this.toUnbind.toImmutable(), Lists.mutable.<SourceState>with(), repoTransactionInstances);
            if (shouldCreateNewRepoTransaction) {
                repoTransaction.commit();
            }
        } catch (Exception e) {
            if (shouldCreateNewRepoTransaction) {
                this.rollBack(repoTransaction, e, repoTransactionInstances);
            }
            throw e;
        }
    } else {
        result = new SourceMutation();
        Multimap<String, ? extends Source> sourcesByRepo = sources.groupBy(s -> PureCodeStorage.getSourceRepoName(s.getId()));
        Multimap<String, ? extends Source> sourcesByRepoNew = sources.groupBy(PureCodeStorage.GET_SOURCE_REPO);
        Multimap<String, SourceState> sourceStatesByRepo = this.oldSourceStates.groupBy(GET_SOURCE_STATE_REPO);
        Multimap<String, CoreInstance> potentialRepos = potentialToProcess.groupBy(GET_COREINSTANCE_REPO_NAME);
        MutableSet<String> allReposToCompile = Sets.mutable.withAll(sourcesByRepo.keysView()).withAll(potentialRepos.keysView());
        ListIterable<String> repoCompileOrder = allReposToCompile.toSortedList(new RepositoryComparator(this.codeStorage.getAllRepositories()));
        MutableList<String> newCompileOrder = Lists.mutable.empty();
        repoCompileOrder.forEach(repo -> {
            if ((repo != null) && repo.startsWith("model")) {
                if (!newCompileOrder.contains("model-all")) {
                    newCompileOrder.add("model-all");
                }
            } else {
                newCompileOrder.add(repo);
            }
        });
        int repoCount = newCompileOrder.size();
        boolean shouldCreateNewRepoTransactions = this.isTransactionalByDefault && (threadLocalTransaction == null);
        newCompileOrder.forEachWithIndex((repo, i) -> {
            IncrementalCompilerTransaction repoTransaction = shouldCreateNewRepoTransactions ? this.newTransaction(true) : threadLocalTransaction;
            RichIterable<? extends Source> repoSources = sourcesByRepoNew.get(repo);
            RichIterable<CoreInstance> toProcessThisRepo = this.toProcess.selectWith(CORE_INSTANCE_IS_FROM_REPO, repo).toImmutable();
            RichIterable<CoreInstance> toUnbindThisRepo = this.toUnbind.selectWith(CORE_INSTANCE_IS_FROM_REPO, repo).toImmutable();
            MutableSet<CoreInstance> repoTransactionInstances = Sets.mutable.empty();
            SourceMutation repoResult;
            try {
                repoResult = this.compileRepoSources(repoTransaction, repo, i + 1, repoCount, repoSources, toProcessThisRepo, toUnbindThisRepo, sourceStatesByRepo.get(repo), repoTransactionInstances);
                if (shouldCreateNewRepoTransactions) {
                    repoTransaction.commit();
                }
            } catch (Exception e) {
                if (shouldCreateNewRepoTransactions) {
                    this.rollBack(repoTransaction, e, repoTransactionInstances);
                }
                throw e;
            }
            result.merge(repoResult);
            if (repo == null) {
                compiledSourcesByRepo.putAll(repo, repoSources);
            } else if ("model-all".equals(repo)) {
                compiledSourcesByRepo.putAll(repoSources.groupBy(source -> PureCodeStorage.getSourceRepoName(source.getId())));
            } else {
                compiledSourcesByRepo.putAll(repo, repoSources);
            }
        });
    }
    this.runEventHandlers(compilerEventHandlers, this.processed, compiledSourcesByRepo);
    this.processed.clear();
    return result;
}

Example 2 with SourceState

use of org.finos.legend.pure.m3.serialization.runtime.SourceState in project legend-pure by finos.

the class DiagramParser method parse.

@Override
public void parse(String code, String sourceName, boolean addLines, int offset, ModelRepository repository, MutableList<CoreInstance> coreInstancesResult, M3M4StateListener listener, Context context, int count, SourceState oldState) throws PureParserException {
    String result = this.parseDefinition(true, code, sourceName, addLines, offset, repository, listener, context, count);
    new M3AntlrParser(false).parse(result, sourceName, false, offset, repository, coreInstancesResult, listener, context, count, null);
}

Example 3 with SourceState

use of org.finos.legend.pure.m3.serialization.runtime.SourceState in project legend-pure by finos.

the class IncrementalCompiler_New method compileRepoSources.

private SourceMutation compileRepoSources(IncrementalCompilerTransaction transaction, String repoName, int repoNum, int repoTotalCount, RichIterable<? extends Source> sources, RichIterable<CoreInstance> instancesToProcess, RichIterable<CoreInstance> instancesToUnbind, RichIterable<SourceState> sourceStates, MutableSet<CoreInstance> repoTransactionInstances) throws PureCompilationException, PureParserException {
    String repoDisplayName = repoName == null ? "non-repository" : repoName;
    try (ThreadLocalTransactionContext ignored = transaction != null ? transaction.openInCurrentThread() : null) {
        AtomicInteger sourceNum = this.message == null ? null : new AtomicInteger(0);
        int sourceTotalCount = sources.size();
        Procedure<Source> parseSource = source -> {
            try (ThreadLocalTransactionContext ignored1 = transaction != null ? transaction.openInCurrentThread() : null) {
                if (this.message != null) {
                    int thisSourceNum = sourceNum.incrementAndGet();
                    StringBuilder message = new StringBuilder("Parsing ").append(repoDisplayName);
                    if (repoTotalCount > 1) {
                        message.append(" (").append(repoNum).append('/').append(repoTotalCount).append(')');
                    }
                    message.append(" sources (").append(thisSourceNum).append('/').append(sourceTotalCount).append(')');
                    this.message.setMessage(message.toString());
                }
                MutableList<SourceState> oldState = sourceStates.select(sourceState -> source.equals(sourceState.getSource()), Lists.mutable.empty());
                ListMultimap<Parser, CoreInstance> newInstancesByParser = new TopParser().parse(source.getContent(), source.getId(), this.modelRepository, this.library, VoidM3M4StateListener.VOID_M3_M4_STATE_LISTENER, this.context, oldState.size() == 1 ? oldState.get(0) : null);
                this.updateSource(source, newInstancesByParser);
                if (transaction != null) {
                    transaction.noteSourceCompiled(source);
                }
            } catch (RuntimeException e) {
                if (PureException.canFindPureException(e)) {
                    throw e;
                }
                throw new PureParserException(new SourceInformation(source.getId(), -1, -1, -1, -1), "Error parsing " + source.getId(), e);
            }
        };
        if (this.shouldParallelize(sourceTotalCount, PARSE_SOURCES_THRESHOLD)) {
            ForkJoinTools.forEach(this.forkJoinPool, ListHelper.wrapListIterable(sources), parseSource, PARSE_SOURCES_THRESHOLD);
        } else {
            sources.forEach(parseSource);
        }
        // Parsing for repo completed successfully
        // New Instances in the sources from the repo
        MutableSet<CoreInstance> newInstances = sources.flatCollect(Source::getNewInstances, Sets.mutable.empty());
        // Old Instances in the sources from repo
        MutableSet<CoreInstance> oldInstances = sourceStates.flatCollect(SourceState::getInstances, Sets.mutable.empty());
        // Instances which are newly created (added or modified)
        MutableSet<CoreInstance> newButNotOld = newInstances.difference(oldInstances);
        // Instances which are not retained (deleted or modified)
        MutableSet<CoreInstance> oldButNotNew = oldInstances.difference(newInstances);
        // Source Ids in the repo within scope
        MutableSet<String> sourcesInScope = Sets.mutable.empty();
        sources.collect(Source.SOURCE_ID, sourcesInScope);
        // Unload Walk on only instances which are not retained and generate toUnbindGenerated (There can be instances from following repos as well)
        MutableSet<CoreInstance> toUnbindGenerated = this.walkTheGraphForUnload(oldButNotNew);
        // Filter the instances which are within the repo
        MutableSet<CoreInstance> toUnbindWithinRepo = toUnbindGenerated.selectWith(CORE_INSTANCE_IS_FROM_REPO, repoName);
        // Total Unbind set is ( generated here + obtained through call - non retained )
        MutableSet<CoreInstance> hereUnbind = toUnbindWithinRepo.union(oldButNotNew).union(instancesToUnbind.toSet());
        Unbinder.process(hereUnbind, this.modelRepository, this.library, this.dslLibrary, this.context, this.processorSupport, new UnbindState(this.context, this.urlPatternLibrary, this.processorSupport), this.message);
        // Invalidate the unbound instances
        if (hereUnbind.notEmpty()) {
            this.compilerEventHandlers.forEach(eh -> eh.invalidate(hereUnbind));
        }
        // ToProcessGenerated - If the instances is within the scope of resources, it should be in the new instances - This can contain instances from following repos as well
        MutableSet<CoreInstance> toProcessGenerated = toUnbindGenerated.select(each -> !sourcesInScope.contains(each.getSourceInformation().getSourceId()) || newInstances.contains(each));
        // Filter the instances which are within this repo
        MutableSet<CoreInstance> toProcessWithinRepoGenerated = toProcessGenerated.selectWith(CORE_INSTANCE_IS_FROM_REPO, repoName);
        // ToProcess from call is filtered for the existence in new instances if source is within the sourcesInScope
        MutableSet<CoreInstance> instancesToProcessFiltered = instancesToProcess.select(each -> !sourcesInScope.contains(each.getSourceInformation().getSourceId()) || newInstances.contains(each)).toSet();
        // ToUnbind filtered for the existence in new instances if source is within the sourcesInScope
        MutableSet<CoreInstance> instancesToUnbindFiltered = instancesToUnbind.select(each -> !sourcesInScope.contains(each.getSourceInformation().getSourceId()) || newInstances.contains(each)).toSet();
        // Final instances to be processed is (generated within repo + Filtered call parameters (toProcess, toUnbind))
        MutableList<CoreInstance> newInstancesConsolidated = this.removeNodesFromRemovedSources(toProcessWithinRepoGenerated.union(instancesToProcessFiltered).union(instancesToUnbindFiltered)).toSet().difference(newButNotOld).toList();
        // Maintain ordering to avoid errors in unit tests
        sources.forEach(source -> source.getNewInstances().select(newButNotOld::contains, newInstancesConsolidated));
        // Collect all instances to be registered (Even retained instances are unregistered at start)
        MutableList<CoreInstance> allInstances = this.removeNodesFromRemovedSources(toProcessWithinRepoGenerated.union(instancesToProcessFiltered).union(instancesToUnbindFiltered)).toSet().difference(newInstances).toList();
        sources.forEach(source -> allInstances.addAllIterable(source.getNewInstances()));
        // Store the to be processed set for rollback
        repoTransactionInstances.addAllIterable(newInstancesConsolidated);
        // Do postprocessing, validation - can throw an error
        SourceMutation result = this.finishRepoCompilation(repoDisplayName, allInstances, newInstancesConsolidated, ValidationType.SHALLOW);
        // Repo compilation Successful
        // Remove any redundant packages of old instances
        oldInstances.forEach(this::tryRemovePackage);
        // Remove source states
        this.oldSourceStates.removeAllIterable(sourceStates);
        // Remove old instances from toUnload
        this.toUnload.removeAllIterable(oldInstances);
        // Update toProcess
        this.toProcess = this.toProcess.difference(oldButNotNew);
        this.toProcess.addAllIterable(this.removeNodesFromRemovedSources(toProcessGenerated.difference(toProcessWithinRepoGenerated).difference(oldButNotNew).difference(newInstancesConsolidated.toSet())));
        // Update toUnbind
        this.toUnbind.removeAllIterable(hereUnbind);
        this.toUnbind.addAllIterable(toUnbindGenerated.difference(toUnbindWithinRepo));
        // Update processed to be used by compiler event handlers
        this.processed.addAllIterable(toProcessWithinRepoGenerated.withAll(instancesToProcessFiltered).withAll(instancesToUnbindFiltered));
        return result;
    }
}

Example 4 with SourceState

use of org.finos.legend.pure.m3.serialization.runtime.SourceState in project legend-pure by finos.

the class TopParser method parseDefinition.

private MutableListMultimap<Parser, CoreInstance> parseDefinition(boolean useFastParser, String code, String sourceName, ModelRepository repository, ParserLibrary parserLibrary, M3M4StateListener listener, Context context, SourceState oldState) {
    TopAntlrParser parser = this.initAntlrParser(useFastParser, "\u005cn###Pure\u005cn" + code, sourceName);
    TopGraphBuilder visitor = new TopGraphBuilder(sourceName, repository, listener, context, parserLibrary, oldState);
    TopAntlrParser.DefinitionContext c = parser.definition();
    return visitor.visitDefinition(c);
}

Example 5 with SourceState

use of org.finos.legend.pure.m3.serialization.runtime.SourceState in project legend-pure by finos.

the class MappingParser method parse.

@Override
public void parse(String string, String sourceName, boolean addLines, int offset, ModelRepository repository, MutableList<CoreInstance> coreInstancesResult, M3M4StateListener listener, Context context, int count, SourceState oldState) throws PureParserException {
    String result = parseDefinition(true, string, sourceName, addLines, offset, repository, listener, context, count);
    new M3AntlrParser(false).parse(result, sourceName, false, offset, repository, coreInstancesResult, listener, context, count, null);
}