Examples with TopParser org.finos.legend.pure.m3.serialization.grammar.top.TopParser used on opensource projects

Example 1 with TopParser

use of org.finos.legend.pure.m3.serialization.grammar.top.TopParser in project legend-pure by finos.

the class Loader method parseM3.

public static void parseM3(String code, ModelRepository repository, ParserLibrary library, ValidationType validationType, M3M4StateListener stateListener, Context context) {
    try {
        ListMultimap<Parser, CoreInstance> newInstancesByParser = new TopParser().parse(code, "fromString.pure", repository, library, stateListener, context, null);
        ListIterable<CoreInstance> newInstances = newInstancesByParser.valuesView().toList();
        updateStateFullContextCache(newInstances, context);
        ProcessorSupport processorSupport = new M3ProcessorSupport(context, repository);
        PostProcessor.process(newInstances, repository, library, new InlineDSLLibrary(), null, context, processorSupport, null, null);
        processExcludes(repository, processorSupport, stateListener);
        if (validationType == ValidationType.DEEP) {
            repository.validate(stateListener);
        }
        // Validate M3
        stateListener.startValidation();
        Validator.validateM3(newInstances, validationType, library, new InlineDSLLibrary(), new PureCodeStorage(null), repository, context, processorSupport);
        stateListener.finishedValidation();
    } catch (PureException exception) {
        String space = "      ";
        throw new RuntimeException(exception + " in\n" + space + code.replace("\n", "\n" + space), exception);
    }
}

Example 2 with TopParser

use of org.finos.legend.pure.m3.serialization.grammar.top.TopParser in project legend-pure by finos.

the class IncrementalCompiler_New method compileRepoSources.

private SourceMutation compileRepoSources(IncrementalCompilerTransaction transaction, String repoName, int repoNum, int repoTotalCount, RichIterable<? extends Source> sources, RichIterable<CoreInstance> instancesToProcess, RichIterable<CoreInstance> instancesToUnbind, RichIterable<SourceState> sourceStates, MutableSet<CoreInstance> repoTransactionInstances) throws PureCompilationException, PureParserException {
    String repoDisplayName = repoName == null ? "non-repository" : repoName;
    try (ThreadLocalTransactionContext ignored = transaction != null ? transaction.openInCurrentThread() : null) {
        AtomicInteger sourceNum = this.message == null ? null : new AtomicInteger(0);
        int sourceTotalCount = sources.size();
        Procedure<Source> parseSource = source -> {
            try (ThreadLocalTransactionContext ignored1 = transaction != null ? transaction.openInCurrentThread() : null) {
                if (this.message != null) {
                    int thisSourceNum = sourceNum.incrementAndGet();
                    StringBuilder message = new StringBuilder("Parsing ").append(repoDisplayName);
                    if (repoTotalCount > 1) {
                        message.append(" (").append(repoNum).append('/').append(repoTotalCount).append(')');
                    }
                    message.append(" sources (").append(thisSourceNum).append('/').append(sourceTotalCount).append(')');
                    this.message.setMessage(message.toString());
                }
                MutableList<SourceState> oldState = sourceStates.select(sourceState -> source.equals(sourceState.getSource()), Lists.mutable.empty());
                ListMultimap<Parser, CoreInstance> newInstancesByParser = new TopParser().parse(source.getContent(), source.getId(), this.modelRepository, this.library, VoidM3M4StateListener.VOID_M3_M4_STATE_LISTENER, this.context, oldState.size() == 1 ? oldState.get(0) : null);
                this.updateSource(source, newInstancesByParser);
                if (transaction != null) {
                    transaction.noteSourceCompiled(source);
                }
            } catch (RuntimeException e) {
                if (PureException.canFindPureException(e)) {
                    throw e;
                }
                throw new PureParserException(new SourceInformation(source.getId(), -1, -1, -1, -1), "Error parsing " + source.getId(), e);
            }
        };
        if (this.shouldParallelize(sourceTotalCount, PARSE_SOURCES_THRESHOLD)) {
            ForkJoinTools.forEach(this.forkJoinPool, ListHelper.wrapListIterable(sources), parseSource, PARSE_SOURCES_THRESHOLD);
        } else {
            sources.forEach(parseSource);
        }
        // Parsing for repo completed successfully
        // New Instances in the sources from the repo
        MutableSet<CoreInstance> newInstances = sources.flatCollect(Source::getNewInstances, Sets.mutable.empty());
        // Old Instances in the sources from repo
        MutableSet<CoreInstance> oldInstances = sourceStates.flatCollect(SourceState::getInstances, Sets.mutable.empty());
        // Instances which are newly created (added or modified)
        MutableSet<CoreInstance> newButNotOld = newInstances.difference(oldInstances);
        // Instances which are not retained (deleted or modified)
        MutableSet<CoreInstance> oldButNotNew = oldInstances.difference(newInstances);
        // Source Ids in the repo within scope
        MutableSet<String> sourcesInScope = Sets.mutable.empty();
        sources.collect(Source.SOURCE_ID, sourcesInScope);
        // Unload Walk on only instances which are not retained and generate toUnbindGenerated (There can be instances from following repos as well)
        MutableSet<CoreInstance> toUnbindGenerated = this.walkTheGraphForUnload(oldButNotNew);
        // Filter the instances which are within the repo
        MutableSet<CoreInstance> toUnbindWithinRepo = toUnbindGenerated.selectWith(CORE_INSTANCE_IS_FROM_REPO, repoName);
        // Total Unbind set is ( generated here + obtained through call - non retained )
        MutableSet<CoreInstance> hereUnbind = toUnbindWithinRepo.union(oldButNotNew).union(instancesToUnbind.toSet());
        Unbinder.process(hereUnbind, this.modelRepository, this.library, this.dslLibrary, this.context, this.processorSupport, new UnbindState(this.context, this.urlPatternLibrary, this.processorSupport), this.message);
        // Invalidate the unbound instances
        if (hereUnbind.notEmpty()) {
            this.compilerEventHandlers.forEach(eh -> eh.invalidate(hereUnbind));
        }
        // ToProcessGenerated - If the instances is within the scope of resources, it should be in the new instances - This can contain instances from following repos as well
        MutableSet<CoreInstance> toProcessGenerated = toUnbindGenerated.select(each -> !sourcesInScope.contains(each.getSourceInformation().getSourceId()) || newInstances.contains(each));
        // Filter the instances which are within this repo
        MutableSet<CoreInstance> toProcessWithinRepoGenerated = toProcessGenerated.selectWith(CORE_INSTANCE_IS_FROM_REPO, repoName);
        // ToProcess from call is filtered for the existence in new instances if source is within the sourcesInScope
        MutableSet<CoreInstance> instancesToProcessFiltered = instancesToProcess.select(each -> !sourcesInScope.contains(each.getSourceInformation().getSourceId()) || newInstances.contains(each)).toSet();
        // ToUnbind filtered for the existence in new instances if source is within the sourcesInScope
        MutableSet<CoreInstance> instancesToUnbindFiltered = instancesToUnbind.select(each -> !sourcesInScope.contains(each.getSourceInformation().getSourceId()) || newInstances.contains(each)).toSet();
        // Final instances to be processed is (generated within repo + Filtered call parameters (toProcess, toUnbind))
        MutableList<CoreInstance> newInstancesConsolidated = this.removeNodesFromRemovedSources(toProcessWithinRepoGenerated.union(instancesToProcessFiltered).union(instancesToUnbindFiltered)).toSet().difference(newButNotOld).toList();
        // Maintain ordering to avoid errors in unit tests
        sources.forEach(source -> source.getNewInstances().select(newButNotOld::contains, newInstancesConsolidated));
        // Collect all instances to be registered (Even retained instances are unregistered at start)
        MutableList<CoreInstance> allInstances = this.removeNodesFromRemovedSources(toProcessWithinRepoGenerated.union(instancesToProcessFiltered).union(instancesToUnbindFiltered)).toSet().difference(newInstances).toList();
        sources.forEach(source -> allInstances.addAllIterable(source.getNewInstances()));
        // Store the to be processed set for rollback
        repoTransactionInstances.addAllIterable(newInstancesConsolidated);
        // Do postprocessing, validation - can throw an error
        SourceMutation result = this.finishRepoCompilation(repoDisplayName, allInstances, newInstancesConsolidated, ValidationType.SHALLOW);
        // Repo compilation Successful
        // Remove any redundant packages of old instances
        oldInstances.forEach(this::tryRemovePackage);
        // Remove source states
        this.oldSourceStates.removeAllIterable(sourceStates);
        // Remove old instances from toUnload
        this.toUnload.removeAllIterable(oldInstances);
        // Update toProcess
        this.toProcess = this.toProcess.difference(oldButNotNew);
        this.toProcess.addAllIterable(this.removeNodesFromRemovedSources(toProcessGenerated.difference(toProcessWithinRepoGenerated).difference(oldButNotNew).difference(newInstancesConsolidated.toSet())));
        // Update toUnbind
        this.toUnbind.removeAllIterable(hereUnbind);
        this.toUnbind.addAllIterable(toUnbindGenerated.difference(toUnbindWithinRepo));
        // Update processed to be used by compiler event handlers
        this.processed.addAllIterable(toProcessWithinRepoGenerated.withAll(instancesToProcessFiltered).withAll(instancesToUnbindFiltered));
        return result;
    }
}

Example 3 with TopParser

use of org.finos.legend.pure.m3.serialization.grammar.top.TopParser in project legend-pure by finos.

the class IncrementalCompiler_Old method compileRepoSources.

private SourceMutation compileRepoSources(IncrementalCompilerTransaction transaction, String repoName, int repoNum, int repoTotalCount, RichIterable<? extends Source> sources, RichIterable<CoreInstance> instancesToProcess) throws PureCompilationException, PureParserException {
    String repoDisplayName = repoName == null ? "non-repository" : repoName;
    try (ThreadLocalTransactionContext ignored = transaction != null ? transaction.openInCurrentThread() : null) {
        AtomicInteger sourceNum = this.message == null ? null : new AtomicInteger(0);
        int sourceTotalCount = sources.size();
        Procedure<Source> parseSource = source -> {
            try (ThreadLocalTransactionContext ignore = transaction != null ? transaction.openInCurrentThread() : null) {
                if (this.message != null) {
                    int thisSourceNum = sourceNum.incrementAndGet();
                    StringBuilder message = new StringBuilder("Parsing ").append(repoDisplayName);
                    if (repoTotalCount > 1) {
                        message.append(" (").append(repoNum).append('/').append(repoTotalCount).append(')');
                    }
                    message.append(" sources (").append(thisSourceNum).append('/').append(sourceTotalCount).append(')');
                    this.message.setMessage(message.toString());
                }
                ListMultimap<Parser, CoreInstance> newInstancesByParser = new TopParser().parse(source.getContent(), source.getId(), this.modelRepository, this.library, VoidM3M4StateListener.VOID_M3_M4_STATE_LISTENER, this.context, null);
                updateSource(source, newInstancesByParser);
                if (transaction != null) {
                    transaction.noteSourceCompiled(source);
                }
            } catch (RuntimeException e) {
                if (PureException.canFindPureException(e)) {
                    throw e;
                }
                throw new PureParserException(new SourceInformation(source.getId(), -1, -1, -1, -1), "Error parsing " + source.getId(), e);
            }
        };
        if (this.shouldParallelize(sourceTotalCount, PARSE_SOURCES_THRESHOLD)) {
            ForkJoinTools.forEach(this.forkJoinPool, ListHelper.wrapListIterable(sources), parseSource, PARSE_SOURCES_THRESHOLD);
        } else {
            sources.forEach(parseSource);
        }
        MutableList<CoreInstance> newInstancesConsolidated = sources.flatCollect(Source::getNewInstances, instancesToProcess.toList());
        return this.finishRepoCompilation(repoDisplayName, newInstancesConsolidated, ValidationType.SHALLOW);
    }
}