Examples with DFAState org.antlr.v4.runtime.dfa.DFAState used on opensource projects

Example 26 with DFAState

use of org.antlr.v4.runtime.dfa.DFAState in project antlr4 by antlr.

the class TestPerformance method parseSources.

protected void parseSources(final int currentPass, final ParserFactory factory, Collection<InputDescriptor> sources, boolean shuffleSources) throws InterruptedException {
    if (shuffleSources) {
        List<InputDescriptor> sourcesList = new ArrayList<InputDescriptor>(sources);
        synchronized (RANDOM) {
            Collections.shuffle(sourcesList, RANDOM);
        }
        sources = sourcesList;
    }
    long startTime = System.nanoTime();
    tokenCount.set(currentPass, 0);
    int inputSize = 0;
    int inputCount = 0;
    Collection<Future<FileParseResult>> results = new ArrayList<Future<FileParseResult>>();
    ExecutorService executorService;
    if (FILE_GRANULARITY) {
        executorService = Executors.newFixedThreadPool(FILE_GRANULARITY ? NUMBER_OF_THREADS : 1, new NumberedThreadFactory());
    } else {
        executorService = Executors.newSingleThreadExecutor(new FixedThreadNumberFactory(((NumberedThread) Thread.currentThread()).getThreadNumber()));
    }
    for (InputDescriptor inputDescriptor : sources) {
        if (inputCount >= MAX_FILES_PER_PARSE_ITERATION) {
            break;
        }
        final CharStream input = inputDescriptor.getInputStream();
        input.seek(0);
        inputSize += input.size();
        inputCount++;
        Future<FileParseResult> futureChecksum = executorService.submit(new Callable<FileParseResult>() {

            @Override
            public FileParseResult call() {
                // System.out.format("Parsing file %s\n", input.getSourceName());
                try {
                    return factory.parseFile(input, currentPass, ((NumberedThread) Thread.currentThread()).getThreadNumber());
                } catch (IllegalStateException ex) {
                    ex.printStackTrace(System.err);
                } catch (Throwable t) {
                    t.printStackTrace(System.err);
                }
                return null;
            }
        });
        results.add(futureChecksum);
    }
    MurmurHashChecksum checksum = new MurmurHashChecksum();
    int currentIndex = -1;
    for (Future<FileParseResult> future : results) {
        currentIndex++;
        int fileChecksum = 0;
        try {
            FileParseResult fileResult = future.get();
            if (COMPUTE_TRANSITION_STATS) {
                totalTransitionsPerFile[currentPass][currentIndex] = sum(fileResult.parserTotalTransitions);
                computedTransitionsPerFile[currentPass][currentIndex] = sum(fileResult.parserComputedTransitions);
                if (DETAILED_DFA_STATE_STATS) {
                    decisionInvocationsPerFile[currentPass][currentIndex] = fileResult.decisionInvocations;
                    fullContextFallbackPerFile[currentPass][currentIndex] = fileResult.fullContextFallback;
                    nonSllPerFile[currentPass][currentIndex] = fileResult.nonSll;
                    totalTransitionsPerDecisionPerFile[currentPass][currentIndex] = fileResult.parserTotalTransitions;
                    computedTransitionsPerDecisionPerFile[currentPass][currentIndex] = fileResult.parserComputedTransitions;
                    fullContextTransitionsPerDecisionPerFile[currentPass][currentIndex] = fileResult.parserFullContextTransitions;
                }
            }
            if (COMPUTE_TIMING_STATS) {
                timePerFile[currentPass][currentIndex] = fileResult.endTime - fileResult.startTime;
                tokensPerFile[currentPass][currentIndex] = fileResult.tokenCount;
            }
            fileChecksum = fileResult.checksum;
        } catch (ExecutionException ex) {
            Logger.getLogger(TestPerformance.class.getName()).log(Level.SEVERE, null, ex);
        }
        if (COMPUTE_CHECKSUM) {
            updateChecksum(checksum, fileChecksum);
        }
    }
    executorService.shutdown();
    executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
    System.out.format("%d. Total parse time for %d files (%d KB, %d tokens%s): %.0fms%n", currentPass + 1, inputCount, inputSize / 1024, tokenCount.get(currentPass), COMPUTE_CHECKSUM ? String.format(", checksum 0x%8X", checksum.getValue()) : "", (double) (System.nanoTime() - startTime) / 1000000.0);
    if (sharedLexers.length > 0) {
        int index = FILE_GRANULARITY ? 0 : ((NumberedThread) Thread.currentThread()).getThreadNumber();
        Lexer lexer = sharedLexers[index];
        final LexerATNSimulator lexerInterpreter = lexer.getInterpreter();
        final DFA[] modeToDFA = lexerInterpreter.decisionToDFA;
        if (SHOW_DFA_STATE_STATS) {
            int states = 0;
            int configs = 0;
            Set<ATNConfig> uniqueConfigs = new HashSet<ATNConfig>();
            for (int i = 0; i < modeToDFA.length; i++) {
                DFA dfa = modeToDFA[i];
                if (dfa == null) {
                    continue;
                }
                states += dfa.states.size();
                for (DFAState state : dfa.states.values()) {
                    configs += state.configs.size();
                    uniqueConfigs.addAll(state.configs);
                }
            }
            System.out.format("There are %d lexer DFAState instances, %d configs (%d unique).%n", states, configs, uniqueConfigs.size());
            if (DETAILED_DFA_STATE_STATS) {
                System.out.format("\tMode\tStates\tConfigs\tMode%n");
                for (int i = 0; i < modeToDFA.length; i++) {
                    DFA dfa = modeToDFA[i];
                    if (dfa == null || dfa.states.isEmpty()) {
                        continue;
                    }
                    int modeConfigs = 0;
                    for (DFAState state : dfa.states.values()) {
                        modeConfigs += state.configs.size();
                    }
                    String modeName = lexer.getModeNames()[i];
                    System.out.format("\t%d\t%d\t%d\t%s%n", dfa.decision, dfa.states.size(), modeConfigs, modeName);
                }
            }
        }
    }
    if (RUN_PARSER && sharedParsers.length > 0) {
        int index = FILE_GRANULARITY ? 0 : ((NumberedThread) Thread.currentThread()).getThreadNumber();
        Parser parser = sharedParsers[index];
        // make sure the individual DFAState objects actually have unique ATNConfig arrays
        final ParserATNSimulator interpreter = parser.getInterpreter();
        final DFA[] decisionToDFA = interpreter.decisionToDFA;
        if (SHOW_DFA_STATE_STATS) {
            int states = 0;
            int configs = 0;
            Set<ATNConfig> uniqueConfigs = new HashSet<ATNConfig>();
            for (int i = 0; i < decisionToDFA.length; i++) {
                DFA dfa = decisionToDFA[i];
                if (dfa == null) {
                    continue;
                }
                states += dfa.states.size();
                for (DFAState state : dfa.states.values()) {
                    configs += state.configs.size();
                    uniqueConfigs.addAll(state.configs);
                }
            }
            System.out.format("There are %d parser DFAState instances, %d configs (%d unique).%n", states, configs, uniqueConfigs.size());
            if (DETAILED_DFA_STATE_STATS) {
                if (COMPUTE_TRANSITION_STATS) {
                    System.out.format("\tDecision\tStates\tConfigs\tPredict (ALL)\tPredict (LL)\tNon-SLL\tTransitions\tTransitions (ATN)\tTransitions (LL)\tLA (SLL)\tLA (LL)\tRule%n");
                } else {
                    System.out.format("\tDecision\tStates\tConfigs\tRule%n");
                }
                for (int i = 0; i < decisionToDFA.length; i++) {
                    DFA dfa = decisionToDFA[i];
                    if (dfa == null || dfa.states.isEmpty()) {
                        continue;
                    }
                    int decisionConfigs = 0;
                    for (DFAState state : dfa.states.values()) {
                        decisionConfigs += state.configs.size();
                    }
                    String ruleName = parser.getRuleNames()[parser.getATN().decisionToState.get(dfa.decision).ruleIndex];
                    long calls = 0;
                    long fullContextCalls = 0;
                    long nonSllCalls = 0;
                    long transitions = 0;
                    long computedTransitions = 0;
                    long fullContextTransitions = 0;
                    double lookahead = 0;
                    double fullContextLookahead = 0;
                    String formatString;
                    if (COMPUTE_TRANSITION_STATS) {
                        for (long[] data : decisionInvocationsPerFile[currentPass]) {
                            calls += data[i];
                        }
                        for (long[] data : fullContextFallbackPerFile[currentPass]) {
                            fullContextCalls += data[i];
                        }
                        for (long[] data : nonSllPerFile[currentPass]) {
                            nonSllCalls += data[i];
                        }
                        for (long[] data : totalTransitionsPerDecisionPerFile[currentPass]) {
                            transitions += data[i];
                        }
                        for (long[] data : computedTransitionsPerDecisionPerFile[currentPass]) {
                            computedTransitions += data[i];
                        }
                        for (long[] data : fullContextTransitionsPerDecisionPerFile[currentPass]) {
                            fullContextTransitions += data[i];
                        }
                        if (calls > 0) {
                            lookahead = (double) (transitions - fullContextTransitions) / (double) calls;
                        }
                        if (fullContextCalls > 0) {
                            fullContextLookahead = (double) fullContextTransitions / (double) fullContextCalls;
                        }
                        formatString = "\t%1$d\t%2$d\t%3$d\t%4$d\t%5$d\t%6$d\t%7$d\t%8$d\t%9$d\t%10$f\t%11$f\t%12$s%n";
                    } else {
                        calls = 0;
                        formatString = "\t%1$d\t%2$d\t%3$d\t%12$s%n";
                    }
                    System.out.format(formatString, dfa.decision, dfa.states.size(), decisionConfigs, calls, fullContextCalls, nonSllCalls, transitions, computedTransitions, fullContextTransitions, lookahead, fullContextLookahead, ruleName);
                }
            }
        }
        int localDfaCount = 0;
        int globalDfaCount = 0;
        int localConfigCount = 0;
        int globalConfigCount = 0;
        int[] contextsInDFAState = new int[0];
        for (int i = 0; i < decisionToDFA.length; i++) {
            DFA dfa = decisionToDFA[i];
            if (dfa == null) {
                continue;
            }
            if (SHOW_CONFIG_STATS) {
                for (DFAState state : dfa.states.keySet()) {
                    if (state.configs.size() >= contextsInDFAState.length) {
                        contextsInDFAState = Arrays.copyOf(contextsInDFAState, state.configs.size() + 1);
                    }
                    if (state.isAcceptState) {
                        boolean hasGlobal = false;
                        for (ATNConfig config : state.configs) {
                            if (config.reachesIntoOuterContext > 0) {
                                globalConfigCount++;
                                hasGlobal = true;
                            } else {
                                localConfigCount++;
                            }
                        }
                        if (hasGlobal) {
                            globalDfaCount++;
                        } else {
                            localDfaCount++;
                        }
                    }
                    contextsInDFAState[state.configs.size()]++;
                }
            }
        }
        if (SHOW_CONFIG_STATS && currentPass == 0) {
            System.out.format("  DFA accept states: %d total, %d with only local context, %d with a global context%n", localDfaCount + globalDfaCount, localDfaCount, globalDfaCount);
            System.out.format("  Config stats: %d total, %d local, %d global%n", localConfigCount + globalConfigCount, localConfigCount, globalConfigCount);
            if (SHOW_DFA_STATE_STATS) {
                for (int i = 0; i < contextsInDFAState.length; i++) {
                    if (contextsInDFAState[i] != 0) {
                        System.out.format("  %d configs = %d%n", i, contextsInDFAState[i]);
                    }
                }
            }
        }
    }
    if (COMPUTE_TIMING_STATS) {
        System.out.format("File\tTokens\tTime%n");
        for (int i = 0; i < timePerFile[currentPass].length; i++) {
            System.out.format("%d\t%d\t%d%n", i + 1, tokensPerFile[currentPass][i], timePerFile[currentPass][i]);
        }
    }
}

Example 27 with DFAState

use of org.antlr.v4.runtime.dfa.DFAState in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method addDFAState.

/**
 * See comment on LexerInterpreter.addDFAState.
 */
@NotNull
protected DFAState addDFAState(@NotNull DFA dfa, @NotNull ATNConfigSet configs, PredictionContextCache contextCache) {
    final boolean enableDfa = enable_global_context_dfa || !configs.isOutermostConfigSet();
    if (enableDfa) {
        if (!configs.isReadOnly()) {
            configs.optimizeConfigs(this);
        }
        DFAState proposed = createDFAState(dfa, configs);
        DFAState existing = dfa.states.get(proposed);
        if (existing != null)
            return existing;
    }
    if (!configs.isReadOnly()) {
        if (configs.getConflictInfo() == null) {
            configs.setConflictInfo(isConflicted(configs, contextCache));
        }
    }
    DFAState newState = createDFAState(dfa, configs.clone(true));
    DecisionState decisionState = atn.getDecisionState(dfa.decision);
    int predictedAlt = getUniqueAlt(configs);
    if (predictedAlt != ATN.INVALID_ALT_NUMBER) {
        newState.setAcceptState(new AcceptStateInfo(predictedAlt));
    } else if (configs.getConflictingAlts() != null) {
        newState.setAcceptState(new AcceptStateInfo(newState.configs.getConflictingAlts().nextSetBit(0)));
    }
    if (newState.isAcceptState() && configs.hasSemanticContext()) {
        predicateDFAState(newState, configs, decisionState.getNumberOfTransitions());
    }
    if (!enableDfa) {
        return newState;
    }
    DFAState added = dfa.addState(newState);
    if (debug && added == newState)
        System.out.println("adding new DFA state: " + newState);
    return added;
}

Example 28 with DFAState

use of org.antlr.v4.runtime.dfa.DFAState in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method computeReachSet.

protected SimulatorState computeReachSet(DFA dfa, SimulatorState previous, int t, PredictionContextCache contextCache) {
    final boolean useContext = previous.useContext;
    ParserRuleContext remainingGlobalContext = previous.remainingOuterContext;
    DFAState s = previous.s0;
    if (useContext) {
        while (s.isContextSymbol(t)) {
            DFAState next = null;
            if (remainingGlobalContext != null) {
                remainingGlobalContext = skipTailCalls(remainingGlobalContext);
                next = s.getContextTarget(getReturnState(remainingGlobalContext));
            }
            if (next == null) {
                break;
            }
            assert remainingGlobalContext != null;
            remainingGlobalContext = remainingGlobalContext.getParent();
            s = next;
        }
    }
    assert !isAcceptState(s, useContext);
    if (isAcceptState(s, useContext)) {
        return new SimulatorState(previous.outerContext, s, useContext, remainingGlobalContext);
    }
    final DFAState s0 = s;
    DFAState target = getExistingTargetState(s0, t);
    if (target == null) {
        Tuple2<DFAState, ParserRuleContext> result = computeTargetState(dfa, s0, remainingGlobalContext, t, useContext, contextCache);
        target = result.getItem1();
        remainingGlobalContext = result.getItem2();
    }
    if (target == ERROR) {
        return null;
    }
    assert !useContext || !target.configs.getDipsIntoOuterContext();
    return new SimulatorState(previous.outerContext, target, useContext, remainingGlobalContext);
}

Example 29 with DFAState

use of org.antlr.v4.runtime.dfa.DFAState in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method computeTargetState.

/**
 * Compute a target state for an edge in the DFA, and attempt to add the
 * computed state and corresponding edge to the DFA.
 *
 * @param dfa
 * @param s The current DFA state
 * @param remainingGlobalContext
 * @param t The next input symbol
 * @param useContext
 * @param contextCache
 *
 * @return The computed target DFA state for the given input symbol
 * {@code t}. If {@code t} does not lead to a valid DFA state, this method
 * returns {@link #ERROR}.
 */
@NotNull
protected Tuple2<DFAState, ParserRuleContext> computeTargetState(@NotNull DFA dfa, @NotNull DFAState s, ParserRuleContext remainingGlobalContext, int t, boolean useContext, PredictionContextCache contextCache) {
    List<ATNConfig> closureConfigs = new ArrayList<ATNConfig>(s.configs);
    IntegerList contextElements = null;
    ATNConfigSet reach = new ATNConfigSet();
    boolean stepIntoGlobal;
    do {
        boolean hasMoreContext = !useContext || remainingGlobalContext != null;
        if (!hasMoreContext) {
            reach.setOutermostConfigSet(true);
        }
        ATNConfigSet reachIntermediate = new ATNConfigSet();
        /* Configurations already in a rule stop state indicate reaching the end
			 * of the decision rule (local context) or end of the start rule (full
			 * context). Once reached, these configurations are never updated by a
			 * closure operation, so they are handled separately for the performance
			 * advantage of having a smaller intermediate set when calling closure.
			 *
			 * For full-context reach operations, separate handling is required to
			 * ensure that the alternative matching the longest overall sequence is
			 * chosen when multiple such configurations can match the input.
			 */
        List<ATNConfig> skippedStopStates = null;
        for (ATNConfig c : closureConfigs) {
            if (debug)
                System.out.println("testing " + getTokenName(t) + " at " + c.toString());
            if (c.getState() instanceof RuleStopState) {
                assert c.getContext().isEmpty();
                if (useContext && !c.getReachesIntoOuterContext() || t == IntStream.EOF) {
                    if (skippedStopStates == null) {
                        skippedStopStates = new ArrayList<ATNConfig>();
                    }
                    skippedStopStates.add(c);
                }
                continue;
            }
            int n = c.getState().getNumberOfOptimizedTransitions();
            for (int ti = 0; ti < n; ti++) {
                // for each optimized transition
                Transition trans = c.getState().getOptimizedTransition(ti);
                ATNState target = getReachableTarget(c, trans, t);
                if (target != null) {
                    reachIntermediate.add(c.transform(target, false), contextCache);
                }
            }
        }
        /* This block optimizes the reach operation for intermediate sets which
			 * trivially indicate a termination state for the overall
			 * adaptivePredict operation.
			 *
			 * The conditions assume that intermediate
			 * contains all configurations relevant to the reach set, but this
			 * condition is not true when one or more configurations have been
			 * withheld in skippedStopStates, or when the current symbol is EOF.
			 */
        if (optimize_unique_closure && skippedStopStates == null && t != Token.EOF && reachIntermediate.getUniqueAlt() != ATN.INVALID_ALT_NUMBER) {
            reachIntermediate.setOutermostConfigSet(reach.isOutermostConfigSet());
            reach = reachIntermediate;
            break;
        }
        /* If the reach set could not be trivially determined, perform a closure
			 * operation on the intermediate set to compute its initial value.
			 */
        final boolean collectPredicates = false;
        boolean treatEofAsEpsilon = t == Token.EOF;
        closure(reachIntermediate, reach, collectPredicates, hasMoreContext, contextCache, treatEofAsEpsilon);
        stepIntoGlobal = reach.getDipsIntoOuterContext();
        if (t == IntStream.EOF) {
            /* After consuming EOF no additional input is possible, so we are
				 * only interested in configurations which reached the end of the
				 * decision rule (local context) or end of the start rule (full
				 * context). Update reach to contain only these configurations. This
				 * handles both explicit EOF transitions in the grammar and implicit
				 * EOF transitions following the end of the decision or start rule.
				 *
				 * This is handled before the configurations in skippedStopStates,
				 * because any configurations potentially added from that list are
				 * already guaranteed to meet this condition whether or not it's
				 * required.
				 */
            reach = removeAllConfigsNotInRuleStopState(reach, contextCache);
        }
        /* If skippedStopStates is not null, then it contains at least one
			 * configuration. For full-context reach operations, these
			 * configurations reached the end of the start rule, in which case we
			 * only add them back to reach if no configuration during the current
			 * closure operation reached such a state. This ensures adaptivePredict
			 * chooses an alternative matching the longest overall sequence when
			 * multiple alternatives are viable.
			 */
        if (skippedStopStates != null && (!useContext || !PredictionMode.hasConfigInRuleStopState(reach))) {
            assert !skippedStopStates.isEmpty();
            for (ATNConfig c : skippedStopStates) {
                reach.add(c, contextCache);
            }
        }
        if (useContext && stepIntoGlobal) {
            reach.clear();
            remainingGlobalContext = skipTailCalls(remainingGlobalContext);
            int nextContextElement = getReturnState(remainingGlobalContext);
            if (contextElements == null) {
                contextElements = new IntegerList();
            }
            if (remainingGlobalContext.isEmpty()) {
                remainingGlobalContext = null;
            } else {
                remainingGlobalContext = remainingGlobalContext.getParent();
            }
            contextElements.add(nextContextElement);
            if (nextContextElement != PredictionContext.EMPTY_FULL_STATE_KEY) {
                for (int i = 0; i < closureConfigs.size(); i++) {
                    closureConfigs.set(i, closureConfigs.get(i).appendContext(nextContextElement, contextCache));
                }
            }
        }
    } while (useContext && stepIntoGlobal);
    if (reach.isEmpty()) {
        addDFAEdge(s, t, ERROR);
        return Tuple.create(ERROR, remainingGlobalContext);
    }
    DFAState result = addDFAEdge(dfa, s, t, contextElements, reach, contextCache);
    return Tuple.create(result, remainingGlobalContext);
}

Example 30 with DFAState

use of org.antlr.v4.runtime.dfa.DFAState in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method reportAmbiguity.

/**
 * If context sensitive parsing, we know it's ambiguity not conflict
 */
protected void reportAmbiguity(@NotNull DFA dfa, // the DFA state from execATN() that had SLL conflicts
DFAState D, int startIndex, int stopIndex, boolean exact, @NotNull BitSet ambigAlts, // configs that LL not SLL considered conflicting
@NotNull ATNConfigSet configs) {
    if (debug || retry_debug) {
        Interval interval = Interval.of(startIndex, stopIndex);
        System.out.println("reportAmbiguity " + ambigAlts + ":" + configs + ", input=" + parser.getInputStream().getText(interval));
    }
    if (parser != null)
        parser.getErrorListenerDispatch().reportAmbiguity(parser, dfa, startIndex, stopIndex, exact, ambigAlts, configs);
}