Search in sources :

Example 6 with ATNConfig

use of org.antlr.v4.runtime.atn.ATNConfig in project antlr4 by antlr.

the class TestPerformance method parseSources.

protected void parseSources(final int currentPass, final ParserFactory factory, Collection<InputDescriptor> sources, boolean shuffleSources) throws InterruptedException {
    if (shuffleSources) {
        List<InputDescriptor> sourcesList = new ArrayList<InputDescriptor>(sources);
        synchronized (RANDOM) {
            Collections.shuffle(sourcesList, RANDOM);
        }
        sources = sourcesList;
    }
    long startTime = System.nanoTime();
    tokenCount.set(currentPass, 0);
    int inputSize = 0;
    int inputCount = 0;
    Collection<Future<FileParseResult>> results = new ArrayList<Future<FileParseResult>>();
    ExecutorService executorService;
    if (FILE_GRANULARITY) {
        executorService = Executors.newFixedThreadPool(FILE_GRANULARITY ? NUMBER_OF_THREADS : 1, new NumberedThreadFactory());
    } else {
        executorService = Executors.newSingleThreadExecutor(new FixedThreadNumberFactory(((NumberedThread) Thread.currentThread()).getThreadNumber()));
    }
    for (InputDescriptor inputDescriptor : sources) {
        if (inputCount >= MAX_FILES_PER_PARSE_ITERATION) {
            break;
        }
        final CharStream input = inputDescriptor.getInputStream();
        input.seek(0);
        inputSize += input.size();
        inputCount++;
        Future<FileParseResult> futureChecksum = executorService.submit(new Callable<FileParseResult>() {

            @Override
            public FileParseResult call() {
                //System.out.format("Parsing file %s\n", input.getSourceName());
                try {
                    return factory.parseFile(input, currentPass, ((NumberedThread) Thread.currentThread()).getThreadNumber());
                } catch (IllegalStateException ex) {
                    ex.printStackTrace(System.err);
                } catch (Throwable t) {
                    t.printStackTrace(System.err);
                }
                return null;
            }
        });
        results.add(futureChecksum);
    }
    MurmurHashChecksum checksum = new MurmurHashChecksum();
    int currentIndex = -1;
    for (Future<FileParseResult> future : results) {
        currentIndex++;
        int fileChecksum = 0;
        try {
            FileParseResult fileResult = future.get();
            if (COMPUTE_TRANSITION_STATS) {
                totalTransitionsPerFile[currentPass][currentIndex] = sum(fileResult.parserTotalTransitions);
                computedTransitionsPerFile[currentPass][currentIndex] = sum(fileResult.parserComputedTransitions);
                if (DETAILED_DFA_STATE_STATS) {
                    decisionInvocationsPerFile[currentPass][currentIndex] = fileResult.decisionInvocations;
                    fullContextFallbackPerFile[currentPass][currentIndex] = fileResult.fullContextFallback;
                    nonSllPerFile[currentPass][currentIndex] = fileResult.nonSll;
                    totalTransitionsPerDecisionPerFile[currentPass][currentIndex] = fileResult.parserTotalTransitions;
                    computedTransitionsPerDecisionPerFile[currentPass][currentIndex] = fileResult.parserComputedTransitions;
                    fullContextTransitionsPerDecisionPerFile[currentPass][currentIndex] = fileResult.parserFullContextTransitions;
                }
            }
            if (COMPUTE_TIMING_STATS) {
                timePerFile[currentPass][currentIndex] = fileResult.endTime - fileResult.startTime;
                tokensPerFile[currentPass][currentIndex] = fileResult.tokenCount;
            }
            fileChecksum = fileResult.checksum;
        } catch (ExecutionException ex) {
            Logger.getLogger(TestPerformance.class.getName()).log(Level.SEVERE, null, ex);
        }
        if (COMPUTE_CHECKSUM) {
            updateChecksum(checksum, fileChecksum);
        }
    }
    executorService.shutdown();
    executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
    System.out.format("%d. Total parse time for %d files (%d KB, %d tokens%s): %.0fms%n", currentPass + 1, inputCount, inputSize / 1024, tokenCount.get(currentPass), COMPUTE_CHECKSUM ? String.format(", checksum 0x%8X", checksum.getValue()) : "", (double) (System.nanoTime() - startTime) / 1000000.0);
    if (sharedLexers.length > 0) {
        int index = FILE_GRANULARITY ? 0 : ((NumberedThread) Thread.currentThread()).getThreadNumber();
        Lexer lexer = sharedLexers[index];
        final LexerATNSimulator lexerInterpreter = lexer.getInterpreter();
        final DFA[] modeToDFA = lexerInterpreter.decisionToDFA;
        if (SHOW_DFA_STATE_STATS) {
            int states = 0;
            int configs = 0;
            Set<ATNConfig> uniqueConfigs = new HashSet<ATNConfig>();
            for (int i = 0; i < modeToDFA.length; i++) {
                DFA dfa = modeToDFA[i];
                if (dfa == null) {
                    continue;
                }
                states += dfa.states.size();
                for (DFAState state : dfa.states.values()) {
                    configs += state.configs.size();
                    uniqueConfigs.addAll(state.configs);
                }
            }
            System.out.format("There are %d lexer DFAState instances, %d configs (%d unique).%n", states, configs, uniqueConfigs.size());
            if (DETAILED_DFA_STATE_STATS) {
                System.out.format("\tMode\tStates\tConfigs\tMode%n");
                for (int i = 0; i < modeToDFA.length; i++) {
                    DFA dfa = modeToDFA[i];
                    if (dfa == null || dfa.states.isEmpty()) {
                        continue;
                    }
                    int modeConfigs = 0;
                    for (DFAState state : dfa.states.values()) {
                        modeConfigs += state.configs.size();
                    }
                    String modeName = lexer.getModeNames()[i];
                    System.out.format("\t%d\t%d\t%d\t%s%n", dfa.decision, dfa.states.size(), modeConfigs, modeName);
                }
            }
        }
    }
    if (RUN_PARSER && sharedParsers.length > 0) {
        int index = FILE_GRANULARITY ? 0 : ((NumberedThread) Thread.currentThread()).getThreadNumber();
        Parser parser = sharedParsers[index];
        // make sure the individual DFAState objects actually have unique ATNConfig arrays
        final ParserATNSimulator interpreter = parser.getInterpreter();
        final DFA[] decisionToDFA = interpreter.decisionToDFA;
        if (SHOW_DFA_STATE_STATS) {
            int states = 0;
            int configs = 0;
            Set<ATNConfig> uniqueConfigs = new HashSet<ATNConfig>();
            for (int i = 0; i < decisionToDFA.length; i++) {
                DFA dfa = decisionToDFA[i];
                if (dfa == null) {
                    continue;
                }
                states += dfa.states.size();
                for (DFAState state : dfa.states.values()) {
                    configs += state.configs.size();
                    uniqueConfigs.addAll(state.configs);
                }
            }
            System.out.format("There are %d parser DFAState instances, %d configs (%d unique).%n", states, configs, uniqueConfigs.size());
            if (DETAILED_DFA_STATE_STATS) {
                if (COMPUTE_TRANSITION_STATS) {
                    System.out.format("\tDecision\tStates\tConfigs\tPredict (ALL)\tPredict (LL)\tNon-SLL\tTransitions\tTransitions (ATN)\tTransitions (LL)\tLA (SLL)\tLA (LL)\tRule%n");
                } else {
                    System.out.format("\tDecision\tStates\tConfigs\tRule%n");
                }
                for (int i = 0; i < decisionToDFA.length; i++) {
                    DFA dfa = decisionToDFA[i];
                    if (dfa == null || dfa.states.isEmpty()) {
                        continue;
                    }
                    int decisionConfigs = 0;
                    for (DFAState state : dfa.states.values()) {
                        decisionConfigs += state.configs.size();
                    }
                    String ruleName = parser.getRuleNames()[parser.getATN().decisionToState.get(dfa.decision).ruleIndex];
                    long calls = 0;
                    long fullContextCalls = 0;
                    long nonSllCalls = 0;
                    long transitions = 0;
                    long computedTransitions = 0;
                    long fullContextTransitions = 0;
                    double lookahead = 0;
                    double fullContextLookahead = 0;
                    String formatString;
                    if (COMPUTE_TRANSITION_STATS) {
                        for (long[] data : decisionInvocationsPerFile[currentPass]) {
                            calls += data[i];
                        }
                        for (long[] data : fullContextFallbackPerFile[currentPass]) {
                            fullContextCalls += data[i];
                        }
                        for (long[] data : nonSllPerFile[currentPass]) {
                            nonSllCalls += data[i];
                        }
                        for (long[] data : totalTransitionsPerDecisionPerFile[currentPass]) {
                            transitions += data[i];
                        }
                        for (long[] data : computedTransitionsPerDecisionPerFile[currentPass]) {
                            computedTransitions += data[i];
                        }
                        for (long[] data : fullContextTransitionsPerDecisionPerFile[currentPass]) {
                            fullContextTransitions += data[i];
                        }
                        if (calls > 0) {
                            lookahead = (double) (transitions - fullContextTransitions) / (double) calls;
                        }
                        if (fullContextCalls > 0) {
                            fullContextLookahead = (double) fullContextTransitions / (double) fullContextCalls;
                        }
                        formatString = "\t%1$d\t%2$d\t%3$d\t%4$d\t%5$d\t%6$d\t%7$d\t%8$d\t%9$d\t%10$f\t%11$f\t%12$s%n";
                    } else {
                        calls = 0;
                        formatString = "\t%1$d\t%2$d\t%3$d\t%12$s%n";
                    }
                    System.out.format(formatString, dfa.decision, dfa.states.size(), decisionConfigs, calls, fullContextCalls, nonSllCalls, transitions, computedTransitions, fullContextTransitions, lookahead, fullContextLookahead, ruleName);
                }
            }
        }
        int localDfaCount = 0;
        int globalDfaCount = 0;
        int localConfigCount = 0;
        int globalConfigCount = 0;
        int[] contextsInDFAState = new int[0];
        for (int i = 0; i < decisionToDFA.length; i++) {
            DFA dfa = decisionToDFA[i];
            if (dfa == null) {
                continue;
            }
            if (SHOW_CONFIG_STATS) {
                for (DFAState state : dfa.states.keySet()) {
                    if (state.configs.size() >= contextsInDFAState.length) {
                        contextsInDFAState = Arrays.copyOf(contextsInDFAState, state.configs.size() + 1);
                    }
                    if (state.isAcceptState) {
                        boolean hasGlobal = false;
                        for (ATNConfig config : state.configs) {
                            if (config.reachesIntoOuterContext > 0) {
                                globalConfigCount++;
                                hasGlobal = true;
                            } else {
                                localConfigCount++;
                            }
                        }
                        if (hasGlobal) {
                            globalDfaCount++;
                        } else {
                            localDfaCount++;
                        }
                    }
                    contextsInDFAState[state.configs.size()]++;
                }
            }
        }
        if (SHOW_CONFIG_STATS && currentPass == 0) {
            System.out.format("  DFA accept states: %d total, %d with only local context, %d with a global context%n", localDfaCount + globalDfaCount, localDfaCount, globalDfaCount);
            System.out.format("  Config stats: %d total, %d local, %d global%n", localConfigCount + globalConfigCount, localConfigCount, globalConfigCount);
            if (SHOW_DFA_STATE_STATS) {
                for (int i = 0; i < contextsInDFAState.length; i++) {
                    if (contextsInDFAState[i] != 0) {
                        System.out.format("  %d configs = %d%n", i, contextsInDFAState[i]);
                    }
                }
            }
        }
    }
    if (COMPUTE_TIMING_STATS) {
        System.out.format("File\tTokens\tTime%n");
        for (int i = 0; i < timePerFile[currentPass].length; i++) {
            System.out.format("%d\t%d\t%d%n", i + 1, tokensPerFile[currentPass][i], timePerFile[currentPass][i]);
        }
    }
}
Also used : ArrayList(java.util.ArrayList) CharStream(org.antlr.v4.runtime.CharStream) ATNConfig(org.antlr.v4.runtime.atn.ATNConfig) ExecutionException(java.util.concurrent.ExecutionException) HashSet(java.util.HashSet) DFAState(org.antlr.v4.runtime.dfa.DFAState) Parser(org.antlr.v4.runtime.Parser) Lexer(org.antlr.v4.runtime.Lexer) ExecutorService(java.util.concurrent.ExecutorService) LexerATNSimulator(org.antlr.v4.runtime.atn.LexerATNSimulator) Future(java.util.concurrent.Future) ParserATNSimulator(org.antlr.v4.runtime.atn.ParserATNSimulator) DFA(org.antlr.v4.runtime.dfa.DFA)

Aggregations

IntervalSet (org.antlr.v4.runtime.misc.IntervalSet)3 ArrayList (java.util.ArrayList)2 BitSet (java.util.BitSet)2 HashSet (java.util.HashSet)2 ATNConfig (org.antlr.v4.runtime.atn.ATNConfig)2 DFA (org.antlr.v4.runtime.dfa.DFA)2 DFAState (org.antlr.v4.runtime.dfa.DFAState)2 ExecutionException (java.util.concurrent.ExecutionException)1 ExecutorService (java.util.concurrent.ExecutorService)1 Future (java.util.concurrent.Future)1 CharStream (org.antlr.v4.runtime.CharStream)1 Lexer (org.antlr.v4.runtime.Lexer)1 Parser (org.antlr.v4.runtime.Parser)1 LexerATNSimulator (org.antlr.v4.runtime.atn.LexerATNSimulator)1 ParserATNSimulator (org.antlr.v4.runtime.atn.ParserATNSimulator)1 IntegerList (org.antlr.v4.runtime.misc.IntegerList)1