Examples with LexerInterpreter org.antlr.v4.runtime.LexerInterpreter used on opensource projects

Example 56 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestTokenStreamRewriter method testLeaveAloneDisjointInsert2.

@Test
public void testLeaveAloneDisjointInsert2() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar T;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n");
    String input = "abcc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(2, 3, "foo");
    tokens.insertBefore(1, "x");
    String result = tokens.getText();
    String expecting = "axbfoo";
    assertEquals(expecting, result);
}

Example 57 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestTokenStreamRewriter method testCombine3Inserts.

@Test
public void testCombine3Inserts() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar T;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.insertBefore(1, "x");
    tokens.insertBefore(0, "y");
    tokens.insertBefore(1, "z");
    String result = tokens.getText();
    String expecting = "yazxbc";
    assertEquals(expecting, result);
}

Example 58 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestPerformance method parseSources.

protected void parseSources(final int currentPass, final ParserFactory factory, Collection<InputDescriptor> sources, boolean shuffleSources) throws InterruptedException {
    if (shuffleSources) {
        List<InputDescriptor> sourcesList = new ArrayList<InputDescriptor>(sources);
        synchronized (RANDOM) {
            Collections.shuffle(sourcesList, RANDOM);
        }
        sources = sourcesList;
    }
    long startTime = System.nanoTime();
    tokenCount.set(currentPass, 0);
    int inputSize = 0;
    int inputCount = 0;
    Collection<Future<FileParseResult>> results = new ArrayList<Future<FileParseResult>>();
    ExecutorService executorService;
    if (FILE_GRANULARITY) {
        executorService = Executors.newFixedThreadPool(FILE_GRANULARITY ? NUMBER_OF_THREADS : 1, new NumberedThreadFactory());
    } else {
        executorService = Executors.newSingleThreadExecutor(new FixedThreadNumberFactory(((NumberedThread) Thread.currentThread()).getThreadNumber()));
    }
    for (InputDescriptor inputDescriptor : sources) {
        if (inputCount >= MAX_FILES_PER_PARSE_ITERATION) {
            break;
        }
        final CharStream input = inputDescriptor.getInputStream();
        input.seek(0);
        inputSize += input.size();
        inputCount++;
        Future<FileParseResult> futureChecksum = executorService.submit(new Callable<FileParseResult>() {

            @Override
            public FileParseResult call() {
                //System.out.format("Parsing file %s\n", input.getSourceName());
                try {
                    return factory.parseFile(input, currentPass, ((NumberedThread) Thread.currentThread()).getThreadNumber());
                } catch (IllegalStateException ex) {
                    ex.printStackTrace(System.err);
                } catch (Throwable t) {
                    t.printStackTrace(System.err);
                }
                return null;
            }
        });
        results.add(futureChecksum);
    }
    MurmurHashChecksum checksum = new MurmurHashChecksum();
    int currentIndex = -1;
    for (Future<FileParseResult> future : results) {
        currentIndex++;
        int fileChecksum = 0;
        try {
            FileParseResult fileResult = future.get();
            if (COMPUTE_TRANSITION_STATS) {
                totalTransitionsPerFile[currentPass][currentIndex] = sum(fileResult.parserTotalTransitions);
                computedTransitionsPerFile[currentPass][currentIndex] = sum(fileResult.parserComputedTransitions);
                if (DETAILED_DFA_STATE_STATS) {
                    decisionInvocationsPerFile[currentPass][currentIndex] = fileResult.decisionInvocations;
                    fullContextFallbackPerFile[currentPass][currentIndex] = fileResult.fullContextFallback;
                    nonSllPerFile[currentPass][currentIndex] = fileResult.nonSll;
                    totalTransitionsPerDecisionPerFile[currentPass][currentIndex] = fileResult.parserTotalTransitions;
                    computedTransitionsPerDecisionPerFile[currentPass][currentIndex] = fileResult.parserComputedTransitions;
                    fullContextTransitionsPerDecisionPerFile[currentPass][currentIndex] = fileResult.parserFullContextTransitions;
                }
            }
            if (COMPUTE_TIMING_STATS) {
                timePerFile[currentPass][currentIndex] = fileResult.endTime - fileResult.startTime;
                tokensPerFile[currentPass][currentIndex] = fileResult.tokenCount;
            }
            fileChecksum = fileResult.checksum;
        } catch (ExecutionException ex) {
            Logger.getLogger(TestPerformance.class.getName()).log(Level.SEVERE, null, ex);
        }
        if (COMPUTE_CHECKSUM) {
            updateChecksum(checksum, fileChecksum);
        }
    }
    executorService.shutdown();
    executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
    System.out.format("%d. Total parse time for %d files (%d KB, %d tokens%s): %.0fms%n", currentPass + 1, inputCount, inputSize / 1024, tokenCount.get(currentPass), COMPUTE_CHECKSUM ? String.format(", checksum 0x%8X", checksum.getValue()) : "", (double) (System.nanoTime() - startTime) / 1000000.0);
    if (sharedLexers.length > 0) {
        int index = FILE_GRANULARITY ? 0 : ((NumberedThread) Thread.currentThread()).getThreadNumber();
        Lexer lexer = sharedLexers[index];
        final LexerATNSimulator lexerInterpreter = lexer.getInterpreter();
        final DFA[] modeToDFA = lexerInterpreter.decisionToDFA;
        if (SHOW_DFA_STATE_STATS) {
            int states = 0;
            int configs = 0;
            Set<ATNConfig> uniqueConfigs = new HashSet<ATNConfig>();
            for (int i = 0; i < modeToDFA.length; i++) {
                DFA dfa = modeToDFA[i];
                if (dfa == null) {
                    continue;
                }
                states += dfa.states.size();
                for (DFAState state : dfa.states.values()) {
                    configs += state.configs.size();
                    uniqueConfigs.addAll(state.configs);
                }
            }
            System.out.format("There are %d lexer DFAState instances, %d configs (%d unique).%n", states, configs, uniqueConfigs.size());
            if (DETAILED_DFA_STATE_STATS) {
                System.out.format("\tMode\tStates\tConfigs\tMode%n");
                for (int i = 0; i < modeToDFA.length; i++) {
                    DFA dfa = modeToDFA[i];
                    if (dfa == null || dfa.states.isEmpty()) {
                        continue;
                    }
                    int modeConfigs = 0;
                    for (DFAState state : dfa.states.values()) {
                        modeConfigs += state.configs.size();
                    }
                    String modeName = lexer.getModeNames()[i];
                    System.out.format("\t%d\t%d\t%d\t%s%n", dfa.decision, dfa.states.size(), modeConfigs, modeName);
                }
            }
        }
    }
    if (RUN_PARSER && sharedParsers.length > 0) {
        int index = FILE_GRANULARITY ? 0 : ((NumberedThread) Thread.currentThread()).getThreadNumber();
        Parser parser = sharedParsers[index];
        // make sure the individual DFAState objects actually have unique ATNConfig arrays
        final ParserATNSimulator interpreter = parser.getInterpreter();
        final DFA[] decisionToDFA = interpreter.decisionToDFA;
        if (SHOW_DFA_STATE_STATS) {
            int states = 0;
            int configs = 0;
            Set<ATNConfig> uniqueConfigs = new HashSet<ATNConfig>();
            for (int i = 0; i < decisionToDFA.length; i++) {
                DFA dfa = decisionToDFA[i];
                if (dfa == null) {
                    continue;
                }
                states += dfa.states.size();
                for (DFAState state : dfa.states.values()) {
                    configs += state.configs.size();
                    uniqueConfigs.addAll(state.configs);
                }
            }
            System.out.format("There are %d parser DFAState instances, %d configs (%d unique).%n", states, configs, uniqueConfigs.size());
            if (DETAILED_DFA_STATE_STATS) {
                if (COMPUTE_TRANSITION_STATS) {
                    System.out.format("\tDecision\tStates\tConfigs\tPredict (ALL)\tPredict (LL)\tNon-SLL\tTransitions\tTransitions (ATN)\tTransitions (LL)\tLA (SLL)\tLA (LL)\tRule%n");
                } else {
                    System.out.format("\tDecision\tStates\tConfigs\tRule%n");
                }
                for (int i = 0; i < decisionToDFA.length; i++) {
                    DFA dfa = decisionToDFA[i];
                    if (dfa == null || dfa.states.isEmpty()) {
                        continue;
                    }
                    int decisionConfigs = 0;
                    for (DFAState state : dfa.states.values()) {
                        decisionConfigs += state.configs.size();
                    }
                    String ruleName = parser.getRuleNames()[parser.getATN().decisionToState.get(dfa.decision).ruleIndex];
                    long calls = 0;
                    long fullContextCalls = 0;
                    long nonSllCalls = 0;
                    long transitions = 0;
                    long computedTransitions = 0;
                    long fullContextTransitions = 0;
                    double lookahead = 0;
                    double fullContextLookahead = 0;
                    String formatString;
                    if (COMPUTE_TRANSITION_STATS) {
                        for (long[] data : decisionInvocationsPerFile[currentPass]) {
                            calls += data[i];
                        }
                        for (long[] data : fullContextFallbackPerFile[currentPass]) {
                            fullContextCalls += data[i];
                        }
                        for (long[] data : nonSllPerFile[currentPass]) {
                            nonSllCalls += data[i];
                        }
                        for (long[] data : totalTransitionsPerDecisionPerFile[currentPass]) {
                            transitions += data[i];
                        }
                        for (long[] data : computedTransitionsPerDecisionPerFile[currentPass]) {
                            computedTransitions += data[i];
                        }
                        for (long[] data : fullContextTransitionsPerDecisionPerFile[currentPass]) {
                            fullContextTransitions += data[i];
                        }
                        if (calls > 0) {
                            lookahead = (double) (transitions - fullContextTransitions) / (double) calls;
                        }
                        if (fullContextCalls > 0) {
                            fullContextLookahead = (double) fullContextTransitions / (double) fullContextCalls;
                        }
                        formatString = "\t%1$d\t%2$d\t%3$d\t%4$d\t%5$d\t%6$d\t%7$d\t%8$d\t%9$d\t%10$f\t%11$f\t%12$s%n";
                    } else {
                        calls = 0;
                        formatString = "\t%1$d\t%2$d\t%3$d\t%12$s%n";
                    }
                    System.out.format(formatString, dfa.decision, dfa.states.size(), decisionConfigs, calls, fullContextCalls, nonSllCalls, transitions, computedTransitions, fullContextTransitions, lookahead, fullContextLookahead, ruleName);
                }
            }
        }
        int localDfaCount = 0;
        int globalDfaCount = 0;
        int localConfigCount = 0;
        int globalConfigCount = 0;
        int[] contextsInDFAState = new int[0];
        for (int i = 0; i < decisionToDFA.length; i++) {
            DFA dfa = decisionToDFA[i];
            if (dfa == null) {
                continue;
            }
            if (SHOW_CONFIG_STATS) {
                for (DFAState state : dfa.states.keySet()) {
                    if (state.configs.size() >= contextsInDFAState.length) {
                        contextsInDFAState = Arrays.copyOf(contextsInDFAState, state.configs.size() + 1);
                    }
                    if (state.isAcceptState) {
                        boolean hasGlobal = false;
                        for (ATNConfig config : state.configs) {
                            if (config.reachesIntoOuterContext > 0) {
                                globalConfigCount++;
                                hasGlobal = true;
                            } else {
                                localConfigCount++;
                            }
                        }
                        if (hasGlobal) {
                            globalDfaCount++;
                        } else {
                            localDfaCount++;
                        }
                    }
                    contextsInDFAState[state.configs.size()]++;
                }
            }
        }
        if (SHOW_CONFIG_STATS && currentPass == 0) {
            System.out.format("  DFA accept states: %d total, %d with only local context, %d with a global context%n", localDfaCount + globalDfaCount, localDfaCount, globalDfaCount);
            System.out.format("  Config stats: %d total, %d local, %d global%n", localConfigCount + globalConfigCount, localConfigCount, globalConfigCount);
            if (SHOW_DFA_STATE_STATS) {
                for (int i = 0; i < contextsInDFAState.length; i++) {
                    if (contextsInDFAState[i] != 0) {
                        System.out.format("  %d configs = %d%n", i, contextsInDFAState[i]);
                    }
                }
            }
        }
    }
    if (COMPUTE_TIMING_STATS) {
        System.out.format("File\tTokens\tTime%n");
        for (int i = 0; i < timePerFile[currentPass].length; i++) {
            System.out.format("%d\t%d\t%d%n", i + 1, tokensPerFile[currentPass][i], timePerFile[currentPass][i]);
        }
    }
}

Example 59 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class Grammar method createLexerInterpreter.

public LexerInterpreter createLexerInterpreter(CharStream input) {
    if (this.isParser()) {
        throw new IllegalStateException("A lexer interpreter can only be created for a lexer or combined grammar.");
    }
    if (this.isCombined()) {
        return implicitLexer.createLexerInterpreter(input);
    }
    char[] serializedAtn = ATNSerializer.getSerializedAsChars(atn);
    ATN deserialized = new ATNDeserializer().deserialize(serializedAtn);
    List<String> allChannels = new ArrayList<String>();
    allChannels.add("DEFAULT_TOKEN_CHANNEL");
    allChannels.add("HIDDEN");
    allChannels.addAll(channelValueToNameList);
    return new LexerInterpreter(fileName, getVocabulary(), Arrays.asList(getRuleNames()), allChannels, ((LexerGrammar) this).modes.keySet(), deserialized, input);
}

Example 60 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestUnbufferedTokenStream method testMarkStart.

@Test
public void testMarkStart() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 302;
    CharStream input = new ANTLRInputStream(new StringReader("x = 302;"));
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine);
    int m = tokens.mark();
    assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString());
    assertEquals("x", tokens.LT(1).getText());
    // consume x
    tokens.consume();
    assertEquals("[[@0,0:0='x',<1>,1:0], [@1,1:1=' ',<7>,1:1]]", tokens.getBuffer().toString());
    // ' '
    tokens.consume();
    // =
    tokens.consume();
    // ' '
    tokens.consume();
    // 302
    tokens.consume();
    // ;
    tokens.consume();
    assertEquals("[[@0,0:0='x',<1>,1:0], [@1,1:1=' ',<7>,1:1]," + " [@2,2:2='=',<4>,1:2], [@3,3:3=' ',<7>,1:3]," + " [@4,4:6='302',<2>,1:4], [@5,7:7=';',<3>,1:7]," + " [@6,8:7='<EOF>',<-1>,1:8]]", tokens.getBuffer().toString());
}