Examples with TokenSource org.antlr.v4.runtime.TokenSource used on opensource projects

Example 11 with TokenSource

use of org.antlr.v4.runtime.TokenSource in project ANNIS by korpling.

the class AnnisParserAntlr method parse.

public QueryData parse(String aql, List<Long> corpusList) {
    final List<AqlParseError> errors = new LinkedList<>();
    AqlLexer lexerNonDNF = new AqlLexer(new ANTLRInputStream(aql));
    lexerNonDNF.removeErrorListeners();
    lexerNonDNF.addErrorListener(new AqlLexerErrorListener(errors));
    // bring first into DNF
    RawAqlPreParser rawParser = new RawAqlPreParser(new CommonTokenStream(lexerNonDNF));
    rawParser.removeErrorListeners();
    rawParser.addErrorListener(new AqlParseErrorListener(errors));
    RawAqlPreParser.StartContext treeRaw = rawParser.start();
    if (!errors.isEmpty()) {
        throw new AnnisQLSyntaxException(Joiner.on("\n").join(errors), errors);
    }
    // treeRaw.inspect(rawParser);
    ParseTreeWalker walkerRaw = new ParseTreeWalker();
    RawAqlListener listenerRaw = new RawAqlListener();
    walkerRaw.walk(listenerRaw, treeRaw);
    LogicClause topNode = listenerRaw.getRoot();
    DNFTransformer.toDNF(topNode);
    // use the DNF form and parse it again
    TokenSource source = new ListTokenSource(topNode.getCoveredToken());
    AqlParser parserDNF = new AqlParser(new CommonTokenStream(source));
    parserDNF.removeErrorListeners();
    parserDNF.addErrorListener(new AqlParseErrorListener(errors));
    AqlParser.StartContext treeDNF = parserDNF.start();
    if (!errors.isEmpty()) {
        throw new AnnisQLSyntaxException(Joiner.on("\n").join(errors), errors);
    }
    ParseTreeWalker walker = new ParseTreeWalker();
    NodeIDListener idListener = new NodeIDListener();
    walker.walk(idListener, treeDNF);
    QueryNodeListener nodeListener = new QueryNodeListener(idListener.getNodeIntervalToID());
    try {
        walker.walk(nodeListener, treeDNF);
        QueryData data = nodeListener.getQueryData();
        data.setCorpusList(corpusList);
        data.addMetaAnnotations(nodeListener.getMetaData());
        JoinListener joinListener = new JoinListener(data, precedenceBound, nodeListener.getTokenPositions());
        walker.walk(joinListener, treeDNF);
        if (postProcessors != null) {
            for (QueryDataTransformer transformer : postProcessors) {
                data = transformer.transform(data);
            }
        }
        return data;
    } catch (NullPointerException ex) {
        log.warn("Null pointer exception occured during parsing", ex);
        throw new AnnisQLSemanticsException(ex.getMessage());
    } catch (IllegalArgumentException ex) {
        throw new AnnisQLSemanticsException(ex.getMessage());
    }
}

Example 12 with TokenSource

use of org.antlr.v4.runtime.TokenSource in project bmoth by hhu-stups.

the class SubstitutionVisitorExceptionsTest method init.

@Before
public void init() {
    visitor = new SubstitutionVisitor<Object, Object>() {

        @Override
        public Object visitSkipSubstitutionNode(SkipSubstitutionNode node, Object expected) {
            return null;
        }

        @Override
        public Object visitIfSubstitutionNode(IfSubstitutionNode node, Object expected) {
            return null;
        }

        @Override
        public Object visitConditionSubstitutionNode(ConditionSubstitutionNode node, Object expected) {
            return null;
        }

        @Override
        public Object visitAnySubstitution(AnySubstitutionNode node, Object expected) {
            return null;
        }

        @Override
        public Object visitSelectSubstitutionNode(SelectSubstitutionNode node, Object expected) {
            return null;
        }

        @Override
        public Object visitSingleAssignSubstitution(SingleAssignSubstitutionNode node, Object expected) {
            return null;
        }

        @Override
        public Object visitParallelSubstitutionNode(ParallelSubstitutionNode node, Object expected) {
            return null;
        }

        @Override
        public Object visitBecomesElementOfSubstitutionNode(BecomesElementOfSubstitutionNode node, Object expected) {
            return null;
        }

        @Override
        public Object visitBecomesSuchThatSubstitutionNode(BecomesSuchThatSubstitutionNode node, Object expected) {
            return null;
        }

        @Override
        public Object visitLTLPrefixOperatorNode(LTLPrefixOperatorNode node, Object expected) {
            return null;
        }

        @Override
        public Object visitLTLKeywordNode(LTLKeywordNode node, Object expected) {
            return null;
        }

        @Override
        public Object visitLTLInfixOperatorNode(LTLInfixOperatorNode node, Object expected) {
            return null;
        }

        @Override
        public Object visitLTLBPredicateNode(LTLBPredicateNode node, Object expected) {
            return null;
        }
    };
    terminalNode = new TerminalNode() {

        @Override
        public Token getSymbol() {
            return new Token() {

                @Override
                public String getText() {
                    return "adsf";
                }

                @Override
                public int getType() {
                    return 0;
                }

                @Override
                public int getLine() {
                    return 0;
                }

                @Override
                public int getCharPositionInLine() {
                    return 0;
                }

                @Override
                public int getChannel() {
                    return 0;
                }

                @Override
                public int getTokenIndex() {
                    return 0;
                }

                @Override
                public int getStartIndex() {
                    return 0;
                }

                @Override
                public int getStopIndex() {
                    return 0;
                }

                @Override
                public TokenSource getTokenSource() {
                    return null;
                }

                @Override
                public CharStream getInputStream() {
                    return null;
                }
            };
        }

        @Override
        public ParseTree getParent() {
            return null;
        }

        @Override
        public ParseTree getChild(int i) {
            return null;
        }

        @Override
        public <T> T accept(ParseTreeVisitor<? extends T> parseTreeVisitor) {
            return null;
        }

        @Override
        public String getText() {
            return null;
        }

        @Override
        public String toStringTree(Parser parser) {
            return null;
        }

        @Override
        public Interval getSourceInterval() {
            return null;
        }

        @Override
        public Object getPayload() {
            return null;
        }

        @Override
        public int getChildCount() {
            return 0;
        }

        @Override
        public String toStringTree() {
            return null;
        }

        @Override
        public void setParent(RuleContext parent) {
        // not needed in test stub
        }
    };
}

Example 13 with TokenSource

use of org.antlr.v4.runtime.TokenSource in project presto by prestodb.

the class StatementSplitter method squeezeStatement.

public static String squeezeStatement(String sql) {
    TokenSource tokens = getLexer(sql, ImmutableSet.of());
    StringBuilder sb = new StringBuilder();
    while (true) {
        Token token = tokens.nextToken();
        if (token.getType() == Token.EOF) {
            break;
        }
        if (token.getType() == SqlBaseLexer.WS) {
            sb.append(' ');
        } else {
            sb.append(token.getText());
        }
    }
    return sb.toString().trim();
}

Example 14 with TokenSource

use of org.antlr.v4.runtime.TokenSource in project antlr4 by tunnelvisionlabs.

the class TestCommonTokenStream method testCannotConsumeEOF.

@Test(expected = IllegalStateException.class)
public void testCannotConsumeEOF() throws Exception {
    TokenSource lexer = new TokenSource() {

        @Override
        public Token nextToken() {
            return new CommonToken(Token.EOF);
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public String getSourceName() {
            return IntStream.UNKNOWN_SOURCE_NAME;
        }

        @Override
        public TokenFactory getTokenFactory() {
            throw new UnsupportedOperationException("Not supported yet.");
        }

        @Override
        public void setTokenFactory(TokenFactory factory) {
            throw new UnsupportedOperationException("Not supported yet.");
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    assertEquals(Token.EOF, tokens.LA(1));
    assertEquals(0, tokens.index());
    assertEquals(1, tokens.size());
    tokens.consume();
}

Example 15 with TokenSource

use of org.antlr.v4.runtime.TokenSource in project titan.EclipsePlug-ins by eclipse.

the class PPDirectiveTokenFactory method create.

@Override
public CommonToken create(final Pair<TokenSource, CharStream> source, final int type, final String text, final int channel, final int start, final int stop, final int line, final int charPositionInLine) {
    CommonToken t = new CommonToken(source, type, channel, start, stop);
    t.setLine(line);
    t.setStartIndex(start + token.getStartIndex());
    t.setStopIndex(stop + token.getStartIndex() + 1);
    t.setCharPositionInLine(charPositionInLine);
    if (text != null) {
        t.setText(text);
    } else if (copyText && source.b != null) {
        t.setText(source.b.getText(Interval.of(start, stop)));
    }
    return t;
}