Examples with Lexer org.antlr.v4.runtime.Lexer used on opensource projects

Example 16 with Lexer

use of org.antlr.v4.runtime.Lexer in project nikita-noark5-core by HiOA-ABI.

the class TestODataApp method main.

public static void main(String[] args) throws Exception {
    System.out.println("Starting OData parser test");
    System.out.println("Picks first line from odata_samples.txt in " + "resources folder.");
    try {
        AfterApplicationStartup afterApplicationStartup = new AfterApplicationStartup(null);
        afterApplicationStartup.populateTranslatedNames();
        TestODataApp app = new TestODataApp();
        ODataLexer lexer = new ODataLexer(CharStreams.fromStream(app.getInputStreamForParseFile("odata" + File.separator + "odata_samples.txt")));
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        ODataParser parser = new ODataParser(tokens);
        ParseTree tree = parser.odataURL();
        ParseTreeWalker walker = new ParseTreeWalker();
        // Make the SQL Statement
        NikitaODataToSQLWalker sqlWalker = new NikitaODataToSQLWalker();
        walker.walk(sqlWalker, tree);
        System.out.println(sqlWalker.getSqlStatement());
    } catch (RecognitionException e) {
        throw new IllegalStateException("Recognition exception");
    }
}

Example 17 with Lexer

use of org.antlr.v4.runtime.Lexer in project nikita-noark5-core by HiOA-ABI.

the class OdataTest method testOdata.

@RequestMapping(method = RequestMethod.GET, value = "arkivstruktur/{\\w*}")
public ResponseEntity<String> testOdata(final UriComponentsBuilder uriBuilder, HttpServletRequest request, final HttpServletResponse response) throws Exception {
    String uqueryString = request.getQueryString();
    String decoded = URLDecoder.decode(uqueryString, UTF_8);
    StringBuffer originalRequest = request.getRequestURL();
    originalRequest.append("?" + decoded);
    CharStream stream = CharStreams.fromString(originalRequest.toString());
    ODataLexer lexer = new ODataLexer(stream);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    ODataParser parser = new ODataParser(tokens);
    ParseTree tree = parser.odataURL();
    ParseTreeWalker walker = new ParseTreeWalker();
    // Make the HQL Statement
    NikitaODataToHQLWalker hqlWalker = new NikitaODataToHQLWalker();
    walker.walk(hqlWalker, tree);
    Session session = entityManager.unwrap(org.hibernate.Session.class);
    Query query = hqlWalker.getHqlStatment(session);
    String queryString = query.getQueryString();
    System.out.println(queryString);
    List<NoarkEntity> list = query.getResultList();
    return ResponseEntity.status(HttpStatus.CREATED).body(list.toString());
}

Example 18 with Lexer

use of org.antlr.v4.runtime.Lexer in project grakn by graknlabs.

the class QueryParserImpl method parseList.

/**
 * @param reader a reader representing several queries
 * @return a list of queries
 */
@Override
public <T extends Query<?>> Stream<T> parseList(Reader reader) {
    UnbufferedCharStream charStream = new UnbufferedCharStream(reader);
    GraqlErrorListener errorListener = GraqlErrorListener.withoutQueryString();
    GraqlLexer lexer = createLexer(charStream, errorListener);
    /*
            We tell the lexer to copy the text into each generated token.
            Normally when calling `Token#getText`, it will look into the underlying `TokenStream` and call
            `TokenStream#size` to check it is in-bounds. However, `UnbufferedTokenStream#size` is not supported
            (because then it would have to read the entire input). To avoid this issue, we set this flag which will
            copy over the text into each `Token`, s.t. that `Token#getText` will just look up the copied text field.
        */
    lexer.setTokenFactory(new CommonTokenFactory(true));
    // Use an unbuffered token stream so we can handle extremely large input strings
    UnbufferedTokenStream tokenStream = new UnbufferedTokenStream(ChannelTokenSource.of(lexer));
    GraqlParser parser = createParser(tokenStream, errorListener);
    /*
            The "bail" error strategy prevents us reading all the way to the end of the input, e.g.

            ```
            match $x isa person; insert $x has name "Bob"; match $x isa movie; get;
                                                           ^
            ```

            In this example, when ANTLR reaches the indicated `match`, it considers two possibilities:

            1. this is the end of the query
            2. the user has made a mistake. Maybe they accidentally pasted the `match` here.

            Because of case 2, ANTLR will parse beyond the `match` in order to produce a more helpful error message.
            This causes memory issues for very large queries, so we use the simpler "bail" strategy that will
            immediately stop when it hits `match`.
        */
    parser.setErrorHandler(new BailErrorStrategy());
    // This is a lazy iterator that will only consume a single query at a time, without parsing any further.
    // This means it can pass arbitrarily long streams of queries in constant memory!
    Iterable<T> queryIterator = () -> new AbstractIterator<T>() {

        @Nullable
        @Override
        protected T computeNext() {
            int latestToken = tokenStream.LA(1);
            if (latestToken == Token.EOF) {
                endOfData();
                return null;
            } else {
                // When we next run it, it will start where it left off in the stream
                return (T) QUERY.parse(parser, errorListener);
            }
        }
    };
    return StreamSupport.stream(queryIterator.spliterator(), false);
}

Example 19 with Lexer

use of org.antlr.v4.runtime.Lexer in project grakn by graknlabs.

the class Autocomplete method getTokens.

/**
 * @param query a graql query
 * @return a list of tokens from running the lexer on the query
 */
private static List<? extends Token> getTokens(String query) {
    ANTLRInputStream input = new ANTLRInputStream(query);
    GraqlLexer lexer = new GraqlLexer(input);
    // Ignore syntax errors
    lexer.removeErrorListeners();
    lexer.addErrorListener(new BaseErrorListener());
    return lexer.getAllTokens();
}

Example 20 with Lexer

use of org.antlr.v4.runtime.Lexer in project drools by kiegroup.

the class FEELParser method parse.

public static FEEL_1_1Parser parse(FEELEventListenersManager eventsManager, String source, Map<String, Type> inputVariableTypes, Map<String, Object> inputVariables, Collection<FEELFunction> additionalFunctions, List<FEELProfile> profiles) {
    ANTLRInputStream input = new ANTLRInputStream(source);
    FEEL_1_1Lexer lexer = new FEEL_1_1Lexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    FEEL_1_1Parser parser = new FEEL_1_1Parser(tokens);
    ParserHelper parserHelper = new ParserHelper(eventsManager);
    additionalFunctions.forEach(f -> parserHelper.getSymbolTable().getBuiltInScope().define(f.getSymbol()));
    profiles.stream().filter(KieExtendedFEELProfile.class::isInstance).forEach(dc -> parserHelper.setFeatDMN12EnhancedForLoopEnabled(true));
    parser.setHelper(parserHelper);
    parser.setErrorHandler(new FEELErrorHandler());
    // removes the error listener that prints to the console
    parser.removeErrorListeners();
    parser.addErrorListener(new FEELParserErrorListener(eventsManager));
    // pre-loads the parser with symbols
    defineVariables(inputVariableTypes, inputVariables, parser);
    return parser;
}