Search in sources :

Example 51 with ANTLRInputStream

use of org.antlr.v4.runtime.ANTLRInputStream in project SSM by Intel-bigdata.

the class TestSmartRuleParser method parseAndExecuteRule.

private void parseAndExecuteRule(String rule) throws Exception {
    System.out.println("--> " + rule);
    InputStream input = new ByteArrayInputStream(rule.getBytes());
    ANTLRInputStream antlrInput = new ANTLRInputStream(input);
    SmartRuleLexer lexer = new SmartRuleLexer(antlrInput);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SmartRuleParser parser = new SmartRuleParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(new SSMRuleErrorListener());
    ParseTree tree = parser.ssmrule();
    System.out.println("Parser tree: " + tree.toStringTree(parser));
    System.out.println("Total number of errors: " + parseErrors.size());
    SmartRuleVisitTranslator visitor = new SmartRuleVisitTranslator();
    visitor.visit(tree);
    System.out.println("\nQuery:");
    TranslateResult result = visitor.generateSql();
    int index = 1;
    for (String sql : result.getSqlStatements()) {
        System.out.println("" + index + ". " + sql);
        index++;
    }
    if (parseErrors.size() > 0) {
        throw new IOException("Error while parse rule");
    }
    ExecutionContext ctx = new ExecutionContext();
    ctx.setProperty(ExecutionContext.RULE_ID, 2016);
    DBAdapter dbAdapter = new DBAdapter(TestDBUtil.getTestDBInstance());
    RuleQueryExecutor qe = new RuleQueryExecutor(null, ctx, result, dbAdapter);
    List<String> paths = qe.executeFileRuleQuery();
    index = 1;
    System.out.println("\nFiles:");
    for (String path : paths) {
        System.out.println("" + index + ". " + path);
        index++;
    }
}
Also used : CommonTokenStream(org.antlr.v4.runtime.CommonTokenStream) SmartRuleParser(org.smartdata.server.rule.parser.SmartRuleParser) ANTLRInputStream(org.antlr.v4.runtime.ANTLRInputStream) ByteArrayInputStream(java.io.ByteArrayInputStream) InputStream(java.io.InputStream) SmartRuleLexer(org.smartdata.server.rule.parser.SmartRuleLexer) SmartRuleVisitTranslator(org.smartdata.server.rule.parser.SmartRuleVisitTranslator) IOException(java.io.IOException) DBAdapter(org.smartdata.server.metastore.DBAdapter) ExecutionContext(org.smartdata.server.metastore.ExecutionContext) ByteArrayInputStream(java.io.ByteArrayInputStream) TranslateResult(org.smartdata.server.rule.parser.TranslateResult) ANTLRInputStream(org.antlr.v4.runtime.ANTLRInputStream) ParseTree(org.antlr.v4.runtime.tree.ParseTree)

Example 52 with ANTLRInputStream

use of org.antlr.v4.runtime.ANTLRInputStream in project SSM by Intel-bigdata.

the class RuleStringParser method translate.

public TranslateResult translate() throws IOException {
    InputStream input = new ByteArrayInputStream(rule.getBytes());
    ANTLRInputStream antlrInput = new ANTLRInputStream(input);
    SmartRuleLexer lexer = new SmartRuleLexer(antlrInput);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SmartRuleParser parser = new SmartRuleParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(new SSMRuleErrorListener());
    tree = parser.ssmrule();
    if (parseErrors.size() > 0) {
        throw new IOException(parserErrorMessage);
    }
    SmartRuleVisitTranslator visitor = new SmartRuleVisitTranslator(ctx);
    try {
        visitor.visit(tree);
    } catch (RuntimeException e) {
        throw new IOException(e.getMessage());
    }
    TranslateResult result = visitor.generateSql();
    return result;
}
Also used : CommonTokenStream(org.antlr.v4.runtime.CommonTokenStream) ByteArrayInputStream(java.io.ByteArrayInputStream) ANTLRInputStream(org.antlr.v4.runtime.ANTLRInputStream) ByteArrayInputStream(java.io.ByteArrayInputStream) InputStream(java.io.InputStream) IOException(java.io.IOException) ANTLRInputStream(org.antlr.v4.runtime.ANTLRInputStream)

Example 53 with ANTLRInputStream

use of org.antlr.v4.runtime.ANTLRInputStream in project compiler by boalang.

the class BaseTest method lex.

protected CommonTokenStream lex(final String input, final int[] ids, final String[] strings, final String[] errors) throws IOException {
    final List<String> foundErr = new ArrayList<String>();
    final BoaLexer lexer = new BoaLexer(new ANTLRInputStream(new StringReader(input)));
    lexer.removeErrorListeners();
    lexer.addErrorListener(new BaseErrorListener() {

        @Override
        public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line, final int charPositionInLine, final String msg, final RecognitionException e) {
            foundErr.add(line + "," + charPositionInLine + ": " + msg);
        }
    });
    final CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    if (ids.length > 0 && strings.length > 0)
        assertEquals("ids != strings", ids.length, strings.length);
    if (ids.length > 0) {
        final List<Token> t = tokens.getTokens();
        if (DEBUG) {
            for (int i = 0; i < t.size(); i++) {
                final Token token = t.get(i);
                System.out.print(token.getType() + ", ");
            }
            System.out.println();
            for (int i = 0; i < t.size(); i++) {
                final Token token = t.get(i);
                System.out.print(token.getText() + ", ");
            }
            System.out.println();
            System.out.println();
        }
        assertEquals("wrong number of tokens", ids.length, t.size());
        for (int i = 0; i < t.size(); i++) assertEquals("wrong token type", ids[i], t.get(i).getType());
    }
    if (strings.length > 0) {
        final List<Token> t = tokens.getTokens();
        assertEquals("wrong number of tokens", strings.length, t.size());
        for (int i = 0; i < t.size(); i++) assertEquals("wrong token type", strings[i], t.get(i).getText());
    }
    assertEquals("wrong number of errors: " + input, errors.length, foundErr.size());
    for (int i = 0; i < foundErr.size(); i++) assertEquals("wrong error", errors[i], foundErr.get(i));
    return tokens;
}
Also used : CommonTokenStream(org.antlr.v4.runtime.CommonTokenStream) BaseErrorListener(org.antlr.v4.runtime.BaseErrorListener) ArrayList(java.util.ArrayList) Token(org.antlr.v4.runtime.Token) StringReader(java.io.StringReader) JavaFileObject(javax.tools.JavaFileObject) BoaLexer(boa.parser.BoaLexer) ANTLRInputStream(org.antlr.v4.runtime.ANTLRInputStream) RecognitionException(org.antlr.v4.runtime.RecognitionException)

Example 54 with ANTLRInputStream

use of org.antlr.v4.runtime.ANTLRInputStream in project jabref by JabRef.

the class GrammarBasedSearchRule method init.

private void init(String query) throws ParseCancellationException {
    if (Objects.equals(this.query, query)) {
        return;
    }
    SearchLexer lexer = new SearchLexer(new ANTLRInputStream(query));
    // no infos on file system
    lexer.removeErrorListeners();
    lexer.addErrorListener(ThrowingErrorListener.INSTANCE);
    SearchParser parser = new SearchParser(new CommonTokenStream(lexer));
    // no infos on file system
    parser.removeErrorListeners();
    parser.addErrorListener(ThrowingErrorListener.INSTANCE);
    // ParseCancelationException on parse errors
    parser.setErrorHandler(new BailErrorStrategy());
    tree = parser.start();
    this.query = query;
}
Also used : CommonTokenStream(org.antlr.v4.runtime.CommonTokenStream) BailErrorStrategy(org.antlr.v4.runtime.BailErrorStrategy) SearchLexer(org.jabref.search.SearchLexer) SearchParser(org.jabref.search.SearchParser) ANTLRInputStream(org.antlr.v4.runtime.ANTLRInputStream)

Example 55 with ANTLRInputStream

use of org.antlr.v4.runtime.ANTLRInputStream in project incubator-systemml by apache.

the class PyDMLParserWrapper method doParse.

/**
 * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import'
 * @param fileName script file name
 * @param dmlScript script file contents
 * @param sourceNamespace namespace from source statement
 * @param argVals script arguments
 * @return dml program, or null if at least one error
 */
public DMLProgram doParse(String fileName, String dmlScript, String sourceNamespace, Map<String, String> argVals) {
    DMLProgram dmlPgm = null;
    ANTLRInputStream in;
    try {
        if (dmlScript == null) {
            dmlScript = readDMLScript(fileName, LOG);
        }
        InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
        in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
    } catch (FileNotFoundException e) {
        throw new ParseException("Cannot find file/resource: " + fileName, e);
    } catch (IOException e) {
        throw new ParseException("Cannot open file: " + fileName, e);
    } catch (LanguageException e) {
        throw new ParseException(e.getMessage(), e);
    }
    ProgramrootContext ast = null;
    CustomErrorListener errorListener = new CustomErrorListener();
    try {
        PydmlLexer lexer = new PydmlLexer(in);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        PydmlParser antlr4Parser = new PydmlParser(tokens);
        // For now no optimization, since it is not able to parse integer value.
        boolean tryOptimizedParsing = false;
        if (tryOptimizedParsing) {
            // Try faster and simpler SLL
            antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
            antlr4Parser.removeErrorListeners();
            antlr4Parser.setErrorHandler(new BailErrorStrategy());
            try {
                ast = antlr4Parser.programroot();
            // If successful, no need to try out full LL(*) ... SLL was enough
            } catch (ParseCancellationException ex) {
                // Error occurred, so now try full LL(*) for better error messages
                tokens.reset();
                antlr4Parser.reset();
                if (fileName != null) {
                    errorListener.setCurrentFileName(fileName);
                } else {
                    errorListener.setCurrentFileName("MAIN_SCRIPT");
                }
                // Set our custom error listener
                antlr4Parser.addErrorListener(errorListener);
                antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
                antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
                ast = antlr4Parser.programroot();
            }
        } else {
            // Set our custom error listener
            antlr4Parser.removeErrorListeners();
            antlr4Parser.addErrorListener(errorListener);
            errorListener.setCurrentFileName(fileName);
            // Now do the parsing
            ast = antlr4Parser.programroot();
        }
    } catch (Exception e) {
        throw new ParseException("ERROR: Cannot parse the program:" + fileName, e);
    }
    // Now convert the parse tree into DMLProgram
    // Do syntactic validation while converting
    ParseTree tree = ast;
    // And also do syntactic validation
    ParseTreeWalker walker = new ParseTreeWalker();
    // Get list of function definitions which take precedence over built-in functions if same name
    PydmlPreprocessor prep = new PydmlPreprocessor(errorListener);
    walker.walk(prep, tree);
    // Syntactic validation
    PydmlSyntacticValidator validator = new PydmlSyntacticValidator(errorListener, argVals, sourceNamespace, prep.getFunctionDefs());
    walker.walk(validator, tree);
    errorListener.unsetCurrentFileName();
    this.parseIssues = errorListener.getParseIssues();
    this.atLeastOneWarning = errorListener.isAtLeastOneWarning();
    this.atLeastOneError = errorListener.isAtLeastOneError();
    if (atLeastOneError) {
        throw new ParseException(parseIssues, dmlScript);
    }
    if (atLeastOneWarning) {
        LOG.warn(CustomErrorListener.generateParseIssuesMessage(dmlScript, parseIssues));
    }
    dmlPgm = createDMLProgram(ast, sourceNamespace);
    return dmlPgm;
}
Also used : FileNotFoundException(java.io.FileNotFoundException) LanguageException(org.apache.sysml.parser.LanguageException) DefaultErrorStrategy(org.antlr.v4.runtime.DefaultErrorStrategy) DMLProgram(org.apache.sysml.parser.DMLProgram) ParseTreeWalker(org.antlr.v4.runtime.tree.ParseTreeWalker) CommonTokenStream(org.antlr.v4.runtime.CommonTokenStream) CustomErrorListener(org.apache.sysml.parser.common.CustomErrorListener) ANTLRInputStream(org.antlr.v4.runtime.ANTLRInputStream) ByteArrayInputStream(java.io.ByteArrayInputStream) ANTLRInputStream(org.antlr.v4.runtime.ANTLRInputStream) InputStream(java.io.InputStream) BailErrorStrategy(org.antlr.v4.runtime.BailErrorStrategy) ProgramrootContext(org.apache.sysml.parser.pydml.PydmlParser.ProgramrootContext) IOException(java.io.IOException) ParseCancellationException(org.antlr.v4.runtime.misc.ParseCancellationException) LanguageException(org.apache.sysml.parser.LanguageException) IOException(java.io.IOException) FileNotFoundException(java.io.FileNotFoundException) ParseException(org.apache.sysml.parser.ParseException) ByteArrayInputStream(java.io.ByteArrayInputStream) ParseCancellationException(org.antlr.v4.runtime.misc.ParseCancellationException) ParseException(org.apache.sysml.parser.ParseException) ANTLRInputStream(org.antlr.v4.runtime.ANTLRInputStream) ParseTree(org.antlr.v4.runtime.tree.ParseTree)

Aggregations

ANTLRInputStream (org.antlr.v4.runtime.ANTLRInputStream)110 CommonTokenStream (org.antlr.v4.runtime.CommonTokenStream)86 LexerInterpreter (org.antlr.v4.runtime.LexerInterpreter)59 Test (org.junit.Test)59 LexerGrammar (org.antlr.v4.tool.LexerGrammar)52 TokenStreamRewriter (org.antlr.v4.runtime.TokenStreamRewriter)43 BaseJavaTest (org.antlr.v4.test.runtime.java.BaseJavaTest)43 CharStream (org.antlr.v4.runtime.CharStream)15 ParseTree (org.antlr.v4.runtime.tree.ParseTree)15 Token (org.antlr.v4.runtime.Token)12 TokenStream (org.antlr.v4.runtime.TokenStream)9 ByteArrayInputStream (java.io.ByteArrayInputStream)8 IOException (java.io.IOException)7 InputStream (java.io.InputStream)7 StringReader (java.io.StringReader)7 BailErrorStrategy (org.antlr.v4.runtime.BailErrorStrategy)7 ParseCancellationException (org.antlr.v4.runtime.misc.ParseCancellationException)7 BufferedTokenStream (org.antlr.v4.runtime.BufferedTokenStream)6 IntegerList (org.antlr.v4.runtime.misc.IntegerList)6 ParseTreeWalker (org.antlr.v4.runtime.tree.ParseTreeWalker)5