Examples with CommonTokenStream org.antlr.v4.runtime.CommonTokenStream used on opensource projects

Example 61 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project incubator-systemml by apache.

the class DMLParserWrapper method doParse.

/**
 * This function is supposed to be called directly only from DmlSyntacticValidator when it encounters 'import'
 * @param fileName script file name
 * @param dmlScript script file contents
 * @param sourceNamespace namespace from source statement
 * @param argVals script arguments
 * @return dml program, or null if at least one error
 */
public DMLProgram doParse(String fileName, String dmlScript, String sourceNamespace, Map<String, String> argVals) {
    DMLProgram dmlPgm = null;
    ANTLRInputStream in;
    try {
        if (dmlScript == null) {
            dmlScript = readDMLScript(fileName, LOG);
        }
        InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
        in = new ANTLRInputStream(stream);
    } catch (FileNotFoundException e) {
        throw new ParseException("Cannot find file/resource: " + fileName, e);
    } catch (IOException e) {
        throw new ParseException("Cannot open file: " + fileName, e);
    } catch (LanguageException e) {
        throw new ParseException(e.getMessage(), e);
    }
    ProgramrootContext ast = null;
    CustomErrorListener errorListener = new CustomErrorListener();
    try {
        DmlLexer lexer = new DmlLexer(in);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        DmlParser antlr4Parser = new DmlParser(tokens);
        // For now no optimization, since it is not able to parse integer value.
        boolean tryOptimizedParsing = false;
        if (tryOptimizedParsing) {
            // Try faster and simpler SLL
            antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
            antlr4Parser.removeErrorListeners();
            antlr4Parser.setErrorHandler(new BailErrorStrategy());
            try {
                ast = antlr4Parser.programroot();
            // If successful, no need to try out full LL(*) ... SLL was enough
            } catch (ParseCancellationException ex) {
                // Error occurred, so now try full LL(*) for better error messages
                tokens.reset();
                antlr4Parser.reset();
                if (fileName != null) {
                    errorListener.setCurrentFileName(fileName);
                } else {
                    errorListener.setCurrentFileName("MAIN_SCRIPT");
                }
                // Set our custom error listener
                antlr4Parser.addErrorListener(errorListener);
                antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
                antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
                ast = antlr4Parser.programroot();
            }
        } else {
            // Set our custom error listener
            antlr4Parser.removeErrorListeners();
            antlr4Parser.addErrorListener(errorListener);
            errorListener.setCurrentFileName(fileName);
            // Now do the parsing
            ast = antlr4Parser.programroot();
        }
    } catch (Exception e) {
        throw new ParseException("ERROR: Cannot parse the program:" + fileName, e);
    }
    // Now convert the parse tree into DMLProgram
    // Do syntactic validation while converting
    ParseTree tree = ast;
    // And also do syntactic validation
    ParseTreeWalker walker = new ParseTreeWalker();
    // Get list of function definitions which take precedence over built-in functions if same name
    DmlPreprocessor prep = new DmlPreprocessor(errorListener);
    walker.walk(prep, tree);
    // Syntactic validation
    DmlSyntacticValidator validator = new DmlSyntacticValidator(errorListener, argVals, sourceNamespace, prep.getFunctionDefs());
    walker.walk(validator, tree);
    errorListener.unsetCurrentFileName();
    this.parseIssues = errorListener.getParseIssues();
    this.atLeastOneWarning = errorListener.isAtLeastOneWarning();
    this.atLeastOneError = errorListener.isAtLeastOneError();
    if (atLeastOneError) {
        throw new ParseException(parseIssues, dmlScript);
    }
    if (atLeastOneWarning) {
        LOG.warn(CustomErrorListener.generateParseIssuesMessage(dmlScript, parseIssues));
    }
    dmlPgm = createDMLProgram(ast, sourceNamespace);
    return dmlPgm;
}

Example 62 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project rest.li by linkedin.

the class PdlSchemaParser method parse.

/**
   * Parse a JSON representation of a schema from a {{java.io.Reader}}.
   *
   * The top level {{DataSchema}}'s parsed are in {{#topLevelDataSchemas}}.
   * These are the types that are not defined within other types.
   * Parse errors are in {{#errorMessageBuilder}} and indicated
   * by {{#hasError()}}.
   *
   * @param reader with the JSON representation of the schema.
   */
public void parse(Reader reader) {
    try {
        ErrorRecorder errorRecorder = new ErrorRecorder();
        PdlLexer lexer;
        try {
            lexer = new PdlLexer(new ANTLRInputStream(reader));
        } catch (IOException e) {
            ParseError error = new ParseError(new ParseErrorLocation(0, 0), e.getMessage());
            startErrorMessage(error).append(error.message).append(NEWLINE);
            return;
        }
        lexer.removeErrorListeners();
        lexer.addErrorListener(errorRecorder);
        PdlParser parser = new PdlParser(new CommonTokenStream(lexer));
        parser.removeErrorListeners();
        parser.addErrorListener(errorRecorder);
        DocumentContext antlrDocument = parser.document();
        parse(antlrDocument);
        if (errorRecorder.errors.size() > 0) {
            for (ParseError error : errorRecorder.errors) {
                startErrorMessage(error).append(error.message).append(NEWLINE);
            }
        }
    } catch (ParseException e) {
        startErrorMessage(e.error).append(e.getMessage()).append(NEWLINE);
    } catch (Throwable t) {
        ParseError parseError = new ParseError(new ParseErrorLocation(0, 0), null);
        startErrorMessage(parseError).append("Unexpected parser error: ").append(ExceptionUtils.getStackTrace(t)).append(NEWLINE);
    }
}

Example 63 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project druid by druid-io.

the class Parser method parse.

public static Expr parse(String in, boolean withFlatten) {
    ExprLexer lexer = new ExprLexer(new ANTLRInputStream(in));
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    ExprParser parser = new ExprParser(tokens);
    parser.setBuildParseTree(true);
    ParseTree parseTree = parser.expr();
    ParseTreeWalker walker = new ParseTreeWalker();
    ExprListenerImpl listener = new ExprListenerImpl(parseTree);
    walker.walk(listener, parseTree);
    return withFlatten ? flatten(listener.getAST()) : listener.getAST();
}

Example 64 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project checkstyle by checkstyle.

the class JavadocParseTreeTest method parseJavadoc.

private ParseTree parseJavadoc(String aBlockComment) throws IOException {
    final Charset utf8Charset = Charset.forName("UTF-8");
    final InputStream in = new ByteArrayInputStream(aBlockComment.getBytes(utf8Charset));
    final ANTLRInputStream input = new ANTLRInputStream(in);
    final JavadocLexer lexer = new JavadocLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(errorListener);
    final CommonTokenStream tokens = new CommonTokenStream(lexer);
    parser = new JavadocParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(errorListener);
    return parser.javadoc();
}

Example 65 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project checkstyle by checkstyle.

the class JavadocDetailNodeParser method parseJavadocAsParseTree.

/**
     * Parses block comment content as javadoc comment.
     * @param blockComment
     *        block comment content.
     * @return parse tree
     */
private ParseTree parseJavadocAsParseTree(String blockComment) {
    final ANTLRInputStream input = new ANTLRInputStream(blockComment);
    final JavadocLexer lexer = new JavadocLexer(input);
    // remove default error listeners
    lexer.removeErrorListeners();
    // add custom error listener that logs parsing errors
    lexer.addErrorListener(errorListener);
    final CommonTokenStream tokens = new CommonTokenStream(lexer);
    final JavadocParser parser = new JavadocParser(tokens);
    // remove default error listeners
    parser.removeErrorListeners();
    // add custom error listener that logs syntax errors
    parser.addErrorListener(errorListener);
    // This strategy stops parsing when parser error occurs.
    // By default it uses Error Recover Strategy which is slow and useless.
    parser.setErrorHandler(new BailErrorStrategy());
    return parser.javadoc();
}