Examples with org.antlr.v4.runtime.tree org.antlr.v4.runtime.tree used on opensource projects

Example 11 with org.antlr.v4.runtime.tree

use of org.antlr.v4.runtime.tree in project systemml by apache.

the class PyDMLParserWrapper method doParse.

/**
 * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import'
 * @param fileName script file name
 * @param dmlScript script file contents
 * @param sourceNamespace namespace from source statement
 * @param argVals script arguments
 * @return dml program, or null if at least one error
 */
public DMLProgram doParse(String fileName, String dmlScript, String sourceNamespace, Map<String, String> argVals) {
    DMLProgram dmlPgm = null;
    ANTLRInputStream in;
    try {
        if (dmlScript == null) {
            dmlScript = readDMLScript(fileName, LOG);
        }
        InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
        in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
    } catch (FileNotFoundException e) {
        throw new ParseException("Cannot find file/resource: " + fileName, e);
    } catch (IOException e) {
        throw new ParseException("Cannot open file: " + fileName, e);
    } catch (LanguageException e) {
        throw new ParseException(e.getMessage(), e);
    }
    ProgramrootContext ast = null;
    CustomErrorListener errorListener = new CustomErrorListener();
    try {
        PydmlLexer lexer = new PydmlLexer(in);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        PydmlParser antlr4Parser = new PydmlParser(tokens);
        // For now no optimization, since it is not able to parse integer value.
        boolean tryOptimizedParsing = false;
        if (tryOptimizedParsing) {
            // Try faster and simpler SLL
            antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
            antlr4Parser.removeErrorListeners();
            antlr4Parser.setErrorHandler(new BailErrorStrategy());
            try {
                ast = antlr4Parser.programroot();
            // If successful, no need to try out full LL(*) ... SLL was enough
            } catch (ParseCancellationException ex) {
                // Error occurred, so now try full LL(*) for better error messages
                tokens.reset();
                antlr4Parser.reset();
                if (fileName != null) {
                    errorListener.setCurrentFileName(fileName);
                } else {
                    errorListener.setCurrentFileName("MAIN_SCRIPT");
                }
                // Set our custom error listener
                antlr4Parser.addErrorListener(errorListener);
                antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
                antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
                ast = antlr4Parser.programroot();
            }
        } else {
            // Set our custom error listener
            antlr4Parser.removeErrorListeners();
            antlr4Parser.addErrorListener(errorListener);
            errorListener.setCurrentFileName(fileName);
            // Now do the parsing
            ast = antlr4Parser.programroot();
        }
    } catch (Exception e) {
        throw new ParseException("ERROR: Cannot parse the program:" + fileName, e);
    }
    // Now convert the parse tree into DMLProgram
    // Do syntactic validation while converting
    ParseTree tree = ast;
    // And also do syntactic validation
    ParseTreeWalker walker = new ParseTreeWalker();
    // Get list of function definitions which take precedence over built-in functions if same name
    PydmlPreprocessor prep = new PydmlPreprocessor(errorListener);
    walker.walk(prep, tree);
    // Syntactic validation
    PydmlSyntacticValidator validator = new PydmlSyntacticValidator(errorListener, argVals, sourceNamespace, prep.getFunctionDefs());
    walker.walk(validator, tree);
    errorListener.unsetCurrentFileName();
    this.parseIssues = errorListener.getParseIssues();
    this.atLeastOneWarning = errorListener.isAtLeastOneWarning();
    this.atLeastOneError = errorListener.isAtLeastOneError();
    if (atLeastOneError) {
        throw new ParseException(parseIssues, dmlScript);
    }
    if (atLeastOneWarning) {
        LOG.warn(CustomErrorListener.generateParseIssuesMessage(dmlScript, parseIssues));
    }
    dmlPgm = createDMLProgram(ast, sourceNamespace);
    return dmlPgm;
}

Example 12 with org.antlr.v4.runtime.tree

use of org.antlr.v4.runtime.tree in project CFLint by cflint.

the class CFLint method syntaxError.

@Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, int line, int charPositionInLine, final String msg, final org.antlr.v4.runtime.RecognitionException re) {
    String expression = null;
    int offset = charPositionInLine;
    int startLine = 0;
    int startOffset = 0;
    final Context context = new Context(currentFile, currentElement, null, true, null, null);
    if (currentElement != null) {
        startOffset = context.offset();
        if (context.startLine() != 1) {
            startLine = currentElement.getSource().getRow(startOffset) - 1;
        }
    }
    if (offendingSymbol instanceof Token && re != null) {
        // grab the first non-comment previous token, which is actually the cause of the syntax error theoretically
        CommonTokenStream tokenStream = (CommonTokenStream) recognizer.getInputStream();
        Token previousToken = tokenStream.get(re.getOffendingToken().getTokenIndex() - 1);
        if (previousToken != null) {
            while (previousToken.getChannel() == Token.HIDDEN_CHANNEL && tokenStream.get(previousToken.getTokenIndex() - 1) != null) {
                previousToken = tokenStream.get(previousToken.getTokenIndex() - 1);
            }
            line = previousToken.getLine();
            offset = previousToken.getStopIndex();
            expression = previousToken.getText();
        } else {
            expression = re.getOffendingToken().getText();
        }
        if (expression.length() > 50) {
            expression = expression.substring(1, 40) + "...";
        }
    }
    offset += startOffset;
    line += startLine;
    if (recognizer instanceof Parser && ((Parser) recognizer).isExpectedToken(CFSCRIPTParser.SEMICOLON)) {
        final ContextMessage cm = new ContextMessage(MISSING_SEMI, expression, null, line, offset);
        reportRule(currentElement, null, context, null, cm);
    } else {
        final ContextMessage cm = new ContextMessage(PARSE_ERROR, expression, null, line, offset);
        reportRule(currentElement, null, context, null, cm);
    }
}

Example 13 with org.antlr.v4.runtime.tree

use of org.antlr.v4.runtime.tree in project antlr4 by tunnelvisionlabs.

the class TimeLexerSpeed method load_legacy_java_ascii_file.

@SuppressWarnings("deprecation")
public void load_legacy_java_ascii_file(String resourceName, int n) throws Exception {
    URL sampleJavaFile = TimeLexerSpeed.class.getClassLoader().getResource(resourceName);
    if (sampleJavaFile == null) {
        System.err.println("Can't run load_legacy_java_ascii_file from jar (or can't find " + resourceName + ")");
        // cannot find resource
        return;
    }
    if (!new File(sampleJavaFile.getFile()).exists()) {
        System.err.println("Can't run load_legacy_java_ascii_file from jar (or can't find " + resourceName + ")");
        return;
    }
    long start = System.nanoTime();
    // keep refs around so we can average memory
    CharStream[] input = new CharStream[n];
    for (int i = 0; i < n; i++) {
        input[i] = new org.antlr.v4.runtime.ANTLRFileStream(sampleJavaFile.getFile());
    }
    long stop = System.nanoTime();
    long tus = (stop - start) / 1000;
    int size = input[0].size();
    String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
    GraphLayout olayout = GraphLayout.parseInstance((Object) input[0]);
    long streamSize = olayout.totalSize();
    streamFootprints.add(basename(resourceName) + " (" + size + " char): " + olayout.toFootprint());
    if (output)
        System.out.printf("%27s average time %5dus size %6db over %4d loads of %5d symbols from %s\n", currentMethodName, tus / n, streamSize, n, size, basename(resourceName));
}

Example 14 with org.antlr.v4.runtime.tree

use of org.antlr.v4.runtime.tree in project antlr4 by tunnelvisionlabs.

the class TimeLexerSpeed method load_legacy_java_utf8.

@SuppressWarnings("deprecation")
public void load_legacy_java_utf8(String resourceName, int n) throws Exception {
    // keep refs around so we can average memory
    CharStream[] input = new CharStream[n];
    ClassLoader loader = TimeLexerSpeed.class.getClassLoader();
    InputStream[] streams = new InputStream[n];
    for (int i = 0; i < n; i++) {
        streams[i] = loader.getResourceAsStream(resourceName);
    }
    // track only time to suck data out of stream
    long start = System.nanoTime();
    for (int i = 0; i < n; i++) {
        InputStream is = streams[i];
        try {
            InputStreamReader isr = new InputStreamReader(is, Charset.forName("UTF-8"));
            try {
                BufferedReader br = new BufferedReader(isr);
                try {
                    input[i] = new org.antlr.v4.runtime.ANTLRInputStream(br);
                } finally {
                    br.close();
                }
            } finally {
                isr.close();
            }
        } finally {
            is.close();
        }
    }
    long stop = System.nanoTime();
    long tus = (stop - start) / 1000;
    int size = input[0].size();
    long streamSize = GraphLayout.parseInstance((Object) input[0]).totalSize();
    streamFootprints.add(basename(resourceName) + " (" + size + " char): " + GraphLayout.parseInstance((Object) input[0]).toFootprint());
    String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
    if (output)
        System.out.printf("%27s average time %5dus size %6db over %4d loads of %5d symbols from %s\n", currentMethodName, tus / n, streamSize, n, size, basename(resourceName));
}

Example 15 with org.antlr.v4.runtime.tree

use of org.antlr.v4.runtime.tree in project antlr4 by tunnelvisionlabs.

the class TimeLexerSpeed method load_legacy_java_ascii.

@SuppressWarnings("deprecation")
public void load_legacy_java_ascii(String resourceName, int n) throws Exception {
    // keep refs around so we can average memory
    CharStream[] input = new CharStream[n];
    ClassLoader loader = TimeLexerSpeed.class.getClassLoader();
    InputStream[] streams = new InputStream[n];
    for (int i = 0; i < n; i++) {
        streams[i] = loader.getResourceAsStream(resourceName);
    }
    // track only time to suck data out of stream
    long start = System.nanoTime();
    for (int i = 0; i < n; i++) {
        InputStream is = streams[i];
        try {
            input[i] = new org.antlr.v4.runtime.ANTLRInputStream(is);
        } finally {
            is.close();
        }
    }
    long stop = System.nanoTime();
    long tus = (stop - start) / 1000;
    int size = input[0].size();
    long streamSize = GraphLayout.parseInstance((Object) input[0]).totalSize();
    streamFootprints.add(basename(resourceName) + " (" + size + " char): " + GraphLayout.parseInstance((Object) input[0]).toFootprint());
    String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
    if (output)
        System.out.printf("%27s average time %5dus size %6db over %4d loads of %5d symbols from %s\n", currentMethodName, tus / n, streamSize, n, size, basename(resourceName));
}