Examples with CommonTokenStream org.antlr.v4.runtime.CommonTokenStream used on opensource projects

Example 11 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project compiler by boalang.

the class BoaCompiler method main.

public static void main(final String[] args) throws IOException {
    CommandLine cl = processCommandLineOptions(args);
    if (cl == null)
        return;
    final ArrayList<File> inputFiles = BoaCompiler.inputFiles;
    // get the name of the generated class
    final String className = getGeneratedClass(cl);
    // get the filename of the jar we will be writing
    final String jarName;
    if (cl.hasOption('o'))
        jarName = cl.getOptionValue('o');
    else
        jarName = className + ".jar";
    // make the output directory
    File outputRoot = null;
    if (cl.hasOption("cd")) {
        outputRoot = new File(cl.getOptionValue("cd"));
    } else {
        outputRoot = new File(new File(System.getProperty("java.io.tmpdir")), UUID.randomUUID().toString());
    }
    final File outputSrcDir = new File(outputRoot, "boa");
    if (!outputSrcDir.mkdirs())
        throw new IOException("unable to mkdir " + outputSrcDir);
    // find custom libs to load
    final List<URL> libs = new ArrayList<URL>();
    if (cl.hasOption('l'))
        for (final String lib : cl.getOptionValues('l')) libs.add(new File(lib).toURI().toURL());
    final File outputFile = new File(outputSrcDir, className + ".java");
    final BufferedOutputStream o = new BufferedOutputStream(new FileOutputStream(outputFile));
    try {
        final List<String> jobnames = new ArrayList<String>();
        final List<String> jobs = new ArrayList<String>();
        final List<Integer> seeds = new ArrayList<Integer>();
        boolean isSimple = true;
        final List<Program> visitorPrograms = new ArrayList<Program>();
        SymbolTable.initialize(libs);
        final int maxVisitors;
        if (cl.hasOption('v'))
            maxVisitors = Integer.parseInt(cl.getOptionValue('v'));
        else
            maxVisitors = Integer.MAX_VALUE;
        for (int i = 0; i < inputFiles.size(); i++) {
            final File f = inputFiles.get(i);
            try {
                final BoaLexer lexer = new BoaLexer(new ANTLRFileStream(f.getAbsolutePath()));
                // use the whole input string to seed the RNG
                seeds.add(lexer._input.getText(new Interval(0, lexer._input.size())).hashCode());
                lexer.removeErrorListeners();
                lexer.addErrorListener(new LexerErrorListener());
                final CommonTokenStream tokens = new CommonTokenStream(lexer);
                final BoaParser parser = new BoaParser(tokens);
                parser.removeErrorListeners();
                parser.addErrorListener(new BaseErrorListener() {

                    @Override
                    public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) throws ParseCancellationException {
                        throw new ParseCancellationException(e);
                    }
                });
                final BoaErrorListener parserErrorListener = new ParserErrorListener();
                final Start p = parse(tokens, parser, parserErrorListener);
                if (cl.hasOption("ast"))
                    new ASTPrintingVisitor().start(p);
                final String jobName = "" + i;
                try {
                    if (!parserErrorListener.hasError) {
                        new TypeCheckingVisitor().start(p, new SymbolTable());
                        final TaskClassifyingVisitor simpleVisitor = new TaskClassifyingVisitor();
                        simpleVisitor.start(p);
                        LOG.info(f.getName() + ": task complexity: " + (!simpleVisitor.isComplex() ? "simple" : "complex"));
                        isSimple &= !simpleVisitor.isComplex();
                        new InheritedAttributeTransformer().start(p);
                        new LocalAggregationTransformer().start(p);
                        // also let jobs have own methods if visitor merging is disabled
                        if (!simpleVisitor.isComplex() || maxVisitors < 2 || inputFiles.size() == 1) {
                            new VisitorOptimizingTransformer().start(p);
                            if (cl.hasOption("pp"))
                                new PrettyPrintVisitor().start(p);
                            if (cl.hasOption("ast2"))
                                new ASTPrintingVisitor().start(p);
                            final CodeGeneratingVisitor cg = new CodeGeneratingVisitor(jobName);
                            cg.start(p);
                            jobs.add(cg.getCode());
                            jobnames.add(jobName);
                        } else // if a job has visitors, fuse them all together into a single program
                        {
                            p.getProgram().jobName = jobName;
                            visitorPrograms.add(p.getProgram());
                        }
                    }
                } catch (final TypeCheckException e) {
                    parserErrorListener.error("typecheck", lexer, null, e.n.beginLine, e.n.beginColumn, e.n2.endColumn - e.n.beginColumn + 1, e.getMessage(), e);
                }
            } catch (final Exception e) {
                System.err.print(f.getName() + ": compilation failed: ");
                e.printStackTrace();
            }
        }
        if (!visitorPrograms.isEmpty())
            try {
                for (final Program p : new VisitorMergingTransformer().mergePrograms(visitorPrograms, maxVisitors)) {
                    new VisitorOptimizingTransformer().start(p);
                    if (cl.hasOption("pp"))
                        new PrettyPrintVisitor().start(p);
                    if (cl.hasOption("ast2"))
                        new ASTPrintingVisitor().start(p);
                    final CodeGeneratingVisitor cg = new CodeGeneratingVisitor(p.jobName);
                    cg.start(p);
                    jobs.add(cg.getCode());
                    jobnames.add(p.jobName);
                }
            } catch (final Exception e) {
                System.err.println("error fusing visitors - falling back: " + e);
                e.printStackTrace();
                for (final Program p : visitorPrograms) {
                    new VisitorOptimizingTransformer().start(p);
                    if (cl.hasOption("pp"))
                        new PrettyPrintVisitor().start(p);
                    if (cl.hasOption("ast2"))
                        new ASTPrintingVisitor().start(p);
                    final CodeGeneratingVisitor cg = new CodeGeneratingVisitor(p.jobName);
                    cg.start(p);
                    jobs.add(cg.getCode());
                    jobnames.add(p.jobName);
                }
            }
        if (jobs.size() == 0)
            throw new RuntimeException("no files compiled without error");
        final ST st = AbstractCodeGeneratingVisitor.stg.getInstanceOf("Program");
        st.add("name", className);
        st.add("numreducers", inputFiles.size());
        st.add("jobs", jobs);
        st.add("jobnames", jobnames);
        st.add("combineTables", CodeGeneratingVisitor.combineAggregatorStrings);
        st.add("reduceTables", CodeGeneratingVisitor.reduceAggregatorStrings);
        st.add("splitsize", isSimple ? 64 * 1024 * 1024 : 10 * 1024 * 1024);
        st.add("seeds", seeds);
        if (DefaultProperties.localDataPath != null) {
            st.add("isLocal", true);
        }
        o.write(st.render().getBytes());
    } finally {
        o.close();
    }
    compileGeneratedSrc(cl, jarName, outputRoot, outputFile);
}

Example 12 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project compiler by boalang.

the class BaseTest method lex.

protected CommonTokenStream lex(final String input, final int[] ids, final String[] strings, final String[] errors) throws IOException {
    final List<String> foundErr = new ArrayList<String>();
    final BoaLexer lexer = new BoaLexer(new ANTLRInputStream(new StringReader(input)));
    lexer.removeErrorListeners();
    lexer.addErrorListener(new BaseErrorListener() {

        @Override
        public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line, final int charPositionInLine, final String msg, final RecognitionException e) {
            foundErr.add(line + "," + charPositionInLine + ": " + msg);
        }
    });
    final CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    if (ids.length > 0 && strings.length > 0)
        assertEquals("ids != strings", ids.length, strings.length);
    if (ids.length > 0) {
        final List<Token> t = tokens.getTokens();
        if (DEBUG) {
            for (int i = 0; i < t.size(); i++) {
                final Token token = t.get(i);
                System.out.print(token.getType() + ", ");
            }
            System.out.println();
            for (int i = 0; i < t.size(); i++) {
                final Token token = t.get(i);
                System.out.print(token.getText() + ", ");
            }
            System.out.println();
            System.out.println();
        }
        assertEquals("wrong number of tokens", ids.length, t.size());
        for (int i = 0; i < t.size(); i++) assertEquals("wrong token type", ids[i], t.get(i).getType());
    }
    if (strings.length > 0) {
        final List<Token> t = tokens.getTokens();
        assertEquals("wrong number of tokens", strings.length, t.size());
        for (int i = 0; i < t.size(); i++) assertEquals("wrong token type", strings[i], t.get(i).getText());
    }
    assertEquals("wrong number of errors: " + input, errors.length, foundErr.size());
    for (int i = 0; i < foundErr.size(); i++) assertEquals("wrong error", errors[i], foundErr.get(i));
    return tokens;
}

Example 13 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project compiler by boalang.

the class BaseTest method parse.

protected StartContext parse(final String input, final String[] errors) throws IOException {
    final CommonTokenStream tokens = lex(input);
    final BoaParser parser = new BoaParser(tokens);
    final List<String> foundErr = new ArrayList<String>();
    parser.removeErrorListeners();
    parser.addErrorListener(new BaseErrorListener() {

        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) throws ParseCancellationException {
            throw new ParseCancellationException(e);
        }
    });
    parser.setBuildParseTree(false);
    parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
    StartContext p;
    try {
        p = parser.start();
    } catch (final Exception e) {
        // fall-back to LL mode parsing if SLL fails
        tokens.reset();
        parser.reset();
        parser.removeErrorListeners();
        parser.addErrorListener(new BaseErrorListener() {

            @Override
            public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line, final int charPositionInLine, final String msg, final RecognitionException e) {
                foundErr.add(line + "," + charPositionInLine + ": " + msg);
            }
        });
        parser.getInterpreter().setPredictionMode(PredictionMode.LL);
        p = parser.start();
    }
    if (!DEBUG)
        assertEquals("wrong number of errors", errors.length, foundErr.size());
    for (int i = 0; i < foundErr.size(); i++) {
        if (DEBUG)
            System.out.println(foundErr.get(i));
        else
            assertEquals("wrong error", errors[i], foundErr.get(i));
    }
    return p;
}

Example 14 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project jabref by JabRef.

the class GrammarBasedSearchRule method init.

private void init(String query) throws ParseCancellationException {
    if (Objects.equals(this.query, query)) {
        return;
    }
    SearchLexer lexer = new SearchLexer(new ANTLRInputStream(query));
    // no infos on file system
    lexer.removeErrorListeners();
    lexer.addErrorListener(ThrowingErrorListener.INSTANCE);
    SearchParser parser = new SearchParser(new CommonTokenStream(lexer));
    // no infos on file system
    parser.removeErrorListeners();
    parser.addErrorListener(ThrowingErrorListener.INSTANCE);
    // ParseCancelationException on parse errors
    parser.setErrorHandler(new BailErrorStrategy());
    tree = parser.start();
    this.query = query;
}

Example 15 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project incubator-systemml by apache.

the class PyDMLParserWrapper method doParse.

/**
 * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import'
 * @param fileName script file name
 * @param dmlScript script file contents
 * @param sourceNamespace namespace from source statement
 * @param argVals script arguments
 * @return dml program, or null if at least one error
 */
public DMLProgram doParse(String fileName, String dmlScript, String sourceNamespace, Map<String, String> argVals) {
    DMLProgram dmlPgm = null;
    ANTLRInputStream in;
    try {
        if (dmlScript == null) {
            dmlScript = readDMLScript(fileName, LOG);
        }
        InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
        in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
    } catch (FileNotFoundException e) {
        throw new ParseException("Cannot find file/resource: " + fileName, e);
    } catch (IOException e) {
        throw new ParseException("Cannot open file: " + fileName, e);
    } catch (LanguageException e) {
        throw new ParseException(e.getMessage(), e);
    }
    ProgramrootContext ast = null;
    CustomErrorListener errorListener = new CustomErrorListener();
    try {
        PydmlLexer lexer = new PydmlLexer(in);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        PydmlParser antlr4Parser = new PydmlParser(tokens);
        // For now no optimization, since it is not able to parse integer value.
        boolean tryOptimizedParsing = false;
        if (tryOptimizedParsing) {
            // Try faster and simpler SLL
            antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
            antlr4Parser.removeErrorListeners();
            antlr4Parser.setErrorHandler(new BailErrorStrategy());
            try {
                ast = antlr4Parser.programroot();
            // If successful, no need to try out full LL(*) ... SLL was enough
            } catch (ParseCancellationException ex) {
                // Error occurred, so now try full LL(*) for better error messages
                tokens.reset();
                antlr4Parser.reset();
                if (fileName != null) {
                    errorListener.setCurrentFileName(fileName);
                } else {
                    errorListener.setCurrentFileName("MAIN_SCRIPT");
                }
                // Set our custom error listener
                antlr4Parser.addErrorListener(errorListener);
                antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
                antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
                ast = antlr4Parser.programroot();
            }
        } else {
            // Set our custom error listener
            antlr4Parser.removeErrorListeners();
            antlr4Parser.addErrorListener(errorListener);
            errorListener.setCurrentFileName(fileName);
            // Now do the parsing
            ast = antlr4Parser.programroot();
        }
    } catch (Exception e) {
        throw new ParseException("ERROR: Cannot parse the program:" + fileName, e);
    }
    // Now convert the parse tree into DMLProgram
    // Do syntactic validation while converting
    ParseTree tree = ast;
    // And also do syntactic validation
    ParseTreeWalker walker = new ParseTreeWalker();
    // Get list of function definitions which take precedence over built-in functions if same name
    PydmlPreprocessor prep = new PydmlPreprocessor(errorListener);
    walker.walk(prep, tree);
    // Syntactic validation
    PydmlSyntacticValidator validator = new PydmlSyntacticValidator(errorListener, argVals, sourceNamespace, prep.getFunctionDefs());
    walker.walk(validator, tree);
    errorListener.unsetCurrentFileName();
    this.parseIssues = errorListener.getParseIssues();
    this.atLeastOneWarning = errorListener.isAtLeastOneWarning();
    this.atLeastOneError = errorListener.isAtLeastOneError();
    if (atLeastOneError) {
        throw new ParseException(parseIssues, dmlScript);
    }
    if (atLeastOneWarning) {
        LOG.warn(CustomErrorListener.generateParseIssuesMessage(dmlScript, parseIssues));
    }
    dmlPgm = createDMLProgram(ast, sourceNamespace);
    return dmlPgm;
}