Search in sources :

Example 1 with TaskClassifyingVisitor

use of boa.compiler.visitors.TaskClassifyingVisitor in project compiler by boalang.

the class BoaCompiler method main.

public static void main(final String[] args) throws IOException {
    CommandLine cl = processCommandLineOptions(args);
    if (cl == null)
        return;
    final ArrayList<File> inputFiles = BoaCompiler.inputFiles;
    // get the name of the generated class
    final String className = getGeneratedClass(cl);
    // get the filename of the jar we will be writing
    final String jarName;
    if (cl.hasOption('o'))
        jarName = cl.getOptionValue('o');
    else
        jarName = className + ".jar";
    // make the output directory
    File outputRoot = null;
    if (cl.hasOption("cd")) {
        outputRoot = new File(cl.getOptionValue("cd"));
    } else {
        outputRoot = new File(new File(System.getProperty("java.io.tmpdir")), UUID.randomUUID().toString());
    }
    final File outputSrcDir = new File(outputRoot, "boa");
    if (!outputSrcDir.mkdirs())
        throw new IOException("unable to mkdir " + outputSrcDir);
    // find custom libs to load
    final List<URL> libs = new ArrayList<URL>();
    if (cl.hasOption('l'))
        for (final String lib : cl.getOptionValues('l')) libs.add(new File(lib).toURI().toURL());
    final File outputFile = new File(outputSrcDir, className + ".java");
    final BufferedOutputStream o = new BufferedOutputStream(new FileOutputStream(outputFile));
    try {
        final List<String> jobnames = new ArrayList<String>();
        final List<String> jobs = new ArrayList<String>();
        final List<Integer> seeds = new ArrayList<Integer>();
        boolean isSimple = true;
        final List<Program> visitorPrograms = new ArrayList<Program>();
        SymbolTable.initialize(libs);
        final int maxVisitors;
        if (cl.hasOption('v'))
            maxVisitors = Integer.parseInt(cl.getOptionValue('v'));
        else
            maxVisitors = Integer.MAX_VALUE;
        for (int i = 0; i < inputFiles.size(); i++) {
            final File f = inputFiles.get(i);
            try {
                final BoaLexer lexer = new BoaLexer(new ANTLRFileStream(f.getAbsolutePath()));
                // use the whole input string to seed the RNG
                seeds.add(lexer._input.getText(new Interval(0, lexer._input.size())).hashCode());
                lexer.removeErrorListeners();
                lexer.addErrorListener(new LexerErrorListener());
                final CommonTokenStream tokens = new CommonTokenStream(lexer);
                final BoaParser parser = new BoaParser(tokens);
                parser.removeErrorListeners();
                parser.addErrorListener(new BaseErrorListener() {

                    @Override
                    public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) throws ParseCancellationException {
                        throw new ParseCancellationException(e);
                    }
                });
                final BoaErrorListener parserErrorListener = new ParserErrorListener();
                final Start p = parse(tokens, parser, parserErrorListener);
                if (cl.hasOption("ast"))
                    new ASTPrintingVisitor().start(p);
                final String jobName = "" + i;
                try {
                    if (!parserErrorListener.hasError) {
                        new TypeCheckingVisitor().start(p, new SymbolTable());
                        final TaskClassifyingVisitor simpleVisitor = new TaskClassifyingVisitor();
                        simpleVisitor.start(p);
                        LOG.info(f.getName() + ": task complexity: " + (!simpleVisitor.isComplex() ? "simple" : "complex"));
                        isSimple &= !simpleVisitor.isComplex();
                        new InheritedAttributeTransformer().start(p);
                        new LocalAggregationTransformer().start(p);
                        // also let jobs have own methods if visitor merging is disabled
                        if (!simpleVisitor.isComplex() || maxVisitors < 2 || inputFiles.size() == 1) {
                            new VisitorOptimizingTransformer().start(p);
                            if (cl.hasOption("pp"))
                                new PrettyPrintVisitor().start(p);
                            if (cl.hasOption("ast2"))
                                new ASTPrintingVisitor().start(p);
                            final CodeGeneratingVisitor cg = new CodeGeneratingVisitor(jobName);
                            cg.start(p);
                            jobs.add(cg.getCode());
                            jobnames.add(jobName);
                        } else // if a job has visitors, fuse them all together into a single program
                        {
                            p.getProgram().jobName = jobName;
                            visitorPrograms.add(p.getProgram());
                        }
                    }
                } catch (final TypeCheckException e) {
                    parserErrorListener.error("typecheck", lexer, null, e.n.beginLine, e.n.beginColumn, e.n2.endColumn - e.n.beginColumn + 1, e.getMessage(), e);
                }
            } catch (final Exception e) {
                System.err.print(f.getName() + ": compilation failed: ");
                e.printStackTrace();
            }
        }
        if (!visitorPrograms.isEmpty())
            try {
                for (final Program p : new VisitorMergingTransformer().mergePrograms(visitorPrograms, maxVisitors)) {
                    new VisitorOptimizingTransformer().start(p);
                    if (cl.hasOption("pp"))
                        new PrettyPrintVisitor().start(p);
                    if (cl.hasOption("ast2"))
                        new ASTPrintingVisitor().start(p);
                    final CodeGeneratingVisitor cg = new CodeGeneratingVisitor(p.jobName);
                    cg.start(p);
                    jobs.add(cg.getCode());
                    jobnames.add(p.jobName);
                }
            } catch (final Exception e) {
                System.err.println("error fusing visitors - falling back: " + e);
                e.printStackTrace();
                for (final Program p : visitorPrograms) {
                    new VisitorOptimizingTransformer().start(p);
                    if (cl.hasOption("pp"))
                        new PrettyPrintVisitor().start(p);
                    if (cl.hasOption("ast2"))
                        new ASTPrintingVisitor().start(p);
                    final CodeGeneratingVisitor cg = new CodeGeneratingVisitor(p.jobName);
                    cg.start(p);
                    jobs.add(cg.getCode());
                    jobnames.add(p.jobName);
                }
            }
        if (jobs.size() == 0)
            throw new RuntimeException("no files compiled without error");
        final ST st = AbstractCodeGeneratingVisitor.stg.getInstanceOf("Program");
        st.add("name", className);
        st.add("numreducers", inputFiles.size());
        st.add("jobs", jobs);
        st.add("jobnames", jobnames);
        st.add("combineTables", CodeGeneratingVisitor.combineAggregatorStrings);
        st.add("reduceTables", CodeGeneratingVisitor.reduceAggregatorStrings);
        st.add("splitsize", isSimple ? 64 * 1024 * 1024 : 10 * 1024 * 1024);
        st.add("seeds", seeds);
        if (DefaultProperties.localDataPath != null) {
            st.add("isLocal", true);
        }
        o.write(st.render().getBytes());
    } finally {
        o.close();
    }
    compileGeneratedSrc(cl, jarName, outputRoot, outputFile);
}
Also used : BoaParser(boa.parser.BoaParser) Start(boa.compiler.ast.Start) ArrayList(java.util.ArrayList) URL(java.net.URL) VisitorOptimizingTransformer(boa.compiler.transforms.VisitorOptimizingTransformer) BoaErrorListener(boa.compiler.listeners.BoaErrorListener) TypeCheckingVisitor(boa.compiler.visitors.TypeCheckingVisitor) BufferedOutputStream(java.io.BufferedOutputStream) BoaLexer(boa.parser.BoaLexer) PrettyPrintVisitor(boa.compiler.visitors.PrettyPrintVisitor) CommonTokenStream(org.antlr.v4.runtime.CommonTokenStream) ST(org.stringtemplate.v4.ST) Program(boa.compiler.ast.Program) LexerErrorListener(boa.compiler.listeners.LexerErrorListener) BaseErrorListener(org.antlr.v4.runtime.BaseErrorListener) InheritedAttributeTransformer(boa.compiler.transforms.InheritedAttributeTransformer) VisitorMergingTransformer(boa.compiler.transforms.VisitorMergingTransformer) LocalAggregationTransformer(boa.compiler.transforms.LocalAggregationTransformer) IOException(java.io.IOException) TaskClassifyingVisitor(boa.compiler.visitors.TaskClassifyingVisitor) FileNotFoundException(java.io.FileNotFoundException) ParseCancellationException(org.antlr.v4.runtime.misc.ParseCancellationException) IOException(java.io.IOException) RecognitionException(org.antlr.v4.runtime.RecognitionException) ParserErrorListener(boa.compiler.listeners.ParserErrorListener) CommandLine(org.apache.commons.cli.CommandLine) ANTLRFileStream(org.antlr.v4.runtime.ANTLRFileStream) ParseCancellationException(org.antlr.v4.runtime.misc.ParseCancellationException) FileOutputStream(java.io.FileOutputStream) AbstractCodeGeneratingVisitor(boa.compiler.visitors.AbstractCodeGeneratingVisitor) CodeGeneratingVisitor(boa.compiler.visitors.CodeGeneratingVisitor) ASTPrintingVisitor(boa.compiler.visitors.ASTPrintingVisitor) File(java.io.File) RecognitionException(org.antlr.v4.runtime.RecognitionException) Interval(org.antlr.v4.runtime.misc.Interval)

Example 2 with TaskClassifyingVisitor

use of boa.compiler.visitors.TaskClassifyingVisitor in project compiler by boalang.

the class BoaCompiler method parseOnly.

public static void parseOnly(final String[] args) throws IOException {
    final CommandLine cl = processParseCommandLineOptions(args);
    if (cl == null)
        return;
    final ArrayList<File> inputFiles = BoaCompiler.inputFiles;
    // find custom libs to load
    final List<URL> libs = new ArrayList<URL>();
    if (cl.hasOption('l'))
        for (final String lib : cl.getOptionValues('l')) libs.add(new File(lib).toURI().toURL());
    SymbolTable.initialize(libs);
    final int maxVisitors;
    if (cl.hasOption('v'))
        maxVisitors = Integer.parseInt(cl.getOptionValue('v'));
    else
        maxVisitors = Integer.MAX_VALUE;
    for (int i = 0; i < inputFiles.size(); i++) {
        final File f = inputFiles.get(i);
        try {
            final BoaLexer lexer = new BoaLexer(new ANTLRFileStream(f.getAbsolutePath()));
            lexer.removeErrorListeners();
            lexer.addErrorListener(new LexerErrorListener());
            final CommonTokenStream tokens = new CommonTokenStream(lexer);
            final BoaParser parser = new BoaParser(tokens);
            parser.removeErrorListeners();
            parser.addErrorListener(new BaseErrorListener() {

                @Override
                public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) throws ParseCancellationException {
                    throw new ParseCancellationException(e);
                }
            });
            final BoaErrorListener parserErrorListener = new ParserErrorListener();
            final Start p = parse(tokens, parser, parserErrorListener);
            try {
                if (!parserErrorListener.hasError) {
                    new TypeCheckingVisitor().start(p, new SymbolTable());
                    final TaskClassifyingVisitor simpleVisitor = new TaskClassifyingVisitor();
                    simpleVisitor.start(p);
                    LOG.info(f.getName() + ": task complexity: " + (!simpleVisitor.isComplex() ? "simple" : "complex"));
                }
            } catch (final TypeCheckException e) {
                parserErrorListener.error("typecheck", lexer, null, e.n.beginLine, e.n.beginColumn, e.n2.endColumn - e.n.beginColumn + 1, e.getMessage(), e);
            }
        } catch (final Exception e) {
            System.err.print(f.getName() + ": parsing failed: ");
            e.printStackTrace();
        }
    }
}
Also used : BoaParser(boa.parser.BoaParser) Start(boa.compiler.ast.Start) ArrayList(java.util.ArrayList) URL(java.net.URL) BoaErrorListener(boa.compiler.listeners.BoaErrorListener) TypeCheckingVisitor(boa.compiler.visitors.TypeCheckingVisitor) BoaLexer(boa.parser.BoaLexer) CommonTokenStream(org.antlr.v4.runtime.CommonTokenStream) LexerErrorListener(boa.compiler.listeners.LexerErrorListener) BaseErrorListener(org.antlr.v4.runtime.BaseErrorListener) TaskClassifyingVisitor(boa.compiler.visitors.TaskClassifyingVisitor) FileNotFoundException(java.io.FileNotFoundException) ParseCancellationException(org.antlr.v4.runtime.misc.ParseCancellationException) IOException(java.io.IOException) RecognitionException(org.antlr.v4.runtime.RecognitionException) ParserErrorListener(boa.compiler.listeners.ParserErrorListener) CommandLine(org.apache.commons.cli.CommandLine) ANTLRFileStream(org.antlr.v4.runtime.ANTLRFileStream) ParseCancellationException(org.antlr.v4.runtime.misc.ParseCancellationException) File(java.io.File) RecognitionException(org.antlr.v4.runtime.RecognitionException)

Aggregations

Start (boa.compiler.ast.Start)2 BoaErrorListener (boa.compiler.listeners.BoaErrorListener)2 LexerErrorListener (boa.compiler.listeners.LexerErrorListener)2 ParserErrorListener (boa.compiler.listeners.ParserErrorListener)2 TaskClassifyingVisitor (boa.compiler.visitors.TaskClassifyingVisitor)2 TypeCheckingVisitor (boa.compiler.visitors.TypeCheckingVisitor)2 BoaLexer (boa.parser.BoaLexer)2 BoaParser (boa.parser.BoaParser)2 File (java.io.File)2 FileNotFoundException (java.io.FileNotFoundException)2 IOException (java.io.IOException)2 URL (java.net.URL)2 ArrayList (java.util.ArrayList)2 ANTLRFileStream (org.antlr.v4.runtime.ANTLRFileStream)2 BaseErrorListener (org.antlr.v4.runtime.BaseErrorListener)2 CommonTokenStream (org.antlr.v4.runtime.CommonTokenStream)2 RecognitionException (org.antlr.v4.runtime.RecognitionException)2 ParseCancellationException (org.antlr.v4.runtime.misc.ParseCancellationException)2 CommandLine (org.apache.commons.cli.CommandLine)2 Program (boa.compiler.ast.Program)1