Examples with TokenRewriteStream org.antlr.runtime.TokenRewriteStream used on opensource projects

Example 6 with TokenRewriteStream

use of org.antlr.runtime.TokenRewriteStream in project hive by apache.

the class ParseDriver method parseHint.

/*
   * Parse a string as a query hint.
   */
public ASTNode parseHint(String command) throws ParseException {
    LOG.info("Parsing hint: " + command);
    HiveLexerX lexer = new HiveLexerX(new ANTLRNoCaseStringStream(command));
    TokenRewriteStream tokens = new TokenRewriteStream(lexer);
    HintParser parser = new HintParser(tokens);
    parser.setTreeAdaptor(adaptor);
    HintParser.hint_return r = null;
    try {
        r = parser.hint();
    } catch (RecognitionException e) {
        e.printStackTrace();
        throw new ParseException(parser.errors);
    }
    if (lexer.getErrors().size() == 0 && parser.errors.size() == 0) {
        LOG.info("Parse Completed");
    } else if (lexer.getErrors().size() != 0) {
        throw new ParseException(lexer.getErrors());
    } else {
        throw new ParseException(parser.errors);
    }
    return (ASTNode) r.getTree();
}

Example 7 with TokenRewriteStream

use of org.antlr.runtime.TokenRewriteStream in project hive by apache.

the class ParseDriver method parse.

/**
   * Parses a command, optionally assigning the parser's token stream to the
   * given context.
   *
   * @param command
   *          command to parse
   *
   * @param ctx
   *          context with which to associate this parser's token stream, or
   *          null if either no context is available or the context already has
   *          an existing stream
   *
   * @return parsed AST
   */
public ASTNode parse(String command, Context ctx, boolean setTokenRewriteStream) throws ParseException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Parsing command: " + command);
    }
    HiveLexerX lexer = new HiveLexerX(new ANTLRNoCaseStringStream(command));
    TokenRewriteStream tokens = new TokenRewriteStream(lexer);
    if (ctx != null) {
        if (setTokenRewriteStream) {
            ctx.setTokenRewriteStream(tokens);
        }
        lexer.setHiveConf(ctx.getConf());
    }
    HiveParser parser = new HiveParser(tokens);
    if (ctx != null) {
        parser.setHiveConf(ctx.getConf());
    }
    parser.setTreeAdaptor(adaptor);
    HiveParser.statement_return r = null;
    try {
        r = parser.statement();
    } catch (RecognitionException e) {
        e.printStackTrace();
        throw new ParseException(parser.errors);
    }
    if (lexer.getErrors().size() == 0 && parser.errors.size() == 0) {
        LOG.debug("Parse Completed");
    } else if (lexer.getErrors().size() != 0) {
        throw new ParseException(lexer.getErrors());
    } else {
        throw new ParseException(parser.errors);
    }
    ASTNode tree = (ASTNode) r.getTree();
    tree.setUnknownTokenBoundaries();
    return tree;
}

Example 8 with TokenRewriteStream

use of org.antlr.runtime.TokenRewriteStream in project hive by apache.

the class ExplainSQRewriteTask method execute.

@Override
public int execute(DriverContext driverContext) {
    PrintStream out = null;
    try {
        Path resFile = new Path(work.getResFile());
        OutputStream outS = resFile.getFileSystem(conf).create(resFile);
        out = new PrintStream(outS);
        QB qb = work.getQb();
        TokenRewriteStream stream = work.getCtx().getTokenRewriteStream();
        String program = "sq rewrite";
        ASTNode ast = work.getAst();
        try {
            addRewrites(stream, qb, program, out);
            out.println("\nRewritten Query:\n" + stream.toString(program, ast.getTokenStartIndex(), ast.getTokenStopIndex()));
        } finally {
            stream.deleteProgram(program);
        }
        out.close();
        out = null;
        return (0);
    } catch (Exception e) {
        console.printError("Failed with exception " + e.getMessage(), "\n" + StringUtils.stringifyException(e));
        return (1);
    } finally {
        IOUtils.closeStream(out);
    }
}

Example 9 with TokenRewriteStream

use of org.antlr.runtime.TokenRewriteStream in project ceylon-compiler by ceylon.

the class CeylonVersionTool method updateModuleVersion.

private boolean updateModuleVersion(Module module, Map<String, String> updatedModuleVersions) throws IOException, RecognitionException {
    String moduleDescriptorPath = module.getUnit().getFullPath();
    CeylonLexer lexer = new CeylonLexer(new ANTLRFileStream(moduleDescriptorPath, encoding));
    TokenRewriteStream tokenStream = new TokenRewriteStream(lexer);
    CeylonParser parser = new CeylonParser(tokenStream);
    Tree.CompilationUnit cu = parser.compilationUnit();
    String v = this.confirm == Confirm.dependencies ? this.newVersion : confirm("update.module.version", module.getNameAsString(), module.getVersion(), this.newVersion);
    if (v == null) {
        return false;
    } else if (!v.isEmpty()) {
        // record the new version for this module
        updatedModuleVersions.put(module.getNameAsString(), v);
        updateModuleVersion(moduleDescriptorPath, tokenStream, cu, v);
    }
    return true;
}

Example 10 with TokenRewriteStream

use of org.antlr.runtime.TokenRewriteStream in project hive by apache.

the class ParseDriver method parseSelect.

/*
   * parse a String as a Select List. This allows table functions to be passed expression Strings
   * that are translated in
   * the context they define at invocation time. Currently used by NPath to allow users to specify
   * what output they want.
   * NPath allows expressions n 'tpath' a column that represents the matched set of rows. This
   * column doesn't exist in
   * the input schema and hence the Result Expression cannot be analyzed by the regular Hive
   * translation process.
   */
public ASTNode parseSelect(String command, Context ctx) throws ParseException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Parsing command: " + command);
    }
    HiveLexerX lexer = new HiveLexerX(new ANTLRNoCaseStringStream(command));
    TokenRewriteStream tokens = new TokenRewriteStream(lexer);
    if (ctx != null) {
        ctx.setTokenRewriteStream(tokens);
    }
    HiveParser parser = new HiveParser(tokens);
    parser.setTreeAdaptor(adaptor);
    HiveParser_SelectClauseParser.selectClause_return r = null;
    try {
        r = parser.selectClause();
    } catch (RecognitionException e) {
        e.printStackTrace();
        throw new ParseException(parser.errors);
    }
    if (lexer.getErrors().size() == 0 && parser.errors.size() == 0) {
        LOG.debug("Parse Completed");
    } else if (lexer.getErrors().size() != 0) {
        throw new ParseException(lexer.getErrors());
    } else {
        throw new ParseException(parser.errors);
    }
    return r.getTree();
}