Examples with ClassicToken org.antlr.runtime.ClassicToken used on opensource projects

Example 1 with ClassicToken

use of org.antlr.runtime.ClassicToken in project hive by apache.

the class SemanticAnalyzer method materializeCTE.

Table materializeCTE(String cteName, CTEClause cte) throws HiveException {
    ASTNode createTable = new ASTNode(new ClassicToken(HiveParser.TOK_CREATETABLE));
    ASTNode tableName = new ASTNode(new ClassicToken(HiveParser.TOK_TABNAME));
    tableName.addChild(new ASTNode(new ClassicToken(HiveParser.Identifier, cteName)));
    ASTNode temporary = new ASTNode(new ClassicToken(HiveParser.KW_TEMPORARY, MATERIALIZATION_MARKER));
    createTable.addChild(tableName);
    createTable.addChild(temporary);
    createTable.addChild(cte.cteNode);
    SemanticAnalyzer analyzer = new SemanticAnalyzer(queryState);
    analyzer.initCtx(ctx);
    analyzer.init(false);
    // should share cte contexts
    analyzer.aliasToCTEs.putAll(aliasToCTEs);
    HiveOperation operation = queryState.getHiveOperation();
    try {
        analyzer.analyzeInternal(createTable);
    } finally {
        queryState.setCommandType(operation);
    }
    Table table = analyzer.tableDesc.toTable(conf);
    Path location = table.getDataLocation();
    try {
        location.getFileSystem(conf).mkdirs(location);
    } catch (IOException e) {
        throw new HiveException(e);
    }
    table.setMaterializedTable(true);
    LOG.info(cteName + " will be materialized into " + location);
    cte.table = table;
    cte.source = analyzer;
    ctx.addMaterializedTable(cteName, table);
    return table;
}

Example 2 with ClassicToken

use of org.antlr.runtime.ClassicToken in project hive by apache.

the class CalcitePlanner method materializeCTE.

@Override
Table materializeCTE(String cteName, CTEClause cte) throws HiveException {
    ASTNode createTable = new ASTNode(new ClassicToken(HiveParser.TOK_CREATETABLE));
    ASTNode tableName = new ASTNode(new ClassicToken(HiveParser.TOK_TABNAME));
    tableName.addChild(new ASTNode(new ClassicToken(HiveParser.Identifier, cteName)));
    ASTNode temporary = new ASTNode(new ClassicToken(HiveParser.KW_TEMPORARY, MATERIALIZATION_MARKER));
    createTable.addChild(tableName);
    createTable.addChild(temporary);
    createTable.addChild(cte.cteNode);
    CalcitePlanner analyzer = new CalcitePlanner(queryState);
    analyzer.initCtx(ctx);
    analyzer.init(false);
    // should share cte contexts
    analyzer.aliasToCTEs.putAll(aliasToCTEs);
    HiveOperation operation = queryState.getHiveOperation();
    try {
        analyzer.analyzeInternal(createTable);
    } finally {
        queryState.setCommandType(operation);
    }
    Table table = analyzer.tableDesc.toTable(conf);
    Path location = table.getDataLocation();
    try {
        location.getFileSystem(conf).mkdirs(location);
    } catch (IOException e) {
        throw new HiveException(e);
    }
    table.setMaterializedTable(true);
    LOG.info(cteName + " will be materialized into " + location);
    cte.table = table;
    cte.source = analyzer;
    ctx.addMaterializedTable(cteName, table);
    // For CalcitePlanner, store qualified name too
    ctx.addMaterializedTable(table.getDbName() + "." + table.getTableName(), table);
    return table;
}

Example 3 with ClassicToken

use of org.antlr.runtime.ClassicToken in project hive by apache.

the class SemanticAnalyzer method genValuesTempTable.

/**
   * Generate a temp table out of a values clause
   * See also {@link #preProcessForInsert(ASTNode, QB)}
   */
private ASTNode genValuesTempTable(ASTNode originalFrom, QB qb) throws SemanticException {
    Path dataDir = null;
    if (!qb.getEncryptedTargetTablePaths().isEmpty()) {
        //currently only Insert into T values(...) is supported thus only 1 values clause
        //and only 1 target table are possible.  If/when support for
        //select ... from values(...) is added an insert statement may have multiple
        //encrypted target tables.
        dataDir = ctx.getMRTmpPath(qb.getEncryptedTargetTablePaths().get(0).toUri());
    }
    // Pick a name for the table
    SessionState ss = SessionState.get();
    String tableName = VALUES_TMP_TABLE_NAME_PREFIX + ss.getNextValuesTempTableSuffix();
    // Step 1, parse the values clause we were handed
    List<? extends Node> fromChildren = originalFrom.getChildren();
    // First child should be the virtual table ref
    ASTNode virtualTableRef = (ASTNode) fromChildren.get(0);
    assert virtualTableRef.getToken().getType() == HiveParser.TOK_VIRTUAL_TABREF : "Expected first child of TOK_VIRTUAL_TABLE to be TOK_VIRTUAL_TABREF but was " + virtualTableRef.getName();
    List<? extends Node> virtualTableRefChildren = virtualTableRef.getChildren();
    // First child of this should be the table name.  If it's anonymous,
    // then we don't have a table name.
    ASTNode tabName = (ASTNode) virtualTableRefChildren.get(0);
    if (tabName.getToken().getType() != HiveParser.TOK_ANONYMOUS) {
        // you need to parse this list of columns names and build it into the table
        throw new SemanticException(ErrorMsg.VALUES_TABLE_CONSTRUCTOR_NOT_SUPPORTED.getMsg());
    }
    // The second child of the TOK_VIRTUAL_TABLE should be TOK_VALUES_TABLE
    ASTNode valuesTable = (ASTNode) fromChildren.get(1);
    assert valuesTable.getToken().getType() == HiveParser.TOK_VALUES_TABLE : "Expected second child of TOK_VIRTUAL_TABLE to be TOK_VALUE_TABLE but was " + valuesTable.getName();
    // Each of the children of TOK_VALUES_TABLE will be a TOK_VALUE_ROW
    List<? extends Node> valuesTableChildren = valuesTable.getChildren();
    // Now that we're going to start reading through the rows, open a file to write the rows too
    // If we leave this method before creating the temporary table we need to be sure to clean up
    // this file.
    Path tablePath = null;
    FileSystem fs = null;
    FSDataOutputStream out = null;
    try {
        if (dataDir == null) {
            tablePath = Warehouse.getDnsPath(new Path(ss.getTempTableSpace(), tableName), conf);
        } else {
            //if target table of insert is encrypted, make sure temporary table data is stored
            //similarly encrypted
            tablePath = Warehouse.getDnsPath(new Path(dataDir, tableName), conf);
        }
        fs = tablePath.getFileSystem(conf);
        fs.mkdirs(tablePath);
        Path dataFile = new Path(tablePath, "data_file");
        out = fs.create(dataFile);
        List<FieldSchema> fields = new ArrayList<FieldSchema>();
        boolean firstRow = true;
        for (Node n : valuesTableChildren) {
            ASTNode valuesRow = (ASTNode) n;
            assert valuesRow.getToken().getType() == HiveParser.TOK_VALUE_ROW : "Expected child of TOK_VALUE_TABLE to be TOK_VALUE_ROW but was " + valuesRow.getName();
            // Each of the children of this should be a literal
            List<? extends Node> valuesRowChildren = valuesRow.getChildren();
            boolean isFirst = true;
            int nextColNum = 1;
            for (Node n1 : valuesRowChildren) {
                ASTNode value = (ASTNode) n1;
                if (firstRow) {
                    fields.add(new FieldSchema("tmp_values_col" + nextColNum++, "string", ""));
                }
                if (isFirst)
                    isFirst = false;
                else
                    writeAsText("
", out);
                writeAsText(unparseExprForValuesClause(value), out);
            }
            writeAsText("\n", out);
            firstRow = false;
        }
        // Step 2, create a temp table, using the created file as the data
        StorageFormat format = new StorageFormat(conf);
        format.processStorageFormat("TextFile");
        Table table = db.newTable(tableName);
        table.setSerializationLib(format.getSerde());
        table.setFields(fields);
        table.setDataLocation(tablePath);
        table.getTTable().setTemporary(true);
        table.setStoredAsSubDirectories(false);
        table.setInputFormatClass(format.getInputFormat());
        table.setOutputFormatClass(format.getOutputFormat());
        db.createTable(table, false);
    } catch (Exception e) {
        String errMsg = ErrorMsg.INSERT_CANNOT_CREATE_TEMP_FILE.getMsg() + e.getMessage();
        LOG.error(errMsg);
        // Try to delete the file
        if (fs != null && tablePath != null) {
            try {
                fs.delete(tablePath, false);
            } catch (IOException swallowIt) {
            }
        }
        throw new SemanticException(errMsg, e);
    } finally {
        IOUtils.closeStream(out);
    }
    // Step 3, return a new subtree with a from clause built around that temp table
    // The form of the tree is TOK_TABREF->TOK_TABNAME->identifier(tablename)
    Token t = new ClassicToken(HiveParser.TOK_TABREF);
    ASTNode tabRef = new ASTNode(t);
    t = new ClassicToken(HiveParser.TOK_TABNAME);
    ASTNode tabNameNode = new ASTNode(t);
    tabRef.addChild(tabNameNode);
    t = new ClassicToken(HiveParser.Identifier, tableName);
    ASTNode identifier = new ASTNode(t);
    tabNameNode.addChild(identifier);
    return tabRef;
}