Examples with QueryOperation org.apache.flink.table.operations.QueryOperation used on opensource projects

Example 6 with QueryOperation

use of org.apache.flink.table.operations.QueryOperation in project flink by apache.

the class OperationTreeBuilder method windowTableAggregate.

public QueryOperation windowTableAggregate(List<Expression> groupingExpressions, GroupWindow window, List<Expression> windowProperties, Expression tableAggFunction, QueryOperation child) {
    // Step1: add a default name to the call in the grouping expressions, e.g., groupBy(a % 5)
    // to
    // groupBy(a % 5 as TMP_0). We need a name for every column so that to perform alias for the
    // table aggregate function in Step4.
    List<Expression> newGroupingExpressions = addAliasToTheCallInAggregate(child.getResolvedSchema().getColumnNames(), groupingExpressions);
    // Step2: resolve expressions, including grouping, aggregates and window properties.
    ExpressionResolver resolver = getAggResolver(child, groupingExpressions);
    ResolvedGroupWindow resolvedWindow = aggregateOperationFactory.createResolvedWindow(window, resolver);
    ExpressionResolver resolverWithWindowReferences = getResolverBuilder(child).withLocalReferences(localRef(resolvedWindow.getAlias(), resolvedWindow.getTimeAttribute().getOutputDataType())).build();
    List<ResolvedExpression> convertedGroupings = resolverWithWindowReferences.resolve(newGroupingExpressions);
    List<ResolvedExpression> convertedAggregates = resolverWithWindowReferences.resolve(Collections.singletonList(tableAggFunction));
    List<ResolvedExpression> convertedProperties = resolverWithWindowReferences.resolve(windowProperties);
    Tuple2<ResolvedExpression, List<String>> resolvedFunctionAndAlias = aggregateOperationFactory.extractTableAggFunctionAndAliases(convertedAggregates.get(0));
    // Step3: create window table agg operation
    QueryOperation tableAggOperation = aggregateOperationFactory.createWindowAggregate(convertedGroupings, Collections.singletonList(resolvedFunctionAndAlias.f0), convertedProperties, resolvedWindow, child);
    // window attribute.
    return aliasBackwardFields(tableAggOperation, resolvedFunctionAndAlias.f1, groupingExpressions.size());
}

Example 7 with QueryOperation

use of org.apache.flink.table.operations.QueryOperation in project flink by apache.

the class AliasOperationUtils method createAliasList.

/**
 * Creates a list of valid alias expressions. Resulting expression might still contain {@link
 * UnresolvedReferenceExpression}.
 *
 * @param aliases aliases to validate
 * @param child relational operation on top of which to apply the aliases
 * @return validated list of aliases
 */
static List<Expression> createAliasList(List<Expression> aliases, QueryOperation child) {
    ResolvedSchema childSchema = child.getResolvedSchema();
    if (aliases.size() > childSchema.getColumnCount()) {
        throw new ValidationException("Aliasing more fields than we actually have.");
    }
    List<ValueLiteralExpression> fieldAliases = aliases.stream().map(f -> f.accept(aliasLiteralValidator)).collect(Collectors.toList());
    List<String> childNames = childSchema.getColumnNames();
    return IntStream.range(0, childNames.size()).mapToObj(idx -> {
        UnresolvedReferenceExpression oldField = unresolvedRef(childNames.get(idx));
        if (idx < fieldAliases.size()) {
            ValueLiteralExpression alias = fieldAliases.get(idx);
            return unresolvedCall(BuiltInFunctionDefinitions.AS, oldField, alias);
        } else {
            return oldField;
        }
    }).collect(Collectors.toList());
}

Example 8 with QueryOperation

use of org.apache.flink.table.operations.QueryOperation in project flink by apache.

the class HiveParserDMLHelper method createInsertOperation.

public Operation createInsertOperation(HiveParserCalcitePlanner analyzer, RelNode queryRelNode) throws SemanticException {
    HiveParserQB topQB = analyzer.getQB();
    QBMetaData qbMetaData = topQB.getMetaData();
    // decide the dest table
    Map<String, Table> nameToDestTable = qbMetaData.getNameToDestTable();
    Map<String, Partition> nameToDestPart = qbMetaData.getNameToDestPartition();
    // for now we only support inserting to a single table
    Preconditions.checkState(nameToDestTable.size() <= 1 && nameToDestPart.size() <= 1, "Only support inserting to 1 table");
    Table destTable;
    String insClauseName;
    if (!nameToDestTable.isEmpty()) {
        insClauseName = nameToDestTable.keySet().iterator().next();
        destTable = nameToDestTable.values().iterator().next();
    } else if (!nameToDestPart.isEmpty()) {
        insClauseName = nameToDestPart.keySet().iterator().next();
        destTable = nameToDestPart.values().iterator().next().getTable();
    } else {
        // happens for INSERT DIRECTORY
        throw new SemanticException("INSERT DIRECTORY is not supported");
    }
    // decide static partition specs
    Map<String, String> staticPartSpec = new LinkedHashMap<>();
    if (destTable.isPartitioned()) {
        List<String> partCols = HiveCatalog.getFieldNames(destTable.getTTable().getPartitionKeys());
        if (!nameToDestPart.isEmpty()) {
            // static partition
            Partition destPart = nameToDestPart.values().iterator().next();
            Preconditions.checkState(partCols.size() == destPart.getValues().size(), "Part cols and static spec doesn't match");
            for (int i = 0; i < partCols.size(); i++) {
                staticPartSpec.put(partCols.get(i), destPart.getValues().get(i));
            }
        } else {
            // dynamic partition
            Map<String, String> spec = qbMetaData.getPartSpecForAlias(insClauseName);
            if (spec != null) {
                for (String partCol : partCols) {
                    String val = spec.get(partCol);
                    if (val != null) {
                        staticPartSpec.put(partCol, val);
                    }
                }
            }
        }
    }
    // decide whether it's overwrite
    boolean overwrite = topQB.getParseInfo().getInsertOverwriteTables().keySet().stream().map(String::toLowerCase).collect(Collectors.toSet()).contains(destTable.getDbName() + "." + destTable.getTableName());
    Tuple4<ObjectIdentifier, QueryOperation, Map<String, String>, Boolean> insertOperationInfo = createInsertOperationInfo(queryRelNode, destTable, staticPartSpec, analyzer.getDestSchemaForClause(insClauseName), overwrite);
    return new SinkModifyOperation(catalogManager.getTableOrError(insertOperationInfo.f0), insertOperationInfo.f1, insertOperationInfo.f2, insertOperationInfo.f3, Collections.emptyMap());
}

Example 9 with QueryOperation

use of org.apache.flink.table.operations.QueryOperation in project zeppelin by apache.

the class Flink113Shims method parseBySqlParser.

private SqlCommandCall parseBySqlParser(Parser sqlParser, String stmt) throws Exception {
    List<Operation> operations;
    try {
        operations = sqlParser.parse(stmt);
    } catch (Throwable e) {
        throw new Exception("Invalidate SQL statement.", e);
    }
    if (operations.size() != 1) {
        throw new Exception("Only single statement is supported now.");
    }
    final SqlCommand cmd;
    String[] operands = new String[] { stmt };
    Operation operation = operations.get(0);
    if (operation instanceof CatalogSinkModifyOperation) {
        boolean overwrite = ((CatalogSinkModifyOperation) operation).isOverwrite();
        cmd = overwrite ? SqlCommand.INSERT_OVERWRITE : SqlCommand.INSERT_INTO;
    } else if (operation instanceof CreateTableOperation) {
        cmd = SqlCommand.CREATE_TABLE;
    } else if (operation instanceof DropTableOperation) {
        cmd = SqlCommand.DROP_TABLE;
    } else if (operation instanceof AlterTableOperation) {
        cmd = SqlCommand.ALTER_TABLE;
    } else if (operation instanceof CreateViewOperation) {
        cmd = SqlCommand.CREATE_VIEW;
    } else if (operation instanceof DropViewOperation) {
        cmd = SqlCommand.DROP_VIEW;
    } else if (operation instanceof CreateDatabaseOperation) {
        cmd = SqlCommand.CREATE_DATABASE;
    } else if (operation instanceof DropDatabaseOperation) {
        cmd = SqlCommand.DROP_DATABASE;
    } else if (operation instanceof AlterDatabaseOperation) {
        cmd = SqlCommand.ALTER_DATABASE;
    } else if (operation instanceof CreateCatalogOperation) {
        cmd = SqlCommand.CREATE_CATALOG;
    } else if (operation instanceof DropCatalogOperation) {
        cmd = SqlCommand.DROP_CATALOG;
    } else if (operation instanceof UseCatalogOperation) {
        cmd = SqlCommand.USE_CATALOG;
        operands = new String[] { ((UseCatalogOperation) operation).getCatalogName() };
    } else if (operation instanceof UseDatabaseOperation) {
        cmd = SqlCommand.USE;
        operands = new String[] { ((UseDatabaseOperation) operation).getDatabaseName() };
    } else if (operation instanceof ShowCatalogsOperation) {
        cmd = SqlCommand.SHOW_CATALOGS;
        operands = new String[0];
    } else if (operation instanceof ShowDatabasesOperation) {
        cmd = SqlCommand.SHOW_DATABASES;
        operands = new String[0];
    } else if (operation instanceof ShowTablesOperation) {
        cmd = SqlCommand.SHOW_TABLES;
        operands = new String[0];
    } else if (operation instanceof ShowFunctionsOperation) {
        cmd = SqlCommand.SHOW_FUNCTIONS;
        operands = new String[0];
    } else if (operation instanceof CreateCatalogFunctionOperation || operation instanceof CreateTempSystemFunctionOperation) {
        cmd = SqlCommand.CREATE_FUNCTION;
    } else if (operation instanceof DropCatalogFunctionOperation || operation instanceof DropTempSystemFunctionOperation) {
        cmd = SqlCommand.DROP_FUNCTION;
    } else if (operation instanceof AlterCatalogFunctionOperation) {
        cmd = SqlCommand.ALTER_FUNCTION;
    } else if (operation instanceof ExplainOperation) {
        cmd = SqlCommand.EXPLAIN;
    } else if (operation instanceof DescribeTableOperation) {
        cmd = SqlCommand.DESCRIBE;
        operands = new String[] { ((DescribeTableOperation) operation).getSqlIdentifier().asSerializableString() };
    } else if (operation instanceof QueryOperation) {
        cmd = SqlCommand.SELECT;
    } else {
        throw new Exception("Unknown operation: " + operation.asSummaryString());
    }
    return new SqlCommandCall(cmd, operands, stmt);
}

Example 10 with QueryOperation

use of org.apache.flink.table.operations.QueryOperation in project zeppelin by apache.

the class Flink114Shims method parseBySqlParser.

private SqlCommandCall parseBySqlParser(Parser sqlParser, String stmt) throws Exception {
    List<Operation> operations;
    try {
        operations = sqlParser.parse(stmt);
    } catch (Throwable e) {
        throw new Exception("Invalidate SQL statement.", e);
    }
    if (operations.size() != 1) {
        throw new Exception("Only single statement is supported now.");
    }
    final SqlCommand cmd;
    String[] operands = new String[] { stmt };
    Operation operation = operations.get(0);
    if (operation instanceof CatalogSinkModifyOperation) {
        boolean overwrite = ((CatalogSinkModifyOperation) operation).isOverwrite();
        cmd = overwrite ? SqlCommand.INSERT_OVERWRITE : SqlCommand.INSERT_INTO;
    } else if (operation instanceof CreateTableOperation) {
        cmd = SqlCommand.CREATE_TABLE;
    } else if (operation instanceof DropTableOperation) {
        cmd = SqlCommand.DROP_TABLE;
    } else if (operation instanceof AlterTableOperation) {
        cmd = SqlCommand.ALTER_TABLE;
    } else if (operation instanceof CreateViewOperation) {
        cmd = SqlCommand.CREATE_VIEW;
    } else if (operation instanceof DropViewOperation) {
        cmd = SqlCommand.DROP_VIEW;
    } else if (operation instanceof CreateDatabaseOperation) {
        cmd = SqlCommand.CREATE_DATABASE;
    } else if (operation instanceof DropDatabaseOperation) {
        cmd = SqlCommand.DROP_DATABASE;
    } else if (operation instanceof AlterDatabaseOperation) {
        cmd = SqlCommand.ALTER_DATABASE;
    } else if (operation instanceof CreateCatalogOperation) {
        cmd = SqlCommand.CREATE_CATALOG;
    } else if (operation instanceof DropCatalogOperation) {
        cmd = SqlCommand.DROP_CATALOG;
    } else if (operation instanceof UseCatalogOperation) {
        cmd = SqlCommand.USE_CATALOG;
        operands = new String[] { ((UseCatalogOperation) operation).getCatalogName() };
    } else if (operation instanceof UseDatabaseOperation) {
        cmd = SqlCommand.USE;
        operands = new String[] { ((UseDatabaseOperation) operation).getDatabaseName() };
    } else if (operation instanceof ShowCatalogsOperation) {
        cmd = SqlCommand.SHOW_CATALOGS;
        operands = new String[0];
    } else if (operation instanceof ShowDatabasesOperation) {
        cmd = SqlCommand.SHOW_DATABASES;
        operands = new String[0];
    } else if (operation instanceof ShowTablesOperation) {
        cmd = SqlCommand.SHOW_TABLES;
        operands = new String[0];
    } else if (operation instanceof ShowFunctionsOperation) {
        cmd = SqlCommand.SHOW_FUNCTIONS;
        operands = new String[0];
    } else if (operation instanceof CreateCatalogFunctionOperation || operation instanceof CreateTempSystemFunctionOperation) {
        cmd = SqlCommand.CREATE_FUNCTION;
    } else if (operation instanceof DropCatalogFunctionOperation || operation instanceof DropTempSystemFunctionOperation) {
        cmd = SqlCommand.DROP_FUNCTION;
    } else if (operation instanceof AlterCatalogFunctionOperation) {
        cmd = SqlCommand.ALTER_FUNCTION;
    } else if (operation instanceof ExplainOperation) {
        cmd = SqlCommand.EXPLAIN;
    } else if (operation instanceof DescribeTableOperation) {
        cmd = SqlCommand.DESCRIBE;
        operands = new String[] { ((DescribeTableOperation) operation).getSqlIdentifier().asSerializableString() };
    } else if (operation instanceof QueryOperation) {
        cmd = SqlCommand.SELECT;
    } else {
        throw new Exception("Unknown operation: " + operation.asSummaryString());
    }
    return new SqlCommandCall(cmd, operands, stmt);
}