use of org.apache.flink.table.operations.QueryOperation in project flink by apache.
the class OperationTreeBuilder method windowAggregate.
public QueryOperation windowAggregate(List<Expression> groupingExpressions, GroupWindow window, List<Expression> windowProperties, Expression aggregateFunction, QueryOperation child) {
ExpressionResolver resolver = getAggResolver(child, groupingExpressions);
Expression resolvedAggregate = aggregateFunction.accept(lookupResolver);
AggregateWithAlias aggregateWithAlias = resolvedAggregate.accept(new ExtractAliasAndAggregate(true, resolver));
List<Expression> groupsAndAggregate = new ArrayList<>(groupingExpressions);
groupsAndAggregate.add(aggregateWithAlias.aggregate);
List<Expression> namedGroupsAndAggregate = addAliasToTheCallInAggregate(child.getResolvedSchema().getColumnNames(), groupsAndAggregate);
// Step1: add a default name to the call in the grouping expressions, e.g., groupBy(a % 5)
// to
// groupBy(a % 5 as TMP_0). We need a name for every column so that to perform alias for the
// table aggregate function in Step6.
List<Expression> newGroupingExpressions = namedGroupsAndAggregate.subList(0, groupingExpressions.size());
// Step2: turn agg to a named agg, because it will be verified later.
Expression aggregateRenamed = namedGroupsAndAggregate.get(groupingExpressions.size());
// Step3: resolve expressions, including grouping, aggregates and window properties.
ResolvedGroupWindow resolvedWindow = aggregateOperationFactory.createResolvedWindow(window, resolver);
ExpressionResolver resolverWithWindowReferences = getResolverBuilder(child).withLocalReferences(localRef(resolvedWindow.getAlias(), resolvedWindow.getTimeAttribute().getOutputDataType())).build();
List<ResolvedExpression> convertedGroupings = resolverWithWindowReferences.resolve(newGroupingExpressions);
List<ResolvedExpression> convertedAggregates = resolverWithWindowReferences.resolve(Collections.singletonList(aggregateRenamed));
List<ResolvedExpression> convertedProperties = resolverWithWindowReferences.resolve(windowProperties);
// Step4: create window agg operation
QueryOperation aggregateOperation = aggregateOperationFactory.createWindowAggregate(convertedGroupings, Collections.singletonList(convertedAggregates.get(0)), convertedProperties, resolvedWindow, child);
// Step5: flatten the aggregate function
List<String> aggNames = aggregateOperation.getResolvedSchema().getColumnNames();
List<Expression> flattenedExpressions = aggNames.stream().map(ApiExpressionUtils::unresolvedRef).collect(Collectors.toCollection(ArrayList::new));
flattenedExpressions.set(groupingExpressions.size(), unresolvedCall(BuiltInFunctionDefinitions.FLATTEN, unresolvedRef(aggNames.get(groupingExpressions.size()))));
QueryOperation flattenedProjection = this.project(flattenedExpressions, aggregateOperation);
// window attribute.
return aliasBackwardFields(flattenedProjection, aggregateWithAlias.aliases, groupingExpressions.size());
}
use of org.apache.flink.table.operations.QueryOperation in project flink by apache.
the class OperationTreeBuilder method tableAggregate.
public QueryOperation tableAggregate(List<Expression> groupingExpressions, Expression tableAggFunction, QueryOperation child) {
// Step1: add a default name to the call in the grouping expressions, e.g., groupBy(a % 5)
// to
// groupBy(a % 5 as TMP_0). We need a name for every column so that to perform alias for the
// table aggregate function in Step4.
List<Expression> newGroupingExpressions = addAliasToTheCallInAggregate(child.getResolvedSchema().getColumnNames(), groupingExpressions);
// Step2: resolve expressions
ExpressionResolver resolver = getAggResolver(child, groupingExpressions);
List<ResolvedExpression> resolvedGroupings = resolver.resolve(newGroupingExpressions);
Tuple2<ResolvedExpression, List<String>> resolvedFunctionAndAlias = aggregateOperationFactory.extractTableAggFunctionAndAliases(resolveSingleExpression(tableAggFunction, resolver));
// Step3: create table agg operation
QueryOperation tableAggOperation = aggregateOperationFactory.createAggregate(resolvedGroupings, Collections.singletonList(resolvedFunctionAndAlias.f0), child);
// Step4: add a top project to alias the output fields of the table aggregate.
return aliasBackwardFields(tableAggOperation, resolvedFunctionAndAlias.f1, groupingExpressions.size());
}
use of org.apache.flink.table.operations.QueryOperation in project flink by apache.
the class AbstractStreamTableEnvironmentImpl method toStreamInternal.
protected <T> DataStream<T> toStreamInternal(Table table, SchemaTranslator.ProducingResult schemaTranslationResult, @Nullable ChangelogMode changelogMode) {
final CatalogManager catalogManager = getCatalogManager();
final OperationTreeBuilder operationTreeBuilder = getOperationTreeBuilder();
final QueryOperation projectOperation = schemaTranslationResult.getProjections().map(projections -> operationTreeBuilder.project(projections.stream().map(ApiExpressionUtils::unresolvedRef).collect(Collectors.toList()), table.getQueryOperation())).orElseGet(table::getQueryOperation);
final ResolvedCatalogTable resolvedCatalogTable = catalogManager.resolveCatalogTable(new ExternalCatalogTable(schemaTranslationResult.getSchema()));
final ExternalModifyOperation modifyOperation = new ExternalModifyOperation(ContextResolvedTable.anonymous("datastream_sink", resolvedCatalogTable), projectOperation, changelogMode, schemaTranslationResult.getPhysicalDataType().orElseGet(() -> resolvedCatalogTable.getResolvedSchema().toPhysicalRowDataType()));
return toStreamInternal(table, modifyOperation);
}
use of org.apache.flink.table.operations.QueryOperation in project flink by apache.
the class AbstractStreamTableEnvironmentImpl method fromStreamInternal.
protected <T> Table fromStreamInternal(DataStream<T> dataStream, @Nullable Schema schema, @Nullable String viewPath, ChangelogMode changelogMode) {
Preconditions.checkNotNull(dataStream, "Data stream must not be null.");
Preconditions.checkNotNull(changelogMode, "Changelog mode must not be null.");
if (dataStream.getExecutionEnvironment() != executionEnvironment) {
throw new ValidationException("The DataStream's StreamExecutionEnvironment must be identical to the one that " + "has been passed to the StreamTableEnvironment during instantiation.");
}
final CatalogManager catalogManager = getCatalogManager();
final OperationTreeBuilder operationTreeBuilder = getOperationTreeBuilder();
final SchemaTranslator.ConsumingResult schemaTranslationResult = SchemaTranslator.createConsumingResult(catalogManager.getDataTypeFactory(), dataStream.getType(), schema);
final ResolvedCatalogTable resolvedCatalogTable = catalogManager.resolveCatalogTable(new ExternalCatalogTable(schemaTranslationResult.getSchema()));
final ContextResolvedTable contextResolvedTable;
if (viewPath != null) {
UnresolvedIdentifier unresolvedIdentifier = getParser().parseIdentifier(viewPath);
final ObjectIdentifier objectIdentifier = catalogManager.qualifyIdentifier(unresolvedIdentifier);
contextResolvedTable = ContextResolvedTable.temporary(objectIdentifier, resolvedCatalogTable);
} else {
contextResolvedTable = ContextResolvedTable.anonymous("datastream_source", resolvedCatalogTable);
}
final QueryOperation scanOperation = new ExternalQueryOperation<>(contextResolvedTable, dataStream, schemaTranslationResult.getPhysicalDataType(), schemaTranslationResult.isTopLevelRecord(), changelogMode);
final List<String> projections = schemaTranslationResult.getProjections();
if (projections == null) {
return createTable(scanOperation);
}
final QueryOperation projectOperation = operationTreeBuilder.project(projections.stream().map(ApiExpressionUtils::unresolvedRef).collect(Collectors.toList()), scanOperation);
return createTable(projectOperation);
}
use of org.apache.flink.table.operations.QueryOperation in project flink by apache.
the class TableEnvironmentImpl method sqlQuery.
@Override
public Table sqlQuery(String query) {
List<Operation> operations = getParser().parse(query);
if (operations.size() != 1) {
throw new ValidationException("Unsupported SQL query! sqlQuery() only accepts a single SQL query.");
}
Operation operation = operations.get(0);
if (operation instanceof QueryOperation && !(operation instanceof ModifyOperation)) {
return createTable((QueryOperation) operation);
} else {
throw new ValidationException("Unsupported SQL query! sqlQuery() only accepts a single SQL query of type " + "SELECT, UNION, INTERSECT, EXCEPT, VALUES, and ORDER_BY.");
}
}
Aggregations