Examples with ExecEdge org.apache.flink.table.planner.plan.nodes.exec.ExecEdge used on opensource projects

Example 16 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class BatchExecPythonGroupAggregate method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputRowType = (RowType) inputEdge.getOutputType();
    final RowType outputRowType = InternalTypeInfo.of(getOutputType()).toRowType();
    Configuration pythonConfig = CommonPythonUtil.getMergedConfig(planner.getExecEnv(), config.getTableConfig());
    OneInputTransformation<RowData, RowData> transform = createPythonOneInputTransformation(inputTransform, inputRowType, outputRowType, pythonConfig, config);
    if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(pythonConfig)) {
        transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
    }
    return transform;
}

Example 17 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class BatchExecPythonOverAggregate method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputType = (RowType) inputEdge.getOutputType();
    List<OverSpec.GroupSpec> groups = overSpec.getGroups();
    boolean[] isRangeWindows = new boolean[groups.size()];
    for (int i = 0; i < groups.size(); i++) {
        OverSpec.GroupSpec group = groups.get(i);
        List<AggregateCall> groupAggCalls = group.getAggCalls();
        aggCalls.addAll(groupAggCalls);
        for (int j = 0; j < groupAggCalls.size(); j++) {
            aggWindowIndex.add(i);
        }
        OverWindowMode mode = inferGroupMode(group);
        if (mode == OverWindowMode.ROW) {
            isRangeWindows[i] = false;
            if (isUnboundedWindow(group)) {
                lowerBoundary.add(Long.MIN_VALUE);
                upperBoundary.add(Long.MAX_VALUE);
            } else if (isUnboundedPrecedingWindow(group)) {
                lowerBoundary.add(Long.MIN_VALUE);
                upperBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getUpperBound()));
            } else if (isUnboundedFollowingWindow(group)) {
                lowerBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getLowerBound()));
                upperBoundary.add(Long.MAX_VALUE);
            } else if (isSlidingWindow(group)) {
                lowerBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getLowerBound()));
                upperBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getUpperBound()));
            } else {
                throw new TableException("Unsupported row window group spec " + group);
            }
        } else {
            isRangeWindows[i] = true;
            if (isUnboundedWindow(group)) {
                lowerBoundary.add(Long.MIN_VALUE);
                upperBoundary.add(Long.MAX_VALUE);
            } else if (isUnboundedPrecedingWindow(group)) {
                lowerBoundary.add(Long.MIN_VALUE);
                upperBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getUpperBound()));
            } else if (isUnboundedFollowingWindow(group)) {
                lowerBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getLowerBound()));
                upperBoundary.add(Long.MAX_VALUE);
            } else if (isSlidingWindow(group)) {
                lowerBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getLowerBound()));
                upperBoundary.add(OverAggregateUtil.getLongBoundary(overSpec, group.getUpperBound()));
            } else {
                throw new TableException("Unsupported range window group spec " + group);
            }
        }
    }
    Configuration pythonConfig = CommonPythonUtil.getMergedConfig(planner.getExecEnv(), config.getTableConfig());
    OneInputTransformation<RowData, RowData> transform = createPythonOneInputTransformation(inputTransform, inputType, InternalTypeInfo.of(getOutputType()).toRowType(), isRangeWindows, pythonConfig, config);
    if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(pythonConfig)) {
        transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
    }
    return transform;
}

Example 18 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class CommonExecLegacySink method translateToTransformation.

/**
 * Translates {@link TableSink} into a {@link Transformation}.
 *
 * @param withChangeFlag Set to true to emit records with change flags.
 * @return The {@link Transformation} that corresponds to the translated {@link TableSink}.
 */
@SuppressWarnings("unchecked")
private Transformation<T> translateToTransformation(PlannerBase planner, ExecNodeConfig config, boolean withChangeFlag) {
    // if no change flags are requested, verify table is an insert-only (append-only) table.
    if (!withChangeFlag && needRetraction) {
        throw new TableException("Table is not an append-only table. " + "Use the toRetractStream() in order to handle add and retract messages.");
    }
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputRowType = (RowType) inputEdge.getOutputType();
    final RowType convertedInputRowType = checkAndConvertInputTypeIfNeeded(inputRowType);
    final DataType resultDataType = tableSink.getConsumedDataType();
    if (CodeGenUtils.isInternalClass(resultDataType)) {
        return (Transformation<T>) inputTransform;
    } else {
        final int rowtimeIndex = getRowtimeIndex(inputRowType);
        final DataType physicalOutputType = TableSinkUtils.inferSinkPhysicalDataType(resultDataType, convertedInputRowType, withChangeFlag);
        final TypeInformation<T> outputTypeInfo = SinkCodeGenerator.deriveSinkOutputTypeInfo(tableSink, physicalOutputType, withChangeFlag);
        final CodeGenOperatorFactory<T> converterOperator = SinkCodeGenerator.generateRowConverterOperator(new CodeGeneratorContext(config.getTableConfig()), convertedInputRowType, tableSink, physicalOutputType, withChangeFlag, "SinkConversion", rowtimeIndex);
        final String description = "SinkConversion To " + resultDataType.getConversionClass().getSimpleName();
        return ExecNodeUtil.createOneInputTransformation(inputTransform, createFormattedTransformationName(description, "SinkConversion", config), createFormattedTransformationDescription(description, config), converterOperator, outputTypeInfo, inputTransform.getParallelism());
    }
}

Example 19 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class CommonExecPythonCorrelate method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final Configuration pythonConfig = CommonPythonUtil.getMergedConfig(planner.getExecEnv(), config.getTableConfig());
    OneInputTransformation<RowData, RowData> transform = createPythonOneInputTransformation(inputTransform, config, pythonConfig);
    if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(pythonConfig)) {
        transform.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
    }
    return transform;
}

Example 20 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class BatchExecMultipleInput method translateToPlanInternal.

@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final List<Transformation<?>> inputTransforms = new ArrayList<>();
    for (ExecEdge inputEdge : getInputEdges()) {
        inputTransforms.add(inputEdge.translateToPlan(planner));
    }
    final Transformation<?> outputTransform = rootNode.translateToPlan(planner);
    final int[] readOrders = getInputProperties().stream().map(InputProperty::getPriority).mapToInt(i -> i).toArray();
    final TableOperatorWrapperGenerator generator = new TableOperatorWrapperGenerator(inputTransforms, outputTransform, readOrders);
    generator.generate();
    final List<Pair<Transformation<?>, InputSpec>> inputTransformAndInputSpecPairs = generator.getInputTransformAndInputSpecPairs();
    final MultipleInputTransformation<RowData> multipleInputTransform = new MultipleInputTransformation<>(createTransformationName(config), new BatchMultipleInputStreamOperatorFactory(inputTransformAndInputSpecPairs.stream().map(Pair::getValue).collect(Collectors.toList()), generator.getHeadWrappers(), generator.getTailWrapper()), InternalTypeInfo.of(getOutputType()), generator.getParallelism());
    multipleInputTransform.setDescription(createTransformationDescription(config));
    inputTransformAndInputSpecPairs.forEach(input -> multipleInputTransform.addInput(input.getKey()));
    if (generator.getMaxParallelism() > 0) {
        multipleInputTransform.setMaxParallelism(generator.getMaxParallelism());
    }
    // set resources
    multipleInputTransform.setResources(generator.getMinResources(), generator.getPreferredResources());
    final int memoryWeight = generator.getManagedMemoryWeight();
    final long memoryBytes = (long) memoryWeight << 20;
    ExecNodeUtil.setManagedMemoryWeight(multipleInputTransform, memoryBytes);
    // set chaining strategy for source chaining
    multipleInputTransform.setChainingStrategy(ChainingStrategy.HEAD_WITH_SOURCES);
    return multipleInputTransform;
}