use of org.apache.flink.streaming.api.transformations.UnionTransformation in project flink by apache.
the class PythonOperatorChainingOptimizer method replaceInput.
private static void replaceInput(Transformation<?> transformation, Transformation<?> oldInput, Transformation<?> newInput) {
try {
if (transformation instanceof OneInputTransformation || transformation instanceof FeedbackTransformation || transformation instanceof SideOutputTransformation || transformation instanceof ReduceTransformation || transformation instanceof SinkTransformation || transformation instanceof LegacySinkTransformation || transformation instanceof TimestampsAndWatermarksTransformation || transformation instanceof PartitionTransformation) {
final Field inputField = transformation.getClass().getDeclaredField("input");
inputField.setAccessible(true);
inputField.set(transformation, newInput);
} else if (transformation instanceof TwoInputTransformation) {
final Field inputField;
if (((TwoInputTransformation<?, ?, ?>) transformation).getInput1() == oldInput) {
inputField = transformation.getClass().getDeclaredField("input1");
} else {
inputField = transformation.getClass().getDeclaredField("input2");
}
inputField.setAccessible(true);
inputField.set(transformation, newInput);
} else if (transformation instanceof UnionTransformation || transformation instanceof AbstractMultipleInputTransformation) {
final Field inputsField = transformation.getClass().getDeclaredField("inputs");
inputsField.setAccessible(true);
List<Transformation<?>> newInputs = Lists.newArrayList();
newInputs.addAll(transformation.getInputs());
newInputs.remove(oldInput);
newInputs.add(newInput);
inputsField.set(transformation, newInputs);
} else if (transformation instanceof AbstractBroadcastStateTransformation) {
final Field inputField;
if (((AbstractBroadcastStateTransformation<?, ?, ?>) transformation).getRegularInput() == oldInput) {
inputField = transformation.getClass().getDeclaredField("regularInput");
} else {
inputField = transformation.getClass().getDeclaredField("broadcastInput");
}
inputField.setAccessible(true);
inputField.set(transformation, newInput);
} else {
throw new RuntimeException("Unsupported transformation: " + transformation);
}
} catch (NoSuchFieldException | IllegalAccessException e) {
// This should never happen
throw new RuntimeException(e);
}
}
use of org.apache.flink.streaming.api.transformations.UnionTransformation in project flink by apache.
the class StreamExecIntervalJoin method createNegativeWindowSizeJoin.
private Transformation<RowData> createNegativeWindowSizeJoin(JoinSpec joinSpec, Transformation<RowData> leftInputTransform, Transformation<RowData> rightInputTransform, int leftArity, int rightArity, InternalTypeInfo<RowData> returnTypeInfo, ReadableConfig config) {
// We filter all records instead of adding an empty source to preserve the watermarks.
FilterAllFlatMapFunction allFilter = new FilterAllFlatMapFunction(returnTypeInfo);
OuterJoinPaddingUtil paddingUtil = new OuterJoinPaddingUtil(leftArity, rightArity);
PaddingLeftMapFunction leftPadder = new PaddingLeftMapFunction(paddingUtil, returnTypeInfo);
PaddingRightMapFunction rightPadder = new PaddingRightMapFunction(paddingUtil, returnTypeInfo);
int leftParallelism = leftInputTransform.getParallelism();
int rightParallelism = rightInputTransform.getParallelism();
OneInputTransformation<RowData, RowData> filterAllLeftStream = new OneInputTransformation<>(leftInputTransform, "FilterLeft", new StreamFlatMap<>(allFilter), returnTypeInfo, leftParallelism);
filterAllLeftStream.setUid(createTransformationUid(FILTER_LEFT_TRANSFORMATION));
filterAllLeftStream.setDescription(createFormattedTransformationDescription("filter all left input transformation", config));
filterAllLeftStream.setName(createFormattedTransformationName(filterAllLeftStream.getDescription(), "FilterLeft", config));
OneInputTransformation<RowData, RowData> filterAllRightStream = new OneInputTransformation<>(rightInputTransform, "FilterRight", new StreamFlatMap<>(allFilter), returnTypeInfo, rightParallelism);
filterAllRightStream.setUid(createTransformationUid(FILTER_RIGHT_TRANSFORMATION));
filterAllRightStream.setDescription(createFormattedTransformationDescription("filter all right input transformation", config));
filterAllRightStream.setName(createFormattedTransformationName(filterAllRightStream.getDescription(), "FilterRight", config));
OneInputTransformation<RowData, RowData> padLeftStream = new OneInputTransformation<>(leftInputTransform, "PadLeft", new StreamMap<>(leftPadder), returnTypeInfo, leftParallelism);
padLeftStream.setUid(createTransformationUid(PAD_LEFT_TRANSFORMATION));
padLeftStream.setDescription(createFormattedTransformationDescription("pad left input transformation", config));
padLeftStream.setName(createFormattedTransformationName(padLeftStream.getDescription(), "PadLeft", config));
OneInputTransformation<RowData, RowData> padRightStream = new OneInputTransformation<>(rightInputTransform, "PadRight", new StreamMap<>(rightPadder), returnTypeInfo, rightParallelism);
padRightStream.setUid(createTransformationUid(PAD_RIGHT_TRANSFORMATION));
padRightStream.setDescription(createFormattedTransformationDescription("pad right input transformation", config));
padRightStream.setName(createFormattedTransformationName(padRightStream.getDescription(), "PadRight", config));
switch(joinSpec.getJoinType()) {
case INNER:
return new UnionTransformation<>(Lists.newArrayList(filterAllLeftStream, filterAllRightStream));
case LEFT:
return new UnionTransformation<>(Lists.newArrayList(padLeftStream, filterAllRightStream));
case RIGHT:
return new UnionTransformation<>(Lists.newArrayList(filterAllLeftStream, padRightStream));
case FULL:
return new UnionTransformation<>(Lists.newArrayList(padLeftStream, padRightStream));
default:
throw new TableException("should no reach here");
}
}
use of org.apache.flink.streaming.api.transformations.UnionTransformation in project flink by apache.
the class MultipleInputITCase method testNonKeyed.
public void testNonKeyed(boolean withUnion) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
TestListResultSink<Long> resultSink = new TestListResultSink<>();
DataStream<Long> source1 = env.fromElements(1L, 10L);
DataStream<Long> source2 = env.fromElements(2L, 11L);
DataStream<String> source3 = env.fromElements("42", "44");
MultipleInputTransformation<Long> multipleInput = new MultipleInputTransformation<>("My Operator", new SumAllInputOperatorFactory(withUnion ? 2 : 3), BasicTypeInfo.LONG_TYPE_INFO, 1);
MultipleInputTransformation<Long> multipleInputTransformation;
if (withUnion) {
UnionTransformation<Long> union = new UnionTransformation<>(Arrays.asList(source1.getTransformation(), source2.getTransformation()));
multipleInputTransformation = multipleInput.addInput(union);
} else {
multipleInputTransformation = multipleInput.addInput(source1.getTransformation()).addInput(source2.getTransformation());
}
env.addOperator(multipleInputTransformation.addInput(source3.getTransformation()));
new MultipleConnectedStreams(env).transform(multipleInput).addSink(resultSink);
env.execute();
List<Long> result = resultSink.getResult();
Collections.sort(result);
long actualSum = result.get(result.size() - 1);
assertEquals(1 + 10 + 2 + 11 + 42 + 44, actualSum);
}
Aggregations