use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.
the class PartitioningReusageTest method checkValidJoinInputProperties.
private void checkValidJoinInputProperties(DualInputPlanNode join) {
GlobalProperties inProps1 = join.getInput1().getGlobalProperties();
GlobalProperties inProps2 = join.getInput2().getGlobalProperties();
if (inProps1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED && inProps2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED) {
// check that both inputs are hash partitioned on the same fields
FieldList pFields1 = inProps1.getPartitioningFields();
FieldList pFields2 = inProps2.getPartitioningFields();
assertTrue("Inputs are not the same number of fields. Input 1: " + pFields1 + ", Input 2: " + pFields2, pFields1.size() == pFields2.size());
FieldList reqPFields1 = join.getKeysForInput1();
FieldList reqPFields2 = join.getKeysForInput2();
for (int i = 0; i < pFields1.size(); i++) {
// get fields
int f1 = pFields1.get(i);
int f2 = pFields2.get(i);
// check that field positions in original key field list are identical
int pos1 = getPosInFieldList(f1, reqPFields1);
int pos2 = getPosInFieldList(f2, reqPFields2);
if (pos1 < 0) {
fail("Input 1 is partitioned on field " + f1 + " which is not contained in the key set " + reqPFields1);
}
if (pos2 < 0) {
fail("Input 2 is partitioned on field " + f2 + " which is not contained in the key set " + reqPFields2);
}
if (pos1 != pos2) {
fail("Inputs are not partitioned on the same key fields");
}
}
} else if (inProps1.getPartitioning() == PartitioningProperty.FULL_REPLICATION && inProps2.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED) {
// we are good. No need to check for fields
} else if (inProps1.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED && inProps2.getPartitioning() == PartitioningProperty.FULL_REPLICATION) {
// we are good. No need to check for fields
} else {
throw new UnsupportedOperationException("This method has only been implemented to check for hash partitioned coGroupinputs");
}
}
use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.
the class RangePartitionRewriter method rewriteRangePartitionChannel.
private List<Channel> rewriteRangePartitionChannel(Channel channel) {
final List<Channel> sourceNewOutputChannels = new ArrayList<>();
final PlanNode sourceNode = channel.getSource();
final PlanNode targetNode = channel.getTarget();
final int sourceParallelism = sourceNode.getParallelism();
final int targetParallelism = targetNode.getParallelism();
final Costs defaultZeroCosts = new Costs(0, 0, 0);
final TypeComparatorFactory<?> comparator = Utils.getShipComparator(channel, this.plan.getOriginalPlan().getExecutionConfig());
// 1. Fixed size sample in each partitions.
final int sampleSize = SAMPLES_PER_PARTITION * targetParallelism;
final SampleInPartition sampleInPartition = new SampleInPartition(false, sampleSize, SEED);
final TypeInformation<?> sourceOutputType = sourceNode.getOptimizerNode().getOperator().getOperatorInfo().getOutputType();
final TypeInformation<IntermediateSampleData> isdTypeInformation = TypeExtractor.getForClass(IntermediateSampleData.class);
final UnaryOperatorInformation sipOperatorInformation = new UnaryOperatorInformation(sourceOutputType, isdTypeInformation);
final MapPartitionOperatorBase sipOperatorBase = new MapPartitionOperatorBase(sampleInPartition, sipOperatorInformation, SIP_NAME);
final MapPartitionNode sipNode = new MapPartitionNode(sipOperatorBase);
final Channel sipChannel = new Channel(sourceNode, TempMode.NONE);
sipChannel.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
final SingleInputPlanNode sipPlanNode = new SingleInputPlanNode(sipNode, SIP_NAME, sipChannel, DriverStrategy.MAP_PARTITION);
sipNode.setParallelism(sourceParallelism);
sipPlanNode.setParallelism(sourceParallelism);
sipPlanNode.initProperties(new GlobalProperties(), new LocalProperties());
sipPlanNode.setCosts(defaultZeroCosts);
sipChannel.setTarget(sipPlanNode);
this.plan.getAllNodes().add(sipPlanNode);
sourceNewOutputChannels.add(sipChannel);
// 2. Fixed size sample in a single coordinator.
final SampleInCoordinator sampleInCoordinator = new SampleInCoordinator(false, sampleSize, SEED);
final UnaryOperatorInformation sicOperatorInformation = new UnaryOperatorInformation(isdTypeInformation, sourceOutputType);
final GroupReduceOperatorBase sicOperatorBase = new GroupReduceOperatorBase(sampleInCoordinator, sicOperatorInformation, SIC_NAME);
final GroupReduceNode sicNode = new GroupReduceNode(sicOperatorBase);
final Channel sicChannel = new Channel(sipPlanNode, TempMode.NONE);
sicChannel.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
final SingleInputPlanNode sicPlanNode = new SingleInputPlanNode(sicNode, SIC_NAME, sicChannel, DriverStrategy.ALL_GROUP_REDUCE);
sicNode.setParallelism(1);
sicPlanNode.setParallelism(1);
sicPlanNode.initProperties(new GlobalProperties(), new LocalProperties());
sicPlanNode.setCosts(defaultZeroCosts);
sicChannel.setTarget(sicPlanNode);
sipPlanNode.addOutgoingChannel(sicChannel);
this.plan.getAllNodes().add(sicPlanNode);
// 3. Use sampled data to build range boundaries.
final RangeBoundaryBuilder rangeBoundaryBuilder = new RangeBoundaryBuilder(comparator, targetParallelism);
final TypeInformation<CommonRangeBoundaries> rbTypeInformation = TypeExtractor.getForClass(CommonRangeBoundaries.class);
final UnaryOperatorInformation rbOperatorInformation = new UnaryOperatorInformation(sourceOutputType, rbTypeInformation);
final MapPartitionOperatorBase rbOperatorBase = new MapPartitionOperatorBase(rangeBoundaryBuilder, rbOperatorInformation, RB_NAME);
final MapPartitionNode rbNode = new MapPartitionNode(rbOperatorBase);
final Channel rbChannel = new Channel(sicPlanNode, TempMode.NONE);
rbChannel.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
final SingleInputPlanNode rbPlanNode = new SingleInputPlanNode(rbNode, RB_NAME, rbChannel, DriverStrategy.MAP_PARTITION);
rbNode.setParallelism(1);
rbPlanNode.setParallelism(1);
rbPlanNode.initProperties(new GlobalProperties(), new LocalProperties());
rbPlanNode.setCosts(defaultZeroCosts);
rbChannel.setTarget(rbPlanNode);
sicPlanNode.addOutgoingChannel(rbChannel);
this.plan.getAllNodes().add(rbPlanNode);
// 4. Take range boundaries as broadcast input and take the tuple of partition id and record
// as output.
final AssignRangeIndex assignRangeIndex = new AssignRangeIndex(comparator);
final TypeInformation<Tuple2> ariOutputTypeInformation = new TupleTypeInfo<>(BasicTypeInfo.INT_TYPE_INFO, sourceOutputType);
final UnaryOperatorInformation ariOperatorInformation = new UnaryOperatorInformation(sourceOutputType, ariOutputTypeInformation);
final MapPartitionOperatorBase ariOperatorBase = new MapPartitionOperatorBase(assignRangeIndex, ariOperatorInformation, ARI_NAME);
final MapPartitionNode ariNode = new MapPartitionNode(ariOperatorBase);
final Channel ariChannel = new Channel(sourceNode, TempMode.NONE);
// To avoid deadlock, set the DataExchangeMode of channel between source node and this to
// Batch.
ariChannel.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.BATCH);
final SingleInputPlanNode ariPlanNode = new SingleInputPlanNode(ariNode, ARI_NAME, ariChannel, DriverStrategy.MAP_PARTITION);
ariNode.setParallelism(sourceParallelism);
ariPlanNode.setParallelism(sourceParallelism);
ariPlanNode.initProperties(new GlobalProperties(), new LocalProperties());
ariPlanNode.setCosts(defaultZeroCosts);
ariChannel.setTarget(ariPlanNode);
this.plan.getAllNodes().add(ariPlanNode);
sourceNewOutputChannels.add(ariChannel);
final NamedChannel broadcastChannel = new NamedChannel("RangeBoundaries", rbPlanNode);
broadcastChannel.setShipStrategy(ShipStrategyType.BROADCAST, DataExchangeMode.PIPELINED);
broadcastChannel.setTarget(ariPlanNode);
List<NamedChannel> broadcastChannels = new ArrayList<>(1);
broadcastChannels.add(broadcastChannel);
ariPlanNode.setBroadcastInputs(broadcastChannels);
// 5. Remove the partition id.
final Channel partChannel = new Channel(ariPlanNode, TempMode.NONE);
final FieldList keys = new FieldList(0);
partChannel.setShipStrategy(ShipStrategyType.PARTITION_CUSTOM, keys, idPartitioner, DataExchangeMode.PIPELINED);
ariPlanNode.addOutgoingChannel(partChannel);
final RemoveRangeIndex partitionIDRemoveWrapper = new RemoveRangeIndex();
final UnaryOperatorInformation prOperatorInformation = new UnaryOperatorInformation(ariOutputTypeInformation, sourceOutputType);
final MapOperatorBase prOperatorBase = new MapOperatorBase(partitionIDRemoveWrapper, prOperatorInformation, PR_NAME);
final MapNode prRemoverNode = new MapNode(prOperatorBase);
final SingleInputPlanNode prPlanNode = new SingleInputPlanNode(prRemoverNode, PR_NAME, partChannel, DriverStrategy.MAP);
partChannel.setTarget(prPlanNode);
prRemoverNode.setParallelism(targetParallelism);
prPlanNode.setParallelism(targetParallelism);
GlobalProperties globalProperties = new GlobalProperties();
globalProperties.setRangePartitioned(new Ordering(0, null, Order.ASCENDING));
prPlanNode.initProperties(globalProperties, new LocalProperties());
prPlanNode.setCosts(defaultZeroCosts);
this.plan.getAllNodes().add(prPlanNode);
// 6. Connect to target node.
channel.setSource(prPlanNode);
channel.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
prPlanNode.addOutgoingChannel(channel);
return sourceNewOutputChannels;
}
use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.
the class FeedbackPropertiesMatchTest method testSingleInputOperators.
@Test
public void testSingleInputOperators() {
try {
SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
Channel toMap1 = new Channel(target);
toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap1.setLocalStrategy(LocalStrategy.NONE);
SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
Channel toMap2 = new Channel(map1);
toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap2.setLocalStrategy(LocalStrategy.NONE);
SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
// no feedback properties and none are ever required and present
{
GlobalProperties gp = new GlobalProperties();
LocalProperties lp = new LocalProperties();
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some global feedback properties and none are ever required and present
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(2, 5));
LocalProperties lp = new LocalProperties();
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some local feedback properties and none are ever required and present
{
GlobalProperties gp = new GlobalProperties();
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some global and local feedback properties and none are ever required and present
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(2, 5));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// --------------------------- requirements on channel 1 -----------------------
// some required global properties, which are matched exactly
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(2, 5));
LocalProperties lp = new LocalProperties();
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setHashPartitioned(new FieldList(2, 5));
toMap1.setRequiredGlobalProps(reqGp);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some required local properties, which are matched exactly
{
GlobalProperties gp = new GlobalProperties();
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(1, 2));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(reqLp);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some required global and local properties, which are matched exactly
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(2, 5));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setHashPartitioned(new FieldList(2, 5));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(1, 2));
toMap1.setRequiredGlobalProps(reqGp);
toMap1.setRequiredLocalProps(reqLp);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some required global and local properties, which are over-fulfilled
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(2));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setHashPartitioned(new FieldSet(2, 5));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(1));
toMap1.setRequiredGlobalProps(reqGp);
toMap1.setRequiredLocalProps(reqLp);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some required global properties that are not met
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(2, 1));
LocalProperties lp = new LocalProperties();
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setHashPartitioned(new FieldList(2, 5));
toMap1.setRequiredGlobalProps(reqGp);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
// some required local properties that are not met
{
GlobalProperties gp = new GlobalProperties();
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(2, 1));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(reqLp);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
// some required global and local properties where the global properties are not met
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(2, 1));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setAnyPartitioning(new FieldList(2, 5));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(1));
toMap1.setRequiredGlobalProps(reqGp);
toMap1.setRequiredLocalProps(reqLp);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
// some required global and local properties where the local properties are not met
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setAnyPartitioning(new FieldList(1));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(2));
toMap1.setRequiredGlobalProps(reqGp);
toMap1.setRequiredLocalProps(reqLp);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
// --------------------------- requirements on channel 2 -----------------------
// some required global properties, which are matched exactly
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(2, 5));
LocalProperties lp = new LocalProperties();
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setHashPartitioned(new FieldList(2, 5));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(reqGp);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some required local properties, which are matched exactly
{
GlobalProperties gp = new GlobalProperties();
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(1, 2));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(reqLp);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some required global and local properties, which are matched exactly
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(2, 5));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setHashPartitioned(new FieldList(2, 5));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(1, 2));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(reqGp);
toMap2.setRequiredLocalProps(reqLp);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some required global and local properties, which are over-fulfilled
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(2));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setHashPartitioned(new FieldSet(2, 5));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(1));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(reqGp);
toMap2.setRequiredLocalProps(reqLp);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some required global properties that are not met
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(2, 1));
LocalProperties lp = new LocalProperties();
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setHashPartitioned(new FieldSet(2, 5));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(reqGp);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
// some required local properties that are not met
{
GlobalProperties gp = new GlobalProperties();
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(2, 1));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(reqLp);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
// some required global and local properties where the global properties are not met
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(2, 1));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setAnyPartitioning(new FieldSet(2, 5));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(1));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(reqGp);
toMap2.setRequiredLocalProps(reqLp);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
// some required global and local properties where the local properties are not met
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setAnyPartitioning(new FieldList(1));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(2));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(reqGp);
toMap2.setRequiredLocalProps(reqLp);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
// ---------------------- requirements mixed on 1 and 2 -----------------------
// some required global properties at step one and some more at step 2
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1, 2));
LocalProperties lp = LocalProperties.EMPTY;
RequestedGlobalProperties reqGp1 = new RequestedGlobalProperties();
reqGp1.setAnyPartitioning(new FieldList(1, 2));
RequestedGlobalProperties reqGp2 = new RequestedGlobalProperties();
reqGp2.setHashPartitioned(new FieldList(1, 2));
toMap1.setRequiredGlobalProps(reqGp1);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(reqGp2);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some required local properties at step one and some more at step 2
{
GlobalProperties gp = new GlobalProperties();
LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
RequestedLocalProperties reqLp1 = new RequestedLocalProperties();
reqLp1.setGroupedFields(new FieldList(3, 1));
RequestedLocalProperties reqLp2 = new RequestedLocalProperties();
reqLp2.setOrdering(new Ordering(3, null, Order.ANY).appendOrdering(1, null, Order.ANY));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(reqLp1);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(reqLp2);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some required global properties at step one and some local ones at step 2
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1, 2));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setAnyPartitioning(new FieldList(1, 2));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(2));
toMap1.setRequiredGlobalProps(reqGp);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(reqLp);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some required local properties at step one and some global ones at step 2
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1, 2));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setAnyPartitioning(new FieldList(1, 2));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(2));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(reqLp);
toMap2.setRequiredGlobalProps(reqGp);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// some fulfilled global properties at step one and some non-fulfilled local ones at
// step 2
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1, 2));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setAnyPartitioning(new FieldList(1, 2));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(2, 3));
toMap1.setRequiredGlobalProps(reqGp);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(reqLp);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
// some fulfilled local properties at step one and some non-fulfilled global ones at
// step 2
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1, 2));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setAnyPartitioning(new FieldList(2, 3));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(2, 1));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(reqLp);
toMap2.setRequiredGlobalProps(reqGp);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
// some non-fulfilled global properties at step one and some fulfilled local ones at
// step 2
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1, 2));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setAnyPartitioning(new FieldList(2, 3));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(2, 1));
toMap1.setRequiredGlobalProps(reqGp);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(reqLp);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
// some non-fulfilled local properties at step one and some fulfilled global ones at
// step 2
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1, 2));
LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setAnyPartitioning(new FieldList(1, 2));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(2, 1, 3));
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(reqLp);
toMap2.setRequiredGlobalProps(reqGp);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.
the class FeedbackPropertiesMatchTest method testSingleInputOperatorsWithReCreation.
@Test
public void testSingleInputOperatorsWithReCreation() {
try {
SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
Channel toMap1 = new Channel(target);
SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
Channel toMap2 = new Channel(map1);
SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
// set ship strategy in first channel, so later non matching global properties do not
// matter
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1, 2));
LocalProperties lp = LocalProperties.EMPTY;
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setAnyPartitioning(new FieldSet(2, 5));
toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 5), DataExchangeMode.PIPELINED);
toMap1.setLocalStrategy(LocalStrategy.NONE);
toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap2.setLocalStrategy(LocalStrategy.NONE);
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(reqGp);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(MET, report);
}
// set ship strategy in second channel, so previous non matching global properties void
// the match
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1, 2));
LocalProperties lp = LocalProperties.EMPTY;
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setAnyPartitioning(new FieldSet(2, 5));
toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap1.setLocalStrategy(LocalStrategy.NONE);
toMap2.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 5), DataExchangeMode.PIPELINED);
toMap2.setLocalStrategy(LocalStrategy.NONE);
toMap1.setRequiredGlobalProps(reqGp);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
// set local strategy in first channel, so later non matching local properties do not
// matter
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1, 2));
LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(4, 1));
toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] { false, false });
toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap2.setLocalStrategy(LocalStrategy.NONE);
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(null);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(reqLp);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
}
// set local strategy in second channel, so previous non matching local properties void
// the match
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1, 2));
LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(4, 1));
toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap1.setLocalStrategy(LocalStrategy.NONE);
toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap2.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] { false, false });
toMap1.setRequiredGlobalProps(null);
toMap1.setRequiredLocalProps(reqLp);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(NOT_MET, report);
}
// create the properties on the same node as the requirement
{
GlobalProperties gp = new GlobalProperties();
gp.setHashPartitioned(new FieldList(1, 2));
LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
reqGp.setAnyPartitioning(new FieldSet(5, 7));
RequestedLocalProperties reqLp = new RequestedLocalProperties();
reqLp.setGroupedFields(new FieldList(5, 7));
toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(5, 7), DataExchangeMode.PIPELINED);
toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] { false, false });
toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap2.setLocalStrategy(LocalStrategy.NONE);
toMap1.setRequiredGlobalProps(reqGp);
toMap1.setRequiredLocalProps(reqLp);
toMap2.setRequiredGlobalProps(null);
toMap2.setRequiredLocalProps(null);
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
assertEquals(MET, report);
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.
the class WordCountCompilerTest method checkWordCount.
private void checkWordCount(boolean estimates) {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(DEFAULT_PARALLELISM);
// get input data
DataSet<String> lines = env.readTextFile(IN_FILE).name("Input Lines");
lines.map(new MapFunction<String, Tuple2<String, Integer>>() {
private static final long serialVersionUID = -3952739820618875030L;
@Override
public Tuple2<String, Integer> map(String v) throws Exception {
return new Tuple2<>(v, 1);
}
}).name("Tokenize Lines").groupBy(0).sum(1).name("Count Words").output(new DiscardingOutputFormat<Tuple2<String, Integer>>()).name("Word Counts");
// get the plan and compile it
Plan p = env.createProgramPlan();
p.setExecutionConfig(new ExecutionConfig());
OptimizedPlan plan;
if (estimates) {
GenericDataSourceBase<?, ?> source = getContractResolver(p).getNode("Input Lines");
setSourceStatistics(source, 1024 * 1024 * 1024 * 1024L, 24f);
plan = compileWithStats(p);
} else {
plan = compileNoStats(p);
}
// get the optimizer plan nodes
OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(plan);
SinkPlanNode sink = resolver.getNode("Word Counts");
SingleInputPlanNode reducer = resolver.getNode("Count Words");
SingleInputPlanNode mapper = resolver.getNode("Tokenize Lines");
// verify the strategies
Assert.assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy());
Assert.assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy());
Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
Channel c = reducer.getInput();
Assert.assertEquals(LocalStrategy.COMBININGSORT, c.getLocalStrategy());
FieldList l = new FieldList(0);
Assert.assertEquals(l, c.getShipStrategyKeys());
Assert.assertEquals(l, c.getLocalStrategyKeys());
Assert.assertTrue(Arrays.equals(c.getLocalStrategySortOrder(), reducer.getSortOrders(0)));
// check the combiner
SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getPredecessor();
Assert.assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy());
Assert.assertEquals(l, combiner.getKeys(0));
Assert.assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
}
Aggregations