Examples with ResultPartitionType org.apache.flink.runtime.io.network.partition.ResultPartitionType used on opensource projects

Example 1 with ResultPartitionType

use of org.apache.flink.runtime.io.network.partition.ResultPartitionType in project flink by apache.

the class ResultPartitionDeploymentDescriptorTest method testSerialization.

/**
	 * Tests simple de/serialization.
	 */
@Test
public void testSerialization() throws Exception {
    // Expected values
    IntermediateDataSetID resultId = new IntermediateDataSetID();
    IntermediateResultPartitionID partitionId = new IntermediateResultPartitionID();
    ResultPartitionType partitionType = ResultPartitionType.PIPELINED;
    int numberOfSubpartitions = 24;
    ResultPartitionDeploymentDescriptor orig = new ResultPartitionDeploymentDescriptor(resultId, partitionId, partitionType, numberOfSubpartitions, numberOfSubpartitions, true);
    ResultPartitionDeploymentDescriptor copy = CommonTestUtils.createCopySerializable(orig);
    assertEquals(resultId, copy.getResultId());
    assertEquals(partitionId, copy.getPartitionId());
    assertEquals(partitionType, copy.getPartitionType());
    assertEquals(numberOfSubpartitions, copy.getNumberOfSubpartitions());
    assertTrue(copy.sendScheduleOrUpdateConsumersMessage());
}

Example 2 with ResultPartitionType

use of org.apache.flink.runtime.io.network.partition.ResultPartitionType in project flink by apache.

the class UnionClosedBranchingTest method testUnionClosedBranchingTest.

@Test
public void testUnionClosedBranchingTest() throws Exception {
    // -----------------------------------------------------------------------------------------
    // Build test program
    // -----------------------------------------------------------------------------------------
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setExecutionMode(executionMode);
    env.setParallelism(4);
    DataSet<Tuple1<Integer>> src1 = env.fromElements(new Tuple1<>(0), new Tuple1<>(1));
    DataSet<Tuple1<Integer>> src2 = env.fromElements(new Tuple1<>(0), new Tuple1<>(1));
    DataSet<Tuple1<Integer>> union = src1.union(src2);
    DataSet<Tuple2<Integer, Integer>> join = union.join(union).where(0).equalTo(0).projectFirst(0).projectSecond(0);
    join.output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());
    // -----------------------------------------------------------------------------------------
    // Verify optimized plan
    // -----------------------------------------------------------------------------------------
    OptimizedPlan optimizedPlan = compileNoStats(env.createProgramPlan());
    SinkPlanNode sinkNode = optimizedPlan.getDataSinks().iterator().next();
    DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
    // Verify that the compiler correctly sets the expected data exchange modes.
    for (Channel channel : joinNode.getInputs()) {
        assertEquals("Unexpected data exchange mode between union and join node.", unionToJoin, channel.getDataExchangeMode());
        assertEquals("Unexpected ship strategy between union and join node.", unionToJoinStrategy, channel.getShipStrategy());
    }
    for (SourcePlanNode src : optimizedPlan.getDataSources()) {
        for (Channel channel : src.getOutgoingChannels()) {
            assertEquals("Unexpected data exchange mode between source and union node.", sourceToUnion, channel.getDataExchangeMode());
            assertEquals("Unexpected ship strategy between source and union node.", sourceToUnionStrategy, channel.getShipStrategy());
        }
    }
    // -----------------------------------------------------------------------------------------
    // Verify generated JobGraph
    // -----------------------------------------------------------------------------------------
    JobGraphGenerator jgg = new JobGraphGenerator();
    JobGraph jobGraph = jgg.compileJobGraph(optimizedPlan);
    List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
    // Sanity check for the test setup
    assertEquals("Unexpected number of vertices created.", 4, vertices.size());
    // Verify all sources
    JobVertex[] sources = new JobVertex[] { vertices.get(0), vertices.get(1) };
    for (JobVertex src : sources) {
        // Sanity check
        assertTrue("Unexpected vertex type. Test setup is broken.", src.isInputVertex());
        // The union is not translated to an extra union task, but the join uses a union
        // input gate to read multiple inputs. The source create a single result per consumer.
        assertEquals("Unexpected number of created results.", 2, src.getNumberOfProducedIntermediateDataSets());
        for (IntermediateDataSet dataSet : src.getProducedDataSets()) {
            ResultPartitionType dsType = dataSet.getResultType();
            // Ensure batch exchange unless PIPELINED_FORCE is enabled.
            if (!executionMode.equals(ExecutionMode.PIPELINED_FORCED)) {
                assertTrue("Expected batch exchange, but result type is " + dsType + ".", dsType.isBlocking());
            } else {
                assertFalse("Expected non-batch exchange, but result type is " + dsType + ".", dsType.isBlocking());
            }
        }
    }
}

Example 3 with ResultPartitionType

use of org.apache.flink.runtime.io.network.partition.ResultPartitionType in project flink by apache.

the class JobGraphGenerator method connectJobVertices.

// ------------------------------------------------------------------------
// Connecting Vertices
// ------------------------------------------------------------------------
/**
 * NOTE: The channel for global and local strategies are different if we connect a union. The
 * global strategy channel is then the channel into the union node, the local strategy channel
 * the one from the union to the actual target operator.
 *
 * @throws CompilerException
 */
private DistributionPattern connectJobVertices(Channel channel, int inputNumber, final JobVertex sourceVertex, final TaskConfig sourceConfig, final JobVertex targetVertex, final TaskConfig targetConfig, boolean isBroadcast) throws CompilerException {
    // ------------ connect the vertices to the job graph --------------
    final DistributionPattern distributionPattern;
    switch(channel.getShipStrategy()) {
        case FORWARD:
            distributionPattern = DistributionPattern.POINTWISE;
            break;
        case PARTITION_RANDOM:
        case BROADCAST:
        case PARTITION_HASH:
        case PARTITION_CUSTOM:
        case PARTITION_RANGE:
        case PARTITION_FORCED_REBALANCE:
            distributionPattern = DistributionPattern.ALL_TO_ALL;
            break;
        default:
            throw new RuntimeException("Unknown runtime ship strategy: " + channel.getShipStrategy());
    }
    final ResultPartitionType resultType;
    switch(channel.getDataExchangeMode()) {
        case PIPELINED:
            resultType = ResultPartitionType.PIPELINED;
            break;
        case BATCH:
            // BLOCKING results are currently not supported in closed loop iterations
            // 
            // See https://issues.apache.org/jira/browse/FLINK-1713 for details
            resultType = channel.getSource().isOnDynamicPath() ? ResultPartitionType.PIPELINED : ResultPartitionType.BLOCKING;
            break;
        case PIPELINE_WITH_BATCH_FALLBACK:
            throw new UnsupportedOperationException("Data exchange mode " + channel.getDataExchangeMode() + " currently not supported.");
        default:
            throw new UnsupportedOperationException("Unknown data exchange mode.");
    }
    JobEdge edge = targetVertex.connectNewDataSetAsInput(sourceVertex, distributionPattern, resultType);
    // -------------- configure the source task's ship strategy strategies in task config
    // --------------
    final int outputIndex = sourceConfig.getNumOutputs();
    sourceConfig.addOutputShipStrategy(channel.getShipStrategy());
    if (outputIndex == 0) {
        sourceConfig.setOutputSerializer(channel.getSerializer());
    }
    if (channel.getShipStrategyComparator() != null) {
        sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex);
    }
    if (channel.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
        final DataDistribution dataDistribution = channel.getDataDistribution();
        if (dataDistribution != null) {
            sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex);
        } else {
            throw new RuntimeException("Range partitioning requires data distribution.");
        }
    }
    if (channel.getShipStrategy() == ShipStrategyType.PARTITION_CUSTOM) {
        if (channel.getPartitioner() != null) {
            sourceConfig.setOutputPartitioner(channel.getPartitioner(), outputIndex);
        } else {
            throw new CompilerException("The ship strategy was set to custom partitioning, but no partitioner was set.");
        }
    }
    // ---------------- configure the receiver -------------------
    if (isBroadcast) {
        targetConfig.addBroadcastInputToGroup(inputNumber);
    } else {
        targetConfig.addInputToGroup(inputNumber);
    }
    // ---------------- attach the additional infos to the job edge -------------------
    String shipStrategy = JsonMapper.getShipStrategyString(channel.getShipStrategy());
    if (channel.getShipStrategyKeys() != null && channel.getShipStrategyKeys().size() > 0) {
        shipStrategy += " on " + (channel.getShipStrategySortOrder() == null ? channel.getShipStrategyKeys().toString() : Utils.createOrdering(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()).toString());
    }
    String localStrategy;
    if (channel.getLocalStrategy() == null || channel.getLocalStrategy() == LocalStrategy.NONE) {
        localStrategy = null;
    } else {
        localStrategy = JsonMapper.getLocalStrategyString(channel.getLocalStrategy());
        if (localStrategy != null && channel.getLocalStrategyKeys() != null && channel.getLocalStrategyKeys().size() > 0) {
            localStrategy += " on " + (channel.getLocalStrategySortOrder() == null ? channel.getLocalStrategyKeys().toString() : Utils.createOrdering(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder()).toString());
        }
    }
    String caching = channel.getTempMode() == TempMode.NONE ? null : channel.getTempMode().toString();
    edge.setShipStrategyName(shipStrategy);
    edge.setBroadcast(isBroadcast);
    edge.setForward(channel.getShipStrategy() == ShipStrategyType.FORWARD);
    edge.setPreProcessingOperationName(localStrategy);
    edge.setOperatorLevelCachingDescription(caching);
    return distributionPattern;
}

Example 4 with ResultPartitionType

use of org.apache.flink.runtime.io.network.partition.ResultPartitionType in project flink by apache.

the class TaskDeploymentDescriptorFactory method createInputGateDeploymentDescriptors.

private List<InputGateDeploymentDescriptor> createInputGateDeploymentDescriptors() throws IOException {
    List<InputGateDeploymentDescriptor> inputGates = new ArrayList<>(consumedPartitionGroups.size());
    for (ConsumedPartitionGroup consumedPartitionGroup : consumedPartitionGroups) {
        // If the produced partition has multiple consumers registered, we
        // need to request the one matching our sub task index.
        // TODO Refactor after removing the consumers from the intermediate result partitions
        IntermediateResultPartition resultPartition = resultPartitionRetriever.apply(consumedPartitionGroup.getFirst());
        IntermediateResult consumedIntermediateResult = resultPartition.getIntermediateResult();
        SubpartitionIndexRange consumedSubpartitionRange = computeConsumedSubpartitionRange(resultPartition, subtaskIndex);
        IntermediateDataSetID resultId = consumedIntermediateResult.getId();
        ResultPartitionType partitionType = consumedIntermediateResult.getResultType();
        inputGates.add(new InputGateDeploymentDescriptor(resultId, partitionType, consumedSubpartitionRange, getConsumedPartitionShuffleDescriptors(consumedIntermediateResult, consumedPartitionGroup)));
    }
    return inputGates;
}

Example 5 with ResultPartitionType

use of org.apache.flink.runtime.io.network.partition.ResultPartitionType in project flink by apache.

the class SsgNetworkMemoryCalculationUtils method buildTaskInputsOutputsDescriptor.

private static TaskInputsOutputsDescriptor buildTaskInputsOutputsDescriptor(ExecutionJobVertex ejv, Function<JobVertexID, ExecutionJobVertex> ejvs) {
    Map<IntermediateDataSetID, Integer> maxInputChannelNums;
    Map<IntermediateDataSetID, Integer> maxSubpartitionNums;
    if (ejv.getGraph().isDynamic()) {
        maxInputChannelNums = getMaxInputChannelNumsForDynamicGraph(ejv);
        maxSubpartitionNums = getMaxSubpartitionNumsForDynamicGraph(ejv);
    } else {
        maxInputChannelNums = getMaxInputChannelNums(ejv);
        maxSubpartitionNums = getMaxSubpartitionNums(ejv, ejvs);
    }
    JobVertex jv = ejv.getJobVertex();
    Map<IntermediateDataSetID, ResultPartitionType> partitionTypes = getPartitionTypes(jv);
    return TaskInputsOutputsDescriptor.from(maxInputChannelNums, maxSubpartitionNums, partitionTypes);
}