Examples with JobEdge org.apache.flink.runtime.jobgraph.JobEdge used on opensource projects

Example 1 with JobEdge

use of org.apache.flink.runtime.jobgraph.JobEdge in project flink by apache.

the class JobGraphGenerator method connectJobVertices.

// ------------------------------------------------------------------------
// Connecting Vertices
// ------------------------------------------------------------------------
/**
	 * NOTE: The channel for global and local strategies are different if we connect a union. The global strategy
	 * channel is then the channel into the union node, the local strategy channel the one from the union to the
	 * actual target operator.
	 *
	 * @throws CompilerException
	 */
private DistributionPattern connectJobVertices(Channel channel, int inputNumber, final JobVertex sourceVertex, final TaskConfig sourceConfig, final JobVertex targetVertex, final TaskConfig targetConfig, boolean isBroadcast) throws CompilerException {
    // ------------ connect the vertices to the job graph --------------
    final DistributionPattern distributionPattern;
    switch(channel.getShipStrategy()) {
        case FORWARD:
            distributionPattern = DistributionPattern.POINTWISE;
            break;
        case PARTITION_RANDOM:
        case BROADCAST:
        case PARTITION_HASH:
        case PARTITION_CUSTOM:
        case PARTITION_RANGE:
        case PARTITION_FORCED_REBALANCE:
            distributionPattern = DistributionPattern.ALL_TO_ALL;
            break;
        default:
            throw new RuntimeException("Unknown runtime ship strategy: " + channel.getShipStrategy());
    }
    final ResultPartitionType resultType;
    switch(channel.getDataExchangeMode()) {
        case PIPELINED:
            resultType = ResultPartitionType.PIPELINED;
            break;
        case BATCH:
            // BLOCKING results are currently not supported in closed loop iterations
            //
            // See https://issues.apache.org/jira/browse/FLINK-1713 for details
            resultType = channel.getSource().isOnDynamicPath() ? ResultPartitionType.PIPELINED : ResultPartitionType.BLOCKING;
            break;
        case PIPELINE_WITH_BATCH_FALLBACK:
            throw new UnsupportedOperationException("Data exchange mode " + channel.getDataExchangeMode() + " currently not supported.");
        default:
            throw new UnsupportedOperationException("Unknown data exchange mode.");
    }
    JobEdge edge = targetVertex.connectNewDataSetAsInput(sourceVertex, distributionPattern, resultType);
    // -------------- configure the source task's ship strategy strategies in task config --------------
    final int outputIndex = sourceConfig.getNumOutputs();
    sourceConfig.addOutputShipStrategy(channel.getShipStrategy());
    if (outputIndex == 0) {
        sourceConfig.setOutputSerializer(channel.getSerializer());
    }
    if (channel.getShipStrategyComparator() != null) {
        sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex);
    }
    if (channel.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
        final DataDistribution dataDistribution = channel.getDataDistribution();
        if (dataDistribution != null) {
            sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex);
        } else {
            throw new RuntimeException("Range partitioning requires data distribution.");
        }
    }
    if (channel.getShipStrategy() == ShipStrategyType.PARTITION_CUSTOM) {
        if (channel.getPartitioner() != null) {
            sourceConfig.setOutputPartitioner(channel.getPartitioner(), outputIndex);
        } else {
            throw new CompilerException("The ship strategy was set to custom partitioning, but no partitioner was set.");
        }
    }
    // ---------------- configure the receiver -------------------
    if (isBroadcast) {
        targetConfig.addBroadcastInputToGroup(inputNumber);
    } else {
        targetConfig.addInputToGroup(inputNumber);
    }
    // ---------------- attach the additional infos to the job edge -------------------
    String shipStrategy = JsonMapper.getShipStrategyString(channel.getShipStrategy());
    if (channel.getShipStrategyKeys() != null && channel.getShipStrategyKeys().size() > 0) {
        shipStrategy += " on " + (channel.getShipStrategySortOrder() == null ? channel.getShipStrategyKeys().toString() : Utils.createOrdering(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()).toString());
    }
    String localStrategy;
    if (channel.getLocalStrategy() == null || channel.getLocalStrategy() == LocalStrategy.NONE) {
        localStrategy = null;
    } else {
        localStrategy = JsonMapper.getLocalStrategyString(channel.getLocalStrategy());
        if (localStrategy != null && channel.getLocalStrategyKeys() != null && channel.getLocalStrategyKeys().size() > 0) {
            localStrategy += " on " + (channel.getLocalStrategySortOrder() == null ? channel.getLocalStrategyKeys().toString() : Utils.createOrdering(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder()).toString());
        }
    }
    String caching = channel.getTempMode() == TempMode.NONE ? null : channel.getTempMode().toString();
    edge.setShipStrategyName(shipStrategy);
    edge.setPreProcessingOperationName(localStrategy);
    edge.setOperatorLevelCachingDescription(caching);
    return distributionPattern;
}

Example 2 with JobEdge

use of org.apache.flink.runtime.jobgraph.JobEdge in project flink by apache.

the class ExecutionJobVertex method connectToPredecessors.

//---------------------------------------------------------------------------------------------
public void connectToPredecessors(Map<IntermediateDataSetID, IntermediateResult> intermediateDataSets) throws JobException {
    List<JobEdge> inputs = jobVertex.getInputs();
    if (LOG.isDebugEnabled()) {
        LOG.debug(String.format("Connecting ExecutionJobVertex %s (%s) to %d predecessors.", jobVertex.getID(), jobVertex.getName(), inputs.size()));
    }
    for (int num = 0; num < inputs.size(); num++) {
        JobEdge edge = inputs.get(num);
        if (LOG.isDebugEnabled()) {
            if (edge.getSource() == null) {
                LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via ID %s.", num, jobVertex.getID(), jobVertex.getName(), edge.getSourceId()));
            } else {
                LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via predecessor %s (%s).", num, jobVertex.getID(), jobVertex.getName(), edge.getSource().getProducer().getID(), edge.getSource().getProducer().getName()));
            }
        }
        // fetch the intermediate result via ID. if it does not exist, then it either has not been created, or the order
        // in which this method is called for the job vertices is not a topological order
        IntermediateResult ires = intermediateDataSets.get(edge.getSourceId());
        if (ires == null) {
            throw new JobException("Cannot connect this job graph to the previous graph. No previous intermediate result found for ID " + edge.getSourceId());
        }
        this.inputs.add(ires);
        int consumerIndex = ires.registerConsumer();
        for (int i = 0; i < parallelism; i++) {
            ExecutionVertex ev = taskVertices[i];
            ev.connectSource(num, ires, edge, consumerIndex);
        }
    }
}

Example 3 with JobEdge

use of org.apache.flink.runtime.jobgraph.JobEdge in project flink by apache.

the class StreamingJobGraphGenerator method connect.

private void connect(Integer headOfChain, StreamEdge edge) {
    physicalEdgesInOrder.add(edge);
    Integer downStreamvertexID = edge.getTargetId();
    JobVertex headVertex = jobVertices.get(headOfChain);
    JobVertex downStreamVertex = jobVertices.get(downStreamvertexID);
    StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration());
    downStreamConfig.setNumberOfInputs(downStreamConfig.getNumberOfInputs() + 1);
    StreamPartitioner<?> partitioner = edge.getPartitioner();
    JobEdge jobEdge = null;
    if (partitioner instanceof ForwardPartitioner) {
        jobEdge = downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED_BOUNDED);
    } else if (partitioner instanceof RescalePartitioner) {
        jobEdge = downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED_BOUNDED);
    } else {
        jobEdge = downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED_BOUNDED);
    }
    // set strategy name so that web interface can show it.
    jobEdge.setShipStrategyName(partitioner.toString());
    if (LOG.isDebugEnabled()) {
        LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(), headOfChain, downStreamvertexID);
    }
}

Example 4 with JobEdge

use of org.apache.flink.runtime.jobgraph.JobEdge in project flink by apache.

the class JsonPlanGenerator method generatePlan.

public static String generatePlan(JobGraph jg) {
    try {
        final StringWriter writer = new StringWriter(1024);
        final JsonFactory factory = new JsonFactory();
        final JsonGenerator gen = factory.createGenerator(writer);
        // start of everything
        gen.writeStartObject();
        gen.writeStringField("jid", jg.getJobID().toString());
        gen.writeStringField("name", jg.getName());
        gen.writeArrayFieldStart("nodes");
        // info per vertex
        for (JobVertex vertex : jg.getVertices()) {
            String operator = vertex.getOperatorName() != null ? vertex.getOperatorName() : NOT_SET;
            String operatorDescr = vertex.getOperatorDescription() != null ? vertex.getOperatorDescription() : NOT_SET;
            String optimizerProps = vertex.getResultOptimizerProperties() != null ? vertex.getResultOptimizerProperties() : EMPTY;
            String description = vertex.getOperatorPrettyName() != null ? vertex.getOperatorPrettyName() : vertex.getName();
            // make sure the encoding is HTML pretty
            description = StringEscapeUtils.escapeHtml4(description);
            description = description.replace("\n", "<br/>");
            description = description.replace("\\", "&#92;");
            operatorDescr = StringEscapeUtils.escapeHtml4(operatorDescr);
            operatorDescr = operatorDescr.replace("\n", "<br/>");
            gen.writeStartObject();
            // write the core properties
            gen.writeStringField("id", vertex.getID().toString());
            gen.writeNumberField("parallelism", vertex.getParallelism());
            gen.writeStringField("operator", operator);
            gen.writeStringField("operator_strategy", operatorDescr);
            gen.writeStringField("description", description);
            if (!vertex.isInputVertex()) {
                // write the input edge properties
                gen.writeArrayFieldStart("inputs");
                List<JobEdge> inputs = vertex.getInputs();
                for (int inputNum = 0; inputNum < inputs.size(); inputNum++) {
                    JobEdge edge = inputs.get(inputNum);
                    if (edge.getSource() == null) {
                        continue;
                    }
                    JobVertex predecessor = edge.getSource().getProducer();
                    String shipStrategy = edge.getShipStrategyName();
                    String preProcessingOperation = edge.getPreProcessingOperationName();
                    String operatorLevelCaching = edge.getOperatorLevelCachingDescription();
                    gen.writeStartObject();
                    gen.writeNumberField("num", inputNum);
                    gen.writeStringField("id", predecessor.getID().toString());
                    if (shipStrategy != null) {
                        gen.writeStringField("ship_strategy", shipStrategy);
                    }
                    if (preProcessingOperation != null) {
                        gen.writeStringField("local_strategy", preProcessingOperation);
                    }
                    if (operatorLevelCaching != null) {
                        gen.writeStringField("caching", operatorLevelCaching);
                    }
                    gen.writeStringField("exchange", edge.getSource().getResultType().name().toLowerCase());
                    gen.writeEndObject();
                }
                gen.writeEndArray();
            }
            // write the optimizer properties
            gen.writeFieldName("optimizer_properties");
            gen.writeRawValue(optimizerProps);
            gen.writeEndObject();
        }
        // end of everything
        gen.writeEndArray();
        gen.writeEndObject();
        gen.close();
        return writer.toString();
    } catch (Exception e) {
        throw new RuntimeException("Failed to generate plan", e);
    }
}