Examples with InputSplitSource org.apache.flink.core.io.InputSplitSource used on opensource projects

Example 1 with InputSplitSource

use of org.apache.flink.core.io.InputSplitSource in project flink by apache.

the class ExecutionJobVertex method initialize.

protected void initialize(int maxPriorAttemptsHistoryLength, Time timeout, long createTimestamp, SubtaskAttemptNumberStore initialAttemptCounts, CoordinatorStore coordinatorStore) throws JobException {
    checkState(parallelismInfo.getParallelism() > 0);
    checkState(!isInitialized());
    this.taskVertices = new ExecutionVertex[parallelismInfo.getParallelism()];
    this.inputs = new ArrayList<>(jobVertex.getInputs().size());
    // create the intermediate results
    this.producedDataSets = new IntermediateResult[jobVertex.getNumberOfProducedIntermediateDataSets()];
    for (int i = 0; i < jobVertex.getProducedDataSets().size(); i++) {
        final IntermediateDataSet result = jobVertex.getProducedDataSets().get(i);
        this.producedDataSets[i] = new IntermediateResult(result, this, this.parallelismInfo.getParallelism(), result.getResultType());
    }
    // create all task vertices
    for (int i = 0; i < this.parallelismInfo.getParallelism(); i++) {
        ExecutionVertex vertex = new ExecutionVertex(this, i, producedDataSets, timeout, createTimestamp, maxPriorAttemptsHistoryLength, initialAttemptCounts.getAttemptCount(i));
        this.taskVertices[i] = vertex;
    }
    // execution vertices
    for (IntermediateResult ir : this.producedDataSets) {
        if (ir.getNumberOfAssignedPartitions() != this.parallelismInfo.getParallelism()) {
            throw new RuntimeException("The intermediate result's partitions were not correctly assigned.");
        }
    }
    final List<SerializedValue<OperatorCoordinator.Provider>> coordinatorProviders = getJobVertex().getOperatorCoordinators();
    if (coordinatorProviders.isEmpty()) {
        this.operatorCoordinators = Collections.emptyList();
    } else {
        final ArrayList<OperatorCoordinatorHolder> coordinators = new ArrayList<>(coordinatorProviders.size());
        try {
            for (final SerializedValue<OperatorCoordinator.Provider> provider : coordinatorProviders) {
                coordinators.add(OperatorCoordinatorHolder.create(provider, this, graph.getUserClassLoader(), coordinatorStore));
            }
        } catch (Exception | LinkageError e) {
            IOUtils.closeAllQuietly(coordinators);
            throw new JobException("Cannot instantiate the coordinator for operator " + getName(), e);
        }
        this.operatorCoordinators = Collections.unmodifiableList(coordinators);
    }
    // set up the input splits, if the vertex has any
    try {
        @SuppressWarnings("unchecked") InputSplitSource<InputSplit> splitSource = (InputSplitSource<InputSplit>) jobVertex.getInputSplitSource();
        if (splitSource != null) {
            Thread currentThread = Thread.currentThread();
            ClassLoader oldContextClassLoader = currentThread.getContextClassLoader();
            currentThread.setContextClassLoader(graph.getUserClassLoader());
            try {
                inputSplits = splitSource.createInputSplits(this.parallelismInfo.getParallelism());
                if (inputSplits != null) {
                    splitAssigner = splitSource.getInputSplitAssigner(inputSplits);
                }
            } finally {
                currentThread.setContextClassLoader(oldContextClassLoader);
            }
        } else {
            inputSplits = null;
        }
    } catch (Throwable t) {
        throw new JobException("Creating the input splits caused an error: " + t.getMessage(), t);
    }
}