Examples with OperatorCoordinatorHolder org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder used on opensource projects

Example 1 with OperatorCoordinatorHolder

use of org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder in project flink by apache.

the class ExecutionJobVertex method initialize.

protected void initialize(int maxPriorAttemptsHistoryLength, Time timeout, long createTimestamp, SubtaskAttemptNumberStore initialAttemptCounts, CoordinatorStore coordinatorStore) throws JobException {
    checkState(parallelismInfo.getParallelism() > 0);
    checkState(!isInitialized());
    this.taskVertices = new ExecutionVertex[parallelismInfo.getParallelism()];
    this.inputs = new ArrayList<>(jobVertex.getInputs().size());
    // create the intermediate results
    this.producedDataSets = new IntermediateResult[jobVertex.getNumberOfProducedIntermediateDataSets()];
    for (int i = 0; i < jobVertex.getProducedDataSets().size(); i++) {
        final IntermediateDataSet result = jobVertex.getProducedDataSets().get(i);
        this.producedDataSets[i] = new IntermediateResult(result, this, this.parallelismInfo.getParallelism(), result.getResultType());
    }
    // create all task vertices
    for (int i = 0; i < this.parallelismInfo.getParallelism(); i++) {
        ExecutionVertex vertex = new ExecutionVertex(this, i, producedDataSets, timeout, createTimestamp, maxPriorAttemptsHistoryLength, initialAttemptCounts.getAttemptCount(i));
        this.taskVertices[i] = vertex;
    }
    // execution vertices
    for (IntermediateResult ir : this.producedDataSets) {
        if (ir.getNumberOfAssignedPartitions() != this.parallelismInfo.getParallelism()) {
            throw new RuntimeException("The intermediate result's partitions were not correctly assigned.");
        }
    }
    final List<SerializedValue<OperatorCoordinator.Provider>> coordinatorProviders = getJobVertex().getOperatorCoordinators();
    if (coordinatorProviders.isEmpty()) {
        this.operatorCoordinators = Collections.emptyList();
    } else {
        final ArrayList<OperatorCoordinatorHolder> coordinators = new ArrayList<>(coordinatorProviders.size());
        try {
            for (final SerializedValue<OperatorCoordinator.Provider> provider : coordinatorProviders) {
                coordinators.add(OperatorCoordinatorHolder.create(provider, this, graph.getUserClassLoader(), coordinatorStore));
            }
        } catch (Exception | LinkageError e) {
            IOUtils.closeAllQuietly(coordinators);
            throw new JobException("Cannot instantiate the coordinator for operator " + getName(), e);
        }
        this.operatorCoordinators = Collections.unmodifiableList(coordinators);
    }
    // set up the input splits, if the vertex has any
    try {
        @SuppressWarnings("unchecked") InputSplitSource<InputSplit> splitSource = (InputSplitSource<InputSplit>) jobVertex.getInputSplitSource();
        if (splitSource != null) {
            Thread currentThread = Thread.currentThread();
            ClassLoader oldContextClassLoader = currentThread.getContextClassLoader();
            currentThread.setContextClassLoader(graph.getUserClassLoader());
            try {
                inputSplits = splitSource.createInputSplits(this.parallelismInfo.getParallelism());
                if (inputSplits != null) {
                    splitAssigner = splitSource.getInputSplitAssigner(inputSplits);
                }
            } finally {
                currentThread.setContextClassLoader(oldContextClassLoader);
            }
        } else {
            inputSplits = null;
        }
    } catch (Throwable t) {
        throw new JobException("Creating the input splits caused an error: " + t.getMessage(), t);
    }
}

Example 2 with OperatorCoordinatorHolder

use of org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder in project flink by apache.

the class DefaultOperatorCoordinatorHandler method deliverOperatorEventToCoordinator.

@Override
public void deliverOperatorEventToCoordinator(final ExecutionAttemptID taskExecutionId, final OperatorID operatorId, final OperatorEvent evt) throws FlinkException {
    // Failure semantics (as per the javadocs of the method):
    // If the task manager sends an event for a non-running task or an non-existing operator
    // coordinator, then respond with an exception to the call. If task and coordinator exist,
    // then we assume that the call from the TaskManager was valid, and any bubbling exception
    // needs to cause a job failure.
    final Execution exec = executionGraph.getRegisteredExecutions().get(taskExecutionId);
    if (exec == null || exec.getState() != ExecutionState.RUNNING && exec.getState() != ExecutionState.INITIALIZING) {
        // on the safe, we notify the TM that the event could not be delivered.
        throw new TaskNotRunningException("Task is not known or in state running on the JobManager.");
    }
    final OperatorCoordinatorHolder coordinator = coordinatorMap.get(operatorId);
    if (coordinator == null) {
        throw new FlinkException("No coordinator registered for operator " + operatorId);
    }
    try {
        coordinator.handleEventFromOperator(exec.getParallelSubtaskIndex(), evt);
    } catch (Throwable t) {
        ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
        globalFailureHandler.handleGlobalFailure(t);
    }
}

Example 3 with OperatorCoordinatorHolder

use of org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder in project flink by apache.

the class DefaultOperatorCoordinatorHandler method registerAndStartNewCoordinators.

@Override
public void registerAndStartNewCoordinators(Collection<OperatorCoordinatorHolder> coordinators, ComponentMainThreadExecutor mainThreadExecutor) {
    for (OperatorCoordinatorHolder coordinator : coordinators) {
        coordinatorMap.put(coordinator.operatorId(), coordinator);
        coordinator.lazyInitialize(globalFailureHandler, mainThreadExecutor);
    }
    startOperatorCoordinators(coordinators);
}

Example 4 with OperatorCoordinatorHolder

use of org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder in project flink by apache.

the class DefaultOperatorCoordinatorHandler method deliverCoordinationRequestToCoordinator.

@Override
public CompletableFuture<CoordinationResponse> deliverCoordinationRequestToCoordinator(OperatorID operator, CoordinationRequest request) throws FlinkException {
    final OperatorCoordinatorHolder coordinatorHolder = coordinatorMap.get(operator);
    if (coordinatorHolder == null) {
        throw new FlinkException("Coordinator of operator " + operator + " does not exist or the job vertex this operator belongs to is not initialized.");
    }
    final OperatorCoordinator coordinator = coordinatorHolder.coordinator();
    if (coordinator instanceof CoordinationRequestHandler) {
        return ((CoordinationRequestHandler) coordinator).handleCoordinationRequest(request);
    } else {
        throw new FlinkException("Coordinator of operator " + operator + " cannot handle client event");
    }
}

Example 5 with OperatorCoordinatorHolder

use of org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder in project flink by apache.

the class SchedulerBase method notifyCoordinatorsOfSubtaskRestore.

private void notifyCoordinatorsOfSubtaskRestore(final Map<ExecutionJobVertex, IntArrayList> restoredSubtasks, final long checkpointId) {
    for (final Map.Entry<ExecutionJobVertex, IntArrayList> vertexSubtasks : restoredSubtasks.entrySet()) {
        final ExecutionJobVertex jobVertex = vertexSubtasks.getKey();
        final IntArrayList subtasks = vertexSubtasks.getValue();
        final Collection<OperatorCoordinatorHolder> coordinators = jobVertex.getOperatorCoordinators();
        if (coordinators.isEmpty()) {
            continue;
        }
        while (!subtasks.isEmpty()) {
            final int subtask = // this is how IntArrayList implements iterations
            subtasks.removeLast();
            for (final OperatorCoordinatorHolder opCoordinator : coordinators) {
                opCoordinator.subtaskReset(subtask, checkpointId);
            }
        }
    }
}