Examples with OperatorIDPair org.apache.flink.runtime.OperatorIDPair used on opensource projects

Example 11 with OperatorIDPair

use of org.apache.flink.runtime.OperatorIDPair in project flink by apache.

the class VertexFinishedStateCheckerTest method testRestoringPartiallyFinishedChainsFails.

private void testRestoringPartiallyFinishedChainsFails(boolean useUidHash) throws Exception {
    final JobVertexID jobVertexID1 = new JobVertexID();
    final JobVertexID jobVertexID2 = new JobVertexID();
    // The op1 has uidHash set.
    OperatorIDPair op1 = OperatorIDPair.of(new OperatorID(), new OperatorID());
    OperatorIDPair op2 = OperatorIDPair.generatedIDOnly(new OperatorID());
    OperatorIDPair op3 = OperatorIDPair.generatedIDOnly(new OperatorID());
    final ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexID2, 1, 1, singletonList(op3), true).addJobVertex(jobVertexID1, 1, 1, Arrays.asList(op1, op2), true).build();
    Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
    operatorStates.put(useUidHash ? op1.getUserDefinedOperatorID().get() : op1.getGeneratedOperatorID(), new FullyFinishedOperatorState(op1.getGeneratedOperatorID(), 1, 1));
    operatorStates.put(op2.getGeneratedOperatorID(), new OperatorState(op2.getGeneratedOperatorID(), 1, 1));
    Set<ExecutionJobVertex> vertices = new HashSet<>();
    vertices.add(graph.getJobVertex(jobVertexID1));
    VertexFinishedStateChecker finishedStateChecker = new VertexFinishedStateChecker(vertices, operatorStates);
    FlinkRuntimeException exception = assertThrows(FlinkRuntimeException.class, finishedStateChecker::validateOperatorsFinishedState);
    assertThat(exception.getMessage(), is(equalTo("Can not restore vertex " + "anon(" + jobVertexID1 + ")" + " which contain mixed operator finished state: [ALL_RUNNING, FULLY_FINISHED]")));
}

Example 12 with OperatorIDPair

use of org.apache.flink.runtime.OperatorIDPair in project flink by apache.

the class VertexFinishedStateCheckerTest method testAddingOperatorsBeforePartiallyOrFullyFinishedOne.

private void testAddingOperatorsBeforePartiallyOrFullyFinishedOne(JobVertexID firstVertexId, String firstVertexName, VertexFinishedStateChecker.VertexFinishedState firstOperatorFinishedState, JobVertexID secondVertexId, String secondVertexName, VertexFinishedStateChecker.VertexFinishedState secondOperatorFinishedState, DistributionPattern[] distributionPatterns, Class<? extends Throwable> expectedExceptionalClass, String expectedMessage) throws Exception {
    OperatorIDPair op1 = OperatorIDPair.generatedIDOnly(new OperatorID());
    OperatorIDPair op2 = OperatorIDPair.generatedIDOnly(new OperatorID());
    JobVertex vertex1 = new JobVertex(firstVertexName, firstVertexId, singletonList(op1));
    JobVertex vertex2 = new JobVertex(secondVertexName, secondVertexId, singletonList(op2));
    vertex1.setInvokableClass(NoOpInvokable.class);
    vertex2.setInvokableClass(NoOpInvokable.class);
    final ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(vertex1, true).addJobVertex(vertex2, false).setDistributionPattern(distributionPatterns[0]).build();
    // Adds the additional edges
    for (int i = 1; i < distributionPatterns.length; ++i) {
        vertex2.connectNewDataSetAsInput(vertex1, distributionPatterns[i], ResultPartitionType.PIPELINED);
    }
    Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
    operatorStates.put(op1.getGeneratedOperatorID(), createOperatorState(op1.getGeneratedOperatorID(), firstOperatorFinishedState));
    operatorStates.put(op2.getGeneratedOperatorID(), createOperatorState(op2.getGeneratedOperatorID(), secondOperatorFinishedState));
    Set<ExecutionJobVertex> vertices = new HashSet<>();
    vertices.add(graph.getJobVertex(vertex1.getID()));
    vertices.add(graph.getJobVertex(vertex2.getID()));
    VertexFinishedStateChecker finishedStateChecker = new VertexFinishedStateChecker(vertices, operatorStates);
    Throwable exception = assertThrows(expectedExceptionalClass, finishedStateChecker::validateOperatorsFinishedState);
    assertThat(exception.getMessage(), is(equalTo(expectedMessage)));
}

Example 13 with OperatorIDPair

use of org.apache.flink.runtime.OperatorIDPair in project flink by apache.

the class StreamingJobGraphGenerator method createJobVertex.

private StreamConfig createJobVertex(Integer streamNodeId, OperatorChainInfo chainInfo) {
    JobVertex jobVertex;
    StreamNode streamNode = streamGraph.getStreamNode(streamNodeId);
    byte[] hash = chainInfo.getHash(streamNodeId);
    if (hash == null) {
        throw new IllegalStateException("Cannot find node hash. " + "Did you generate them before calling this method?");
    }
    JobVertexID jobVertexId = new JobVertexID(hash);
    List<Tuple2<byte[], byte[]>> chainedOperators = chainInfo.getChainedOperatorHashes(streamNodeId);
    List<OperatorIDPair> operatorIDPairs = new ArrayList<>();
    if (chainedOperators != null) {
        for (Tuple2<byte[], byte[]> chainedOperator : chainedOperators) {
            OperatorID userDefinedOperatorID = chainedOperator.f1 == null ? null : new OperatorID(chainedOperator.f1);
            operatorIDPairs.add(OperatorIDPair.of(new OperatorID(chainedOperator.f0), userDefinedOperatorID));
        }
    }
    if (chainedInputOutputFormats.containsKey(streamNodeId)) {
        jobVertex = new InputOutputFormatVertex(chainedNames.get(streamNodeId), jobVertexId, operatorIDPairs);
        chainedInputOutputFormats.get(streamNodeId).write(new TaskConfig(jobVertex.getConfiguration()));
    } else {
        jobVertex = new JobVertex(chainedNames.get(streamNodeId), jobVertexId, operatorIDPairs);
    }
    for (OperatorCoordinator.Provider coordinatorProvider : chainInfo.getCoordinatorProviders()) {
        try {
            jobVertex.addOperatorCoordinator(new SerializedValue<>(coordinatorProvider));
        } catch (IOException e) {
            throw new FlinkRuntimeException(String.format("Coordinator Provider for node %s is not serializable.", chainedNames.get(streamNodeId)), e);
        }
    }
    jobVertex.setResources(chainedMinResources.get(streamNodeId), chainedPreferredResources.get(streamNodeId));
    jobVertex.setInvokableClass(streamNode.getJobVertexClass());
    int parallelism = streamNode.getParallelism();
    if (parallelism > 0) {
        jobVertex.setParallelism(parallelism);
    } else {
        parallelism = jobVertex.getParallelism();
    }
    jobVertex.setMaxParallelism(streamNode.getMaxParallelism());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Parallelism set: {} for {}", parallelism, streamNodeId);
    }
    jobVertices.put(streamNodeId, jobVertex);
    builtVertices.add(streamNodeId);
    jobGraph.addVertex(jobVertex);
    return new StreamConfig(jobVertex.getConfiguration());
}

Example 14 with OperatorIDPair

use of org.apache.flink.runtime.OperatorIDPair in project flink by apache.

the class StateAssignmentOperation method assignStates.

public void assignStates() {
    checkStateMappingCompleteness(allowNonRestoredState, operatorStates, tasks);
    Map<OperatorID, OperatorState> localOperators = new HashMap<>(operatorStates);
    // information in first pass
    for (ExecutionJobVertex executionJobVertex : tasks) {
        List<OperatorIDPair> operatorIDPairs = executionJobVertex.getOperatorIDs();
        Map<OperatorID, OperatorState> operatorStates = new HashMap<>(operatorIDPairs.size());
        for (OperatorIDPair operatorIDPair : operatorIDPairs) {
            OperatorID operatorID = operatorIDPair.getUserDefinedOperatorID().filter(localOperators::containsKey).orElse(operatorIDPair.getGeneratedOperatorID());
            OperatorState operatorState = localOperators.remove(operatorID);
            if (operatorState == null) {
                operatorState = new OperatorState(operatorID, executionJobVertex.getParallelism(), executionJobVertex.getMaxParallelism());
            }
            operatorStates.put(operatorIDPair.getGeneratedOperatorID(), operatorState);
        }
        final TaskStateAssignment stateAssignment = new TaskStateAssignment(executionJobVertex, operatorStates, consumerAssignment, vertexAssignments);
        vertexAssignments.put(executionJobVertex, stateAssignment);
        for (final IntermediateResult producedDataSet : executionJobVertex.getInputs()) {
            consumerAssignment.put(producedDataSet.getId(), stateAssignment);
        }
    }
    // repartition state
    for (TaskStateAssignment stateAssignment : vertexAssignments.values()) {
        if (stateAssignment.hasNonFinishedState) {
            assignAttemptState(stateAssignment);
        }
    }
    // actually assign the state
    for (TaskStateAssignment stateAssignment : vertexAssignments.values()) {
        if (stateAssignment.hasNonFinishedState || stateAssignment.isFullyFinished) {
            assignTaskStateToExecutionJobVertices(stateAssignment);
        }
    }
}

Example 15 with OperatorIDPair

use of org.apache.flink.runtime.OperatorIDPair in project flink by apache.

the class StateAssignmentOperation method assignTaskStateToExecutionJobVertices.

private void assignTaskStateToExecutionJobVertices(TaskStateAssignment assignment) {
    ExecutionJobVertex executionJobVertex = assignment.executionJobVertex;
    List<OperatorIDPair> operatorIDs = executionJobVertex.getOperatorIDs();
    final int newParallelism = executionJobVertex.getParallelism();
    /*
         *  An executionJobVertex's all state handles needed to restore are something like a matrix
         *
         * 		parallelism0 parallelism1 parallelism2 parallelism3
         * op0   sh(0,0)     sh(0,1)       sh(0,2)	    sh(0,3)
         * op1   sh(1,0)	 sh(1,1)	   sh(1,2)	    sh(1,3)
         * op2   sh(2,0)	 sh(2,1)	   sh(2,2)		sh(2,3)
         * op3   sh(3,0)	 sh(3,1)	   sh(3,2)		sh(3,3)
         *
         */
    for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) {
        Execution currentExecutionAttempt = executionJobVertex.getTaskVertices()[subTaskIndex].getCurrentExecutionAttempt();
        if (assignment.isFullyFinished) {
            assignFinishedStateToTask(currentExecutionAttempt);
        } else {
            assignNonFinishedStateToTask(assignment, operatorIDs, subTaskIndex, currentExecutionAttempt);
        }
    }
}