Examples with OperatorID org.apache.flink.runtime.jobgraph.OperatorID used on opensource projects

Example 36 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class TaskStateSnapshotTest method putGetSubtaskStateByOperatorID.

@Test
public void putGetSubtaskStateByOperatorID() {
    TaskStateSnapshot taskStateSnapshot = new TaskStateSnapshot();
    OperatorID operatorID_1 = new OperatorID();
    OperatorID operatorID_2 = new OperatorID();
    OperatorSubtaskState operatorSubtaskState_1 = OperatorSubtaskState.builder().build();
    OperatorSubtaskState operatorSubtaskState_2 = OperatorSubtaskState.builder().build();
    OperatorSubtaskState operatorSubtaskState_1_replace = OperatorSubtaskState.builder().build();
    Assert.assertNull(taskStateSnapshot.getSubtaskStateByOperatorID(operatorID_1));
    Assert.assertNull(taskStateSnapshot.getSubtaskStateByOperatorID(operatorID_2));
    taskStateSnapshot.putSubtaskStateByOperatorID(operatorID_1, operatorSubtaskState_1);
    taskStateSnapshot.putSubtaskStateByOperatorID(operatorID_2, operatorSubtaskState_2);
    Assert.assertEquals(operatorSubtaskState_1, taskStateSnapshot.getSubtaskStateByOperatorID(operatorID_1));
    Assert.assertEquals(operatorSubtaskState_2, taskStateSnapshot.getSubtaskStateByOperatorID(operatorID_2));
    Assert.assertEquals(operatorSubtaskState_1, taskStateSnapshot.putSubtaskStateByOperatorID(operatorID_1, operatorSubtaskState_1_replace));
    Assert.assertEquals(operatorSubtaskState_1_replace, taskStateSnapshot.getSubtaskStateByOperatorID(operatorID_1));
}

Example 37 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class FullyFinishedOperatorStateTest method testFullyFinishedOperatorState.

@Test
public void testFullyFinishedOperatorState() {
    OperatorState operatorState = new FullyFinishedOperatorState(new OperatorID(), 5, 256);
    assertTrue(operatorState.isFullyFinished());
    assertEquals(0, operatorState.getSubtaskStates().size());
    assertEquals(0, operatorState.getStates().size());
    assertEquals(0, operatorState.getNumberCollectedStates());
    try {
        operatorState.putState(0, OperatorSubtaskState.builder().build());
        fail("Should not be able to put new subtask states for a fully finished state");
    } catch (UnsupportedOperationException e) {
    // Expected
    }
    try {
        operatorState.setCoordinatorState(new ByteStreamStateHandle("test", new byte[] { 1, 2, 3, 4 }));
        fail("Should not be able to set coordinator states for a fully finished state");
    } catch (UnsupportedOperationException e) {
    // Excepted
    }
}

Example 38 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class RestClusterClientTest method testSendCoordinationRequest.

@Test
public void testSendCoordinationRequest() throws Exception {
    final TestClientCoordinationHandler handler = new TestClientCoordinationHandler();
    try (TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint(handler)) {
        RestClusterClient<?> restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort());
        String payload = "testing payload";
        TestCoordinationRequest<String> request = new TestCoordinationRequest<>(payload);
        try {
            CompletableFuture<CoordinationResponse> future = restClusterClient.sendCoordinationRequest(jobId, new OperatorID(), request);
            TestCoordinationResponse response = (TestCoordinationResponse) future.get();
            assertEquals(payload, response.payload);
        } finally {
            restClusterClient.close();
        }
    }
}

Example 39 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class PendingCheckpoint method acknowledgeCoordinatorState.

public TaskAcknowledgeResult acknowledgeCoordinatorState(OperatorInfo coordinatorInfo, @Nullable ByteStreamStateHandle stateHandle) {
    synchronized (lock) {
        if (disposed) {
            return TaskAcknowledgeResult.DISCARDED;
        }
        final OperatorID operatorId = coordinatorInfo.operatorId();
        OperatorState operatorState = operatorStates.get(operatorId);
        // sanity check for better error reporting
        if (!notYetAcknowledgedOperatorCoordinators.remove(operatorId)) {
            return operatorState != null && operatorState.getCoordinatorState() != null ? TaskAcknowledgeResult.DUPLICATE : TaskAcknowledgeResult.UNKNOWN;
        }
        if (operatorState == null) {
            operatorState = new OperatorState(operatorId, coordinatorInfo.currentParallelism(), coordinatorInfo.maxParallelism());
            operatorStates.put(operatorId, operatorState);
        }
        if (stateHandle != null) {
            operatorState.setCoordinatorState(stateHandle);
        }
        return TaskAcknowledgeResult.SUCCESS;
    }
}

Example 40 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class StateAssignmentOperation method checkStateMappingCompleteness.

/**
 * Verifies that all operator states can be mapped to an execution job vertex.
 *
 * @param allowNonRestoredState if false an exception will be thrown if a state could not be
 *     mapped
 * @param operatorStates operator states to map
 * @param tasks task to map to
 */
private static void checkStateMappingCompleteness(boolean allowNonRestoredState, Map<OperatorID, OperatorState> operatorStates, Set<ExecutionJobVertex> tasks) {
    Set<OperatorID> allOperatorIDs = new HashSet<>();
    for (ExecutionJobVertex executionJobVertex : tasks) {
        for (OperatorIDPair operatorIDPair : executionJobVertex.getOperatorIDs()) {
            allOperatorIDs.add(operatorIDPair.getGeneratedOperatorID());
            operatorIDPair.getUserDefinedOperatorID().ifPresent(allOperatorIDs::add);
        }
    }
    for (Map.Entry<OperatorID, OperatorState> operatorGroupStateEntry : operatorStates.entrySet()) {
        if (!allOperatorIDs.contains(operatorGroupStateEntry.getKey())) {
            OperatorState operatorState = operatorGroupStateEntry.getValue();
            if (allowNonRestoredState) {
                LOG.info("Skipped checkpoint state for operator {}.", operatorState.getOperatorID());
            } else {
                throw new IllegalStateException("There is no operator for the state " + operatorState.getOperatorID());
            }
        }
    }
}