Examples with OperatorStateHandle org.apache.flink.runtime.state.OperatorStateHandle used on opensource projects

Example 56 with OperatorStateHandle

use of org.apache.flink.runtime.state.OperatorStateHandle in project flink by apache.

the class PrioritizedOperatorSubtaskStateTest method generateForConfiguration.

/**
 * Generator for all 3^4 = 81 possible configurations of a OperatorSubtaskState: - 4 different
 * sub-states: managed/raw + operator/keyed. - 3 different options per sub-state: empty
 * (simulate no state), single handle (simulate recovery), 2 handles (simulate e.g. rescaling)
 */
private OperatorSubtaskState generateForConfiguration(int conf) {
    // 3^4
    Preconditions.checkState(conf >= 0 && conf <= 80);
    final int numModes = 3;
    KeyGroupRange keyGroupRange = new KeyGroupRange(0, 4);
    KeyGroupRange keyGroupRange1 = new KeyGroupRange(0, 2);
    KeyGroupRange keyGroupRange2 = new KeyGroupRange(3, 4);
    int div = 1;
    int mode = (conf / div) % numModes;
    StateObjectCollection<OperatorStateHandle> s1 = mode == 0 ? StateObjectCollection.empty() : mode == 1 ? new StateObjectCollection<>(Collections.singletonList(createNewOperatorStateHandle(2, RANDOM))) : new StateObjectCollection<>(Arrays.asList(createNewOperatorStateHandle(2, RANDOM), createNewOperatorStateHandle(2, RANDOM)));
    div *= numModes;
    mode = (conf / div) % numModes;
    StateObjectCollection<OperatorStateHandle> s2 = mode == 0 ? StateObjectCollection.empty() : mode == 1 ? new StateObjectCollection<>(Collections.singletonList(createNewOperatorStateHandle(2, RANDOM))) : new StateObjectCollection<>(Arrays.asList(createNewOperatorStateHandle(2, RANDOM), createNewOperatorStateHandle(2, RANDOM)));
    div *= numModes;
    mode = (conf / div) % numModes;
    StateObjectCollection<KeyedStateHandle> s3 = mode == 0 ? StateObjectCollection.empty() : mode == 1 ? new StateObjectCollection<>(Collections.singletonList(createNewKeyedStateHandle(keyGroupRange))) : new StateObjectCollection<>(Arrays.asList(createNewKeyedStateHandle(keyGroupRange1), createNewKeyedStateHandle(keyGroupRange2)));
    div *= numModes;
    mode = (conf / div) % numModes;
    StateObjectCollection<KeyedStateHandle> s4 = mode == 0 ? StateObjectCollection.empty() : mode == 1 ? new StateObjectCollection<>(Collections.singletonList(createNewKeyedStateHandle(keyGroupRange))) : new StateObjectCollection<>(Arrays.asList(createNewKeyedStateHandle(keyGroupRange1), createNewKeyedStateHandle(keyGroupRange2)));
    return OperatorSubtaskState.builder().setManagedOperatorState(s1).setRawOperatorState(s2).setManagedKeyedState(s3).setRawKeyedState(s4).build();
}

Example 57 with OperatorStateHandle

use of org.apache.flink.runtime.state.OperatorStateHandle in project flink by apache.

the class StreamTaskStateInitializerImpl method operatorStateBackend.

protected OperatorStateBackend operatorStateBackend(String operatorIdentifierText, PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskStates, CloseableRegistry backendCloseableRegistry) throws Exception {
    String logDescription = "operator state backend for " + operatorIdentifierText;
    // Now restore processing is included in backend building/constructing process, so we need
    // to make sure
    // each stream constructed in restore could also be closed in case of task cancel, for
    // example the data
    // input stream opened for serDe during restore.
    CloseableRegistry cancelStreamRegistryForRestore = new CloseableRegistry();
    backendCloseableRegistry.registerCloseable(cancelStreamRegistryForRestore);
    BackendRestorerProcedure<OperatorStateBackend, OperatorStateHandle> backendRestorer = new BackendRestorerProcedure<>((stateHandles) -> stateBackend.createOperatorStateBackend(environment, operatorIdentifierText, stateHandles, cancelStreamRegistryForRestore), backendCloseableRegistry, logDescription);
    try {
        return backendRestorer.createAndRestore(prioritizedOperatorSubtaskStates.getPrioritizedManagedOperatorState());
    } finally {
        if (backendCloseableRegistry.unregisterCloseable(cancelStreamRegistryForRestore)) {
            IOUtils.closeQuietly(cancelStreamRegistryForRestore);
        }
    }
}

Example 58 with OperatorStateHandle

use of org.apache.flink.runtime.state.OperatorStateHandle in project flink by apache.

the class OperatorSnapshotUtil method readStateHandle.

public static OperatorSubtaskState readStateHandle(String path) throws IOException, ClassNotFoundException {
    FileInputStream in = new FileInputStream(path);
    try (DataInputStream dis = new DataInputStream(in)) {
        // required for backwards compatibility.
        final int v = dis.readInt();
        // still required for compatibility to consume the bytes.
        MetadataV3Serializer.deserializeStreamStateHandle(dis);
        List<OperatorStateHandle> rawOperatorState = null;
        int numRawOperatorStates = dis.readInt();
        if (numRawOperatorStates >= 0) {
            rawOperatorState = new ArrayList<>();
            for (int i = 0; i < numRawOperatorStates; i++) {
                OperatorStateHandle operatorState = MetadataV3Serializer.deserializeOperatorStateHandleUtil(dis);
                rawOperatorState.add(operatorState);
            }
        }
        List<OperatorStateHandle> managedOperatorState = null;
        int numManagedOperatorStates = dis.readInt();
        if (numManagedOperatorStates >= 0) {
            managedOperatorState = new ArrayList<>();
            for (int i = 0; i < numManagedOperatorStates; i++) {
                OperatorStateHandle operatorState = MetadataV3Serializer.deserializeOperatorStateHandleUtil(dis);
                managedOperatorState.add(operatorState);
            }
        }
        List<KeyedStateHandle> rawKeyedState = null;
        int numRawKeyedStates = dis.readInt();
        if (numRawKeyedStates >= 0) {
            rawKeyedState = new ArrayList<>();
            for (int i = 0; i < numRawKeyedStates; i++) {
                KeyedStateHandle keyedState = MetadataV3Serializer.deserializeKeyedStateHandleUtil(dis);
                rawKeyedState.add(keyedState);
            }
        }
        List<KeyedStateHandle> managedKeyedState = null;
        int numManagedKeyedStates = dis.readInt();
        if (numManagedKeyedStates >= 0) {
            managedKeyedState = new ArrayList<>();
            for (int i = 0; i < numManagedKeyedStates; i++) {
                KeyedStateHandle keyedState = MetadataV3Serializer.deserializeKeyedStateHandleUtil(dis);
                managedKeyedState.add(keyedState);
            }
        }
        final StateObjectCollection<InputChannelStateHandle> inputChannelStateHandles = v == MetadataV3Serializer.VERSION ? MetadataV3Serializer.deserializeInputChannelStateHandle(dis) : StateObjectCollection.empty();
        final StateObjectCollection<ResultSubpartitionStateHandle> resultSubpartitionStateHandles = v == MetadataV3Serializer.VERSION ? MetadataV3Serializer.INSTANCE.deserializeResultSubpartitionStateHandle(dis) : StateObjectCollection.empty();
        return OperatorSubtaskState.builder().setManagedOperatorState(new StateObjectCollection<>(managedOperatorState)).setRawOperatorState(new StateObjectCollection<>(rawOperatorState)).setManagedKeyedState(new StateObjectCollection<>(managedKeyedState)).setRawKeyedState(new StateObjectCollection<>(rawKeyedState)).setInputChannelState(inputChannelStateHandles).setResultSubpartitionState(resultSubpartitionStateHandles).build();
    }
}

Example 59 with OperatorStateHandle

use of org.apache.flink.runtime.state.OperatorStateHandle in project flink by apache.

the class OperatorSnapshotUtil method writeStateHandle.

public static void writeStateHandle(OperatorSubtaskState state, String path) throws IOException {
    FileOutputStream out = new FileOutputStream(path);
    try (DataOutputStream dos = new DataOutputStream(out)) {
        // required for backwards compatibility.
        dos.writeInt(MetadataV3Serializer.VERSION);
        // still required for compatibility
        MetadataV3Serializer.serializeStreamStateHandle(null, dos);
        Collection<OperatorStateHandle> rawOperatorState = state.getRawOperatorState();
        if (rawOperatorState != null) {
            dos.writeInt(rawOperatorState.size());
            for (OperatorStateHandle operatorStateHandle : rawOperatorState) {
                MetadataV3Serializer.serializeOperatorStateHandleUtil(operatorStateHandle, dos);
            }
        } else {
            // this means no states, not even an empty list
            dos.writeInt(-1);
        }
        Collection<OperatorStateHandle> managedOperatorState = state.getManagedOperatorState();
        if (managedOperatorState != null) {
            dos.writeInt(managedOperatorState.size());
            for (OperatorStateHandle operatorStateHandle : managedOperatorState) {
                MetadataV3Serializer.serializeOperatorStateHandleUtil(operatorStateHandle, dos);
            }
        } else {
            // this means no states, not even an empty list
            dos.writeInt(-1);
        }
        Collection<KeyedStateHandle> rawKeyedState = state.getRawKeyedState();
        if (rawKeyedState != null) {
            dos.writeInt(rawKeyedState.size());
            for (KeyedStateHandle keyedStateHandle : rawKeyedState) {
                MetadataV3Serializer.serializeKeyedStateHandleUtil(keyedStateHandle, dos);
            }
        } else {
            // this means no operator states, not even an empty list
            dos.writeInt(-1);
        }
        Collection<KeyedStateHandle> managedKeyedState = state.getManagedKeyedState();
        if (managedKeyedState != null) {
            dos.writeInt(managedKeyedState.size());
            for (KeyedStateHandle keyedStateHandle : managedKeyedState) {
                MetadataV3Serializer.serializeKeyedStateHandleUtil(keyedStateHandle, dos);
            }
        } else {
            // this means no operator states, not even an empty list
            dos.writeInt(-1);
        }
        Collection<InputChannelStateHandle> inputChannelStateHandles = state.getInputChannelState();
        dos.writeInt(inputChannelStateHandles.size());
        for (InputChannelStateHandle inputChannelStateHandle : inputChannelStateHandles) {
            MetadataV3Serializer.INSTANCE.serializeInputChannelStateHandle(inputChannelStateHandle, dos);
        }
        Collection<ResultSubpartitionStateHandle> resultSubpartitionStateHandles = state.getResultSubpartitionState();
        dos.writeInt(inputChannelStateHandles.size());
        for (ResultSubpartitionStateHandle resultSubpartitionStateHandle : resultSubpartitionStateHandles) {
            MetadataV3Serializer.INSTANCE.serializeResultSubpartitionStateHandle(resultSubpartitionStateHandle, dos);
        }
        dos.flush();
    }
}

Example 60 with OperatorStateHandle

use of org.apache.flink.runtime.state.OperatorStateHandle in project flink by apache.

the class CheckpointCoordinatorFailureTest method testFailingCompletedCheckpointStoreAdd.

/**
 * Tests that a failure while storing a completed checkpoint in the completed checkpoint store
 * will properly fail the originating pending checkpoint and clean upt the completed checkpoint.
 */
@Test
public void testFailingCompletedCheckpointStoreAdd() throws Exception {
    JobVertexID jobVertexId = new JobVertexID();
    final ManuallyTriggeredScheduledExecutor manuallyTriggeredScheduledExecutor = new ManuallyTriggeredScheduledExecutor();
    ExecutionGraph testGraph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexId).build();
    ExecutionVertex vertex = testGraph.getJobVertex(jobVertexId).getTaskVertices()[0];
    // set up the coordinator and validate the initial state
    CheckpointCoordinator coord = new CheckpointCoordinatorBuilder().setExecutionGraph(testGraph).setCompletedCheckpointStore(new FailingCompletedCheckpointStore(new Exception("The failing completed checkpoint store failed again... :-("))).setTimer(manuallyTriggeredScheduledExecutor).build();
    coord.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    assertEquals(1, coord.getNumberOfPendingCheckpoints());
    PendingCheckpoint pendingCheckpoint = coord.getPendingCheckpoints().values().iterator().next();
    assertFalse(pendingCheckpoint.isDisposed());
    final long checkpointId = coord.getPendingCheckpoints().keySet().iterator().next();
    KeyedStateHandle managedKeyedHandle = mock(KeyedStateHandle.class);
    KeyedStateHandle rawKeyedHandle = mock(KeyedStateHandle.class);
    OperatorStateHandle managedOpHandle = mock(OperatorStreamStateHandle.class);
    OperatorStateHandle rawOpHandle = mock(OperatorStreamStateHandle.class);
    InputChannelStateHandle inputChannelStateHandle = new InputChannelStateHandle(new InputChannelInfo(0, 1), mock(StreamStateHandle.class), Collections.singletonList(1L));
    ResultSubpartitionStateHandle resultSubpartitionStateHandle = new ResultSubpartitionStateHandle(new ResultSubpartitionInfo(0, 1), mock(StreamStateHandle.class), Collections.singletonList(1L));
    final OperatorSubtaskState operatorSubtaskState = spy(OperatorSubtaskState.builder().setManagedOperatorState(managedOpHandle).setRawOperatorState(rawOpHandle).setManagedKeyedState(managedKeyedHandle).setRawKeyedState(rawKeyedHandle).setInputChannelState(StateObjectCollection.singleton(inputChannelStateHandle)).setResultSubpartitionState(StateObjectCollection.singleton(resultSubpartitionStateHandle)).build());
    TaskStateSnapshot subtaskState = spy(new TaskStateSnapshot());
    subtaskState.putSubtaskStateByOperatorID(new OperatorID(), operatorSubtaskState);
    when(subtaskState.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(vertex.getJobvertexId()))).thenReturn(operatorSubtaskState);
    AcknowledgeCheckpoint acknowledgeMessage = new AcknowledgeCheckpoint(testGraph.getJobID(), vertex.getCurrentExecutionAttempt().getAttemptId(), checkpointId, new CheckpointMetrics(), subtaskState);
    try {
        coord.receiveAcknowledgeMessage(acknowledgeMessage, "Unknown location");
        fail("Expected a checkpoint exception because the completed checkpoint store could not " + "store the completed checkpoint.");
    } catch (CheckpointException e) {
    // ignore because we expected this exception
    }
    // make sure that the pending checkpoint has been discarded after we could not complete it
    assertTrue(pendingCheckpoint.isDisposed());
    // make sure that the subtask state has been discarded after we could not complete it.
    verify(operatorSubtaskState).discardState();
    verify(operatorSubtaskState.getManagedOperatorState().iterator().next()).discardState();
    verify(operatorSubtaskState.getRawOperatorState().iterator().next()).discardState();
    verify(operatorSubtaskState.getManagedKeyedState().iterator().next()).discardState();
    verify(operatorSubtaskState.getRawKeyedState().iterator().next()).discardState();
    verify(operatorSubtaskState.getInputChannelState().iterator().next().getDelegate()).discardState();
    verify(operatorSubtaskState.getResultSubpartitionState().iterator().next().getDelegate()).discardState();
}