Examples with ResultSubpartitionStateHandle org.apache.flink.runtime.state.ResultSubpartitionStateHandle used on opensource projects

Example 6 with ResultSubpartitionStateHandle

use of org.apache.flink.runtime.state.ResultSubpartitionStateHandle in project flink by apache.

the class OperatorSnapshotUtil method readStateHandle.

public static OperatorSubtaskState readStateHandle(String path) throws IOException, ClassNotFoundException {
    FileInputStream in = new FileInputStream(path);
    try (DataInputStream dis = new DataInputStream(in)) {
        // required for backwards compatibility.
        final int v = dis.readInt();
        // still required for compatibility to consume the bytes.
        MetadataV3Serializer.deserializeStreamStateHandle(dis);
        List<OperatorStateHandle> rawOperatorState = null;
        int numRawOperatorStates = dis.readInt();
        if (numRawOperatorStates >= 0) {
            rawOperatorState = new ArrayList<>();
            for (int i = 0; i < numRawOperatorStates; i++) {
                OperatorStateHandle operatorState = MetadataV3Serializer.deserializeOperatorStateHandleUtil(dis);
                rawOperatorState.add(operatorState);
            }
        }
        List<OperatorStateHandle> managedOperatorState = null;
        int numManagedOperatorStates = dis.readInt();
        if (numManagedOperatorStates >= 0) {
            managedOperatorState = new ArrayList<>();
            for (int i = 0; i < numManagedOperatorStates; i++) {
                OperatorStateHandle operatorState = MetadataV3Serializer.deserializeOperatorStateHandleUtil(dis);
                managedOperatorState.add(operatorState);
            }
        }
        List<KeyedStateHandle> rawKeyedState = null;
        int numRawKeyedStates = dis.readInt();
        if (numRawKeyedStates >= 0) {
            rawKeyedState = new ArrayList<>();
            for (int i = 0; i < numRawKeyedStates; i++) {
                KeyedStateHandle keyedState = MetadataV3Serializer.deserializeKeyedStateHandleUtil(dis);
                rawKeyedState.add(keyedState);
            }
        }
        List<KeyedStateHandle> managedKeyedState = null;
        int numManagedKeyedStates = dis.readInt();
        if (numManagedKeyedStates >= 0) {
            managedKeyedState = new ArrayList<>();
            for (int i = 0; i < numManagedKeyedStates; i++) {
                KeyedStateHandle keyedState = MetadataV3Serializer.deserializeKeyedStateHandleUtil(dis);
                managedKeyedState.add(keyedState);
            }
        }
        final StateObjectCollection<InputChannelStateHandle> inputChannelStateHandles = v == MetadataV3Serializer.VERSION ? MetadataV3Serializer.deserializeInputChannelStateHandle(dis) : StateObjectCollection.empty();
        final StateObjectCollection<ResultSubpartitionStateHandle> resultSubpartitionStateHandles = v == MetadataV3Serializer.VERSION ? MetadataV3Serializer.INSTANCE.deserializeResultSubpartitionStateHandle(dis) : StateObjectCollection.empty();
        return OperatorSubtaskState.builder().setManagedOperatorState(new StateObjectCollection<>(managedOperatorState)).setRawOperatorState(new StateObjectCollection<>(rawOperatorState)).setManagedKeyedState(new StateObjectCollection<>(managedKeyedState)).setRawKeyedState(new StateObjectCollection<>(rawKeyedState)).setInputChannelState(inputChannelStateHandles).setResultSubpartitionState(resultSubpartitionStateHandles).build();
    }
}

Example 7 with ResultSubpartitionStateHandle

use of org.apache.flink.runtime.state.ResultSubpartitionStateHandle in project flink by apache.

the class OperatorSnapshotUtil method writeStateHandle.

public static void writeStateHandle(OperatorSubtaskState state, String path) throws IOException {
    FileOutputStream out = new FileOutputStream(path);
    try (DataOutputStream dos = new DataOutputStream(out)) {
        // required for backwards compatibility.
        dos.writeInt(MetadataV3Serializer.VERSION);
        // still required for compatibility
        MetadataV3Serializer.serializeStreamStateHandle(null, dos);
        Collection<OperatorStateHandle> rawOperatorState = state.getRawOperatorState();
        if (rawOperatorState != null) {
            dos.writeInt(rawOperatorState.size());
            for (OperatorStateHandle operatorStateHandle : rawOperatorState) {
                MetadataV3Serializer.serializeOperatorStateHandleUtil(operatorStateHandle, dos);
            }
        } else {
            // this means no states, not even an empty list
            dos.writeInt(-1);
        }
        Collection<OperatorStateHandle> managedOperatorState = state.getManagedOperatorState();
        if (managedOperatorState != null) {
            dos.writeInt(managedOperatorState.size());
            for (OperatorStateHandle operatorStateHandle : managedOperatorState) {
                MetadataV3Serializer.serializeOperatorStateHandleUtil(operatorStateHandle, dos);
            }
        } else {
            // this means no states, not even an empty list
            dos.writeInt(-1);
        }
        Collection<KeyedStateHandle> rawKeyedState = state.getRawKeyedState();
        if (rawKeyedState != null) {
            dos.writeInt(rawKeyedState.size());
            for (KeyedStateHandle keyedStateHandle : rawKeyedState) {
                MetadataV3Serializer.serializeKeyedStateHandleUtil(keyedStateHandle, dos);
            }
        } else {
            // this means no operator states, not even an empty list
            dos.writeInt(-1);
        }
        Collection<KeyedStateHandle> managedKeyedState = state.getManagedKeyedState();
        if (managedKeyedState != null) {
            dos.writeInt(managedKeyedState.size());
            for (KeyedStateHandle keyedStateHandle : managedKeyedState) {
                MetadataV3Serializer.serializeKeyedStateHandleUtil(keyedStateHandle, dos);
            }
        } else {
            // this means no operator states, not even an empty list
            dos.writeInt(-1);
        }
        Collection<InputChannelStateHandle> inputChannelStateHandles = state.getInputChannelState();
        dos.writeInt(inputChannelStateHandles.size());
        for (InputChannelStateHandle inputChannelStateHandle : inputChannelStateHandles) {
            MetadataV3Serializer.INSTANCE.serializeInputChannelStateHandle(inputChannelStateHandle, dos);
        }
        Collection<ResultSubpartitionStateHandle> resultSubpartitionStateHandles = state.getResultSubpartitionState();
        dos.writeInt(inputChannelStateHandles.size());
        for (ResultSubpartitionStateHandle resultSubpartitionStateHandle : resultSubpartitionStateHandles) {
            MetadataV3Serializer.INSTANCE.serializeResultSubpartitionStateHandle(resultSubpartitionStateHandle, dos);
        }
        dos.flush();
    }
}

Example 8 with ResultSubpartitionStateHandle

use of org.apache.flink.runtime.state.ResultSubpartitionStateHandle in project flink by apache.

the class StateAssignmentOperation method reDistributeResultSubpartitionStates.

public <I, T extends AbstractChannelStateHandle<I>> void reDistributeResultSubpartitionStates(TaskStateAssignment assignment) {
    if (!assignment.hasOutputState) {
        return;
    }
    checkForUnsupportedToplogyChanges(assignment.oldState, OperatorSubtaskState::getResultSubpartitionState, assignment.outputOperatorID);
    final OperatorState outputState = assignment.oldState.get(assignment.outputOperatorID);
    final List<List<ResultSubpartitionStateHandle>> outputOperatorState = splitBySubtasks(outputState, OperatorSubtaskState::getResultSubpartitionState);
    final ExecutionJobVertex executionJobVertex = assignment.executionJobVertex;
    final List<IntermediateDataSet> outputs = executionJobVertex.getJobVertex().getProducedDataSets();
    if (outputState.getParallelism() == executionJobVertex.getParallelism()) {
        assignment.resultSubpartitionStates.putAll(toInstanceMap(assignment.outputOperatorID, outputOperatorState));
        return;
    }
    // according to output mapping.
    for (int partitionIndex = 0; partitionIndex < outputs.size(); partitionIndex++) {
        final List<List<ResultSubpartitionStateHandle>> partitionState = outputs.size() == 1 ? outputOperatorState : getPartitionState(outputOperatorState, ResultSubpartitionInfo::getPartitionIdx, partitionIndex);
        final MappingBasedRepartitioner<ResultSubpartitionStateHandle> repartitioner = new MappingBasedRepartitioner<>(assignment.getOutputMapping(partitionIndex).getRescaleMappings());
        final Map<OperatorInstanceID, List<ResultSubpartitionStateHandle>> repartitioned = applyRepartitioner(assignment.outputOperatorID, repartitioner, partitionState, outputOperatorState.size(), executionJobVertex.getParallelism());
        addToSubtasks(assignment.resultSubpartitionStates, repartitioned);
    }
}

Example 9 with ResultSubpartitionStateHandle

use of org.apache.flink.runtime.state.ResultSubpartitionStateHandle in project flink by apache.

the class CheckpointCoordinatorFailureTest method testFailingCompletedCheckpointStoreAdd.

/**
 * Tests that a failure while storing a completed checkpoint in the completed checkpoint store
 * will properly fail the originating pending checkpoint and clean upt the completed checkpoint.
 */
@Test
public void testFailingCompletedCheckpointStoreAdd() throws Exception {
    JobVertexID jobVertexId = new JobVertexID();
    final ManuallyTriggeredScheduledExecutor manuallyTriggeredScheduledExecutor = new ManuallyTriggeredScheduledExecutor();
    ExecutionGraph testGraph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexId).build();
    ExecutionVertex vertex = testGraph.getJobVertex(jobVertexId).getTaskVertices()[0];
    // set up the coordinator and validate the initial state
    CheckpointCoordinator coord = new CheckpointCoordinatorBuilder().setExecutionGraph(testGraph).setCompletedCheckpointStore(new FailingCompletedCheckpointStore(new Exception("The failing completed checkpoint store failed again... :-("))).setTimer(manuallyTriggeredScheduledExecutor).build();
    coord.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    assertEquals(1, coord.getNumberOfPendingCheckpoints());
    PendingCheckpoint pendingCheckpoint = coord.getPendingCheckpoints().values().iterator().next();
    assertFalse(pendingCheckpoint.isDisposed());
    final long checkpointId = coord.getPendingCheckpoints().keySet().iterator().next();
    KeyedStateHandle managedKeyedHandle = mock(KeyedStateHandle.class);
    KeyedStateHandle rawKeyedHandle = mock(KeyedStateHandle.class);
    OperatorStateHandle managedOpHandle = mock(OperatorStreamStateHandle.class);
    OperatorStateHandle rawOpHandle = mock(OperatorStreamStateHandle.class);
    InputChannelStateHandle inputChannelStateHandle = new InputChannelStateHandle(new InputChannelInfo(0, 1), mock(StreamStateHandle.class), Collections.singletonList(1L));
    ResultSubpartitionStateHandle resultSubpartitionStateHandle = new ResultSubpartitionStateHandle(new ResultSubpartitionInfo(0, 1), mock(StreamStateHandle.class), Collections.singletonList(1L));
    final OperatorSubtaskState operatorSubtaskState = spy(OperatorSubtaskState.builder().setManagedOperatorState(managedOpHandle).setRawOperatorState(rawOpHandle).setManagedKeyedState(managedKeyedHandle).setRawKeyedState(rawKeyedHandle).setInputChannelState(StateObjectCollection.singleton(inputChannelStateHandle)).setResultSubpartitionState(StateObjectCollection.singleton(resultSubpartitionStateHandle)).build());
    TaskStateSnapshot subtaskState = spy(new TaskStateSnapshot());
    subtaskState.putSubtaskStateByOperatorID(new OperatorID(), operatorSubtaskState);
    when(subtaskState.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(vertex.getJobvertexId()))).thenReturn(operatorSubtaskState);
    AcknowledgeCheckpoint acknowledgeMessage = new AcknowledgeCheckpoint(testGraph.getJobID(), vertex.getCurrentExecutionAttempt().getAttemptId(), checkpointId, new CheckpointMetrics(), subtaskState);
    try {
        coord.receiveAcknowledgeMessage(acknowledgeMessage, "Unknown location");
        fail("Expected a checkpoint exception because the completed checkpoint store could not " + "store the completed checkpoint.");
    } catch (CheckpointException e) {
    // ignore because we expected this exception
    }
    // make sure that the pending checkpoint has been discarded after we could not complete it
    assertTrue(pendingCheckpoint.isDisposed());
    // make sure that the subtask state has been discarded after we could not complete it.
    verify(operatorSubtaskState).discardState();
    verify(operatorSubtaskState.getManagedOperatorState().iterator().next()).discardState();
    verify(operatorSubtaskState.getRawOperatorState().iterator().next()).discardState();
    verify(operatorSubtaskState.getManagedKeyedState().iterator().next()).discardState();
    verify(operatorSubtaskState.getRawKeyedState().iterator().next()).discardState();
    verify(operatorSubtaskState.getInputChannelState().iterator().next().getDelegate()).discardState();
    verify(operatorSubtaskState.getResultSubpartitionState().iterator().next().getDelegate()).discardState();
}