Examples with OperatorStreamStateHandle org.apache.flink.runtime.state.OperatorStreamStateHandle used on opensource projects

Example 16 with OperatorStreamStateHandle

use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.

the class StateAssignmentOperationTest method testRepartitionUnionState.

@Test
public void testRepartitionUnionState() {
    OperatorID operatorID = new OperatorID();
    OperatorState operatorState = new OperatorState(operatorID, 2, 4);
    Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
    metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[] { 0 }, OperatorStateHandle.Mode.UNION));
    metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[] { 22, 44 }, OperatorStateHandle.Mode.UNION));
    OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[50]));
    operatorState.putState(0, OperatorSubtaskState.builder().setManagedOperatorState(osh1).build());
    Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(1);
    metaInfoMap2.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[] { 0 }, OperatorStateHandle.Mode.UNION));
    OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[20]));
    operatorState.putState(1, OperatorSubtaskState.builder().setManagedOperatorState(osh2).build());
    verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example 17 with OperatorStreamStateHandle

use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.

the class TestUtils method createOperatorState.

private static Collection<OperatorState> createOperatorState(OperatorID... operatorIds) {
    Random random = new Random();
    Collection<OperatorState> operatorStates = new ArrayList<>(operatorIds.length);
    for (OperatorID operatorId : operatorIds) {
        final OperatorState operatorState = new OperatorState(operatorId, 1, 42);
        final OperatorSubtaskState subtaskState = OperatorSubtaskState.builder().setManagedOperatorState(new OperatorStreamStateHandle(Collections.emptyMap(), new ByteStreamStateHandle("foobar", new byte[0]))).setInputChannelState(singleton(createNewInputChannelStateHandle(10, random))).setResultSubpartitionState(singleton(createNewResultSubpartitionStateHandle(10, random))).build();
        operatorState.putState(0, subtaskState);
        operatorStates.add(operatorState);
    }
    return operatorStates;
}

Example 18 with OperatorStreamStateHandle

use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.

the class CheckpointMetadataLoadingTest method createSavepointWithOperatorSubtaskState.

private static CompletedCheckpointStorageLocation createSavepointWithOperatorSubtaskState(final long checkpointId, final OperatorID operatorId, final int parallelism) throws IOException {
    final Random rnd = new Random();
    final OperatorSubtaskState subtaskState = OperatorSubtaskState.builder().setManagedOperatorState(new OperatorStreamStateHandle(Collections.emptyMap(), new ByteStreamStateHandle("testHandler", new byte[0]))).setInputChannelState(singleton(createNewInputChannelStateHandle(10, rnd))).setResultSubpartitionState(singleton(createNewResultSubpartitionStateHandle(10, rnd))).build();
    final OperatorState state = new OperatorState(operatorId, parallelism, parallelism);
    state.putState(0, subtaskState);
    return createSavepointWithOperatorState(checkpointId, state);
}

Example 19 with OperatorStreamStateHandle

use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.

the class StreamTaskStateInitializerImplTest method testWithRestore.

@SuppressWarnings("unchecked")
@Test
public void testWithRestore() throws Exception {
    StateBackend mockingBackend = spy(new StateBackend() {

        @Override
        public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws Exception {
            return mock(AbstractKeyedStateBackend.class);
        }

        @Override
        public OperatorStateBackend createOperatorStateBackend(Environment env, String operatorIdentifier, @Nonnull Collection<OperatorStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws Exception {
            return mock(OperatorStateBackend.class);
        }
    });
    OperatorID operatorID = new OperatorID(47L, 11L);
    TaskStateSnapshot taskStateSnapshot = new TaskStateSnapshot();
    Random random = new Random(0x42);
    OperatorSubtaskState operatorSubtaskState = OperatorSubtaskState.builder().setManagedOperatorState(new OperatorStreamStateHandle(Collections.singletonMap("a", new OperatorStateHandle.StateMetaInfo(new long[] { 0, 10 }, SPLIT_DISTRIBUTE)), CheckpointTestUtils.createDummyStreamStateHandle(random, null))).setRawOperatorState(new OperatorStreamStateHandle(Collections.singletonMap("_default_", new OperatorStateHandle.StateMetaInfo(new long[] { 0, 20, 30 }, SPLIT_DISTRIBUTE)), CheckpointTestUtils.createDummyStreamStateHandle(random, null))).setManagedKeyedState(CheckpointTestUtils.createDummyKeyGroupStateHandle(random, null)).setRawKeyedState(CheckpointTestUtils.createDummyKeyGroupStateHandle(random, null)).setInputChannelState(singleton(createNewInputChannelStateHandle(10, random))).setResultSubpartitionState(singleton(createNewResultSubpartitionStateHandle(10, random))).build();
    taskStateSnapshot.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
    JobManagerTaskRestore jobManagerTaskRestore = new JobManagerTaskRestore(42L, taskStateSnapshot);
    StreamTaskStateInitializer streamTaskStateManager = streamTaskStateManager(mockingBackend, jobManagerTaskRestore, false);
    AbstractStreamOperator<?> streamOperator = mock(AbstractStreamOperator.class);
    when(streamOperator.getOperatorID()).thenReturn(operatorID);
    TypeSerializer<?> typeSerializer = new IntSerializer();
    CloseableRegistry closeableRegistry = new CloseableRegistry();
    StreamOperatorStateContext stateContext = streamTaskStateManager.streamOperatorStateContext(streamOperator.getOperatorID(), streamOperator.getClass().getSimpleName(), new TestProcessingTimeService(), streamOperator, typeSerializer, closeableRegistry, new UnregisteredMetricsGroup(), 1.0, false);
    OperatorStateBackend operatorStateBackend = stateContext.operatorStateBackend();
    CheckpointableKeyedStateBackend<?> keyedStateBackend = stateContext.keyedStateBackend();
    InternalTimeServiceManager<?> timeServiceManager = stateContext.internalTimerServiceManager();
    CloseableIterable<KeyGroupStatePartitionStreamProvider> keyedStateInputs = stateContext.rawKeyedStateInputs();
    CloseableIterable<StatePartitionStreamProvider> operatorStateInputs = stateContext.rawOperatorStateInputs();
    Assert.assertTrue("Expected the context to be restored", stateContext.isRestored());
    Assert.assertEquals(OptionalLong.of(42L), stateContext.getRestoredCheckpointId());
    Assert.assertNotNull(operatorStateBackend);
    Assert.assertNotNull(keyedStateBackend);
    // this is deactivated on purpose so that it does not attempt to consume the raw keyed
    // state.
    Assert.assertNull(timeServiceManager);
    Assert.assertNotNull(keyedStateInputs);
    Assert.assertNotNull(operatorStateInputs);
    int count = 0;
    for (KeyGroupStatePartitionStreamProvider keyedStateInput : keyedStateInputs) {
        ++count;
    }
    Assert.assertEquals(1, count);
    count = 0;
    for (StatePartitionStreamProvider operatorStateInput : operatorStateInputs) {
        ++count;
    }
    Assert.assertEquals(3, count);
    checkCloseablesRegistered(closeableRegistry, operatorStateBackend, keyedStateBackend, keyedStateInputs, operatorStateInputs);
}