Examples with OperatorStreamStateHandle org.apache.flink.runtime.state.OperatorStreamStateHandle used on opensource projects

Example 1 with OperatorStreamStateHandle

use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.

the class JobMasterTester method createNonEmptyStateSnapshot.

private static TaskStateSnapshot createNonEmptyStateSnapshot(TaskInformation taskInformation) {
    final TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
    checkpointStateHandles.putSubtaskStateByOperatorID(OperatorID.fromJobVertexID(taskInformation.getJobVertexId()), OperatorSubtaskState.builder().setManagedOperatorState(new OperatorStreamStateHandle(Collections.emptyMap(), new ByteStreamStateHandle("foobar", new byte[0]))).build());
    return checkpointStateHandles;
}

Example 2 with OperatorStreamStateHandle

use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.

the class CheckpointCoordinatorTest method doTestPartitionableStateRepartitioning.

private void doTestPartitionableStateRepartitioning(Random r, int oldParallelism, int newParallelism, int numNamedStates, int maxPartitionsPerState) {
    List<List<OperatorStateHandle>> previousParallelOpInstanceStates = new ArrayList<>(oldParallelism);
    for (int i = 0; i < oldParallelism; ++i) {
        Path fakePath = new Path("/fake-" + i);
        Map<String, OperatorStateHandle.StateMetaInfo> namedStatesToOffsets = new HashMap<>();
        int off = 0;
        for (int s = 0; s < numNamedStates - 1; ++s) {
            long[] offs = new long[1 + r.nextInt(maxPartitionsPerState)];
            for (int o = 0; o < offs.length; ++o) {
                offs[o] = off;
                ++off;
            }
            OperatorStateHandle.Mode mode = r.nextInt(10) == 0 ? OperatorStateHandle.Mode.UNION : OperatorStateHandle.Mode.SPLIT_DISTRIBUTE;
            namedStatesToOffsets.put("State-" + s, new OperatorStateHandle.StateMetaInfo(offs, mode));
        }
        if (numNamedStates % 2 == 0) {
            // finally add a broadcast state
            long[] offs = { off + 1, off + 2, off + 3, off + 4 };
            namedStatesToOffsets.put("State-" + (numNamedStates - 1), new OperatorStateHandle.StateMetaInfo(offs, OperatorStateHandle.Mode.BROADCAST));
        }
        previousParallelOpInstanceStates.add(Collections.singletonList(new OperatorStreamStateHandle(namedStatesToOffsets, new FileStateHandle(fakePath, -1))));
    }
    Map<StreamStateHandle, Map<String, List<Long>>> expected = new HashMap<>();
    int taskIndex = 0;
    int expectedTotalPartitions = 0;
    for (List<OperatorStateHandle> previousParallelOpInstanceState : previousParallelOpInstanceStates) {
        Assert.assertEquals(1, previousParallelOpInstanceState.size());
        for (OperatorStateHandle psh : previousParallelOpInstanceState) {
            Map<String, OperatorStateHandle.StateMetaInfo> offsMap = psh.getStateNameToPartitionOffsets();
            Map<String, List<Long>> offsMapWithList = new HashMap<>(offsMap.size());
            for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> e : offsMap.entrySet()) {
                long[] offs = e.getValue().getOffsets();
                int replication;
                switch(e.getValue().getDistributionMode()) {
                    case UNION:
                        replication = newParallelism;
                        break;
                    case BROADCAST:
                        int extra = taskIndex < (newParallelism % oldParallelism) ? 1 : 0;
                        replication = newParallelism / oldParallelism + extra;
                        break;
                    case SPLIT_DISTRIBUTE:
                        replication = 1;
                        break;
                    default:
                        throw new RuntimeException("Unknown distribution mode " + e.getValue().getDistributionMode());
                }
                if (replication > 0) {
                    expectedTotalPartitions += replication * offs.length;
                    List<Long> offsList = new ArrayList<>(offs.length);
                    for (long off : offs) {
                        for (int p = 0; p < replication; ++p) {
                            offsList.add(off);
                        }
                    }
                    offsMapWithList.put(e.getKey(), offsList);
                }
            }
            if (!offsMapWithList.isEmpty()) {
                expected.put(psh.getDelegateStateHandle(), offsMapWithList);
            }
            taskIndex++;
        }
    }
    OperatorStateRepartitioner repartitioner = RoundRobinOperatorStateRepartitioner.INSTANCE;
    List<List<OperatorStateHandle>> pshs = repartitioner.repartitionState(previousParallelOpInstanceStates, oldParallelism, newParallelism);
    Map<StreamStateHandle, Map<String, List<Long>>> actual = new HashMap<>();
    int minCount = Integer.MAX_VALUE;
    int maxCount = 0;
    int actualTotalPartitions = 0;
    for (int p = 0; p < newParallelism; ++p) {
        int partitionCount = 0;
        Collection<OperatorStateHandle> pshc = pshs.get(p);
        for (OperatorStateHandle sh : pshc) {
            for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> namedState : sh.getStateNameToPartitionOffsets().entrySet()) {
                Map<String, List<Long>> stateToOffsets = actual.get(sh.getDelegateStateHandle());
                if (stateToOffsets == null) {
                    stateToOffsets = new HashMap<>();
                    actual.put(sh.getDelegateStateHandle(), stateToOffsets);
                }
                List<Long> actualOffs = stateToOffsets.get(namedState.getKey());
                if (actualOffs == null) {
                    actualOffs = new ArrayList<>();
                    stateToOffsets.put(namedState.getKey(), actualOffs);
                }
                long[] add = namedState.getValue().getOffsets();
                for (long l : add) {
                    actualOffs.add(l);
                }
                partitionCount += namedState.getValue().getOffsets().length;
            }
        }
        minCount = Math.min(minCount, partitionCount);
        maxCount = Math.max(maxCount, partitionCount);
        actualTotalPartitions += partitionCount;
    }
    for (Map<String, List<Long>> v : actual.values()) {
        for (List<Long> l : v.values()) {
            Collections.sort(l);
        }
    }
    // possible.
    if (oldParallelism != newParallelism) {
        int maxLoadDiff = maxCount - minCount;
        Assert.assertTrue("Difference in partition load is > 1 : " + maxLoadDiff, maxLoadDiff <= 1);
    }
    Assert.assertEquals(expectedTotalPartitions, actualTotalPartitions);
    Assert.assertEquals(expected, actual);
}

Example 3 with OperatorStreamStateHandle

use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.

the class StateAssignmentOperationTest method testRepartitionBroadcastStateWithNullSubtaskState.

@Test
public void testRepartitionBroadcastStateWithNullSubtaskState() {
    OperatorID operatorID = new OperatorID();
    OperatorState operatorState = new OperatorState(operatorID, 2, 4);
    // Only the subtask 0 reports the states.
    Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
    metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[] { 0, 10, 20 }, OperatorStateHandle.Mode.BROADCAST));
    metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[] { 30, 40, 50 }, OperatorStateHandle.Mode.BROADCAST));
    OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[60]));
    operatorState.putState(0, OperatorSubtaskState.builder().setManagedOperatorState(osh1).build());
    verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example 4 with OperatorStreamStateHandle

use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.

the class StateAssignmentOperationTest method testRepartitionBroadcastStateWithEmptySubtaskState.

@Test
public void testRepartitionBroadcastStateWithEmptySubtaskState() {
    OperatorID operatorID = new OperatorID();
    OperatorState operatorState = new OperatorState(operatorID, 2, 4);
    // Only the subtask 0 reports the states.
    Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
    metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[] { 0, 10, 20 }, OperatorStateHandle.Mode.BROADCAST));
    metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[] { 30, 40, 50 }, OperatorStateHandle.Mode.BROADCAST));
    OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[60]));
    operatorState.putState(0, OperatorSubtaskState.builder().setManagedOperatorState(osh1).build());
    // The subtask 1 report an empty snapshot.
    operatorState.putState(1, OperatorSubtaskState.builder().build());
    verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example 5 with OperatorStreamStateHandle

use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.

the class StateAssignmentOperationTest method testReDistributeCombinedPartitionableStates.

/**
 * Verify repartition logic on partitionable states with all modes.
 */
@Test
public void testReDistributeCombinedPartitionableStates() {
    OperatorID operatorID = new OperatorID();
    OperatorState operatorState = new OperatorState(operatorID, 2, 4);
    Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(6);
    metaInfoMap1.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[] { 0 }, OperatorStateHandle.Mode.UNION));
    metaInfoMap1.put("t-2", new OperatorStateHandle.StateMetaInfo(new long[] { 22, 44 }, OperatorStateHandle.Mode.UNION));
    metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[] { 52, 63 }, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
    metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[] { 67, 74, 75 }, OperatorStateHandle.Mode.BROADCAST));
    metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[] { 77, 88, 92 }, OperatorStateHandle.Mode.BROADCAST));
    metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[] { 101, 123, 127 }, OperatorStateHandle.Mode.BROADCAST));
    OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[130]));
    operatorState.putState(0, OperatorSubtaskState.builder().setManagedOperatorState(osh1).build());
    Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(3);
    metaInfoMap2.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[] { 0 }, OperatorStateHandle.Mode.UNION));
    metaInfoMap2.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[] { 20, 27, 28 }, OperatorStateHandle.Mode.BROADCAST));
    metaInfoMap2.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[] { 30, 44, 48 }, OperatorStateHandle.Mode.BROADCAST));
    metaInfoMap2.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[] { 57, 79, 83 }, OperatorStateHandle.Mode.BROADCAST));
    OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[86]));
    operatorState.putState(1, OperatorSubtaskState.builder().setManagedOperatorState(osh2).build());
    // rescale up case, parallelism 2 --> 3
    verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 3);
    // rescale down case, parallelism 2 --> 1
    verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 1);
    // not rescale
    verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 2);
}