use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.
the class JobMasterTester method createNonEmptyStateSnapshot.
private static TaskStateSnapshot createNonEmptyStateSnapshot(TaskInformation taskInformation) {
final TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
checkpointStateHandles.putSubtaskStateByOperatorID(OperatorID.fromJobVertexID(taskInformation.getJobVertexId()), OperatorSubtaskState.builder().setManagedOperatorState(new OperatorStreamStateHandle(Collections.emptyMap(), new ByteStreamStateHandle("foobar", new byte[0]))).build());
return checkpointStateHandles;
}
use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.
the class CheckpointCoordinatorTest method doTestPartitionableStateRepartitioning.
private void doTestPartitionableStateRepartitioning(Random r, int oldParallelism, int newParallelism, int numNamedStates, int maxPartitionsPerState) {
List<List<OperatorStateHandle>> previousParallelOpInstanceStates = new ArrayList<>(oldParallelism);
for (int i = 0; i < oldParallelism; ++i) {
Path fakePath = new Path("/fake-" + i);
Map<String, OperatorStateHandle.StateMetaInfo> namedStatesToOffsets = new HashMap<>();
int off = 0;
for (int s = 0; s < numNamedStates - 1; ++s) {
long[] offs = new long[1 + r.nextInt(maxPartitionsPerState)];
for (int o = 0; o < offs.length; ++o) {
offs[o] = off;
++off;
}
OperatorStateHandle.Mode mode = r.nextInt(10) == 0 ? OperatorStateHandle.Mode.UNION : OperatorStateHandle.Mode.SPLIT_DISTRIBUTE;
namedStatesToOffsets.put("State-" + s, new OperatorStateHandle.StateMetaInfo(offs, mode));
}
if (numNamedStates % 2 == 0) {
// finally add a broadcast state
long[] offs = { off + 1, off + 2, off + 3, off + 4 };
namedStatesToOffsets.put("State-" + (numNamedStates - 1), new OperatorStateHandle.StateMetaInfo(offs, OperatorStateHandle.Mode.BROADCAST));
}
previousParallelOpInstanceStates.add(Collections.singletonList(new OperatorStreamStateHandle(namedStatesToOffsets, new FileStateHandle(fakePath, -1))));
}
Map<StreamStateHandle, Map<String, List<Long>>> expected = new HashMap<>();
int taskIndex = 0;
int expectedTotalPartitions = 0;
for (List<OperatorStateHandle> previousParallelOpInstanceState : previousParallelOpInstanceStates) {
Assert.assertEquals(1, previousParallelOpInstanceState.size());
for (OperatorStateHandle psh : previousParallelOpInstanceState) {
Map<String, OperatorStateHandle.StateMetaInfo> offsMap = psh.getStateNameToPartitionOffsets();
Map<String, List<Long>> offsMapWithList = new HashMap<>(offsMap.size());
for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> e : offsMap.entrySet()) {
long[] offs = e.getValue().getOffsets();
int replication;
switch(e.getValue().getDistributionMode()) {
case UNION:
replication = newParallelism;
break;
case BROADCAST:
int extra = taskIndex < (newParallelism % oldParallelism) ? 1 : 0;
replication = newParallelism / oldParallelism + extra;
break;
case SPLIT_DISTRIBUTE:
replication = 1;
break;
default:
throw new RuntimeException("Unknown distribution mode " + e.getValue().getDistributionMode());
}
if (replication > 0) {
expectedTotalPartitions += replication * offs.length;
List<Long> offsList = new ArrayList<>(offs.length);
for (long off : offs) {
for (int p = 0; p < replication; ++p) {
offsList.add(off);
}
}
offsMapWithList.put(e.getKey(), offsList);
}
}
if (!offsMapWithList.isEmpty()) {
expected.put(psh.getDelegateStateHandle(), offsMapWithList);
}
taskIndex++;
}
}
OperatorStateRepartitioner repartitioner = RoundRobinOperatorStateRepartitioner.INSTANCE;
List<List<OperatorStateHandle>> pshs = repartitioner.repartitionState(previousParallelOpInstanceStates, oldParallelism, newParallelism);
Map<StreamStateHandle, Map<String, List<Long>>> actual = new HashMap<>();
int minCount = Integer.MAX_VALUE;
int maxCount = 0;
int actualTotalPartitions = 0;
for (int p = 0; p < newParallelism; ++p) {
int partitionCount = 0;
Collection<OperatorStateHandle> pshc = pshs.get(p);
for (OperatorStateHandle sh : pshc) {
for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> namedState : sh.getStateNameToPartitionOffsets().entrySet()) {
Map<String, List<Long>> stateToOffsets = actual.get(sh.getDelegateStateHandle());
if (stateToOffsets == null) {
stateToOffsets = new HashMap<>();
actual.put(sh.getDelegateStateHandle(), stateToOffsets);
}
List<Long> actualOffs = stateToOffsets.get(namedState.getKey());
if (actualOffs == null) {
actualOffs = new ArrayList<>();
stateToOffsets.put(namedState.getKey(), actualOffs);
}
long[] add = namedState.getValue().getOffsets();
for (long l : add) {
actualOffs.add(l);
}
partitionCount += namedState.getValue().getOffsets().length;
}
}
minCount = Math.min(minCount, partitionCount);
maxCount = Math.max(maxCount, partitionCount);
actualTotalPartitions += partitionCount;
}
for (Map<String, List<Long>> v : actual.values()) {
for (List<Long> l : v.values()) {
Collections.sort(l);
}
}
// possible.
if (oldParallelism != newParallelism) {
int maxLoadDiff = maxCount - minCount;
Assert.assertTrue("Difference in partition load is > 1 : " + maxLoadDiff, maxLoadDiff <= 1);
}
Assert.assertEquals(expectedTotalPartitions, actualTotalPartitions);
Assert.assertEquals(expected, actual);
}
use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.
the class StateAssignmentOperationTest method testRepartitionBroadcastStateWithNullSubtaskState.
@Test
public void testRepartitionBroadcastStateWithNullSubtaskState() {
OperatorID operatorID = new OperatorID();
OperatorState operatorState = new OperatorState(operatorID, 2, 4);
// Only the subtask 0 reports the states.
Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[] { 0, 10, 20 }, OperatorStateHandle.Mode.BROADCAST));
metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[] { 30, 40, 50 }, OperatorStateHandle.Mode.BROADCAST));
OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[60]));
operatorState.putState(0, OperatorSubtaskState.builder().setManagedOperatorState(osh1).build());
verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}
use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.
the class StateAssignmentOperationTest method testRepartitionBroadcastStateWithEmptySubtaskState.
@Test
public void testRepartitionBroadcastStateWithEmptySubtaskState() {
OperatorID operatorID = new OperatorID();
OperatorState operatorState = new OperatorState(operatorID, 2, 4);
// Only the subtask 0 reports the states.
Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[] { 0, 10, 20 }, OperatorStateHandle.Mode.BROADCAST));
metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[] { 30, 40, 50 }, OperatorStateHandle.Mode.BROADCAST));
OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[60]));
operatorState.putState(0, OperatorSubtaskState.builder().setManagedOperatorState(osh1).build());
// The subtask 1 report an empty snapshot.
operatorState.putState(1, OperatorSubtaskState.builder().build());
verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}
use of org.apache.flink.runtime.state.OperatorStreamStateHandle in project flink by apache.
the class StateAssignmentOperationTest method testReDistributeCombinedPartitionableStates.
/**
* Verify repartition logic on partitionable states with all modes.
*/
@Test
public void testReDistributeCombinedPartitionableStates() {
OperatorID operatorID = new OperatorID();
OperatorState operatorState = new OperatorState(operatorID, 2, 4);
Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(6);
metaInfoMap1.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[] { 0 }, OperatorStateHandle.Mode.UNION));
metaInfoMap1.put("t-2", new OperatorStateHandle.StateMetaInfo(new long[] { 22, 44 }, OperatorStateHandle.Mode.UNION));
metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[] { 52, 63 }, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[] { 67, 74, 75 }, OperatorStateHandle.Mode.BROADCAST));
metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[] { 77, 88, 92 }, OperatorStateHandle.Mode.BROADCAST));
metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[] { 101, 123, 127 }, OperatorStateHandle.Mode.BROADCAST));
OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[130]));
operatorState.putState(0, OperatorSubtaskState.builder().setManagedOperatorState(osh1).build());
Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(3);
metaInfoMap2.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[] { 0 }, OperatorStateHandle.Mode.UNION));
metaInfoMap2.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[] { 20, 27, 28 }, OperatorStateHandle.Mode.BROADCAST));
metaInfoMap2.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[] { 30, 44, 48 }, OperatorStateHandle.Mode.BROADCAST));
metaInfoMap2.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[] { 57, 79, 83 }, OperatorStateHandle.Mode.BROADCAST));
OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[86]));
operatorState.putState(1, OperatorSubtaskState.builder().setManagedOperatorState(osh2).build());
// rescale up case, parallelism 2 --> 3
verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 3);
// rescale down case, parallelism 2 --> 1
verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 1);
// not rescale
verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 2);
}
Aggregations