Search in sources :

Example 1 with ARBITRARY

use of org.apache.flink.runtime.io.network.api.writer.SubtaskStateMapper.ARBITRARY in project flink by apache.

the class StateAssignmentOperationTest method testChannelStateAssignmentDownscalingTwoDifferentGates.

@Test
public void testChannelStateAssignmentDownscalingTwoDifferentGates() throws JobException, JobExecutionException {
    JobVertex upstream1 = createJobVertex(new OperatorID(), 2);
    JobVertex upstream2 = createJobVertex(new OperatorID(), 2);
    JobVertex downstream = createJobVertex(new OperatorID(), 2);
    List<OperatorID> operatorIds = Stream.of(upstream1, upstream2, downstream).map(v -> v.getOperatorIDs().get(0).getGeneratedOperatorID()).collect(Collectors.toList());
    Map<OperatorID, OperatorState> states = buildOperatorStates(operatorIds, 3);
    connectVertices(upstream1, downstream, ARBITRARY, RANGE);
    connectVertices(upstream2, downstream, ROUND_ROBIN, ROUND_ROBIN);
    Map<OperatorID, ExecutionJobVertex> vertices = toExecutionVertices(upstream1, upstream2, downstream);
    new StateAssignmentOperation(0, new HashSet<>(vertices.values()), states, false).assignStates();
    assertEquals(new InflightDataRescalingDescriptor(array(gate(to(0, 1), mappings(to(0, 2), to(1)), set(1), RESCALING), gate(to(0, 2), mappings(to(0, 2), to(1)), emptySet(), RESCALING))), getAssignedState(vertices.get(operatorIds.get(2)), operatorIds.get(2), 0).getInputRescalingDescriptor());
    assertEquals(new InflightDataRescalingDescriptor(array(gate(to(0, 1), mappings(to(0, 2), to(1)), set(1), RESCALING), gate(to(0, 2), mappings(to(0, 2), to(1)), emptySet(), RESCALING))), getAssignedState(vertices.get(operatorIds.get(2)), operatorIds.get(2), 0).getInputRescalingDescriptor());
}
Also used : InflightDataRescalingDescriptorUtil.set(org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptorUtil.set) StateHandleDummyUtil.createNewOperatorStateHandle(org.apache.flink.runtime.checkpoint.StateHandleDummyUtil.createNewOperatorStateHandle) CoreMatchers.is(org.hamcrest.CoreMatchers.is) Arrays(java.util.Arrays) TestingDefaultExecutionGraphBuilder(org.apache.flink.runtime.executiongraph.TestingDefaultExecutionGraphBuilder) RANGE(org.apache.flink.runtime.io.network.api.writer.SubtaskStateMapper.RANGE) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) SubtaskStateMapper(org.apache.flink.runtime.io.network.api.writer.SubtaskStateMapper) Random(java.util.Random) RESCALING(org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptor.InflightDataGateOrPartitionRescalingDescriptor.MappingType.RESCALING) Collections.singletonList(java.util.Collections.singletonList) ARBITRARY(org.apache.flink.runtime.io.network.api.writer.SubtaskStateMapper.ARBITRARY) Arrays.asList(java.util.Arrays.asList) Map(java.util.Map) TestLogger(org.apache.flink.util.TestLogger) InflightDataGateOrPartitionRescalingDescriptor(org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptor.InflightDataGateOrPartitionRescalingDescriptor) StateHandleDummyUtil.createNewResultSubpartitionStateHandle(org.apache.flink.runtime.checkpoint.StateHandleDummyUtil.createNewResultSubpartitionStateHandle) KeyGroupRange(org.apache.flink.runtime.state.KeyGroupRange) EnumMap(java.util.EnumMap) KeyedStateHandle(org.apache.flink.runtime.state.KeyedStateHandle) Set(java.util.Set) Collectors(java.util.stream.Collectors) List(java.util.List) StateHandleDummyUtil.createNewInputChannelStateHandle(org.apache.flink.runtime.checkpoint.StateHandleDummyUtil.createNewInputChannelStateHandle) Stream(java.util.stream.Stream) OperatorInstanceID(org.apache.flink.runtime.jobgraph.OperatorInstanceID) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) InflightDataRescalingDescriptorUtil.rescalingDescriptor(org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptorUtil.rescalingDescriptor) IntStream(java.util.stream.IntStream) JobEdge(org.apache.flink.runtime.jobgraph.JobEdge) JobVertex(org.apache.flink.runtime.jobgraph.JobVertex) ExecutionGraph(org.apache.flink.runtime.executiongraph.ExecutionGraph) ResultPartitionType(org.apache.flink.runtime.io.network.partition.ResultPartitionType) HashMap(java.util.HashMap) StateHandleDummyUtil.createNewKeyedStateHandle(org.apache.flink.runtime.checkpoint.StateHandleDummyUtil.createNewKeyedStateHandle) OperatorStreamStateHandle(org.apache.flink.runtime.state.OperatorStreamStateHandle) Function(java.util.function.Function) InflightDataRescalingDescriptorUtil.array(org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptorUtil.array) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) HashSet(java.util.HashSet) JobException(org.apache.flink.runtime.JobException) InflightDataRescalingDescriptorUtil.to(org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptorUtil.to) JobGraphTestUtils(org.apache.flink.runtime.jobgraph.JobGraphTestUtils) OperatorStateHandle(org.apache.flink.runtime.state.OperatorStateHandle) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) JobExecutionException(org.apache.flink.runtime.client.JobExecutionException) DistributionPattern(org.apache.flink.runtime.jobgraph.DistributionPattern) ExecutionJobVertex(org.apache.flink.runtime.executiongraph.ExecutionJobVertex) Matchers.empty(org.hamcrest.Matchers.empty) Collections.emptySet(java.util.Collections.emptySet) ExecutionGraphTestUtils(org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils) ROUND_ROBIN(org.apache.flink.runtime.io.network.api.writer.SubtaskStateMapper.ROUND_ROBIN) Test(org.junit.Test) OperatorIDPair(org.apache.flink.runtime.OperatorIDPair) InflightDataRescalingDescriptorUtil.mappings(org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptorUtil.mappings) ByteStreamStateHandle(org.apache.flink.runtime.state.memory.ByteStreamStateHandle) Assert(org.junit.Assert) ExecutionVertex(org.apache.flink.runtime.executiongraph.ExecutionVertex) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) NoOpInvokable(org.apache.flink.runtime.testtasks.NoOpInvokable) JobVertex(org.apache.flink.runtime.jobgraph.JobVertex) ExecutionJobVertex(org.apache.flink.runtime.executiongraph.ExecutionJobVertex) ExecutionJobVertex(org.apache.flink.runtime.executiongraph.ExecutionJobVertex) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) HashSet(java.util.HashSet) Test(org.junit.Test)

Aggregations

Arrays (java.util.Arrays)1 Arrays.asList (java.util.Arrays.asList)1 Collections (java.util.Collections)1 Collections.emptySet (java.util.Collections.emptySet)1 Collections.singletonList (java.util.Collections.singletonList)1 EnumMap (java.util.EnumMap)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 Random (java.util.Random)1 Set (java.util.Set)1 Function (java.util.function.Function)1 Collectors (java.util.stream.Collectors)1 IntStream (java.util.stream.IntStream)1 Stream (java.util.stream.Stream)1 JobException (org.apache.flink.runtime.JobException)1 OperatorIDPair (org.apache.flink.runtime.OperatorIDPair)1 InflightDataGateOrPartitionRescalingDescriptor (org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptor.InflightDataGateOrPartitionRescalingDescriptor)1 RESCALING (org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptor.InflightDataGateOrPartitionRescalingDescriptor.MappingType.RESCALING)1