Examples with IntermediateResultPartitionID org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID used on opensource projects

Example 31 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class DefaultExecutionVertexTest method setUp.

@Before
public void setUp() throws Exception {
    intermediateResultPartitionId = new IntermediateResultPartitionID();
    DefaultResultPartition schedulingResultPartition = new DefaultResultPartition(intermediateResultPartitionId, new IntermediateDataSetID(), BLOCKING, () -> ResultPartitionState.CREATED, () -> {
        throw new UnsupportedOperationException();
    }, () -> {
        throw new UnsupportedOperationException();
    });
    producerVertex = new DefaultExecutionVertex(new ExecutionVertexID(new JobVertexID(), 0), Collections.singletonList(schedulingResultPartition), stateSupplier, Collections.emptyList(), partitionID -> {
        throw new UnsupportedOperationException();
    });
    schedulingResultPartition.setProducer(producerVertex);
    List<ConsumedPartitionGroup> consumedPartitionGroups = Collections.singletonList(ConsumedPartitionGroup.fromSinglePartition(intermediateResultPartitionId));
    Map<IntermediateResultPartitionID, DefaultResultPartition> resultPartitionById = Collections.singletonMap(intermediateResultPartitionId, schedulingResultPartition);
    consumerVertex = new DefaultExecutionVertex(new ExecutionVertexID(new JobVertexID(), 0), Collections.emptyList(), stateSupplier, consumedPartitionGroups, resultPartitionById::get);
}

Example 32 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class DefaultExecutionVertexTest method testGetConsumedResultPartitions.

@Test
public void testGetConsumedResultPartitions() {
    IntermediateResultPartitionID partitionIds1 = IterableUtils.toStream(consumerVertex.getConsumedResults()).findAny().map(SchedulingResultPartition::getId).orElseThrow(() -> new IllegalArgumentException("can not find result partition"));
    assertEquals(partitionIds1, intermediateResultPartitionId);
}

Example 33 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class DefaultExecutionVertexTest method testGetProducedResultPartitions.

@Test
public void testGetProducedResultPartitions() {
    IntermediateResultPartitionID partitionIds1 = IterableUtils.toStream(producerVertex.getProducedResults()).findAny().map(SchedulingResultPartition::getId).orElseThrow(() -> new IllegalArgumentException("can not find result partition"));
    assertEquals(partitionIds1, intermediateResultPartitionId);
}

Example 34 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class ResultPartitionDeploymentDescriptor method from.

// ------------------------------------------------------------------------
public static ResultPartitionDeploymentDescriptor from(IntermediateResultPartition partition, int maxParallelism, boolean lazyScheduling) {
    final IntermediateDataSetID resultId = partition.getIntermediateResult().getId();
    final IntermediateResultPartitionID partitionId = partition.getPartitionId();
    final ResultPartitionType partitionType = partition.getIntermediateResult().getResultType();
    // The produced data is partitioned among a number of subpartitions.
    //
    // If no consumers are known at this point, we use a single subpartition, otherwise we have
    // one for each consuming sub task.
    int numberOfSubpartitions = 1;
    if (!partition.getConsumers().isEmpty() && !partition.getConsumers().get(0).isEmpty()) {
        if (partition.getConsumers().size() > 1) {
            throw new IllegalStateException("Currently, only a single consumer group per partition is supported.");
        }
        numberOfSubpartitions = partition.getConsumers().get(0).size();
    }
    return new ResultPartitionDeploymentDescriptor(resultId, partitionId, partitionType, numberOfSubpartitions, maxParallelism, lazyScheduling);
}

Example 35 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class NettyMessageSerializationTest method testEncodeDecode.

@Test
public void testEncodeDecode() {
    {
        Buffer buffer = spy(new Buffer(MemorySegmentFactory.allocateUnpooledSegment(1024), mock(BufferRecycler.class)));
        ByteBuffer nioBuffer = buffer.getNioBuffer();
        for (int i = 0; i < 1024; i += 4) {
            nioBuffer.putInt(i);
        }
        NettyMessage.BufferResponse expected = new NettyMessage.BufferResponse(buffer, random.nextInt(), new InputChannelID());
        NettyMessage.BufferResponse actual = encodeAndDecode(expected);
        // Verify recycle has been called on buffer instance
        verify(buffer, times(1)).recycle();
        final ByteBuf retainedSlice = actual.getNettyBuffer();
        // Ensure not recycled and same size as original buffer
        assertEquals(1, retainedSlice.refCnt());
        assertEquals(1024, retainedSlice.readableBytes());
        nioBuffer = retainedSlice.nioBuffer();
        for (int i = 0; i < 1024; i += 4) {
            assertEquals(i, nioBuffer.getInt());
        }
        // Release the retained slice
        actual.releaseBuffer();
        assertEquals(0, retainedSlice.refCnt());
        assertEquals(expected.sequenceNumber, actual.sequenceNumber);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        {
            IllegalStateException expectedError = new IllegalStateException();
            InputChannelID receiverId = new InputChannelID();
            NettyMessage.ErrorResponse expected = new NettyMessage.ErrorResponse(expectedError, receiverId);
            NettyMessage.ErrorResponse actual = encodeAndDecode(expected);
            assertEquals(expected.cause.getClass(), actual.cause.getClass());
            assertEquals(expected.cause.getMessage(), actual.cause.getMessage());
            assertEquals(receiverId, actual.receiverId);
        }
        {
            IllegalStateException expectedError = new IllegalStateException("Illegal illegal illegal");
            InputChannelID receiverId = new InputChannelID();
            NettyMessage.ErrorResponse expected = new NettyMessage.ErrorResponse(expectedError, receiverId);
            NettyMessage.ErrorResponse actual = encodeAndDecode(expected);
            assertEquals(expected.cause.getClass(), actual.cause.getClass());
            assertEquals(expected.cause.getMessage(), actual.cause.getMessage());
            assertEquals(receiverId, actual.receiverId);
        }
        {
            IllegalStateException expectedError = new IllegalStateException("Illegal illegal illegal");
            NettyMessage.ErrorResponse expected = new NettyMessage.ErrorResponse(expectedError);
            NettyMessage.ErrorResponse actual = encodeAndDecode(expected);
            assertEquals(expected.cause.getClass(), actual.cause.getClass());
            assertEquals(expected.cause.getMessage(), actual.cause.getMessage());
            assertNull(actual.receiverId);
            assertTrue(actual.isFatalError());
        }
    }
    {
        NettyMessage.PartitionRequest expected = new NettyMessage.PartitionRequest(new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID()), random.nextInt(), new InputChannelID());
        NettyMessage.PartitionRequest actual = encodeAndDecode(expected);
        assertEquals(expected.partitionId, actual.partitionId);
        assertEquals(expected.queueIndex, actual.queueIndex);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        NettyMessage.TaskEventRequest expected = new NettyMessage.TaskEventRequest(new IntegerTaskEvent(random.nextInt()), new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID()), new InputChannelID());
        NettyMessage.TaskEventRequest actual = encodeAndDecode(expected);
        assertEquals(expected.event, actual.event);
        assertEquals(expected.partitionId, actual.partitionId);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        NettyMessage.CancelPartitionRequest expected = new NettyMessage.CancelPartitionRequest(new InputChannelID());
        NettyMessage.CancelPartitionRequest actual = encodeAndDecode(expected);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        NettyMessage.CloseRequest expected = new NettyMessage.CloseRequest();
        NettyMessage.CloseRequest actual = encodeAndDecode(expected);
        assertEquals(expected.getClass(), actual.getClass());
    }
}