Examples with JobID org.apache.flink.api.common.JobID used on opensource projects

Example 61 with JobID

use of org.apache.flink.api.common.JobID in project flink by apache.

the class InputGateConcurrentTest method testConsumptionWithLocalChannels.

@Test
public void testConsumptionWithLocalChannels() throws Exception {
    final int numChannels = 11;
    final int buffersPerChannel = 1000;
    final ResultPartition resultPartition = mock(ResultPartition.class);
    final PipelinedSubpartition[] partitions = new PipelinedSubpartition[numChannels];
    final Source[] sources = new Source[numChannels];
    final ResultPartitionManager resultPartitionManager = createResultPartitionManager(partitions);
    final SingleInputGate gate = new SingleInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    for (int i = 0; i < numChannels; i++) {
        LocalInputChannel channel = new LocalInputChannel(gate, i, new ResultPartitionID(), resultPartitionManager, mock(TaskEventDispatcher.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
        partitions[i] = new PipelinedSubpartition(0, resultPartition);
        sources[i] = new PipelinedSubpartitionSource(partitions[i]);
    }
    ProducerThread producer = new ProducerThread(sources, numChannels * buffersPerChannel, 4, 10);
    ConsumerThread consumer = new ConsumerThread(gate, numChannels * buffersPerChannel);
    producer.start();
    consumer.start();
    // the 'sync()' call checks for exceptions and failed assertions
    producer.sync();
    consumer.sync();
}

Example 62 with JobID

use of org.apache.flink.api.common.JobID in project flink by apache.

the class InputGateConcurrentTest method testConsumptionWithRemoteChannels.

@Test
public void testConsumptionWithRemoteChannels() throws Exception {
    final int numChannels = 11;
    final int buffersPerChannel = 1000;
    final ConnectionManager connManager = createDummyConnectionManager();
    final Source[] sources = new Source[numChannels];
    final SingleInputGate gate = new SingleInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    for (int i = 0; i < numChannels; i++) {
        RemoteInputChannel channel = new RemoteInputChannel(gate, i, new ResultPartitionID(), mock(ConnectionID.class), connManager, 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
        sources[i] = new RemoteChannelSource(channel);
    }
    ProducerThread producer = new ProducerThread(sources, numChannels * buffersPerChannel, 4, 10);
    ConsumerThread consumer = new ConsumerThread(gate, numChannels * buffersPerChannel);
    producer.start();
    consumer.start();
    // the 'sync()' call checks for exceptions and failed assertions
    producer.sync();
    consumer.sync();
}

Example 63 with JobID

use of org.apache.flink.api.common.JobID in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionRemoteChannels.

@Test
public void testFairConsumptionRemoteChannels() throws Exception {
    final int numChannels = 37;
    final int buffersPerChannel = 27;
    final Buffer mockBuffer = createMockBuffer(42);
    // ----- create some source channels and fill them with buffers -----
    SingleInputGate gate = new FairnessVerifyingInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    final ConnectionManager connManager = createDummyConnectionManager();
    final RemoteInputChannel[] channels = new RemoteInputChannel[numChannels];
    final int[] channelSequenceNums = new int[numChannels];
    for (int i = 0; i < numChannels; i++) {
        RemoteInputChannel channel = new RemoteInputChannel(gate, i, new ResultPartitionID(), mock(ConnectionID.class), connManager, 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        channels[i] = channel;
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
    }
    channels[11].onBuffer(mockBuffer, 0);
    channelSequenceNums[11]++;
    // read all the buffers and the EOF event
    for (int i = 0; i < numChannels * buffersPerChannel; i++) {
        assertNotNull(gate.getNextBufferOrEvent());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (RemoteInputChannel channel : channels) {
            int size = channel.getNumberOfQueuedBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == min + 1);
        if (i % (2 * numChannels) == 0) {
            // add three buffers to each channel, in random order
            fillRandom(channels, channelSequenceNums, 3, mockBuffer);
        }
    }
}

Example 64 with JobID

use of org.apache.flink.api.common.JobID in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionLocalChannelsPreFilled.

@Test
public void testFairConsumptionLocalChannelsPreFilled() throws Exception {
    final int numChannels = 37;
    final int buffersPerChannel = 27;
    final ResultPartition resultPartition = mock(ResultPartition.class);
    final Buffer mockBuffer = createMockBuffer(42);
    // ----- create some source channels and fill them with buffers -----
    final PipelinedSubpartition[] sources = new PipelinedSubpartition[numChannels];
    for (int i = 0; i < numChannels; i++) {
        PipelinedSubpartition partition = new PipelinedSubpartition(0, resultPartition);
        for (int p = 0; p < buffersPerChannel; p++) {
            partition.add(mockBuffer);
        }
        partition.finish();
        sources[i] = partition;
    }
    // ----- create reading side -----
    ResultPartitionManager resultPartitionManager = createResultPartitionManager(sources);
    SingleInputGate gate = new FairnessVerifyingInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    for (int i = 0; i < numChannels; i++) {
        LocalInputChannel channel = new LocalInputChannel(gate, i, new ResultPartitionID(), resultPartitionManager, mock(TaskEventDispatcher.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
    }
    // read all the buffers and the EOF event
    for (int i = numChannels * (buffersPerChannel + 1); i > 0; --i) {
        assertNotNull(gate.getNextBufferOrEvent());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (PipelinedSubpartition source : sources) {
            int size = source.getCurrentNumberOfBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == min + 1);
    }
    assertNull(gate.getNextBufferOrEvent());
}

Example 65 with JobID

use of org.apache.flink.api.common.JobID in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionRemoteChannelsPreFilled.

@Test
public void testFairConsumptionRemoteChannelsPreFilled() throws Exception {
    final int numChannels = 37;
    final int buffersPerChannel = 27;
    final Buffer mockBuffer = createMockBuffer(42);
    // ----- create some source channels and fill them with buffers -----
    SingleInputGate gate = new FairnessVerifyingInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    final ConnectionManager connManager = createDummyConnectionManager();
    final RemoteInputChannel[] channels = new RemoteInputChannel[numChannels];
    for (int i = 0; i < numChannels; i++) {
        RemoteInputChannel channel = new RemoteInputChannel(gate, i, new ResultPartitionID(), mock(ConnectionID.class), connManager, 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        channels[i] = channel;
        for (int p = 0; p < buffersPerChannel; p++) {
            channel.onBuffer(mockBuffer, p);
        }
        channel.onBuffer(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE), buffersPerChannel);
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
    }
    // read all the buffers and the EOF event
    for (int i = numChannels * (buffersPerChannel + 1); i > 0; --i) {
        assertNotNull(gate.getNextBufferOrEvent());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (RemoteInputChannel channel : channels) {
            int size = channel.getNumberOfQueuedBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == min + 1);
    }
    assertNull(gate.getNextBufferOrEvent());
}