Examples with IOManager org.apache.flink.runtime.io.disk.iomanager.IOManager used on opensource projects

Example 1 with IOManager

use of org.apache.flink.runtime.io.disk.iomanager.IOManager in project flink by apache.

the class TaskManagerServices method fromConfiguration.

// --------------------------------------------------------------------------------------------
//  Static factory methods for task manager services
// --------------------------------------------------------------------------------------------
/**
	 * Creates and returns the task manager services.
	 *
	 * @param resourceID resource ID of the task manager
	 * @param taskManagerServicesConfiguration task manager configuration
	 * @return task manager components
	 * @throws Exception
	 */
public static TaskManagerServices fromConfiguration(TaskManagerServicesConfiguration taskManagerServicesConfiguration, ResourceID resourceID) throws Exception {
    // pre-start checks
    checkTempDirs(taskManagerServicesConfiguration.getTmpDirPaths());
    final NetworkEnvironment network = createNetworkEnvironment(taskManagerServicesConfiguration);
    network.start();
    final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(resourceID, taskManagerServicesConfiguration.getTaskManagerAddress(), network.getConnectionManager().getDataPort());
    // this call has to happen strictly after the network stack has been initialized
    final MemoryManager memoryManager = createMemoryManager(taskManagerServicesConfiguration);
    // start the I/O manager, it will create some temp directories.
    final IOManager ioManager = new IOManagerAsync(taskManagerServicesConfiguration.getTmpDirPaths());
    final MetricRegistry metricRegistry = new MetricRegistry(taskManagerServicesConfiguration.getMetricRegistryConfiguration());
    final TaskManagerMetricGroup taskManagerMetricGroup = new TaskManagerMetricGroup(metricRegistry, taskManagerLocation.getHostname(), taskManagerLocation.getResourceID().toString());
    // Initialize the TM metrics
    TaskExecutorMetricsInitializer.instantiateStatusMetrics(taskManagerMetricGroup, network);
    final BroadcastVariableManager broadcastVariableManager = new BroadcastVariableManager();
    final FileCache fileCache = new FileCache(taskManagerServicesConfiguration.getTmpDirPaths());
    final List<ResourceProfile> resourceProfiles = new ArrayList<>(taskManagerServicesConfiguration.getNumberOfSlots());
    for (int i = 0; i < taskManagerServicesConfiguration.getNumberOfSlots(); i++) {
        resourceProfiles.add(new ResourceProfile(1.0, 42));
    }
    final TimerService<AllocationID> timerService = new TimerService<>(new ScheduledThreadPoolExecutor(1), taskManagerServicesConfiguration.getTimerServiceShutdownTimeout());
    final TaskSlotTable taskSlotTable = new TaskSlotTable(resourceProfiles, timerService);
    final JobManagerTable jobManagerTable = new JobManagerTable();
    final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation);
    return new TaskManagerServices(taskManagerLocation, memoryManager, ioManager, network, metricRegistry, taskManagerMetricGroup, broadcastVariableManager, fileCache, taskSlotTable, jobManagerTable, jobLeaderService);
}

Example 2 with IOManager

use of org.apache.flink.runtime.io.disk.iomanager.IOManager in project flink by apache.

the class LocalInputChannelTest method testConcurrentConsumeMultiplePartitions.

/**
	 * Tests the consumption of multiple subpartitions via local input channels.
	 *
	 * <p> Multiple producer tasks produce pipelined partitions, which are consumed by multiple
	 * tasks via local input channels.
	 */
@Test
public void testConcurrentConsumeMultiplePartitions() throws Exception {
    // Config
    final int parallelism = 32;
    final int producerBufferPoolSize = parallelism + 1;
    final int numberOfBuffersPerChannel = 1024;
    checkArgument(parallelism >= 1);
    checkArgument(producerBufferPoolSize >= parallelism);
    checkArgument(numberOfBuffersPerChannel >= 1);
    // Setup
    // One thread per produced partition and one per consumer
    final ExecutorService executor = Executors.newFixedThreadPool(2 * parallelism);
    final NetworkBufferPool networkBuffers = new NetworkBufferPool((parallelism * producerBufferPoolSize) + (parallelism * parallelism), TestBufferFactory.BUFFER_SIZE, MemoryType.HEAP);
    final ResultPartitionConsumableNotifier partitionConsumableNotifier = mock(ResultPartitionConsumableNotifier.class);
    final TaskActions taskActions = mock(TaskActions.class);
    final IOManager ioManager = mock(IOManager.class);
    final JobID jobId = new JobID();
    final ResultPartitionManager partitionManager = new ResultPartitionManager();
    final ResultPartitionID[] partitionIds = new ResultPartitionID[parallelism];
    final TestPartitionProducer[] partitionProducers = new TestPartitionProducer[parallelism];
    // Create all partitions
    for (int i = 0; i < parallelism; i++) {
        partitionIds[i] = new ResultPartitionID();
        final ResultPartition partition = new ResultPartition("Test Name", taskActions, jobId, partitionIds[i], ResultPartitionType.PIPELINED, parallelism, parallelism, partitionManager, partitionConsumableNotifier, ioManager, true);
        // Create a buffer pool for this partition
        partition.registerBufferPool(networkBuffers.createBufferPool(producerBufferPoolSize, producerBufferPoolSize));
        // Create the producer
        partitionProducers[i] = new TestPartitionProducer(partition, false, new TestPartitionProducerBufferSource(parallelism, partition.getBufferProvider(), numberOfBuffersPerChannel));
        // Register with the partition manager in order to allow the local input channels to
        // request their respective partitions.
        partitionManager.registerResultPartition(partition);
    }
    // Test
    try {
        // Submit producer tasks
        List<Future<?>> results = Lists.newArrayListWithCapacity(parallelism + 1);
        for (int i = 0; i < parallelism; i++) {
            results.add(executor.submit(partitionProducers[i]));
        }
        // Submit consumer
        for (int i = 0; i < parallelism; i++) {
            results.add(executor.submit(new TestLocalInputChannelConsumer(i, parallelism, numberOfBuffersPerChannel, networkBuffers.createBufferPool(parallelism, parallelism), partitionManager, new TaskEventDispatcher(), partitionIds)));
        }
        // Wait for all to finish
        for (Future<?> result : results) {
            result.get();
        }
    } finally {
        networkBuffers.destroy();
        executor.shutdown();
    }
}

Example 3 with IOManager

use of org.apache.flink.runtime.io.disk.iomanager.IOManager in project flink by apache.

the class FileChannelStreamsTest method testCloseAndDeleteInputView.

@Test
public void testCloseAndDeleteInputView() {
    final IOManager ioManager = new IOManagerAsync();
    try {
        MemoryManager memMan = new MemoryManager(4 * 16 * 1024, 1, 16 * 1024, MemoryType.HEAP, true);
        List<MemorySegment> memory = new ArrayList<MemorySegment>();
        memMan.allocatePages(new DummyInvokable(), memory, 4);
        FileIOChannel.ID channel = ioManager.createChannel();
        // add some test data
        try (FileWriter wrt = new FileWriter(channel.getPath())) {
            wrt.write("test data");
        }
        BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
        FileChannelInputView in = new FileChannelInputView(reader, memMan, memory, 9);
        // read just something
        in.readInt();
        // close for the first time, make sure all memory returns
        in.close();
        assertTrue(memMan.verifyEmpty());
        // close again, should not cause an exception
        in.close();
        // delete, make sure file is removed
        in.closeAndDelete();
        assertFalse(new File(channel.getPath()).exists());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        ioManager.shutdown();
    }
}

Example 4 with IOManager

use of org.apache.flink.runtime.io.disk.iomanager.IOManager in project flink by apache.

the class SeekableFileChannelInputViewTest method testSeek.

@Test
public void testSeek() {
    final IOManager ioManager = new IOManagerAsync();
    final int PAGE_SIZE = 16 * 1024;
    final int NUM_RECORDS = 120000;
    try {
        MemoryManager memMan = new MemoryManager(4 * PAGE_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
        List<MemorySegment> memory = new ArrayList<MemorySegment>();
        memMan.allocatePages(new DummyInvokable(), memory, 4);
        FileIOChannel.ID channel = ioManager.createChannel();
        BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
        FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
        // write some integers across 7.5 pages (7 pages = 114.688 bytes, 8 pages = 131.072 bytes)
        for (int i = 0; i < NUM_RECORDS; i += 4) {
            out.writeInt(i);
        }
        // close for the first time, make sure all memory returns
        out.close();
        assertTrue(memMan.verifyEmpty());
        memMan.allocatePages(new DummyInvokable(), memory, 4);
        SeekableFileChannelInputView in = new SeekableFileChannelInputView(ioManager, channel, memMan, memory, out.getBytesInLatestSegment());
        // read first, complete
        for (int i = 0; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to the middle of the 3rd page
        int i = 2 * PAGE_SIZE + PAGE_SIZE / 4;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to the end
        i = 120000 - 4;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to the beginning
        i = 0;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to after a page
        i = PAGE_SIZE;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to after a page
        i = 3 * PAGE_SIZE;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to the end
        i = NUM_RECORDS;
        in.seek(i);
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek out of bounds
        try {
            in.seek(-10);
            fail("should throw an exception");
        } catch (IllegalArgumentException ignored) {
        }
        try {
            in.seek(NUM_RECORDS + 1);
            fail("should throw an exception");
        } catch (IllegalArgumentException ignored) {
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        ioManager.shutdown();
    }
}

Example 5 with IOManager

use of org.apache.flink.runtime.io.disk.iomanager.IOManager in project flink by apache.

the class SpillableSubpartitionTest method testConcurrentFinishAndReleaseMemory.

/**
	 * Tests a fix for FLINK-2384.
	 *
	 * @see <a href="https://issues.apache.org/jira/browse/FLINK-2384">FLINK-2384</a>
	 */
@Test
public void testConcurrentFinishAndReleaseMemory() throws Exception {
    // Latches to blocking
    final CountDownLatch doneLatch = new CountDownLatch(1);
    final CountDownLatch blockLatch = new CountDownLatch(1);
    // Blocking spill writer (blocks on the close call)
    AsynchronousBufferFileWriter spillWriter = mock(AsynchronousBufferFileWriter.class);
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            blockLatch.countDown();
            doneLatch.await();
            return null;
        }
    }).when(spillWriter).close();
    // Mock I/O manager returning the blocking spill writer
    IOManager ioManager = mock(IOManager.class);
    when(ioManager.createBufferFileWriter(any(FileIOChannel.ID.class))).thenReturn(spillWriter);
    // The partition
    final SpillableSubpartition partition = new SpillableSubpartition(0, mock(ResultPartition.class), ioManager);
    // Spill the partition initially (creates the spill writer)
    assertEquals(0, partition.releaseMemory());
    ExecutorService executor = Executors.newSingleThreadExecutor();
    // Finish the partition (this blocks because of the mock blocking writer)
    Future<Void> blockingFinish = executor.submit(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            partition.finish();
            return null;
        }
    });
    // Ensure that the blocking call has been made
    blockLatch.await();
    // This call needs to go through. FLINK-2384 discovered a bug, in
    // which the finish call was holding a lock, which was leading to a
    // deadlock when another operation on the partition was happening.
    partition.releaseMemory();
    // Check that the finish call succeeded w/o problems as well to avoid
    // false test successes.
    doneLatch.countDown();
    blockingFinish.get();
}