Examples with IOManagerAsync org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync used on opensource projects

Example 46 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class HashTableTest method testSpillingWhenBuildingTableWithoutOverflow.

/**
 * Tests that the MutableHashTable spills its partitions when creating the initial table without
 * overflow segments in the partitions. This means that the records are large.
 */
@Test
public void testSpillingWhenBuildingTableWithoutOverflow() throws Exception {
    try (final IOManager ioMan = new IOManagerAsync()) {
        final TypeSerializer<byte[]> serializer = BytePrimitiveArraySerializer.INSTANCE;
        final TypeComparator<byte[]> buildComparator = new BytePrimitiveArrayComparator(true);
        final TypeComparator<byte[]> probeComparator = new BytePrimitiveArrayComparator(true);
        @SuppressWarnings("unchecked") final TypePairComparator<byte[], byte[]> pairComparator = new GenericPairComparator<>(new BytePrimitiveArrayComparator(true), new BytePrimitiveArrayComparator(true));
        final int pageSize = 128;
        final int numSegments = 33;
        List<MemorySegment> memory = getMemory(numSegments, pageSize);
        MutableHashTable<byte[], byte[]> table = new MutableHashTable<byte[], byte[]>(serializer, serializer, buildComparator, probeComparator, pairComparator, memory, ioMan, 1, false);
        int numElements = 9;
        table.open(new CombiningIterator<byte[]>(new ByteArrayIterator(numElements, 128, (byte) 0), new ByteArrayIterator(numElements, 128, (byte) 1)), new CombiningIterator<byte[]>(new ByteArrayIterator(1, 128, (byte) 0), new ByteArrayIterator(1, 128, (byte) 1)));
        while (table.nextRecord()) {
            MutableObjectIterator<byte[]> iterator = table.getBuildSideIterator();
            int counter = 0;
            while (iterator.next() != null) {
                counter++;
            }
            // check that we retrieve all our elements
            Assert.assertEquals(numElements, counter);
        }
        table.close();
    }
}

Example 47 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class ReOpenableHashTableTestBase method beforeTest.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Before
public void beforeTest() {
    this.recordSerializer = TestData.getIntStringTupleSerializer();
    this.record1Comparator = TestData.getIntStringTupleComparator();
    this.record2Comparator = TestData.getIntStringTupleComparator();
    this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);
    this.recordBuildSideAccesssor = TestData.getIntIntTupleSerializer();
    this.recordProbeSideAccesssor = TestData.getIntIntTupleSerializer();
    this.recordBuildSideComparator = TestData.getIntIntTupleComparator();
    this.recordProbeSideComparator = TestData.getIntIntTupleComparator();
    this.pactRecordComparator = new GenericPairComparator(this.recordBuildSideComparator, this.recordProbeSideComparator);
    this.memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(MEMORY_SIZE).setPageSize(PAGE_SIZE).build();
    this.ioManager = new IOManagerAsync();
}

Example 48 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class TaskManagerServices method fromConfiguration.

// --------------------------------------------------------------------------------------------
// Static factory methods for task manager services
// --------------------------------------------------------------------------------------------
/**
 * Creates and returns the task manager services.
 *
 * @param taskManagerServicesConfiguration task manager configuration
 * @param permanentBlobService permanentBlobService used by the services
 * @param taskManagerMetricGroup metric group of the task manager
 * @param ioExecutor executor for async IO operations
 * @param fatalErrorHandler to handle class loading OOMs
 * @param workingDirectory the working directory of the process
 * @return task manager components
 * @throws Exception
 */
public static TaskManagerServices fromConfiguration(TaskManagerServicesConfiguration taskManagerServicesConfiguration, PermanentBlobService permanentBlobService, MetricGroup taskManagerMetricGroup, ExecutorService ioExecutor, FatalErrorHandler fatalErrorHandler, WorkingDirectory workingDirectory) throws Exception {
    // pre-start checks
    checkTempDirs(taskManagerServicesConfiguration.getTmpDirPaths());
    final TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
    // start the I/O manager, it will create some temp directories.
    final IOManager ioManager = new IOManagerAsync(taskManagerServicesConfiguration.getTmpDirPaths());
    final ShuffleEnvironment<?, ?> shuffleEnvironment = createShuffleEnvironment(taskManagerServicesConfiguration, taskEventDispatcher, taskManagerMetricGroup, ioExecutor);
    final int listeningDataPort = shuffleEnvironment.start();
    final KvStateService kvStateService = KvStateService.fromConfiguration(taskManagerServicesConfiguration);
    kvStateService.start();
    final UnresolvedTaskManagerLocation unresolvedTaskManagerLocation = new UnresolvedTaskManagerLocation(taskManagerServicesConfiguration.getResourceID(), taskManagerServicesConfiguration.getExternalAddress(), // iff the external data port is not explicitly defined
    taskManagerServicesConfiguration.getExternalDataPort() > 0 ? taskManagerServicesConfiguration.getExternalDataPort() : listeningDataPort);
    final BroadcastVariableManager broadcastVariableManager = new BroadcastVariableManager();
    final TaskSlotTable<Task> taskSlotTable = createTaskSlotTable(taskManagerServicesConfiguration.getNumberOfSlots(), taskManagerServicesConfiguration.getTaskExecutorResourceSpec(), taskManagerServicesConfiguration.getTimerServiceShutdownTimeout(), taskManagerServicesConfiguration.getPageSize(), ioExecutor);
    final JobTable jobTable = DefaultJobTable.create();
    final JobLeaderService jobLeaderService = new DefaultJobLeaderService(unresolvedTaskManagerLocation, taskManagerServicesConfiguration.getRetryingRegistrationConfiguration());
    final TaskExecutorLocalStateStoresManager taskStateManager = new TaskExecutorLocalStateStoresManager(taskManagerServicesConfiguration.isLocalRecoveryEnabled(), taskManagerServicesConfiguration.getLocalRecoveryStateDirectories(), ioExecutor);
    final TaskExecutorStateChangelogStoragesManager changelogStoragesManager = new TaskExecutorStateChangelogStoragesManager();
    final boolean failOnJvmMetaspaceOomError = taskManagerServicesConfiguration.getConfiguration().getBoolean(CoreOptions.FAIL_ON_USER_CLASS_LOADING_METASPACE_OOM);
    final boolean checkClassLoaderLeak = taskManagerServicesConfiguration.getConfiguration().getBoolean(CoreOptions.CHECK_LEAKED_CLASSLOADER);
    final LibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(permanentBlobService, BlobLibraryCacheManager.defaultClassLoaderFactory(taskManagerServicesConfiguration.getClassLoaderResolveOrder(), taskManagerServicesConfiguration.getAlwaysParentFirstLoaderPatterns(), failOnJvmMetaspaceOomError ? fatalErrorHandler : null, checkClassLoaderLeak));
    final SlotAllocationSnapshotPersistenceService slotAllocationSnapshotPersistenceService;
    if (taskManagerServicesConfiguration.isLocalRecoveryEnabled()) {
        slotAllocationSnapshotPersistenceService = new FileSlotAllocationSnapshotPersistenceService(workingDirectory.getSlotAllocationSnapshotDirectory());
    } else {
        slotAllocationSnapshotPersistenceService = NoOpSlotAllocationSnapshotPersistenceService.INSTANCE;
    }
    return new TaskManagerServices(unresolvedTaskManagerLocation, taskManagerServicesConfiguration.getManagedMemorySize().getBytes(), ioManager, shuffleEnvironment, kvStateService, broadcastVariableManager, taskSlotTable, jobTable, jobLeaderService, taskStateManager, changelogStoragesManager, taskEventDispatcher, ioExecutor, libraryCacheManager, slotAllocationSnapshotPersistenceService);
}

Example 49 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class StreamTaskSystemExitTest method createSystemExitTask.

private Task createSystemExitTask(final String invokableClassName, StreamOperator<?> operator) throws Exception {
    final Configuration taskConfiguration = new Configuration();
    final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
    streamConfig.setOperatorID(new OperatorID());
    streamConfig.setStreamOperator(operator);
    // for source run
    streamConfig.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    final JobInformation jobInformation = new JobInformation(new JobID(), "Test Job", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.emptyList(), Collections.emptyList());
    final TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "Test Task", 1, 1, invokableClassName, taskConfiguration);
    final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
    final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    return new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), MemoryManagerBuilder.newBuilder().setMemorySize(32L * 1024L).build(), new IOManagerAsync(), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, new TestTaskStateManager(), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), TestingClassLoaderLease.newBuilder().build(), mock(FileCache.class), taskManagerRuntimeInfo, UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), Executors.directExecutor());
}

Example 50 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class HashTableITCase method setup.

@Before
public void setup() {
    final int[] keyPos = new int[] { 0 };
    @SuppressWarnings("unchecked") final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
    this.recordBuildSideAccesssor = RecordSerializer.get();
    this.recordProbeSideAccesssor = RecordSerializer.get();
    this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
    this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
    this.pactRecordComparator = new RecordPairComparatorFirstInt();
    this.pairBuildSideAccesssor = new IntPairSerializer();
    this.pairProbeSideAccesssor = new IntPairSerializer();
    this.pairBuildSideComparator = new IntPairComparator();
    this.pairProbeSideComparator = new IntPairComparator();
    this.pairComparator = new IntPairPairComparator();
    this.memManager = MemoryManagerBuilder.newBuilder().setMemorySize(32 * 1024 * 1024).build();
    this.ioManager = new IOManagerAsync();
}