Examples with OneShotLatch org.apache.flink.core.testutils.OneShotLatch used on opensource projects

Example 46 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class CheckpointStateOutputStreamTest method testCloseDoesNotLock.

/**
 * This test validates that a close operation can happen even while a 'closeAndGetHandle()' call
 * is in progress.
 *
 * <p>That behavior is essential for fast cancellation (concurrent cleanup).
 */
@Test
public void testCloseDoesNotLock() throws Exception {
    final Path folder = new Path(tmp.newFolder().toURI());
    final String fileName = "this-is-ignored-anyways.file";
    final FileSystem fileSystem = spy(new FsWithoutRecoverableWriter((path) -> new BlockerStream()));
    final FSDataOutputStream checkpointStream = createTestStream(fileSystem, folder, fileName);
    final OneShotLatch sync = new OneShotLatch();
    final CheckedThread thread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            sync.trigger();
            // that call should now block, because it accesses the position
            closeAndGetResult(checkpointStream);
        }
    };
    thread.start();
    sync.await();
    checkpointStream.close();
    // it is not important for this test, important is that the thread does not freeze/lock up
    try {
        thread.sync();
    } catch (IOException ignored) {
    }
}

Example 47 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class StateBackendTestBase method testAsyncSnapshotCancellation.

@Test
public void testAsyncSnapshotCancellation() throws Exception {
    assumeTrue(snapshotUsesStreamFactory());
    OneShotLatch blocker = new OneShotLatch();
    OneShotLatch waiter = new OneShotLatch();
    BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
    streamFactory.setWaiterLatch(waiter);
    streamFactory.setBlockerLatch(blocker);
    streamFactory.setAfterNumberInvocations(10);
    final CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    try {
        if (!supportsAsynchronousSnapshots()) {
            return;
        }
        InternalValueState<Integer, VoidNamespace, Integer> valueState = backend.createInternalState(VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));
        valueState.setCurrentNamespace(VoidNamespace.INSTANCE);
        for (int i = 0; i < 10; ++i) {
            backend.setCurrentKey(i);
            valueState.update(i);
        }
        RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
        Thread runner = new Thread(snapshot);
        runner.start();
        // wait until the code reached some stream read
        waiter.await();
        // close the backend to see if the close is propagated to the stream
        IOUtils.closeQuietly(backend);
        // unblock the stream so that it can run into the IOException
        blocker.trigger();
        runner.join();
        try {
            snapshot.get();
            fail("Close was not propagated.");
        } catch (CancellationException ex) {
        // ignore
        }
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}

Example 48 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class NetworkBufferPoolTest method testRequestMemorySegmentsWithBuffersTaken.

/**
 * Tests {@link NetworkBufferPool#requestUnpooledMemorySegments(int)} with the {@link
 * NetworkBufferPool} currently not containing the number of required free segments (currently
 * occupied by a buffer pool).
 */
@Test
public void testRequestMemorySegmentsWithBuffersTaken() throws IOException, InterruptedException {
    final int numBuffers = 10;
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(numBuffers, 128);
    final List<Buffer> buffers = new ArrayList<>(numBuffers);
    List<MemorySegment> memorySegments = Collections.emptyList();
    Thread bufferRecycler = null;
    BufferPool lbp1 = null;
    try {
        lbp1 = networkBufferPool.createBufferPool(numBuffers / 2, numBuffers);
        // take all buffers (more than the minimum required)
        for (int i = 0; i < numBuffers; ++i) {
            Buffer buffer = lbp1.requestBuffer();
            buffers.add(buffer);
            assertNotNull(buffer);
        }
        // requestMemorySegments() below will wait for buffers
        // this will make sure that enough buffers are freed eventually for it to continue
        final OneShotLatch isRunning = new OneShotLatch();
        bufferRecycler = new Thread(() -> {
            try {
                isRunning.trigger();
                Thread.sleep(100);
            } catch (InterruptedException ignored) {
            }
            for (Buffer buffer : buffers) {
                buffer.recycleBuffer();
            }
        });
        bufferRecycler.start();
        // take more buffers than are freely available at the moment via requestMemorySegments()
        isRunning.await();
        memorySegments = networkBufferPool.requestUnpooledMemorySegments(numBuffers / 2);
        assertThat(memorySegments, not(hasItem(nullValue())));
    } finally {
        if (bufferRecycler != null) {
            bufferRecycler.join();
        }
        if (lbp1 != null) {
            lbp1.lazyDestroy();
        }
        networkBufferPool.recycleUnpooledMemorySegments(memorySegments);
        networkBufferPool.destroy();
    }
}

Example 49 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class NetworkBufferPoolTest method testRequestMemorySegmentsInterruptable.

/**
 * Tests {@link NetworkBufferPool#requestUnpooledMemorySegments(int)}, verifying it may be
 * aborted in case of a concurrent {@link NetworkBufferPool#destroy()} call.
 */
@Test
public void testRequestMemorySegmentsInterruptable() throws Exception {
    final int numBuffers = 10;
    NetworkBufferPool globalPool = new NetworkBufferPool(numBuffers, 128);
    MemorySegment segment = globalPool.requestPooledMemorySegment();
    assertNotNull(segment);
    final OneShotLatch isRunning = new OneShotLatch();
    CheckedThread asyncRequest = new CheckedThread() {

        @Override
        public void go() throws IOException {
            isRunning.trigger();
            globalPool.requestUnpooledMemorySegments(10);
        }
    };
    asyncRequest.start();
    // We want the destroy call inside the blocking part of the
    // globalPool.requestMemorySegments()
    // call above. We cannot guarantee this though but make it highly probable:
    isRunning.await();
    Thread.sleep(10);
    globalPool.destroy();
    segment.free();
    try {
        Exception ex = assertThrows(IllegalStateException.class, asyncRequest::sync);
        assertTrue(ex.getMessage().contains("destroyed"));
    } finally {
        globalPool.destroy();
    }
}

Example 50 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class TaskExecutorSlotLifetimeTest method testUserCodeClassLoaderIsBoundToSlot.

/**
 * Tests that the user code class loader is bound to the lifetime of the slot. This means that
 * it is being reused across a failover, for example. See FLINK-16408.
 */
@Test
public void testUserCodeClassLoaderIsBoundToSlot() throws Exception {
    final Configuration configuration = new Configuration();
    final TestingRpcService rpcService = TESTING_RPC_SERVICE_RESOURCE.getTestingRpcService();
    final TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
    final CompletableFuture<SlotReport> firstSlotReportFuture = new CompletableFuture<>();
    resourceManagerGateway.setSendSlotReportFunction(resourceIDInstanceIDSlotReportTuple3 -> {
        firstSlotReportFuture.complete(resourceIDInstanceIDSlotReportTuple3.f2);
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    final BlockingQueue<TaskExecutionState> taskExecutionStates = new ArrayBlockingQueue<>(3);
    final OneShotLatch slotsOfferedLatch = new OneShotLatch();
    final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().setOfferSlotsFunction((resourceID, slotOffers) -> {
        slotsOfferedLatch.trigger();
        return CompletableFuture.completedFuture(slotOffers);
    }).setUpdateTaskExecutionStateFunction(FunctionUtils.uncheckedFunction(taskExecutionState -> {
        taskExecutionStates.put(taskExecutionState);
        return CompletableFuture.completedFuture(Acknowledge.get());
    })).build();
    final LeaderRetrievalService resourceManagerLeaderRetriever = new SettableLeaderRetrievalService(resourceManagerGateway.getAddress(), resourceManagerGateway.getFencingToken().toUUID());
    final LeaderRetrievalService jobMasterLeaderRetriever = new SettableLeaderRetrievalService(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());
    final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServicesBuilder().setResourceManagerLeaderRetriever(resourceManagerLeaderRetriever).setJobMasterLeaderRetrieverFunction(ignored -> jobMasterLeaderRetriever).build();
    rpcService.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
    rpcService.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
    final LocalUnresolvedTaskManagerLocation unresolvedTaskManagerLocation = new LocalUnresolvedTaskManagerLocation();
    try (final TaskExecutor taskExecutor = createTaskExecutor(configuration, rpcService, haServices, unresolvedTaskManagerLocation)) {
        taskExecutor.start();
        final SlotReport slotReport = firstSlotReportFuture.join();
        final SlotID firstSlotId = slotReport.iterator().next().getSlotID();
        final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
        final JobID jobId = new JobID();
        final AllocationID allocationId = new AllocationID();
        taskExecutorGateway.requestSlot(firstSlotId, jobId, allocationId, ResourceProfile.ZERO, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken(), RpcUtils.INF_TIMEOUT).join();
        final TaskDeploymentDescriptor tdd = TaskDeploymentDescriptorBuilder.newBuilder(jobId, UserClassLoaderExtractingInvokable.class).setAllocationId(allocationId).build();
        slotsOfferedLatch.await();
        taskExecutorGateway.submitTask(tdd, jobMasterGateway.getFencingToken(), RpcUtils.INF_TIMEOUT).join();
        final ClassLoader firstClassLoader = UserClassLoaderExtractingInvokable.take();
        // wait for the first task to finish
        TaskExecutionState taskExecutionState;
        do {
            taskExecutionState = taskExecutionStates.take();
        } while (!taskExecutionState.getExecutionState().isTerminal());
        // check that a second task will re-use the same class loader
        taskExecutorGateway.submitTask(tdd, jobMasterGateway.getFencingToken(), RpcUtils.INF_TIMEOUT).join();
        final ClassLoader secondClassLoader = UserClassLoaderExtractingInvokable.take();
        assertThat(firstClassLoader, sameInstance(secondClassLoader));
    }
}