Examples with BlockerSync org.apache.flink.core.testutils.BlockerSync used on opensource projects

Example 1 with BlockerSync

use of org.apache.flink.core.testutils.BlockerSync in project flink by apache.

the class AvroSerializerConcurrencyTest method testConcurrentUseOfSerializer.

@Test
public void testConcurrentUseOfSerializer() throws Exception {
    final AvroSerializer<String> serializer = new AvroSerializer<>(String.class);
    final BlockerSync sync = new BlockerSync();
    final DataOutputView regularOut = new DataOutputSerializer(32);
    final DataOutputView lockingOut = new LockingView(sync);
    // this thread serializes and gets stuck there
    final CheckedThread thread = new CheckedThread("serializer") {

        @Override
        public void go() throws Exception {
            serializer.serialize("a value", lockingOut);
        }
    };
    thread.start();
    sync.awaitBlocker();
    // this should fail with an exception
    try {
        serializer.serialize("value", regularOut);
        fail("should have failed with an exception");
    } catch (IllegalStateException e) {
    // expected
    } finally {
        // release the thread that serializes
        sync.releaseBlocker();
    }
    // this propagates exceptions from the spawned thread
    thread.sync();
}

Example 2 with BlockerSync

use of org.apache.flink.core.testutils.BlockerSync in project flink by apache.

the class TaskExecutorPartitionLifecycleTest method internalTestPartitionRelease.

private void internalTestPartitionRelease(TaskExecutorPartitionTracker partitionTracker, ShuffleEnvironment<?, ?> shuffleEnvironment, CompletableFuture<ResultPartitionID> startTrackingFuture, TestAction testAction) throws Exception {
    final ResultPartitionDeploymentDescriptor taskResultPartitionDescriptor = PartitionTestUtils.createPartitionDeploymentDescriptor(ResultPartitionType.BLOCKING);
    final ExecutionAttemptID eid1 = taskResultPartitionDescriptor.getShuffleDescriptor().getResultPartitionID().getProducerId();
    final TaskDeploymentDescriptor taskDeploymentDescriptor = TaskExecutorSubmissionTest.createTaskDeploymentDescriptor(jobId, "job", eid1, new SerializedValue<>(new ExecutionConfig()), "Sender", 1, 0, 1, 0, new Configuration(), new Configuration(), TestingInvokable.class.getName(), Collections.singletonList(taskResultPartitionDescriptor), Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
    final TaskSlotTable<Task> taskSlotTable = createTaskSlotTable();
    final TaskExecutorLocalStateStoresManager localStateStoresManager = new TaskExecutorLocalStateStoresManager(false, Reference.owned(new File[] { tmp.newFolder() }), Executors.directExecutor());
    final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(taskSlotTable).setTaskStateManager(localStateStoresManager).setShuffleEnvironment(shuffleEnvironment).build();
    final CompletableFuture<Void> taskFinishedFuture = new CompletableFuture<>();
    final OneShotLatch slotOfferedLatch = new OneShotLatch();
    final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().setRegisterTaskManagerFunction((ignoredJobId, ignoredTaskManagerRegistrationInformation) -> CompletableFuture.completedFuture(new JMTMRegistrationSuccess(ResourceID.generate()))).setOfferSlotsFunction((resourceID, slotOffers) -> {
        slotOfferedLatch.trigger();
        return CompletableFuture.completedFuture(slotOffers);
    }).setUpdateTaskExecutionStateFunction(taskExecutionState -> {
        if (taskExecutionState.getExecutionState() == ExecutionState.FINISHED) {
            taskFinishedFuture.complete(null);
        }
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).build();
    final TestingTaskExecutor taskExecutor = createTestingTaskExecutor(taskManagerServices, partitionTracker);
    final CompletableFuture<SlotReport> initialSlotReportFuture = new CompletableFuture<>();
    final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
    testingResourceManagerGateway.setSendSlotReportFunction(resourceIDInstanceIDSlotReportTuple3 -> {
        initialSlotReportFuture.complete(resourceIDInstanceIDSlotReportTuple3.f2);
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    testingResourceManagerGateway.setRegisterTaskExecutorFunction(input -> CompletableFuture.completedFuture(new TaskExecutorRegistrationSuccess(new InstanceID(), testingResourceManagerGateway.getOwnResourceId(), new ClusterInformation("blobServerHost", 55555))));
    try {
        taskExecutor.start();
        taskExecutor.waitUntilStarted();
        final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
        final String jobMasterAddress = "jm";
        rpc.registerGateway(jobMasterAddress, jobMasterGateway);
        rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
        // inform the task manager about the job leader
        taskManagerServices.getJobLeaderService().addJob(jobId, jobMasterAddress);
        jobManagerLeaderRetriever.notifyListener(jobMasterAddress, UUID.randomUUID());
        resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway.getFencingToken().toUUID());
        final Optional<SlotStatus> slotStatusOptional = StreamSupport.stream(initialSlotReportFuture.get().spliterator(), false).findAny();
        assertTrue(slotStatusOptional.isPresent());
        final SlotStatus slotStatus = slotStatusOptional.get();
        while (true) {
            try {
                taskExecutorGateway.requestSlot(slotStatus.getSlotID(), jobId, taskDeploymentDescriptor.getAllocationId(), ResourceProfile.ZERO, jobMasterAddress, testingResourceManagerGateway.getFencingToken(), timeout).get();
                break;
            } catch (Exception e) {
                // the proper establishment of the RM connection is tracked
                // asynchronously, so we have to poll here until it went through
                // until then, slot requests will fail with an exception
                Thread.sleep(50);
            }
        }
        TestingInvokable.sync = new BlockerSync();
        // Wait till the slot has been successfully offered before submitting the task.
        // This ensures TM has been successfully registered to JM.
        slotOfferedLatch.await();
        taskExecutorGateway.submitTask(taskDeploymentDescriptor, jobMasterGateway.getFencingToken(), timeout).get();
        TestingInvokable.sync.awaitBlocker();
        // the task is still running => the partition is in in-progress and should be tracked
        assertThat(startTrackingFuture.get(), equalTo(taskResultPartitionDescriptor.getShuffleDescriptor().getResultPartitionID()));
        TestingInvokable.sync.releaseBlocker();
        taskFinishedFuture.get(timeout.getSize(), timeout.getUnit());
        testAction.accept(jobId, taskResultPartitionDescriptor, taskExecutor, taskExecutorGateway);
    } finally {
        RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
    }
    // the shutdown of the backing shuffle environment releases all partitions
    // the book-keeping is not aware of this
    assertTrue(shuffleEnvironment.getPartitionsOccupyingLocalResources().isEmpty());
}

Example 3 with BlockerSync

use of org.apache.flink.core.testutils.BlockerSync in project flink by apache.

the class KryoSerializerConcurrencyTest method testConcurrentUseOfSerializer.

@Test
public void testConcurrentUseOfSerializer() throws Exception {
    final KryoSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());
    final BlockerSync sync = new BlockerSync();
    final DataOutputView regularOut = new DataOutputSerializer(32);
    final DataOutputView lockingOut = new LockingView(sync);
    // this thread serializes and gets stuck there
    final CheckedThread thread = new CheckedThread("serializer") {

        @Override
        public void go() throws Exception {
            serializer.serialize("a value", lockingOut);
        }
    };
    thread.start();
    sync.awaitBlocker();
    // this should fail with an exception
    try {
        serializer.serialize("value", regularOut);
        fail("should have failed with an exception");
    } catch (IllegalStateException e) {
    // expected
    } finally {
        // release the thread that serializes
        sync.releaseBlocker();
    }
    // this propagates exceptions from the spawned thread
    thread.sync();
}

Example 4 with BlockerSync

use of org.apache.flink.core.testutils.BlockerSync in project flink by apache.

the class AbstractMetricGroupTest method testGetAllVariablesDoesNotDeadlock.

@Test
public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException {
    final BlockerSync parentSync = new BlockerSync();
    final BlockerSync childSync = new BlockerSync();
    AtomicReference<BlockerSync> syncRef = new AtomicReference<>();
    final MetricRegistry registry = TestingMetricRegistry.builder().setRegisterConsumer((metric, metricName, group) -> {
        syncRef.get().blockNonInterruptible();
        group.getAllVariables();
    }).build();
    final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent");
    final MetricGroup child = parent.addGroup("child");
    final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter"));
    final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter"));
    try {
        // start both threads and have them block in the registry, so they acquire the lock of
        // their respective group
        syncRef.set(childSync);
        childRegisteringThread.start();
        childSync.awaitBlocker();
        syncRef.set(parentSync);
        parentRegisteringThread.start();
        parentSync.awaitBlocker();
        // the parent thread remains blocked to simulate the child thread holding some lock in
        // the registry/reporter
        // the child thread continues execution and calls getAllVariables()
        // in the past this would block indefinitely since the method acquires the locks of all
        // parent groups
        childSync.releaseBlocker();
        // wait with a timeout to ensure the finally block is executed _at some point_,
        // un-blocking the parent
        childRegisteringThread.join(1000 * 10);
        parentSync.releaseBlocker();
        parentRegisteringThread.join();
    } finally {
        parentSync.releaseBlocker();
        childSync.releaseBlocker();
        parentRegisteringThread.join();
        childRegisteringThread.join();
    }
}

Example 5 with BlockerSync

use of org.apache.flink.core.testutils.BlockerSync in project flink by apache.

the class RestServerEndpointITCase method testRequestInterleaving.

/**
 * Tests that request are handled as individual units which don't interfere with each other.
 * This means that request responses can overtake each other.
 */
@Test
public void testRequestInterleaving() throws Exception {
    final BlockerSync sync = new BlockerSync();
    testHandler.handlerBody = id -> {
        if (id == 1) {
            try {
                sync.block();
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
        }
        return CompletableFuture.completedFuture(new TestResponse(id));
    };
    // send first request and wait until the handler blocks
    final CompletableFuture<TestResponse> response1 = sendRequestToTestHandler(new TestRequest(1));
    sync.awaitBlocker();
    // send second request and verify response
    final CompletableFuture<TestResponse> response2 = sendRequestToTestHandler(new TestRequest(2));
    assertEquals(2, response2.get().id);
    // wake up blocked handler
    sync.releaseBlocker();
    // verify response to first request
    assertEquals(1, response1.get().id);
}