Examples with Closer org.apache.flink.shaded.guava30.com.google.common.io.Closer used on opensource projects

Example 46 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project flink by apache.

the class SingleInputGateTest method testBackwardsEventWithUninitializedChannel.

@Test
public void testBackwardsEventWithUninitializedChannel() throws Exception {
    // Setup environment
    TestingTaskEventPublisher taskEventPublisher = new TestingTaskEventPublisher();
    TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(new NoOpResultSubpartitionView());
    // Setup reader with one local and one unknown input channel
    NettyShuffleEnvironment environment = createNettyShuffleEnvironment();
    final SingleInputGate inputGate = createInputGate(environment, 2, ResultPartitionType.PIPELINED);
    final InputChannel[] inputChannels = new InputChannel[2];
    try (Closer closer = Closer.create()) {
        closer.register(environment::close);
        closer.register(inputGate::close);
        // Local
        ResultPartitionID localPartitionId = new ResultPartitionID();
        inputChannels[0] = InputChannelBuilder.newBuilder().setPartitionId(localPartitionId).setPartitionManager(partitionManager).setTaskEventPublisher(taskEventPublisher).buildLocalChannel(inputGate);
        // Unknown
        ResultPartitionID unknownPartitionId = new ResultPartitionID();
        inputChannels[1] = InputChannelBuilder.newBuilder().setChannelIndex(1).setPartitionId(unknownPartitionId).setPartitionManager(partitionManager).setTaskEventPublisher(taskEventPublisher).buildUnknownChannel(inputGate);
        setupInputGate(inputGate, inputChannels);
        // Only the local channel can request
        assertEquals(1, partitionManager.counter);
        // Send event backwards and initialize unknown channel afterwards
        final TaskEvent event = new TestTaskEvent();
        inputGate.sendTaskEvent(event);
        // Only the local channel can send out the event
        assertEquals(1, taskEventPublisher.counter);
        // After the update, the pending event should be send to local channel
        ResourceID location = ResourceID.generate();
        inputGate.updateInputChannel(location, createRemoteWithIdAndLocation(unknownPartitionId.getPartitionId(), location));
        assertEquals(2, partitionManager.counter);
        assertEquals(2, taskEventPublisher.counter);
    }
}

Example 47 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project flink by apache.

the class SingleInputGateTest method testQueuedBuffers.

@Test
public void testQueuedBuffers() throws Exception {
    final NettyShuffleEnvironment network = createNettyShuffleEnvironment();
    final BufferWritingResultPartition resultPartition = (BufferWritingResultPartition) new ResultPartitionBuilder().setResultPartitionManager(network.getResultPartitionManager()).setupBufferPoolFactoryFromNettyShuffleEnvironment(network).build();
    final SingleInputGate inputGate = createInputGate(network, 2, ResultPartitionType.PIPELINED);
    final ResultPartitionID localResultPartitionId = resultPartition.getPartitionId();
    final InputChannel[] inputChannels = new InputChannel[2];
    final RemoteInputChannel remoteInputChannel = InputChannelBuilder.newBuilder().setChannelIndex(1).setupFromNettyShuffleEnvironment(network).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(inputGate);
    inputChannels[0] = remoteInputChannel;
    inputChannels[1] = InputChannelBuilder.newBuilder().setChannelIndex(0).setPartitionId(localResultPartitionId).setupFromNettyShuffleEnvironment(network).setConnectionManager(new TestingConnectionManager()).buildLocalChannel(inputGate);
    try (Closer closer = Closer.create()) {
        closer.register(network::close);
        closer.register(inputGate::close);
        closer.register(resultPartition::release);
        resultPartition.setup();
        setupInputGate(inputGate, inputChannels);
        remoteInputChannel.onBuffer(createBuffer(1), 0, 0);
        assertEquals(1, inputGate.getNumberOfQueuedBuffers());
        resultPartition.emitRecord(ByteBuffer.allocate(1), 0);
        assertEquals(2, inputGate.getNumberOfQueuedBuffers());
    }
}

Example 48 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project flink by apache.

the class CheckpointedInputGateTest method testPriorityBeforeClose.

/**
 * Tests a priority notification happening right before cancellation. The mail would be
 * processed while draining mailbox but can't pull any data anymore.
 */
@Test
public void testPriorityBeforeClose() throws IOException, InterruptedException {
    NetworkBufferPool bufferPool = new NetworkBufferPool(10, 1024);
    try (Closer closer = Closer.create()) {
        closer.register(bufferPool::destroy);
        for (int repeat = 0; repeat < 100; repeat++) {
            setUp();
            SingleInputGate singleInputGate = new SingleInputGateBuilder().setNumberOfChannels(2).setBufferPoolFactory(bufferPool.createBufferPool(2, Integer.MAX_VALUE)).setSegmentProvider(bufferPool).setChannelFactory(InputChannelBuilder::buildRemoteChannel).build();
            singleInputGate.setup();
            ((RemoteInputChannel) singleInputGate.getChannel(0)).requestSubpartition();
            final TaskMailboxImpl mailbox = new TaskMailboxImpl();
            MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(mailbox, 0, StreamTaskActionExecutor.IMMEDIATE);
            ValidatingCheckpointHandler validatingHandler = new ValidatingCheckpointHandler(1);
            SingleCheckpointBarrierHandler barrierHandler = TestBarrierHandlerFactory.forTarget(validatingHandler).create(singleInputGate, new MockChannelStateWriter());
            CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
            final int oldSize = mailbox.size();
            enqueue(checkpointedInputGate, 0, barrier(1));
            // wait for priority mail to be enqueued
            Deadline deadline = Deadline.fromNow(Duration.ofMinutes(1));
            while (deadline.hasTimeLeft() && oldSize >= mailbox.size()) {
                Thread.sleep(1);
            }
            // test the race condition
            // either priority event could be handled, then we expect a checkpoint to be
            // triggered or closing came first in which case we expect a CancelTaskException
            CountDownLatch beforeLatch = new CountDownLatch(2);
            final CheckedThread canceler = new CheckedThread("Canceler") {

                @Override
                public void go() throws IOException {
                    beforeLatch.countDown();
                    singleInputGate.close();
                }
            };
            canceler.start();
            beforeLatch.countDown();
            try {
                while (mailboxExecutor.tryYield()) {
                }
                assertEquals(1L, validatingHandler.triggeredCheckpointCounter);
            } catch (CancelTaskException e) {
            }
            canceler.join();
        }
    }
}

Example 49 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project alluxio by Alluxio.

the class DefaultBlockWorkerClient method close.

@Override
public void close() throws IOException {
    try (Closer closer = Closer.create()) {
        closer.register(() -> {
            if (mStreamingChannel != null) {
                mStreamingChannel.shutdown();
            }
        });
        closer.register(() -> {
            if (mRpcChannel != null) {
                mRpcChannel.shutdown();
            }
        });
        closer.register(() -> {
            if (mTracker != null) {
                mTracker.close(this);
            }
        });
    }
}

Example 50 with Closer

use of org.apache.flink.shaded.guava30.com.google.common.io.Closer in project alluxio by Alluxio.

the class FileSystemContextTest method acquireAtMaxLimit.

/**
 * This test ensures acquiring all the available FileSystem master clients blocks further
 * requests for clients. It also ensures clients are available for reuse after they are released
 * by the previous owners. If the test takes longer than 10 seconds, a deadlock most likely
 * occurred preventing the release of the master clients.
 */
@Test(timeout = 10000)
public void acquireAtMaxLimit() throws Exception {
    Closer closer = Closer.create();
    // Acquire all the clients
    FileSystemContext fsContext = FileSystemContext.create(ClientContext.create(mConf));
    for (int i = 0; i < mConf.getInt(PropertyKey.USER_FILE_MASTER_CLIENT_POOL_SIZE_MAX); i++) {
        closer.register(fsContext.acquireMasterClientResource());
    }
    Thread acquireThread = new Thread(new AcquireClient(fsContext));
    acquireThread.start();
    // Wait for the spawned thread to complete. If it is able to acquire a master client before
    // the defined timeout, fail
    long timeoutMs = Constants.SECOND_MS / 2;
    long start = System.currentTimeMillis();
    acquireThread.join(timeoutMs);
    if (System.currentTimeMillis() - start < timeoutMs) {
        fail("Acquired a master client when the client pool was full.");
    }
    // Release all the clients
    closer.close();
    // Wait for the spawned thread to complete. If it is unable to acquire a master client before
    // the defined timeout, fail.
    timeoutMs = 5 * Constants.SECOND_MS;
    start = System.currentTimeMillis();
    acquireThread.join(timeoutMs);
    if (System.currentTimeMillis() - start >= timeoutMs) {
        fail("Failed to acquire a master client within " + timeoutMs + "ms. Deadlock?");
    }
}