Search in sources :

Example 11 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project flink by apache.

the class StreamingRuntimeContextTest method testListStateInstantiation.

@Test
public void testListStateInstantiation() throws Exception {
    final ExecutionConfig config = new ExecutionConfig();
    config.registerKryoType(Path.class);
    final AtomicReference<Object> descriptorCapture = new AtomicReference<>();
    StreamingRuntimeContext context = new StreamingRuntimeContext(createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap());
    ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class);
    context.getListState(descr);
    ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get();
    TypeSerializer<?> serializer = descrIntercepted.getSerializer();
    // check that the Path class is really registered, i.e., the execution config was applied
    assertTrue(serializer instanceof ListSerializer);
    TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer();
    assertTrue(elementSerializer instanceof KryoSerializer);
    assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}
Also used : Path(org.apache.flink.core.fs.Path) ListSerializer(org.apache.flink.api.common.typeutils.base.ListSerializer) ListStateDescriptor(org.apache.flink.api.common.state.ListStateDescriptor) AtomicReference(java.util.concurrent.atomic.AtomicReference) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) KryoSerializer(org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer) TaskInfo(org.apache.flink.api.common.TaskInfo) Test(org.junit.Test)

Example 12 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project flink by apache.

the class RightOuterJoinTaskTest method testCancelRightOuterJoinTaskWhileBuilding.

@Test
public void testCancelRightOuterJoinTaskWhileBuilding() throws Exception {
    setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
    getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
    getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
    final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
    addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);
    addInput(new UniformIntTupleGenerator(100, 100, true), this.serializer);
    final AtomicReference<Throwable> error = new AtomicReference<>();
    final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {

        @Override
        public void run() {
            try {
                testDriver(testTask, MockJoinStub.class);
            } catch (Throwable t) {
                error.set(t);
            }
        }
    };
    taskRunner.start();
    Thread.sleep(1000);
    cancel();
    taskRunner.join(60000);
    assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());
    final Throwable taskError = error.get();
    if (taskError != null) {
        fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
    }
}
Also used : RuntimePairComparatorFactory(org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory) UniformIntTupleGenerator(org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator) Tuple2(org.apache.flink.api.java.tuple.Tuple2) AtomicReference(java.util.concurrent.atomic.AtomicReference) InfiniteIntTupleIterator(org.apache.flink.runtime.operators.testutils.InfiniteIntTupleIterator) Test(org.junit.Test)

Example 13 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project flink by apache.

the class LeftOuterJoinTaskTest method testCancelLeftOuterJoinTaskWhileProbing.

@Test
public void testCancelLeftOuterJoinTaskWhileProbing() throws Exception {
    setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
    getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
    getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
    final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
    addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);
    addInput(new UniformIntTupleGenerator(1, 1, true), this.serializer);
    final AtomicReference<Throwable> error = new AtomicReference<>();
    final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {

        @Override
        public void run() {
            try {
                testDriver(testTask, MockJoinStub.class);
            } catch (Throwable t) {
                error.set(t);
            }
        }
    };
    taskRunner.start();
    Thread.sleep(1000);
    cancel();
    taskRunner.join(60000);
    assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());
    final Throwable taskError = error.get();
    if (taskError != null) {
        fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
    }
}
Also used : RuntimePairComparatorFactory(org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory) UniformIntTupleGenerator(org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator) Tuple2(org.apache.flink.api.java.tuple.Tuple2) AtomicReference(java.util.concurrent.atomic.AtomicReference) InfiniteIntTupleIterator(org.apache.flink.runtime.operators.testutils.InfiniteIntTupleIterator) Test(org.junit.Test)

Example 14 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project flink by apache.

the class KvStateClientTest method testServerClosesChannel.

/**
	 * Tests that a server channel close, closes the connection and removes it
	 * from the established connections.
	 */
@Test
public void testServerClosesChannel() throws Exception {
    Deadline deadline = TEST_TIMEOUT.fromNow();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    KvStateClient client = null;
    Channel serverChannel = null;
    try {
        client = new KvStateClient(1, stats);
        final AtomicBoolean received = new AtomicBoolean();
        final AtomicReference<Channel> channel = new AtomicReference<>();
        serverChannel = createServerChannel(new ChannelInboundHandlerAdapter() {

            @Override
            public void channelActive(ChannelHandlerContext ctx) throws Exception {
                channel.set(ctx.channel());
            }

            @Override
            public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
                received.set(true);
            }
        });
        KvStateServerAddress serverAddress = getKvStateServerAddress(serverChannel);
        // Requests
        Future<byte[]> future = client.getKvState(serverAddress, new KvStateID(), new byte[0]);
        while (!received.get() && deadline.hasTimeLeft()) {
            Thread.sleep(50);
        }
        assertTrue("Receive timed out", received.get());
        assertEquals(1, stats.getNumConnections());
        channel.get().close().await(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
        try {
            Await.result(future, deadline.timeLeft());
            fail("Did not throw expected server failure");
        } catch (ClosedChannelException ignored) {
        // Expected
        }
        assertEquals(0, stats.getNumConnections());
        // Counts can take some time to propagate
        while (deadline.hasTimeLeft() && (stats.getNumSuccessful() != 0 || stats.getNumFailed() != 1)) {
            Thread.sleep(100);
        }
        assertEquals(1, stats.getNumRequests());
        assertEquals(0, stats.getNumSuccessful());
        assertEquals(1, stats.getNumFailed());
    } finally {
        if (client != null) {
            client.shutDown();
        }
        if (serverChannel != null) {
            serverChannel.close();
        }
        assertEquals("Channel leak", 0, stats.getNumConnections());
    }
}
Also used : ClosedChannelException(java.nio.channels.ClosedChannelException) Deadline(scala.concurrent.duration.Deadline) SocketChannel(io.netty.channel.socket.SocketChannel) NioServerSocketChannel(io.netty.channel.socket.nio.NioServerSocketChannel) Channel(io.netty.channel.Channel) KvStateServerAddress(org.apache.flink.runtime.query.KvStateServerAddress) AtomicReference(java.util.concurrent.atomic.AtomicReference) ChannelHandlerContext(io.netty.channel.ChannelHandlerContext) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) KvStateID(org.apache.flink.runtime.query.KvStateID) ChannelInboundHandlerAdapter(io.netty.channel.ChannelInboundHandlerAdapter) Test(org.junit.Test)

Example 15 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project hadoop by apache.

the class TestRPC method testRPCInterrupted.

@Test(timeout = 30000)
public void testRPCInterrupted() throws Exception {
    Server server;
    RPC.Builder builder = newServerBuilder(conf).setNumHandlers(5).setVerbose(true).setSecretManager(null);
    server = setupTestServer(builder);
    int numConcurrentRPC = 200;
    final CyclicBarrier barrier = new CyclicBarrier(numConcurrentRPC);
    final CountDownLatch latch = new CountDownLatch(numConcurrentRPC);
    final AtomicBoolean leaderRunning = new AtomicBoolean(true);
    final AtomicReference<Throwable> error = new AtomicReference<>();
    Thread leaderThread = null;
    try {
        for (int i = 0; i < numConcurrentRPC; i++) {
            final int num = i;
            final TestRpcService proxy = getClient(addr, conf);
            Thread rpcThread = new Thread(new Runnable() {

                @Override
                public void run() {
                    try {
                        barrier.await();
                        while (num == 0 || leaderRunning.get()) {
                            proxy.slowPing(null, newSlowPingRequest(false));
                        }
                        proxy.slowPing(null, newSlowPingRequest(false));
                    } catch (Exception e) {
                        if (num == 0) {
                            leaderRunning.set(false);
                        } else {
                            error.set(e);
                        }
                        LOG.error("thread " + num, e);
                    } finally {
                        latch.countDown();
                    }
                }
            });
            rpcThread.start();
            if (leaderThread == null) {
                leaderThread = rpcThread;
            }
        }
        // let threads get past the barrier
        Thread.sleep(1000);
        // stop a single thread
        while (leaderRunning.get()) {
            leaderThread.interrupt();
        }
        latch.await();
        // should not cause any other thread to get an error
        assertTrue("rpc got exception " + error.get(), error.get() == null);
    } finally {
        server.stop();
    }
}
Also used : AtomicReference(java.util.concurrent.atomic.AtomicReference) CountDownLatch(java.util.concurrent.CountDownLatch) ServiceException(com.google.protobuf.ServiceException) AuthorizationException(org.apache.hadoop.security.authorize.AuthorizationException) InterruptedIOException(java.io.InterruptedIOException) SocketTimeoutException(java.net.SocketTimeoutException) ConnectException(java.net.ConnectException) HadoopIllegalArgumentException(org.apache.hadoop.HadoopIllegalArgumentException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) AccessControlException(org.apache.hadoop.security.AccessControlException) CyclicBarrier(java.util.concurrent.CyclicBarrier) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Test(org.junit.Test)

Aggregations

AtomicReference (java.util.concurrent.atomic.AtomicReference)1331 Test (org.junit.Test)668 CountDownLatch (java.util.concurrent.CountDownLatch)437 IOException (java.io.IOException)263 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)205 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)159 ArrayList (java.util.ArrayList)108 HashMap (java.util.HashMap)105 List (java.util.List)95 Map (java.util.Map)77 Test (org.testng.annotations.Test)76 File (java.io.File)64 ExecutionException (java.util.concurrent.ExecutionException)60 HashSet (java.util.HashSet)54 URI (java.net.URI)48 TimeoutException (java.util.concurrent.TimeoutException)48 HttpServletRequest (javax.servlet.http.HttpServletRequest)48 HttpServletResponse (javax.servlet.http.HttpServletResponse)46 MockResponse (okhttp3.mockwebserver.MockResponse)46 ByteBuffer (java.nio.ByteBuffer)44