Examples with KvStateRegistry org.apache.flink.runtime.query.KvStateRegistry used on opensource projects

Example 6 with KvStateRegistry

use of org.apache.flink.runtime.query.KvStateRegistry in project flink by apache.

the class KvStateServerHandlerTest method testCloseChannelOnExceptionCaught.

/**
	 * Tests that the channel is closed if an Exception reaches the channel
	 * handler.
	 */
@Test
public void testCloseChannelOnExceptionCaught() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    KvStateServerHandler handler = new KvStateServerHandler(registry, TEST_THREAD_POOL, stats);
    EmbeddedChannel channel = new EmbeddedChannel(handler);
    channel.pipeline().fireExceptionCaught(new RuntimeException("Expected test Exception"));
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(KvStateRequestType.SERVER_FAILURE, KvStateRequestSerializer.deserializeHeader(buf));
    Throwable response = KvStateRequestSerializer.deserializeServerFailure(buf);
    assertTrue(response.getMessage().contains("Expected test Exception"));
    channel.closeFuture().await(READ_TIMEOUT_MILLIS);
    assertFalse(channel.isActive());
}

Example 7 with KvStateRegistry

use of org.apache.flink.runtime.query.KvStateRegistry in project flink by apache.

the class KvStateServerHandlerTest method testUnexpectedMessage.

/**
	 * Tests response on unexpected messages.
	 */
@Test
public void testUnexpectedMessage() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    KvStateServerHandler handler = new KvStateServerHandler(registry, TEST_THREAD_POOL, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    // Write the request and wait for the response
    ByteBuf unexpectedMessage = Unpooled.buffer(8);
    unexpectedMessage.writeInt(4);
    unexpectedMessage.writeInt(123238213);
    channel.writeInbound(unexpectedMessage);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(KvStateRequestType.SERVER_FAILURE, KvStateRequestSerializer.deserializeHeader(buf));
    Throwable response = KvStateRequestSerializer.deserializeServerFailure(buf);
    assertEquals(0, stats.getNumRequests());
    assertEquals(0, stats.getNumFailed());
    unexpectedMessage = KvStateRequestSerializer.serializeKvStateRequestResult(channel.alloc(), 192, new byte[0]);
    channel.writeInbound(unexpectedMessage);
    buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(KvStateRequestType.SERVER_FAILURE, KvStateRequestSerializer.deserializeHeader(buf));
    response = KvStateRequestSerializer.deserializeServerFailure(buf);
    assertTrue("Unexpected failure cause " + response.getClass().getName(), response instanceof IllegalArgumentException);
    assertEquals(0, stats.getNumRequests());
    assertEquals(0, stats.getNumFailed());
}

Example 8 with KvStateRegistry

use of org.apache.flink.runtime.query.KvStateRegistry in project flink by apache.

the class KvStateServerTest method testSimpleRequest.

/**
	 * Tests a simple successful query via a SocketChannel.
	 */
@Test
public void testSimpleRequest() throws Exception {
    KvStateServer server = null;
    Bootstrap bootstrap = null;
    try {
        KvStateRegistry registry = new KvStateRegistry();
        KvStateRequestStats stats = new AtomicKvStateRequestStats();
        server = new KvStateServer(InetAddress.getLocalHost(), 0, 1, 1, registry, stats);
        server.start();
        KvStateServerAddress serverAddress = server.getAddress();
        int numKeyGroups = 1;
        AbstractStateBackend abstractBackend = new MemoryStateBackend();
        DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
        dummyEnv.setKvStateRegistry(registry);
        AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), registry.createTaskRegistry(new JobID(), new JobVertexID()));
        final KvStateServerHandlerTest.TestRegistryListener registryListener = new KvStateServerHandlerTest.TestRegistryListener();
        registry.registerListener(registryListener);
        ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
        desc.setQueryable("vanilla");
        ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        // Update KvState
        int expectedValue = 712828289;
        int key = 99812822;
        backend.setCurrentKey(key);
        state.update(expectedValue);
        // Request
        byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
        // Connect to the server
        final BlockingQueue<ByteBuf> responses = new LinkedBlockingQueue<>();
        bootstrap = createBootstrap(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4), new ChannelInboundHandlerAdapter() {

            @Override
            public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
                responses.add((ByteBuf) msg);
            }
        });
        Channel channel = bootstrap.connect(serverAddress.getHost(), serverAddress.getPort()).sync().channel();
        long requestId = Integer.MAX_VALUE + 182828L;
        assertTrue(registryListener.registrationName.equals("vanilla"));
        ByteBuf request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), requestId, registryListener.kvStateId, serializedKeyAndNamespace);
        channel.writeAndFlush(request);
        ByteBuf buf = responses.poll(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
        assertEquals(KvStateRequestType.REQUEST_RESULT, KvStateRequestSerializer.deserializeHeader(buf));
        KvStateRequestResult response = KvStateRequestSerializer.deserializeKvStateRequestResult(buf);
        assertEquals(requestId, response.getRequestId());
        int actualValue = KvStateRequestSerializer.deserializeValue(response.getSerializedResult(), IntSerializer.INSTANCE);
        assertEquals(expectedValue, actualValue);
    } finally {
        if (server != null) {
            server.shutDown();
        }
        if (bootstrap != null) {
            EventLoopGroup group = bootstrap.group();
            if (group != null) {
                group.shutdownGracefully();
            }
        }
    }
}

Example 9 with KvStateRegistry

use of org.apache.flink.runtime.query.KvStateRegistry in project flink by apache.

the class RocksDBStateBackendConfigTest method testContinueOnSomeDbDirectoriesMissing.

@Test
public void testContinueOnSomeDbDirectoriesMissing() throws Exception {
    File targetDir1 = tempFolder.newFolder();
    File targetDir2 = tempFolder.newFolder();
    String checkpointPath = tempFolder.newFolder().toURI().toString();
    RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);
    try {
        if (!targetDir1.setWritable(false, false)) {
            System.err.println("Cannot execute 'testContinueOnSomeDbDirectoriesMissing' because cannot mark directory non-writable");
            return;
        }
        rocksDbBackend.setDbStoragePaths(targetDir1.getAbsolutePath(), targetDir2.getAbsolutePath());
        try {
            Environment env = getMockEnvironment();
            rocksDbBackend.createKeyedStateBackend(env, env.getJobID(), "foobar", IntSerializer.INSTANCE, 1, new KeyGroupRange(0, 0), new KvStateRegistry().createTaskRegistry(env.getJobID(), new JobVertexID()));
        } catch (Exception e) {
            e.printStackTrace();
            fail("Backend initialization failed even though some paths were available");
        }
    } finally {
        //noinspection ResultOfMethodCallIgnored
        targetDir1.setWritable(true, false);
        FileUtils.deleteDirectory(targetDir1);
        FileUtils.deleteDirectory(targetDir2);
    }
}

Example 10 with KvStateRegistry

use of org.apache.flink.runtime.query.KvStateRegistry in project flink by apache.

the class RocksDBStateBackendConfigTest method testFailWhenNoLocalStorageDir.

// ------------------------------------------------------------------------
//  RocksDB local file directory initialization
// ------------------------------------------------------------------------
@Test
public void testFailWhenNoLocalStorageDir() throws Exception {
    String checkpointPath = tempFolder.newFolder().toURI().toString();
    RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);
    File targetDir = tempFolder.newFolder();
    try {
        if (!targetDir.setWritable(false, false)) {
            System.err.println("Cannot execute 'testFailWhenNoLocalStorageDir' because cannot mark directory non-writable");
            return;
        }
        rocksDbBackend.setDbStoragePath(targetDir.getAbsolutePath());
        boolean hasFailure = false;
        try {
            Environment env = getMockEnvironment();
            rocksDbBackend.createKeyedStateBackend(env, env.getJobID(), "foobar", IntSerializer.INSTANCE, 1, new KeyGroupRange(0, 0), new KvStateRegistry().createTaskRegistry(env.getJobID(), new JobVertexID()));
        } catch (Exception e) {
            assertTrue(e.getMessage().contains("No local storage directories available"));
            assertTrue(e.getMessage().contains(targetDir.getAbsolutePath()));
            hasFailure = true;
        }
        assertTrue("We must see a failure because no storaged directory is feasible.", hasFailure);
    } finally {
        //noinspection ResultOfMethodCallIgnored
        targetDir.setWritable(true, false);
        FileUtils.deleteDirectory(targetDir);
    }
}