Examples with MemoryStateBackend org.apache.flink.runtime.state.memory.MemoryStateBackend used on opensource projects

Example 21 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class StatefulUDFSavepointMigrationITCase method testCreateSavepointOnFlink11.

/**
	 * This has to be manually executed to create the savepoint on Flink 1.1.
	 */
@Test
@Ignore
public void testCreateSavepointOnFlink11() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    // we only test memory state backend yet
    env.setStateBackend(new MemoryStateBackend());
    env.enableCheckpointing(500);
    env.setParallelism(4);
    env.setMaxParallelism(4);
    // create source
    env.addSource(new LegacyCheckpointedSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource").flatMap(new LegacyCheckpointedFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap").keyBy(0).flatMap(new LegacyCheckpointedFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState").keyBy(0).flatMap(new KeyedStateSettingFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap").keyBy(0).transform("custom_operator", new TypeHint<Tuple2<Long, Long>>() {
    }.getTypeInfo(), new CheckpointedUdfOperator(new LegacyCheckpointedFlatMapWithKeyedState())).uid("LegacyCheckpointedOperator").addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>(EXPECTED_ELEMENTS_ACCUMULATOR));
    executeAndSavepoint(env, "src/test/resources/stateful-udf-migration-itcase-flink1.1-savepoint", new Tuple2<>(EXPECTED_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
}

Example 22 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class KvStateClientTest method testClientServerIntegration.

/**
	 * Tests multiple clients querying multiple servers until 100k queries have
	 * been processed. At this point, the client is shut down and its verified
	 * that all ongoing requests are failed.
	 */
@Test
public void testClientServerIntegration() throws Exception {
    // Config
    final int numServers = 2;
    final int numServerEventLoopThreads = 2;
    final int numServerQueryThreads = 2;
    final int numClientEventLoopThreads = 4;
    final int numClientsTasks = 8;
    final int batchSize = 16;
    final int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    KvStateRegistry dummyRegistry = new KvStateRegistry();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(dummyRegistry);
    AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), dummyRegistry.createTaskRegistry(new JobID(), new JobVertexID()));
    final FiniteDuration timeout = new FiniteDuration(10, TimeUnit.SECONDS);
    AtomicKvStateRequestStats clientStats = new AtomicKvStateRequestStats();
    KvStateClient client = null;
    ExecutorService clientTaskExecutor = null;
    final KvStateServer[] server = new KvStateServer[numServers];
    try {
        client = new KvStateClient(numClientEventLoopThreads, clientStats);
        clientTaskExecutor = Executors.newFixedThreadPool(numClientsTasks);
        // Create state
        ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
        desc.setQueryable("any");
        // Create servers
        KvStateRegistry[] registry = new KvStateRegistry[numServers];
        AtomicKvStateRequestStats[] serverStats = new AtomicKvStateRequestStats[numServers];
        final KvStateID[] ids = new KvStateID[numServers];
        for (int i = 0; i < numServers; i++) {
            registry[i] = new KvStateRegistry();
            serverStats[i] = new AtomicKvStateRequestStats();
            server[i] = new KvStateServer(InetAddress.getLocalHost(), 0, numServerEventLoopThreads, numServerQueryThreads, registry[i], serverStats[i]);
            server[i].start();
            backend.setCurrentKey(1010 + i);
            // Value per server
            ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
            state.update(201 + i);
            // we know it must be a KvStat but this is not exposed to the user via State
            InternalKvState<?> kvState = (InternalKvState<?>) state;
            // Register KvState (one state instance for all server)
            ids[i] = registry[i].registerKvState(new JobID(), new JobVertexID(), new KeyGroupRange(0, 0), "any", kvState);
        }
        final KvStateClient finalClient = client;
        Callable<Void> queryTask = new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                while (true) {
                    if (Thread.interrupted()) {
                        throw new InterruptedException();
                    }
                    // Random server permutation
                    List<Integer> random = new ArrayList<>();
                    for (int j = 0; j < batchSize; j++) {
                        random.add(j);
                    }
                    Collections.shuffle(random);
                    // Dispatch queries
                    List<Future<byte[]>> futures = new ArrayList<>(batchSize);
                    for (int j = 0; j < batchSize; j++) {
                        int targetServer = random.get(j) % numServers;
                        byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(1010 + targetServer, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
                        futures.add(finalClient.getKvState(server[targetServer].getAddress(), ids[targetServer], serializedKeyAndNamespace));
                    }
                    // Verify results
                    for (int j = 0; j < batchSize; j++) {
                        int targetServer = random.get(j) % numServers;
                        Future<byte[]> future = futures.get(j);
                        byte[] buf = Await.result(future, timeout);
                        int value = KvStateRequestSerializer.deserializeValue(buf, IntSerializer.INSTANCE);
                        assertEquals(201 + targetServer, value);
                    }
                }
            }
        };
        // Submit tasks
        List<java.util.concurrent.Future<Void>> taskFutures = new ArrayList<>();
        for (int i = 0; i < numClientsTasks; i++) {
            taskFutures.add(clientTaskExecutor.submit(queryTask));
        }
        long numRequests;
        while ((numRequests = clientStats.getNumRequests()) < 100_000) {
            Thread.sleep(100);
            LOG.info("Number of requests {}/100_000", numRequests);
        }
        // Shut down
        client.shutDown();
        for (java.util.concurrent.Future<Void> future : taskFutures) {
            try {
                future.get();
                fail("Did not throw expected Exception after shut down");
            } catch (ExecutionException t) {
                if (t.getCause() instanceof ClosedChannelException || t.getCause() instanceof IllegalStateException) {
                // Expected
                } else {
                    t.printStackTrace();
                    fail("Failed with unexpected Exception type: " + t.getClass().getName());
                }
            }
        }
        assertEquals("Connection leak (client)", 0, clientStats.getNumConnections());
        for (int i = 0; i < numServers; i++) {
            boolean success = false;
            int numRetries = 0;
            while (!success) {
                try {
                    assertEquals("Connection leak (server)", 0, serverStats[i].getNumConnections());
                    success = true;
                } catch (Throwable t) {
                    if (numRetries < 10) {
                        LOG.info("Retrying connection leak check (server)");
                        Thread.sleep((numRetries + 1) * 50);
                        numRetries++;
                    } else {
                        throw t;
                    }
                }
            }
        }
    } finally {
        if (client != null) {
            client.shutDown();
        }
        for (int i = 0; i < numServers; i++) {
            if (server[i] != null) {
                server[i].shutDown();
            }
        }
        if (clientTaskExecutor != null) {
            clientTaskExecutor.shutdown();
        }
    }
}

Example 23 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class KvStateServerHandlerTest method testQueryUnknownKey.

/**
	 * Tests the failure response with {@link UnknownKeyOrNamespace} as cause
	 * on queries for non-existing keys.
	 */
@Test
public void testQueryUnknownKey() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    KvStateServerHandler handler = new KvStateServerHandler(registry, TEST_THREAD_POOL, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(registry);
    KeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()));
    final TestRegistryListener registryListener = new TestRegistryListener();
    registry.registerListener(registryListener);
    // Register state
    ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
    desc.setQueryable("vanilla");
    backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
    byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(1238283, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
    long requestId = Integer.MAX_VALUE + 22982L;
    assertTrue(registryListener.registrationName.equals("vanilla"));
    ByteBuf request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), requestId, registryListener.kvStateId, serializedKeyAndNamespace);
    // Write the request and wait for the response
    channel.writeInbound(request);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(KvStateRequestType.REQUEST_FAILURE, KvStateRequestSerializer.deserializeHeader(buf));
    KvStateRequestFailure response = KvStateRequestSerializer.deserializeKvStateRequestFailure(buf);
    assertEquals(requestId, response.getRequestId());
    assertTrue("Did not respond with expected failure cause", response.getCause() instanceof UnknownKeyOrNamespace);
    assertEquals(1, stats.getNumRequests());
    assertEquals(1, stats.getNumFailed());
}

Example 24 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class KvStateServerHandlerTest method testChunkedResponse.

/**
	 * Tests that large responses are chunked.
	 */
@Test
public void testChunkedResponse() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    KvStateRequestStats stats = new AtomicKvStateRequestStats();
    KvStateServerHandler handler = new KvStateServerHandler(registry, TEST_THREAD_POOL, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(registry);
    AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()));
    final TestRegistryListener registryListener = new TestRegistryListener();
    registry.registerListener(registryListener);
    // Register state
    ValueStateDescriptor<byte[]> desc = new ValueStateDescriptor<>("any", BytePrimitiveArraySerializer.INSTANCE);
    desc.setQueryable("vanilla");
    ValueState<byte[]> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
    // Update KvState
    byte[] bytes = new byte[2 * channel.config().getWriteBufferHighWaterMark()];
    byte current = 0;
    for (int i = 0; i < bytes.length; i++) {
        bytes[i] = current++;
    }
    int key = 99812822;
    backend.setCurrentKey(key);
    state.update(bytes);
    // Request
    byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
    long requestId = Integer.MAX_VALUE + 182828L;
    assertTrue(registryListener.registrationName.equals("vanilla"));
    ByteBuf request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), requestId, registryListener.kvStateId, serializedKeyAndNamespace);
    // Write the request and wait for the response
    channel.writeInbound(request);
    Object msg = readInboundBlocking(channel);
    assertTrue("Not ChunkedByteBuf", msg instanceof ChunkedByteBuf);
}

Example 25 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class KvStateServerHandlerTest method testSerializerMismatch.

/**
	 * Tests the failure response if the serializers don't match.
	 */
@Test
public void testSerializerMismatch() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    KvStateServerHandler handler = new KvStateServerHandler(registry, TEST_THREAD_POOL, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(registry);
    AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()));
    final TestRegistryListener registryListener = new TestRegistryListener();
    registry.registerListener(registryListener);
    // Register state
    ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
    desc.setQueryable("vanilla");
    ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
    int key = 99812822;
    // Update the KvState
    backend.setCurrentKey(key);
    state.update(712828289);
    byte[] wrongKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace("wrong-key-type", StringSerializer.INSTANCE, "wrong-namespace-type", StringSerializer.INSTANCE);
    byte[] wrongNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, "wrong-namespace-type", StringSerializer.INSTANCE);
    assertTrue(registryListener.registrationName.equals("vanilla"));
    ByteBuf request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), 182828, registryListener.kvStateId, wrongKeyAndNamespace);
    // Write the request and wait for the response
    channel.writeInbound(request);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(KvStateRequestType.REQUEST_FAILURE, KvStateRequestSerializer.deserializeHeader(buf));
    KvStateRequestFailure response = KvStateRequestSerializer.deserializeKvStateRequestFailure(buf);
    assertEquals(182828, response.getRequestId());
    assertTrue(response.getCause().getMessage().contains("IOException"));
    // Repeat with wrong namespace only
    request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), 182829, registryListener.kvStateId, wrongNamespace);
    // Write the request and wait for the response
    channel.writeInbound(request);
    buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(KvStateRequestType.REQUEST_FAILURE, KvStateRequestSerializer.deserializeHeader(buf));
    response = KvStateRequestSerializer.deserializeKvStateRequestFailure(buf);
    assertEquals(182829, response.getRequestId());
    assertTrue(response.getCause().getMessage().contains("IOException"));
    assertEquals(2, stats.getNumRequests());
    assertEquals(2, stats.getNumFailed());
}