Examples with MemoryStateBackend org.apache.flink.runtime.state.memory.MemoryStateBackend used on opensource projects

Example 11 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class KeyedOneInputStreamOperatorTestHarness method snapshotLegacy.

/**
	 *
	 */
@Override
public StreamStateHandle snapshotLegacy(long checkpointId, long timestamp) throws Exception {
    // simply use an in-memory handle
    MemoryStateBackend backend = new MemoryStateBackend();
    CheckpointStreamFactory streamFactory = backend.createStreamFactory(new JobID(), "test_op");
    CheckpointStreamFactory.CheckpointStateOutputStream outStream = streamFactory.createCheckpointStateOutputStream(checkpointId, timestamp);
    if (operator instanceof StreamCheckpointedOperator) {
        ((StreamCheckpointedOperator) operator).snapshotState(outStream, checkpointId, timestamp);
    }
    if (keyedStateBackend != null) {
        RunnableFuture<KeyGroupsStateHandle> keyedSnapshotRunnable = keyedStateBackend.snapshot(checkpointId, timestamp, streamFactory, CheckpointOptions.forFullCheckpoint());
        if (!keyedSnapshotRunnable.isDone()) {
            Thread runner = new Thread(keyedSnapshotRunnable);
            runner.start();
        }
        outStream.write(1);
        ObjectOutputStream oos = new ObjectOutputStream(outStream);
        oos.writeObject(keyedSnapshotRunnable.get());
        oos.flush();
    } else {
        outStream.write(0);
    }
    return outStream.closeAndGetHandle();
}

Example 12 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class KvStateServerTest method testSimpleRequest.

/**
	 * Tests a simple successful query via a SocketChannel.
	 */
@Test
public void testSimpleRequest() throws Exception {
    KvStateServer server = null;
    Bootstrap bootstrap = null;
    try {
        KvStateRegistry registry = new KvStateRegistry();
        KvStateRequestStats stats = new AtomicKvStateRequestStats();
        server = new KvStateServer(InetAddress.getLocalHost(), 0, 1, 1, registry, stats);
        server.start();
        KvStateServerAddress serverAddress = server.getAddress();
        int numKeyGroups = 1;
        AbstractStateBackend abstractBackend = new MemoryStateBackend();
        DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
        dummyEnv.setKvStateRegistry(registry);
        AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), registry.createTaskRegistry(new JobID(), new JobVertexID()));
        final KvStateServerHandlerTest.TestRegistryListener registryListener = new KvStateServerHandlerTest.TestRegistryListener();
        registry.registerListener(registryListener);
        ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
        desc.setQueryable("vanilla");
        ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        // Update KvState
        int expectedValue = 712828289;
        int key = 99812822;
        backend.setCurrentKey(key);
        state.update(expectedValue);
        // Request
        byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
        // Connect to the server
        final BlockingQueue<ByteBuf> responses = new LinkedBlockingQueue<>();
        bootstrap = createBootstrap(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4), new ChannelInboundHandlerAdapter() {

            @Override
            public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
                responses.add((ByteBuf) msg);
            }
        });
        Channel channel = bootstrap.connect(serverAddress.getHost(), serverAddress.getPort()).sync().channel();
        long requestId = Integer.MAX_VALUE + 182828L;
        assertTrue(registryListener.registrationName.equals("vanilla"));
        ByteBuf request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), requestId, registryListener.kvStateId, serializedKeyAndNamespace);
        channel.writeAndFlush(request);
        ByteBuf buf = responses.poll(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
        assertEquals(KvStateRequestType.REQUEST_RESULT, KvStateRequestSerializer.deserializeHeader(buf));
        KvStateRequestResult response = KvStateRequestSerializer.deserializeKvStateRequestResult(buf);
        assertEquals(requestId, response.getRequestId());
        int actualValue = KvStateRequestSerializer.deserializeValue(response.getSerializedResult(), IntSerializer.INSTANCE);
        assertEquals(expectedValue, actualValue);
    } finally {
        if (server != null) {
            server.shutDown();
        }
        if (bootstrap != null) {
            EventLoopGroup group = bootstrap.group();
            if (group != null) {
                group.shutdownGracefully();
            }
        }
    }
}

Example 13 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class QueryableStateClientTest method testIntegrationWithKvStateServer.

/**
	 * Tests queries against multiple servers.
	 *
	 * <p>The servers are populated with different keys and the client queries
	 * all available keys from all servers.
	 */
@Test
public void testIntegrationWithKvStateServer() throws Exception {
    // Config
    int numServers = 2;
    int numKeys = 1024;
    int numKeyGroups = 1;
    JobID jobId = new JobID();
    JobVertexID jobVertexId = new JobVertexID();
    KvStateServer[] servers = new KvStateServer[numServers];
    AtomicKvStateRequestStats[] serverStats = new AtomicKvStateRequestStats[numServers];
    QueryableStateClient client = null;
    KvStateClient networkClient = null;
    AtomicKvStateRequestStats networkClientStats = new AtomicKvStateRequestStats();
    MemoryStateBackend backend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    AbstractKeyedStateBackend<Integer> keyedStateBackend = backend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()));
    try {
        KvStateRegistry[] registries = new KvStateRegistry[numServers];
        KvStateID[] kvStateIds = new KvStateID[numServers];
        List<HeapValueState<Integer, VoidNamespace, Integer>> kvStates = new ArrayList<>();
        // Start the servers
        for (int i = 0; i < numServers; i++) {
            registries[i] = new KvStateRegistry();
            serverStats[i] = new AtomicKvStateRequestStats();
            servers[i] = new KvStateServer(InetAddress.getLocalHost(), 0, 1, 1, registries[i], serverStats[i]);
            servers[i].start();
            ValueStateDescriptor<Integer> descriptor = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
            RegisteredBackendStateMetaInfo<VoidNamespace, Integer> registeredBackendStateMetaInfo = new RegisteredBackendStateMetaInfo<>(descriptor.getType(), descriptor.getName(), VoidNamespaceSerializer.INSTANCE, IntSerializer.INSTANCE);
            // Register state
            HeapValueState<Integer, VoidNamespace, Integer> kvState = new HeapValueState<>(descriptor, new NestedMapsStateTable<Integer, VoidNamespace, Integer>(keyedStateBackend, registeredBackendStateMetaInfo), IntSerializer.INSTANCE, VoidNamespaceSerializer.INSTANCE);
            kvStates.add(kvState);
            kvStateIds[i] = registries[i].registerKvState(jobId, new JobVertexID(), new KeyGroupRange(i, i), "choco", kvState);
        }
        int[] expectedRequests = new int[numServers];
        for (int key = 0; key < numKeys; key++) {
            int targetKeyGroupIndex = MathUtils.murmurHash(key) % numServers;
            expectedRequests[targetKeyGroupIndex]++;
            HeapValueState<Integer, VoidNamespace, Integer> kvState = kvStates.get(targetKeyGroupIndex);
            keyedStateBackend.setCurrentKey(key);
            kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
            kvState.update(1337 + key);
        }
        // Location lookup service
        KvStateLocation location = new KvStateLocation(jobId, jobVertexId, numServers, "choco");
        for (int keyGroupIndex = 0; keyGroupIndex < numServers; keyGroupIndex++) {
            location.registerKvState(new KeyGroupRange(keyGroupIndex, keyGroupIndex), kvStateIds[keyGroupIndex], servers[keyGroupIndex].getAddress());
        }
        KvStateLocationLookupService lookupService = mock(KvStateLocationLookupService.class);
        when(lookupService.getKvStateLookupInfo(eq(jobId), eq("choco"))).thenReturn(Futures.successful(location));
        // The client
        networkClient = new KvStateClient(1, networkClientStats);
        client = new QueryableStateClient(lookupService, networkClient, testActorSystem.dispatcher());
        // Send all queries
        List<Future<byte[]>> futures = new ArrayList<>(numKeys);
        for (int key = 0; key < numKeys; key++) {
            byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
            futures.add(client.getKvState(jobId, "choco", key, serializedKeyAndNamespace));
        }
        // Verify results
        Future<Iterable<byte[]>> future = Futures.sequence(futures, testActorSystem.dispatcher());
        Iterable<byte[]> results = Await.result(future, timeout);
        int index = 0;
        for (byte[] buffer : results) {
            int deserializedValue = KvStateRequestSerializer.deserializeValue(buffer, IntSerializer.INSTANCE);
            assertEquals(1337 + index, deserializedValue);
            index++;
        }
        // Verify requests
        for (int i = 0; i < numServers; i++) {
            int numRetries = 10;
            for (int retry = 0; retry < numRetries; retry++) {
                try {
                    assertEquals("Unexpected number of requests", expectedRequests[i], serverStats[i].getNumRequests());
                    assertEquals("Unexpected success requests", expectedRequests[i], serverStats[i].getNumSuccessful());
                    assertEquals("Unexpected failed requests", 0, serverStats[i].getNumFailed());
                    break;
                } catch (Throwable t) {
                    // Retry
                    if (retry == numRetries - 1) {
                        throw t;
                    } else {
                        Thread.sleep(100);
                    }
                }
            }
        }
    } finally {
        if (client != null) {
            client.shutDown();
        }
        if (networkClient != null) {
            networkClient.shutDown();
        }
        for (KvStateServer server : servers) {
            if (server != null) {
                server.shutDown();
            }
        }
    }
}

Example 14 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class RocksDBAsyncSnapshotTest method testFullyAsyncSnapshot.

/**
	 * This ensures that asynchronous state handles are actually materialized asynchronously.
	 *
	 * <p>We use latches to block at various stages and see if the code still continues through
	 * the parts that are not asynchronous. If the checkpoint is not done asynchronously the
	 * test will simply lock forever.
	 */
@Test
public void testFullyAsyncSnapshot() throws Exception {
    LocalFileSystem localFS = new LocalFileSystem();
    localFS.initialize(new URI("file:///"), new Configuration());
    PowerMockito.stub(PowerMockito.method(FileSystem.class, "get", URI.class, Configuration.class)).toReturn(localFS);
    final OneInputStreamTask<String, String> task = new OneInputStreamTask<>();
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(task, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    testHarness.configureForKeyedStream(new KeySelector<String, String>() {

        @Override
        public String getKey(String value) throws Exception {
            return value;
        }
    }, BasicTypeInfo.STRING_TYPE_INFO);
    StreamConfig streamConfig = testHarness.getStreamConfig();
    File dbDir = new File(new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()), "state");
    RocksDBStateBackend backend = new RocksDBStateBackend(new MemoryStateBackend());
    backend.setDbStoragePath(dbDir.getAbsolutePath());
    streamConfig.setStateBackend(backend);
    streamConfig.setStreamOperator(new AsyncCheckpointOperator());
    final OneShotLatch delayCheckpointLatch = new OneShotLatch();
    final OneShotLatch ensureCheckpointLatch = new OneShotLatch();
    StreamMockEnvironment mockEnv = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize) {

        @Override
        public void acknowledgeCheckpoint(long checkpointId, CheckpointMetrics checkpointMetrics, SubtaskState checkpointStateHandles) {
            super.acknowledgeCheckpoint(checkpointId, checkpointMetrics);
            // even though the async checkpoint would not finish
            try {
                delayCheckpointLatch.await();
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
            // should be one k/v state
            assertNotNull(checkpointStateHandles.getManagedKeyedState());
            // we now know that the checkpoint went through
            ensureCheckpointLatch.trigger();
        }
    };
    testHarness.invoke(mockEnv);
    // wait for the task to be running
    for (Field field : StreamTask.class.getDeclaredFields()) {
        if (field.getName().equals("isRunning")) {
            field.setAccessible(true);
            while (!field.getBoolean(task)) {
                Thread.sleep(10);
            }
        }
    }
    task.triggerCheckpoint(new CheckpointMetaData(42, 17), CheckpointOptions.forFullCheckpoint());
    testHarness.processElement(new StreamRecord<>("Wohoo", 0));
    // now we allow the checkpoint
    delayCheckpointLatch.trigger();
    // wait for the checkpoint to go through
    ensureCheckpointLatch.await();
    testHarness.endInput();
    ExecutorService threadPool = task.getAsyncOperationsThreadPool();
    threadPool.shutdown();
    Assert.assertTrue(threadPool.awaitTermination(60_000, TimeUnit.MILLISECONDS));
    testHarness.waitForTaskCompletion();
    if (mockEnv.wasFailedExternally()) {
        fail("Unexpected exception during execution.");
    }
}

Example 15 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project beam by apache.

the class FlinkBroadcastStateInternalsTest method initStateInternals.

@Before
public void initStateInternals() {
    MemoryStateBackend backend = new MemoryStateBackend();
    try {
        OperatorStateBackend operatorStateBackend = backend.createOperatorStateBackend(new DummyEnvironment("test", 1, 0), "");
        underTest = new FlinkBroadcastStateInternals<>(1, operatorStateBackend);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}