Examples with VoidNamespace org.apache.flink.runtime.state.VoidNamespace used on opensource projects

Example 1 with VoidNamespace

use of org.apache.flink.runtime.state.VoidNamespace in project flink by apache.

the class QueryableStateClientTest method testIntegrationWithKvStateServer.

/**
	 * Tests queries against multiple servers.
	 *
	 * <p>The servers are populated with different keys and the client queries
	 * all available keys from all servers.
	 */
@Test
public void testIntegrationWithKvStateServer() throws Exception {
    // Config
    int numServers = 2;
    int numKeys = 1024;
    int numKeyGroups = 1;
    JobID jobId = new JobID();
    JobVertexID jobVertexId = new JobVertexID();
    KvStateServer[] servers = new KvStateServer[numServers];
    AtomicKvStateRequestStats[] serverStats = new AtomicKvStateRequestStats[numServers];
    QueryableStateClient client = null;
    KvStateClient networkClient = null;
    AtomicKvStateRequestStats networkClientStats = new AtomicKvStateRequestStats();
    MemoryStateBackend backend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    AbstractKeyedStateBackend<Integer> keyedStateBackend = backend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()));
    try {
        KvStateRegistry[] registries = new KvStateRegistry[numServers];
        KvStateID[] kvStateIds = new KvStateID[numServers];
        List<HeapValueState<Integer, VoidNamespace, Integer>> kvStates = new ArrayList<>();
        // Start the servers
        for (int i = 0; i < numServers; i++) {
            registries[i] = new KvStateRegistry();
            serverStats[i] = new AtomicKvStateRequestStats();
            servers[i] = new KvStateServer(InetAddress.getLocalHost(), 0, 1, 1, registries[i], serverStats[i]);
            servers[i].start();
            ValueStateDescriptor<Integer> descriptor = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
            RegisteredBackendStateMetaInfo<VoidNamespace, Integer> registeredBackendStateMetaInfo = new RegisteredBackendStateMetaInfo<>(descriptor.getType(), descriptor.getName(), VoidNamespaceSerializer.INSTANCE, IntSerializer.INSTANCE);
            // Register state
            HeapValueState<Integer, VoidNamespace, Integer> kvState = new HeapValueState<>(descriptor, new NestedMapsStateTable<Integer, VoidNamespace, Integer>(keyedStateBackend, registeredBackendStateMetaInfo), IntSerializer.INSTANCE, VoidNamespaceSerializer.INSTANCE);
            kvStates.add(kvState);
            kvStateIds[i] = registries[i].registerKvState(jobId, new JobVertexID(), new KeyGroupRange(i, i), "choco", kvState);
        }
        int[] expectedRequests = new int[numServers];
        for (int key = 0; key < numKeys; key++) {
            int targetKeyGroupIndex = MathUtils.murmurHash(key) % numServers;
            expectedRequests[targetKeyGroupIndex]++;
            HeapValueState<Integer, VoidNamespace, Integer> kvState = kvStates.get(targetKeyGroupIndex);
            keyedStateBackend.setCurrentKey(key);
            kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
            kvState.update(1337 + key);
        }
        // Location lookup service
        KvStateLocation location = new KvStateLocation(jobId, jobVertexId, numServers, "choco");
        for (int keyGroupIndex = 0; keyGroupIndex < numServers; keyGroupIndex++) {
            location.registerKvState(new KeyGroupRange(keyGroupIndex, keyGroupIndex), kvStateIds[keyGroupIndex], servers[keyGroupIndex].getAddress());
        }
        KvStateLocationLookupService lookupService = mock(KvStateLocationLookupService.class);
        when(lookupService.getKvStateLookupInfo(eq(jobId), eq("choco"))).thenReturn(Futures.successful(location));
        // The client
        networkClient = new KvStateClient(1, networkClientStats);
        client = new QueryableStateClient(lookupService, networkClient, testActorSystem.dispatcher());
        // Send all queries
        List<Future<byte[]>> futures = new ArrayList<>(numKeys);
        for (int key = 0; key < numKeys; key++) {
            byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
            futures.add(client.getKvState(jobId, "choco", key, serializedKeyAndNamespace));
        }
        // Verify results
        Future<Iterable<byte[]>> future = Futures.sequence(futures, testActorSystem.dispatcher());
        Iterable<byte[]> results = Await.result(future, timeout);
        int index = 0;
        for (byte[] buffer : results) {
            int deserializedValue = KvStateRequestSerializer.deserializeValue(buffer, IntSerializer.INSTANCE);
            assertEquals(1337 + index, deserializedValue);
            index++;
        }
        // Verify requests
        for (int i = 0; i < numServers; i++) {
            int numRetries = 10;
            for (int retry = 0; retry < numRetries; retry++) {
                try {
                    assertEquals("Unexpected number of requests", expectedRequests[i], serverStats[i].getNumRequests());
                    assertEquals("Unexpected success requests", expectedRequests[i], serverStats[i].getNumSuccessful());
                    assertEquals("Unexpected failed requests", 0, serverStats[i].getNumFailed());
                    break;
                } catch (Throwable t) {
                    // Retry
                    if (retry == numRetries - 1) {
                        throw t;
                    } else {
                        Thread.sleep(100);
                    }
                }
            }
        }
    } finally {
        if (client != null) {
            client.shutDown();
        }
        if (networkClient != null) {
            networkClient.shutDown();
        }
        for (KvStateServer server : servers) {
            if (server != null) {
                server.shutDown();
            }
        }
    }
}

Example 2 with VoidNamespace

use of org.apache.flink.runtime.state.VoidNamespace in project flink by apache.

the class KvStateRequestSerializerTest method testListSerialization.

/**
	 * Tests list serialization utils.
	 */
@Test
public void testListSerialization() throws Exception {
    final long key = 0L;
    // objects for heap state list serialisation
    final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = new HeapKeyedStateBackend<>(mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, ClassLoader.getSystemClassLoader(), 1, new KeyGroupRange(0, 0), async, new ExecutionConfig());
    longHeapKeyedStateBackend.setCurrentKey(key);
    final InternalListState<VoidNamespace, Long> listState = longHeapKeyedStateBackend.createListState(VoidNamespaceSerializer.INSTANCE, new ListStateDescriptor<>("test", LongSerializer.INSTANCE));
    testListSerialization(key, listState);
}

Example 3 with VoidNamespace

use of org.apache.flink.runtime.state.VoidNamespace in project flink by apache.

the class HeapAggregatingStateTest method testAddAndGet.

@Test
public void testAddAndGet() throws Exception {
    final AggregatingStateDescriptor<Long, MutableLong, Long> stateDescr = new AggregatingStateDescriptor<>("my-state", new AddingFunction(), MutableLong.class);
    stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
    final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend();
    try {
        InternalAggregatingState<VoidNamespace, Long, Long> state = keyedBackend.createAggregatingState(VoidNamespaceSerializer.INSTANCE, stateDescr);
        state.setCurrentNamespace(VoidNamespace.INSTANCE);
        keyedBackend.setCurrentKey("abc");
        assertNull(state.get());
        keyedBackend.setCurrentKey("def");
        assertNull(state.get());
        state.add(17L);
        state.add(11L);
        assertEquals(28L, state.get().longValue());
        keyedBackend.setCurrentKey("abc");
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        assertNull(state.get());
        state.add(1L);
        state.add(2L);
        keyedBackend.setCurrentKey("def");
        assertEquals(28L, state.get().longValue());
        state.clear();
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        state.add(3L);
        state.add(2L);
        state.add(1L);
        keyedBackend.setCurrentKey("def");
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        assertEquals(9L, state.get().longValue());
        state.clear();
        // make sure all lists / maps are cleared
        StateTable<String, VoidNamespace, MutableLong> stateTable = ((HeapAggregatingState<String, VoidNamespace, Long, MutableLong, Long>) state).stateTable;
        assertTrue(stateTable.isEmpty());
    } finally {
        keyedBackend.close();
        keyedBackend.dispose();
    }
}

Example 4 with VoidNamespace

use of org.apache.flink.runtime.state.VoidNamespace in project flink by apache.

the class HeapListStateTest method testAddAndGet.

@Test
public void testAddAndGet() throws Exception {
    final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);
    stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
    final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend();
    try {
        InternalListState<VoidNamespace, Long> state = keyedBackend.createListState(VoidNamespaceSerializer.INSTANCE, stateDescr);
        state.setCurrentNamespace(VoidNamespace.INSTANCE);
        keyedBackend.setCurrentKey("abc");
        assertNull(state.get());
        keyedBackend.setCurrentKey("def");
        assertNull(state.get());
        state.add(17L);
        state.add(11L);
        assertEquals(asList(17L, 11L), state.get());
        keyedBackend.setCurrentKey("abc");
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        assertNull(state.get());
        state.add(1L);
        state.add(2L);
        keyedBackend.setCurrentKey("def");
        assertEquals(asList(17L, 11L), state.get());
        state.clear();
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        state.add(3L);
        state.add(2L);
        state.add(1L);
        keyedBackend.setCurrentKey("def");
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        assertEquals(asList(1L, 2L, 3L, 2L, 1L), state.get());
        state.clear();
        // make sure all lists / maps are cleared
        StateTable<String, VoidNamespace, ArrayList<Long>> stateTable = ((HeapListState<String, VoidNamespace, Long>) state).stateTable;
        assertTrue(stateTable.isEmpty());
    } finally {
        keyedBackend.close();
        keyedBackend.dispose();
    }
}

Example 5 with VoidNamespace

use of org.apache.flink.runtime.state.VoidNamespace in project flink by apache.

the class KVStateRequestSerializerRocksDBTest method testListSerialization.

/**
	 * Tests list serialization and deserialization match.
	 *
	 * @see KvStateRequestSerializerTest#testListSerialization()
	 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
	 * test
	 */
@Test
public void testListSerialization() throws Exception {
    final long key = 0L;
    // objects for RocksDB state list serialisation
    DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
    dbOptions.setCreateIfMissing(true);
    ColumnFamilyOptions columnFamilyOptions = PredefinedOptions.DEFAULT.createColumnOptions();
    final RocksDBKeyedStateBackend2<Long> longHeapKeyedStateBackend = new RocksDBKeyedStateBackend2<>(new JobID(), "no-op", ClassLoader.getSystemClassLoader(), temporaryFolder.getRoot(), dbOptions, columnFamilyOptions, mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, 1, new KeyGroupRange(0, 0), new ExecutionConfig());
    longHeapKeyedStateBackend.setCurrentKey(key);
    final InternalListState<VoidNamespace, Long> listState = longHeapKeyedStateBackend.createListState(VoidNamespaceSerializer.INSTANCE, new ListStateDescriptor<>("test", LongSerializer.INSTANCE));
    KvStateRequestSerializerTest.testListSerialization(key, listState);
}