Examples with KeyGroupRange org.apache.flink.runtime.state.KeyGroupRange used on opensource projects

Example 66 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class KvStateServerHandlerTest method testQueryExecutorShutDown.

/**
	 * Tests the failure response on a rejected execution, because the query
	 * executor has been closed.
	 */
@Test
public void testQueryExecutorShutDown() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    ExecutorService closedExecutor = Executors.newSingleThreadExecutor();
    closedExecutor.shutdown();
    assertTrue(closedExecutor.isShutdown());
    KvStateServerHandler handler = new KvStateServerHandler(registry, closedExecutor, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(registry);
    KeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()));
    final TestRegistryListener registryListener = new TestRegistryListener();
    registry.registerListener(registryListener);
    // Register state
    ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
    desc.setQueryable("vanilla");
    backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
    assertTrue(registryListener.registrationName.equals("vanilla"));
    ByteBuf request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), 282872, registryListener.kvStateId, new byte[0]);
    // Write the request and wait for the response
    channel.writeInbound(request);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(KvStateRequestType.REQUEST_FAILURE, KvStateRequestSerializer.deserializeHeader(buf));
    KvStateRequestFailure response = KvStateRequestSerializer.deserializeKvStateRequestFailure(buf);
    assertTrue(response.getCause().getMessage().contains("RejectedExecutionException"));
    assertEquals(1, stats.getNumRequests());
    assertEquals(1, stats.getNumFailed());
}

Example 67 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class KvStateServerHandlerTest method testSimpleQuery.

/**
	 * Tests a simple successful query via an EmbeddedChannel.
	 */
@Test
public void testSimpleQuery() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    KvStateServerHandler handler = new KvStateServerHandler(registry, TEST_THREAD_POOL, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    // Register state
    ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
    desc.setQueryable("vanilla");
    int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(registry);
    AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()));
    final TestRegistryListener registryListener = new TestRegistryListener();
    registry.registerListener(registryListener);
    // Update the KvState and request it
    int expectedValue = 712828289;
    int key = 99812822;
    backend.setCurrentKey(key);
    ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
    state.update(expectedValue);
    byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
    long requestId = Integer.MAX_VALUE + 182828L;
    assertTrue(registryListener.registrationName.equals("vanilla"));
    ByteBuf request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), requestId, registryListener.kvStateId, serializedKeyAndNamespace);
    // Write the request and wait for the response
    channel.writeInbound(request);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(KvStateRequestType.REQUEST_RESULT, KvStateRequestSerializer.deserializeHeader(buf));
    KvStateRequestResult response = KvStateRequestSerializer.deserializeKvStateRequestResult(buf);
    assertEquals(requestId, response.getRequestId());
    int actualValue = KvStateRequestSerializer.deserializeValue(response.getSerializedResult(), IntSerializer.INSTANCE);
    assertEquals(expectedValue, actualValue);
    assertEquals(stats.toString(), 1, stats.getNumRequests());
    // Wait for async successful request report
    long deadline = System.nanoTime() + TimeUnit.NANOSECONDS.convert(30, TimeUnit.SECONDS);
    while (stats.getNumSuccessful() != 1 && System.nanoTime() <= deadline) {
        Thread.sleep(10);
    }
    assertEquals(stats.toString(), 1, stats.getNumSuccessful());
}

Example 68 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class QueryableStateClientTest method testForceLookupOnOutdatedLocation.

/**
	 * All failures should lead to a retry with a forced location lookup.
	 *
	 * UnknownKvStateID, UnknownKvStateKeyGroupLocation, UnknownKvStateLocation,
	 * ConnectException are checked explicitly as these indicate out-of-sync
	 * KvStateLocation.
	 */
@Test
public void testForceLookupOnOutdatedLocation() throws Exception {
    KvStateLocationLookupService lookupService = mock(KvStateLocationLookupService.class);
    KvStateClient networkClient = mock(KvStateClient.class);
    QueryableStateClient client = new QueryableStateClient(lookupService, networkClient, testActorSystem.dispatcher());
    try {
        JobID jobId = new JobID();
        int numKeyGroups = 4;
        //
        // UnknownKvStateLocation
        //
        String query1 = "lucky";
        Future<KvStateLocation> unknownKvStateLocation = Futures.failed(new UnknownKvStateLocation(query1));
        when(lookupService.getKvStateLookupInfo(eq(jobId), eq(query1))).thenReturn(unknownKvStateLocation);
        Future<byte[]> result = client.getKvState(jobId, query1, 0, new byte[0]);
        try {
            Await.result(result, timeout);
            fail("Did not throw expected UnknownKvStateLocation exception");
        } catch (UnknownKvStateLocation ignored) {
        // Expected
        }
        verify(lookupService, times(2)).getKvStateLookupInfo(eq(jobId), eq(query1));
        //
        // UnknownKvStateKeyGroupLocation
        //
        String query2 = "unlucky";
        Future<KvStateLocation> unknownKeyGroupLocation = Futures.successful(new KvStateLocation(jobId, new JobVertexID(), numKeyGroups, query2));
        when(lookupService.getKvStateLookupInfo(eq(jobId), eq(query2))).thenReturn(unknownKeyGroupLocation);
        result = client.getKvState(jobId, query2, 0, new byte[0]);
        try {
            Await.result(result, timeout);
            fail("Did not throw expected UnknownKvStateKeyGroupLocation exception");
        } catch (UnknownKvStateKeyGroupLocation ignored) {
        // Expected
        }
        verify(lookupService, times(2)).getKvStateLookupInfo(eq(jobId), eq(query2));
        //
        // UnknownKvStateID
        //
        String query3 = "water";
        KvStateID kvStateId = new KvStateID();
        Future<byte[]> unknownKvStateId = Futures.failed(new UnknownKvStateID(kvStateId));
        KvStateServerAddress serverAddress = new KvStateServerAddress(InetAddress.getLocalHost(), 12323);
        KvStateLocation location = new KvStateLocation(jobId, new JobVertexID(), numKeyGroups, query3);
        for (int i = 0; i < numKeyGroups; i++) {
            location.registerKvState(new KeyGroupRange(i, i), kvStateId, serverAddress);
        }
        when(lookupService.getKvStateLookupInfo(eq(jobId), eq(query3))).thenReturn(Futures.successful(location));
        when(networkClient.getKvState(eq(serverAddress), eq(kvStateId), any(byte[].class))).thenReturn(unknownKvStateId);
        result = client.getKvState(jobId, query3, 0, new byte[0]);
        try {
            Await.result(result, timeout);
            fail("Did not throw expected UnknownKvStateID exception");
        } catch (UnknownKvStateID ignored) {
        // Expected
        }
        verify(lookupService, times(2)).getKvStateLookupInfo(eq(jobId), eq(query3));
        //
        // ConnectException
        //
        String query4 = "space";
        Future<byte[]> connectException = Futures.failed(new ConnectException());
        kvStateId = new KvStateID();
        serverAddress = new KvStateServerAddress(InetAddress.getLocalHost(), 11123);
        location = new KvStateLocation(jobId, new JobVertexID(), numKeyGroups, query4);
        for (int i = 0; i < numKeyGroups; i++) {
            location.registerKvState(new KeyGroupRange(i, i), kvStateId, serverAddress);
        }
        when(lookupService.getKvStateLookupInfo(eq(jobId), eq(query4))).thenReturn(Futures.successful(location));
        when(networkClient.getKvState(eq(serverAddress), eq(kvStateId), any(byte[].class))).thenReturn(connectException);
        result = client.getKvState(jobId, query4, 0, new byte[0]);
        try {
            Await.result(result, timeout);
            fail("Did not throw expected ConnectException exception");
        } catch (ConnectException ignored) {
        // Expected
        }
        verify(lookupService, times(2)).getKvStateLookupInfo(eq(jobId), eq(query4));
        //
        // Other Exceptions don't lead to a retry no retry
        //
        String query5 = "universe";
        Future<KvStateLocation> exception = Futures.failed(new RuntimeException("Test exception"));
        when(lookupService.getKvStateLookupInfo(eq(jobId), eq(query5))).thenReturn(exception);
        client.getKvState(jobId, query5, 0, new byte[0]);
        verify(lookupService, times(1)).getKvStateLookupInfo(eq(jobId), eq(query5));
    } finally {
        client.shutDown();
    }
}

Example 69 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class QueryableStateClientTest method testLookupMultipleJobIds.

/**
	 * Tests that the QueryableState client correctly caches location lookups
	 * keyed by both job and name. This test is mainly due to a previous bug due
	 * to which cache entries were by name only. This is a problem, because the
	 * same client can be used to query multiple jobs.
	 */
@Test
public void testLookupMultipleJobIds() throws Exception {
    String name = "unique-per-job";
    // Exact contents don't matter here
    KvStateLocation location = new KvStateLocation(new JobID(), new JobVertexID(), 1, name);
    location.registerKvState(new KeyGroupRange(0, 0), new KvStateID(), new KvStateServerAddress(InetAddress.getLocalHost(), 892));
    JobID jobId1 = new JobID();
    JobID jobId2 = new JobID();
    KvStateLocationLookupService lookupService = mock(KvStateLocationLookupService.class);
    when(lookupService.getKvStateLookupInfo(any(JobID.class), anyString())).thenReturn(Futures.successful(location));
    KvStateClient networkClient = mock(KvStateClient.class);
    when(networkClient.getKvState(any(KvStateServerAddress.class), any(KvStateID.class), any(byte[].class))).thenReturn(Futures.successful(new byte[0]));
    QueryableStateClient client = new QueryableStateClient(lookupService, networkClient, testActorSystem.dispatcher());
    // Query ies with same name, but different job IDs should lead to a
    // single lookup per query and job ID.
    client.getKvState(jobId1, name, 0, new byte[0]);
    client.getKvState(jobId2, name, 0, new byte[0]);
    verify(lookupService, times(1)).getKvStateLookupInfo(eq(jobId1), eq(name));
    verify(lookupService, times(1)).getKvStateLookupInfo(eq(jobId2), eq(name));
}

Example 70 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class KVStateRequestSerializerRocksDBTest method testMapSerialization.

/**
	 * Tests map serialization and deserialization match.
	 *
	 * @see KvStateRequestSerializerTest#testMapSerialization()
	 * KvStateRequestSerializerTest#testMapSerialization() using the heap state back-end
	 * test
	 */
@Test
public void testMapSerialization() throws Exception {
    final long key = 0L;
    // objects for RocksDB state list serialisation
    DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
    dbOptions.setCreateIfMissing(true);
    ColumnFamilyOptions columnFamilyOptions = PredefinedOptions.DEFAULT.createColumnOptions();
    final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend = new RocksDBKeyedStateBackend<>(new JobID(), "no-op", ClassLoader.getSystemClassLoader(), temporaryFolder.getRoot(), dbOptions, columnFamilyOptions, mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, 1, new KeyGroupRange(0, 0), new ExecutionConfig());
    longHeapKeyedStateBackend.setCurrentKey(key);
    final InternalMapState<VoidNamespace, Long, String> mapState = (InternalMapState<VoidNamespace, Long, String>) longHeapKeyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));
    KvStateRequestSerializerTest.testMapSerialization(key, mapState);
}