Examples with KeyGroupRange org.apache.flink.runtime.state.KeyGroupRange used on opensource projects

Example 21 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class ClientTest method testClientServerIntegration.

/**
 * Tests multiple clients querying multiple servers until 100k queries have been processed. At
 * this point, the client is shut down and its verified that all ongoing requests are failed.
 */
@Test
public void testClientServerIntegration() throws Throwable {
    // Config
    final int numServers = 2;
    final int numServerEventLoopThreads = 2;
    final int numServerQueryThreads = 2;
    final int numClientEventLoopThreads = 4;
    final int numClientsTasks = 8;
    final int batchSize = 16;
    final int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    KvStateRegistry dummyRegistry = new KvStateRegistry();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(dummyRegistry);
    AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), dummyRegistry.createTaskRegistry(new JobID(), new JobVertexID()), TtlTimeProvider.DEFAULT, new UnregisteredMetricsGroup(), Collections.emptyList(), new CloseableRegistry());
    AtomicKvStateRequestStats clientStats = new AtomicKvStateRequestStats();
    final MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    Client<KvStateInternalRequest, KvStateResponse> client = null;
    ExecutorService clientTaskExecutor = null;
    final KvStateServerImpl[] server = new KvStateServerImpl[numServers];
    try {
        client = new Client<>("Test Client", numClientEventLoopThreads, serializer, clientStats);
        clientTaskExecutor = Executors.newFixedThreadPool(numClientsTasks);
        // Create state
        ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
        desc.setQueryable("any");
        // Create servers
        KvStateRegistry[] registry = new KvStateRegistry[numServers];
        AtomicKvStateRequestStats[] serverStats = new AtomicKvStateRequestStats[numServers];
        final KvStateID[] ids = new KvStateID[numServers];
        for (int i = 0; i < numServers; i++) {
            registry[i] = new KvStateRegistry();
            serverStats[i] = new AtomicKvStateRequestStats();
            server[i] = new KvStateServerImpl(InetAddress.getLocalHost().getHostName(), Collections.singletonList(0).iterator(), numServerEventLoopThreads, numServerQueryThreads, registry[i], serverStats[i]);
            server[i].start();
            backend.setCurrentKey(1010 + i);
            // Value per server
            ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
            state.update(201 + i);
            // we know it must be a KvState but this is not exposed to the user via State
            InternalKvState<Integer, ?, Integer> kvState = (InternalKvState<Integer, ?, Integer>) state;
            // Register KvState (one state instance for all server)
            ids[i] = registry[i].registerKvState(new JobID(), new JobVertexID(), new KeyGroupRange(0, 0), "any", kvState, getClass().getClassLoader());
        }
        final Client<KvStateInternalRequest, KvStateResponse> finalClient = client;
        Callable<Void> queryTask = () -> {
            while (true) {
                if (Thread.interrupted()) {
                    throw new InterruptedException();
                }
                // Random server permutation
                List<Integer> random = new ArrayList<>();
                for (int j = 0; j < batchSize; j++) {
                    random.add(j);
                }
                Collections.shuffle(random);
                // Dispatch queries
                List<CompletableFuture<KvStateResponse>> futures = new ArrayList<>(batchSize);
                for (int j = 0; j < batchSize; j++) {
                    int targetServer = random.get(j) % numServers;
                    byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(1010 + targetServer, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
                    KvStateInternalRequest request = new KvStateInternalRequest(ids[targetServer], serializedKeyAndNamespace);
                    futures.add(finalClient.sendRequest(server[targetServer].getServerAddress(), request));
                }
                // Verify results
                for (int j = 0; j < batchSize; j++) {
                    int targetServer = random.get(j) % numServers;
                    Future<KvStateResponse> future = futures.get(j);
                    byte[] buf = future.get().getContent();
                    int value = KvStateSerializer.deserializeValue(buf, IntSerializer.INSTANCE);
                    assertEquals(201L + targetServer, value);
                }
            }
        };
        // Submit tasks
        List<Future<Void>> taskFutures = new ArrayList<>();
        for (int i = 0; i < numClientsTasks; i++) {
            taskFutures.add(clientTaskExecutor.submit(queryTask));
        }
        long numRequests;
        while ((numRequests = clientStats.getNumRequests()) < 100_000L) {
            Thread.sleep(100L);
            LOG.info("Number of requests {}/100_000", numRequests);
        }
        try {
            client.shutdown().get();
        } catch (Exception e) {
            e.printStackTrace();
        }
        Assert.assertTrue(client.isEventGroupShutdown());
        final CombinableMatcher<Throwable> exceptionMatcher = either(FlinkMatchers.containsCause(ClosedChannelException.class)).or(FlinkMatchers.containsCause(IllegalStateException.class));
        for (Future<Void> future : taskFutures) {
            try {
                future.get();
                fail("Did not throw expected Exception after shut down");
            } catch (ExecutionException t) {
                assertThat(t, exceptionMatcher);
            }
        }
        assertEquals("Connection leak (client)", 0L, clientStats.getNumConnections());
        for (int i = 0; i < numServers; i++) {
            boolean success = false;
            int numRetries = 0;
            while (!success) {
                try {
                    assertEquals("Connection leak (server)", 0L, serverStats[i].getNumConnections());
                    success = true;
                } catch (Throwable t) {
                    if (numRetries < 10) {
                        LOG.info("Retrying connection leak check (server)");
                        Thread.sleep((numRetries + 1) * 50L);
                        numRetries++;
                    } else {
                        throw t;
                    }
                }
            }
        }
    } finally {
        if (client != null) {
            try {
                client.shutdown().get();
            } catch (Exception e) {
                e.printStackTrace();
            }
            Assert.assertTrue(client.isEventGroupShutdown());
        }
        for (int i = 0; i < numServers; i++) {
            if (server[i] != null) {
                server[i].shutdown();
            }
        }
        if (clientTaskExecutor != null) {
            clientTaskExecutor.shutdown();
        }
    }
}

Example 22 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class KvStateRequestSerializerTest method getLongHeapKeyedStateBackend.

private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
    final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
    ExecutionConfig executionConfig = new ExecutionConfig();
    // objects for heap state list serialisation
    final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = new HeapKeyedStateBackendBuilder<>(mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, ClassLoader.getSystemClassLoader(), keyGroupRange.getNumberOfKeyGroups(), keyGroupRange, executionConfig, TtlTimeProvider.DEFAULT, LatencyTrackingStateConfig.disabled(), Collections.emptyList(), AbstractStateBackend.getCompressionDecorator(executionConfig), TestLocalRecoveryConfig.disabled(), new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128), async, new CloseableRegistry()).build();
    longHeapKeyedStateBackend.setCurrentKey(key);
    return longHeapKeyedStateBackend;
}

Example 23 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class BatchExecutionInternalTimeServiceTest method testBatchExecutionManagerCanBeInstantiatedWithBatchStateBackend.

@Test
public void testBatchExecutionManagerCanBeInstantiatedWithBatchStateBackend() throws Exception {
    expectedException.expect(IllegalStateException.class);
    expectedException.expectMessage("Batch execution specific time service can work only with BatchExecutionKeyedStateBackend");
    MockEnvironment mockEnvironment = MockEnvironment.builder().build();
    AbstractKeyedStateBackend<Integer> stateBackend = new MemoryStateBackend().createKeyedStateBackend(mockEnvironment, new JobID(), "dummy", KEY_SERIALIZER, 2, new KeyGroupRange(0, 1), mockEnvironment.getTaskKvStateRegistry(), TtlTimeProvider.DEFAULT, new UnregisteredMetricsGroup(), Collections.emptyList(), new CloseableRegistry());
    BatchExecutionInternalTimeServiceManager.create(stateBackend, this.getClass().getClassLoader(), new DummyKeyContext(), new TestProcessingTimeService(), Collections.emptyList());
}

Example 24 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class BatchExecutionInternalTimeServiceTest method testIgnoringProcessingTimeTimersFromWithinCallback.

@Test
public void testIgnoringProcessingTimeTimersFromWithinCallback() {
    BatchExecutionKeyedStateBackend<Integer> keyedStatedBackend = new BatchExecutionKeyedStateBackend<>(KEY_SERIALIZER, new KeyGroupRange(0, 1));
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    InternalTimeServiceManager<Integer> timeServiceManager = BatchExecutionInternalTimeServiceManager.create(keyedStatedBackend, this.getClass().getClassLoader(), new DummyKeyContext(), processingTimeService, Collections.emptyList());
    List<Long> timers = new ArrayList<>();
    TriggerWithTimerServiceAccess<Integer, VoidNamespace> trigger = TriggerWithTimerServiceAccess.processingTimeTrigger((timer, ts) -> {
        timers.add(timer.getTimestamp());
        ts.registerProcessingTimeTimer(VoidNamespace.INSTANCE, timer.getTimestamp() + 20);
    });
    InternalTimerService<VoidNamespace> timerService = timeServiceManager.getInternalTimerService("test", KEY_SERIALIZER, new VoidNamespaceSerializer(), trigger);
    trigger.setTimerService(timerService);
    keyedStatedBackend.setCurrentKey(1);
    timerService.registerProcessingTimeTimer(VoidNamespace.INSTANCE, 150);
    // we should never register physical timers
    assertThat(processingTimeService.getNumActiveTimers(), equalTo(0));
    // changing the current key fires all timers
    keyedStatedBackend.setCurrentKey(2);
    // We check that the timer from the callback is ignored
    assertThat(timers, equalTo(Collections.singletonList(150L)));
}

Example 25 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class BatchExecutionInternalTimeServiceTest method testCurrentWatermark.

@Test
public void testCurrentWatermark() throws Exception {
    BatchExecutionKeyedStateBackend<Integer> keyedStatedBackend = new BatchExecutionKeyedStateBackend<>(KEY_SERIALIZER, new KeyGroupRange(0, 1));
    InternalTimeServiceManager<Integer> timeServiceManager = BatchExecutionInternalTimeServiceManager.create(keyedStatedBackend, this.getClass().getClassLoader(), new DummyKeyContext(), new TestProcessingTimeService(), Collections.emptyList());
    List<Long> timers = new ArrayList<>();
    TriggerWithTimerServiceAccess<Integer, VoidNamespace> eventTimeTrigger = TriggerWithTimerServiceAccess.eventTimeTrigger((timer, timerService) -> {
        assertThat(timerService.currentWatermark(), equalTo(Long.MAX_VALUE));
        timers.add(timer.getTimestamp());
    });
    InternalTimerService<VoidNamespace> timerService = timeServiceManager.getInternalTimerService("test", KEY_SERIALIZER, new VoidNamespaceSerializer(), eventTimeTrigger);
    eventTimeTrigger.setTimerService(timerService);
    assertThat(timerService.currentWatermark(), equalTo(Long.MIN_VALUE));
    keyedStatedBackend.setCurrentKey(1);
    timerService.registerEventTimeTimer(VoidNamespace.INSTANCE, 123);
    assertThat(timerService.currentWatermark(), equalTo(Long.MIN_VALUE));
    // advancing the watermark to a value different than Long.MAX_VALUE should have no effect
    timeServiceManager.advanceWatermark(new Watermark(1000));
    assertThat(timerService.currentWatermark(), equalTo(Long.MIN_VALUE));
    // changing the current key fires all timers
    keyedStatedBackend.setCurrentKey(2);
    assertThat(timerService.currentWatermark(), equalTo(Long.MIN_VALUE));
    timerService.registerEventTimeTimer(VoidNamespace.INSTANCE, 124);
    // advancing the watermark to Long.MAX_VALUE should fire remaining key
    timeServiceManager.advanceWatermark(Watermark.MAX_WATERMARK);
    assertThat(timers, equalTo(Arrays.asList(123L, 124L)));
}