Examples with KeyGroupRange org.apache.flink.runtime.state.KeyGroupRange used on opensource projects

Example 26 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class BatchExecutionInternalTimeServiceTest method testIgnoringEventTimeTimersFromWithinCallback.

@Test
public void testIgnoringEventTimeTimersFromWithinCallback() {
    BatchExecutionKeyedStateBackend<Integer> keyedStatedBackend = new BatchExecutionKeyedStateBackend<>(KEY_SERIALIZER, new KeyGroupRange(0, 1));
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    InternalTimeServiceManager<Integer> timeServiceManager = BatchExecutionInternalTimeServiceManager.create(keyedStatedBackend, this.getClass().getClassLoader(), new DummyKeyContext(), processingTimeService, Collections.emptyList());
    List<Long> timers = new ArrayList<>();
    TriggerWithTimerServiceAccess<Integer, VoidNamespace> trigger = TriggerWithTimerServiceAccess.eventTimeTrigger((timer, ts) -> {
        timers.add(timer.getTimestamp());
        ts.registerEventTimeTimer(VoidNamespace.INSTANCE, timer.getTimestamp() + 20);
    });
    InternalTimerService<VoidNamespace> timerService = timeServiceManager.getInternalTimerService("test", KEY_SERIALIZER, new VoidNamespaceSerializer(), trigger);
    trigger.setTimerService(timerService);
    keyedStatedBackend.setCurrentKey(1);
    timerService.registerEventTimeTimer(VoidNamespace.INSTANCE, 150);
    // we should never register physical timers
    assertThat(processingTimeService.getNumActiveTimers(), equalTo(0));
    // changing the current key fires all timers
    keyedStatedBackend.setCurrentKey(2);
    // We check that the timer from the callback is ignored
    assertThat(timers, equalTo(Collections.singletonList(150L)));
}

Example 27 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class BatchExecutionInternalTimeServiceTest method testProcessingTimeTimers.

@Test
public void testProcessingTimeTimers() {
    BatchExecutionKeyedStateBackend<Integer> keyedStatedBackend = new BatchExecutionKeyedStateBackend<>(KEY_SERIALIZER, new KeyGroupRange(0, 1));
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    InternalTimeServiceManager<Integer> timeServiceManager = BatchExecutionInternalTimeServiceManager.create(keyedStatedBackend, this.getClass().getClassLoader(), new DummyKeyContext(), processingTimeService, Collections.emptyList());
    List<Long> timers = new ArrayList<>();
    InternalTimerService<VoidNamespace> timerService = timeServiceManager.getInternalTimerService("test", KEY_SERIALIZER, new VoidNamespaceSerializer(), LambdaTrigger.processingTimeTrigger(timer -> timers.add(timer.getTimestamp())));
    keyedStatedBackend.setCurrentKey(1);
    timerService.registerProcessingTimeTimer(VoidNamespace.INSTANCE, 150);
    // we should never register physical timers
    assertThat(processingTimeService.getNumActiveTimers(), equalTo(0));
    // changing the current key fires all timers
    keyedStatedBackend.setCurrentKey(2);
    assertThat(timers, equalTo(Collections.singletonList(150L)));
}

Example 28 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class BatchExecutionStateBackendVerificationTest method verifySnapshotNotSupported.

@Test
public void verifySnapshotNotSupported() {
    expectedException.expect(UnsupportedOperationException.class);
    expectedException.expectMessage("Snapshotting is not supported in BATCH runtime mode.");
    BatchExecutionKeyedStateBackend<Long> stateBackend = new BatchExecutionKeyedStateBackend<>(LONG_SERIALIZER, new KeyGroupRange(0, 9));
    long checkpointId = 0L;
    CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(10);
    stateBackend.snapshot(checkpointId, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
}

Example 29 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class KvStateRegistryTest method testKvStateEntry.

@Test
public void testKvStateEntry() throws InterruptedException {
    final int threads = 10;
    final CountDownLatch latch1 = new CountDownLatch(threads);
    final CountDownLatch latch2 = new CountDownLatch(1);
    final List<KvStateInfo<?, ?, ?>> infos = Collections.synchronizedList(new ArrayList<>());
    final JobID jobID = new JobID();
    final JobVertexID jobVertexId = new JobVertexID();
    final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 1);
    final String registrationName = "foobar";
    final KvStateRegistry kvStateRegistry = new KvStateRegistry();
    final KvStateID stateID = kvStateRegistry.registerKvState(jobID, jobVertexId, keyGroupRange, registrationName, new DummyKvState(), getClass().getClassLoader());
    final AtomicReference<Throwable> exceptionHolder = new AtomicReference<>();
    for (int i = 0; i < threads; i++) {
        new Thread(() -> {
            final KvStateEntry<?, ?, ?> kvState = kvStateRegistry.getKvState(stateID);
            final KvStateInfo<?, ?, ?> stateInfo = kvState.getInfoForCurrentThread();
            infos.add(stateInfo);
            latch1.countDown();
            try {
                latch2.await();
            } catch (InterruptedException e) {
                // compare and set, so that we do not overwrite an exception
                // that was (potentially) already encountered.
                exceptionHolder.compareAndSet(null, e);
            }
        }).start();
    }
    latch1.await();
    final KvStateEntry<?, ?, ?> kvState = kvStateRegistry.getKvState(stateID);
    // verify that all the threads are done correctly.
    Assert.assertEquals(threads, infos.size());
    Assert.assertEquals(threads, kvState.getCacheSize());
    latch2.countDown();
    for (KvStateInfo<?, ?, ?> infoA : infos) {
        boolean instanceAlreadyFound = false;
        for (KvStateInfo<?, ?, ?> infoB : infos) {
            if (infoA == infoB) {
                if (instanceAlreadyFound) {
                    Assert.fail("More than one thread sharing the same serializer instance.");
                }
                instanceAlreadyFound = true;
            } else {
                Assert.assertEquals(infoA, infoB);
            }
        }
    }
    kvStateRegistry.unregisterKvState(jobID, jobVertexId, keyGroupRange, registrationName, stateID);
    Assert.assertEquals(0L, kvState.getCacheSize());
    Throwable t = exceptionHolder.get();
    if (t != null) {
        fail(t.getMessage());
    }
}

Example 30 with KeyGroupRange

use of org.apache.flink.runtime.state.KeyGroupRange in project flink by apache.

the class KvStateRegistryTest method testLegacyCodePathPreference.

/**
 * Tests that {@link KvStateRegistryListener} registered under {@link
 * HighAvailabilityServices#DEFAULT_JOB_ID} will be used for all notifications.
 */
@Test
public void testLegacyCodePathPreference() {
    final KvStateRegistry kvStateRegistry = new KvStateRegistry();
    final ArrayDeque<JobID> stateRegistrationNotifications = new ArrayDeque<>(2);
    final ArrayDeque<JobID> stateDeregistrationNotifications = new ArrayDeque<>(2);
    final TestingKvStateRegistryListener testingListener = new TestingKvStateRegistryListener(stateRegistrationNotifications, stateDeregistrationNotifications);
    final ArrayDeque<JobID> anotherQueue = new ArrayDeque<>(2);
    final TestingKvStateRegistryListener anotherListener = new TestingKvStateRegistryListener(anotherQueue, anotherQueue);
    final JobID jobId = new JobID();
    kvStateRegistry.registerListener(HighAvailabilityServices.DEFAULT_JOB_ID, testingListener);
    kvStateRegistry.registerListener(jobId, anotherListener);
    final JobVertexID jobVertexId = new JobVertexID();
    final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 1);
    final String registrationName = "registrationName";
    final KvStateID kvStateID = kvStateRegistry.registerKvState(jobId, jobVertexId, keyGroupRange, registrationName, new DummyKvState(), getClass().getClassLoader());
    assertThat(stateRegistrationNotifications.poll(), equalTo(jobId));
    // another listener should not have received any notifications
    assertThat(anotherQueue.isEmpty(), is(true));
    kvStateRegistry.unregisterKvState(jobId, jobVertexId, keyGroupRange, registrationName, kvStateID);
    assertThat(stateDeregistrationNotifications.poll(), equalTo(jobId));
    // another listener should not have received any notifications
    assertThat(anotherQueue.isEmpty(), is(true));
}