Examples with StateValue org.apache.beam.runners.samza.runtime.SamzaStoreStateInternals.StateValue used on opensource projects

Example 6 with StateValue

use of org.apache.beam.runners.samza.runtime.SamzaStoreStateInternals.StateValue in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testBufferSizeNotExceedingPipelineOptionValue.

@Test
public void testBufferSizeNotExceedingPipelineOptionValue() {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    pipelineOptions.setEventTimerBufferSize(2);
    final KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey("testKey");
    // timers in store are then timestamped from 0 - 4.
    for (int i = 0; i < 5; i++) {
        timerInternals.setTimer(nameSpace, "timer" + i, "", new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
    }
    // only two timers are supposed to be in event time buffer
    assertEquals(2, timerInternalsFactory.getEventTimeBuffer().size());
    store.close();
}

Example 7 with StateValue

use of org.apache.beam.runners.samza.runtime.SamzaStoreStateInternals.StateValue in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testMaxExpiredEventTimersProcessAtOnce.

private void testMaxExpiredEventTimersProcessAtOnce(int totalNumberOfTimersInStore, int totalNumberOfExpiredTimers, int maxExpiredTimersToProcessOnce, int expectedExpiredTimersToProcess) {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    pipelineOptions.setMaxReadyTimersToProcessOnce(maxExpiredTimersToProcessOnce);
    final KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey("testKey");
    TimerInternals.TimerData timer;
    for (int i = 0; i < totalNumberOfTimersInStore; i++) {
        timer = TimerInternals.TimerData.of("timer" + i, nameSpace, new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
        timerInternals.setTimer(timer);
    }
    // Set the timestamp of the input watermark to be the value of totalNumberOfExpiredTimers
    // so that totalNumberOfExpiredTimers timers are expected be expired with respect to this
    // watermark.
    final Instant inputWatermark = new Instant(totalNumberOfExpiredTimers);
    timerInternalsFactory.setInputWatermark(inputWatermark);
    final Collection<KeyedTimerData<String>> readyTimers = timerInternalsFactory.removeReadyTimers();
    assertEquals(expectedExpiredTimersToProcess, readyTimers.size());
    store.close();
}

Example 8 with StateValue

use of org.apache.beam.runners.samza.runtime.SamzaStoreStateInternals.StateValue in project beam by apache.

the class SamzaStoreStateInternalsTest method testStateValueSerde.

@Test
public void testStateValueSerde() throws IOException {
    StateValueSerdeFactory stateValueSerdeFactory = new StateValueSerdeFactory();
    Serde<StateValue<Integer>> serde = (Serde) stateValueSerdeFactory.getSerde("Test", null);
    int value = 123;
    Coder<Integer> coder = VarIntCoder.of();
    byte[] valueBytes = serde.toBytes(StateValue.of(value, coder));
    StateValue<Integer> stateValue1 = serde.fromBytes(valueBytes);
    StateValue<Integer> stateValue2 = StateValue.of(valueBytes);
    assertEquals(stateValue1.getValue(coder).intValue(), value);
    assertEquals(stateValue2.getValue(coder).intValue(), value);
    Integer nullValue = null;
    byte[] nullBytes = serde.toBytes(StateValue.of(nullValue, coder));
    StateValue<Integer> nullStateValue = serde.fromBytes(nullBytes);
    assertNull(nullBytes);
    assertNull(nullStateValue.getValue(coder));
}

Example 9 with StateValue

use of org.apache.beam.runners.samza.runtime.SamzaStoreStateInternals.StateValue in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testNewTimersAreInsertedInOrder.

@Test
public void testNewTimersAreInsertedInOrder() {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    pipelineOptions.setEventTimerBufferSize(5);
    final KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey("testKey");
    // timers in store now are timestamped from 0 - 9.
    for (int i = 0; i < 10; i++) {
        timerInternals.setTimer(nameSpace, "timer" + i, "", new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
    }
    // fire the first 2 timers.
    // timers in memory now are timestamped from 2 - 4;
    // timers in store now are timestamped from 2 - 9.
    Collection<KeyedTimerData<String>> readyTimers;
    timerInternalsFactory.setInputWatermark(new Instant(1));
    long lastTimestamp = 0;
    readyTimers = timerInternalsFactory.removeReadyTimers();
    for (KeyedTimerData<String> keyedTimerData : readyTimers) {
        final long currentTimeStamp = keyedTimerData.getTimerData().getTimestamp().getMillis();
        assertTrue(lastTimestamp <= currentTimeStamp);
        lastTimestamp = currentTimeStamp;
    }
    assertEquals(2, readyTimers.size());
    // prefixed with timer, timestamp is in order;
    for (int i = 0; i < 3; i++) {
        timerInternals.setTimer(nameSpace, "timer" + i, "", new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
    }
    // there are 11 timers in state now.
    // watermark 5 comes, so 6 timers will be evicted because their timestamp is less than 5.
    // memory will be reloaded once to have 5 to 8 left (reload to have 4 to 8, but 4 is evicted), 5
    // to 9 left in store.
    // all of them are in order for firing.
    timerInternalsFactory.setInputWatermark(new Instant(5));
    lastTimestamp = 0;
    readyTimers = timerInternalsFactory.removeReadyTimers();
    for (KeyedTimerData<String> keyedTimerData : readyTimers) {
        final long currentTimeStamp = keyedTimerData.getTimerData().getTimestamp().getMillis();
        assertTrue(lastTimestamp <= currentTimeStamp);
        lastTimestamp = currentTimeStamp;
    }
    assertEquals(6, readyTimers.size());
    assertEquals(4, timerInternalsFactory.getEventTimeBuffer().size());
    // watermark 10 comes, so all timers will be evicted in order.
    timerInternalsFactory.setInputWatermark(new Instant(10));
    readyTimers = timerInternalsFactory.removeReadyTimers();
    for (KeyedTimerData<String> keyedTimerData : readyTimers) {
        final long currentTimeStamp = keyedTimerData.getTimerData().getTimestamp().getMillis();
        assertTrue(lastTimestamp <= currentTimeStamp);
        lastTimestamp = currentTimeStamp;
    }
    assertEquals(4, readyTimers.size());
    assertEquals(0, timerInternalsFactory.getEventTimeBuffer().size());
    store.close();
}

Example 10 with StateValue

use of org.apache.beam.runners.samza.runtime.SamzaStoreStateInternals.StateValue in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testAllTimersAreFiredWithReload.

@Test
public void testAllTimersAreFiredWithReload() {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    pipelineOptions.setEventTimerBufferSize(2);
    final KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey("testKey");
    // timers in store now are timestamped from 0 - 2.
    for (int i = 0; i < 3; i++) {
        timerInternals.setTimer(nameSpace, "timer" + i, "", new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
    }
    // total number of event time timers to fire equals to the number of timers in store
    Collection<KeyedTimerData<String>> readyTimers;
    timerInternalsFactory.setInputWatermark(new Instant(3));
    readyTimers = timerInternalsFactory.removeReadyTimers();
    // buffer should reload from store and all timers are supposed to be fired.
    assertEquals(3, readyTimers.size());
    store.close();
}