Examples with ByteArray org.apache.beam.runners.samza.runtime.SamzaStoreStateInternals.ByteArray used on opensource projects

Example 6 with ByteArray

use of org.apache.beam.runners.samza.runtime.SamzaStoreStateInternals.ByteArray in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testByteArray.

@Test
public void testByteArray() {
    ByteArray key1 = ByteArray.of("hello world".getBytes(StandardCharsets.UTF_8));
    Serde<ByteArray> serde = new ByteArraySerdeFactory().getSerde("", null);
    byte[] keyBytes = serde.toBytes(key1);
    ByteArray key2 = serde.fromBytes(keyBytes);
    assertEquals(key1, key2);
    Map<ByteArray, String> map = new HashMap<>();
    map.put(key1, "found it");
    assertEquals("found it", map.get(key2));
    map.remove(key1);
    assertTrue(!map.containsKey(key2));
    assertTrue(map.isEmpty());
}

Example 7 with ByteArray

use of org.apache.beam.runners.samza.runtime.SamzaStoreStateInternals.ByteArray in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testBufferSizeNotExceedingPipelineOptionValue.

@Test
public void testBufferSizeNotExceedingPipelineOptionValue() {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    pipelineOptions.setEventTimerBufferSize(2);
    final KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey("testKey");
    // timers in store are then timestamped from 0 - 4.
    for (int i = 0; i < 5; i++) {
        timerInternals.setTimer(nameSpace, "timer" + i, "", new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
    }
    // only two timers are supposed to be in event time buffer
    assertEquals(2, timerInternalsFactory.getEventTimeBuffer().size());
    store.close();
}

Example 8 with ByteArray

use of org.apache.beam.runners.samza.runtime.SamzaStoreStateInternals.ByteArray in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testMaxExpiredEventTimersProcessAtOnce.

private void testMaxExpiredEventTimersProcessAtOnce(int totalNumberOfTimersInStore, int totalNumberOfExpiredTimers, int maxExpiredTimersToProcessOnce, int expectedExpiredTimersToProcess) {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    pipelineOptions.setMaxReadyTimersToProcessOnce(maxExpiredTimersToProcessOnce);
    final KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey("testKey");
    TimerInternals.TimerData timer;
    for (int i = 0; i < totalNumberOfTimersInStore; i++) {
        timer = TimerInternals.TimerData.of("timer" + i, nameSpace, new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
        timerInternals.setTimer(timer);
    }
    // Set the timestamp of the input watermark to be the value of totalNumberOfExpiredTimers
    // so that totalNumberOfExpiredTimers timers are expected be expired with respect to this
    // watermark.
    final Instant inputWatermark = new Instant(totalNumberOfExpiredTimers);
    timerInternalsFactory.setInputWatermark(inputWatermark);
    final Collection<KeyedTimerData<String>> readyTimers = timerInternalsFactory.removeReadyTimers();
    assertEquals(expectedExpiredTimersToProcess, readyTimers.size());
    store.close();
}

Example 9 with ByteArray

use of org.apache.beam.runners.samza.runtime.SamzaStoreStateInternals.ByteArray in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testNewTimersAreInsertedInOrder.

@Test
public void testNewTimersAreInsertedInOrder() {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    pipelineOptions.setEventTimerBufferSize(5);
    final KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey("testKey");
    // timers in store now are timestamped from 0 - 9.
    for (int i = 0; i < 10; i++) {
        timerInternals.setTimer(nameSpace, "timer" + i, "", new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
    }
    // fire the first 2 timers.
    // timers in memory now are timestamped from 2 - 4;
    // timers in store now are timestamped from 2 - 9.
    Collection<KeyedTimerData<String>> readyTimers;
    timerInternalsFactory.setInputWatermark(new Instant(1));
    long lastTimestamp = 0;
    readyTimers = timerInternalsFactory.removeReadyTimers();
    for (KeyedTimerData<String> keyedTimerData : readyTimers) {
        final long currentTimeStamp = keyedTimerData.getTimerData().getTimestamp().getMillis();
        assertTrue(lastTimestamp <= currentTimeStamp);
        lastTimestamp = currentTimeStamp;
    }
    assertEquals(2, readyTimers.size());
    // prefixed with timer, timestamp is in order;
    for (int i = 0; i < 3; i++) {
        timerInternals.setTimer(nameSpace, "timer" + i, "", new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
    }
    // there are 11 timers in state now.
    // watermark 5 comes, so 6 timers will be evicted because their timestamp is less than 5.
    // memory will be reloaded once to have 5 to 8 left (reload to have 4 to 8, but 4 is evicted), 5
    // to 9 left in store.
    // all of them are in order for firing.
    timerInternalsFactory.setInputWatermark(new Instant(5));
    lastTimestamp = 0;
    readyTimers = timerInternalsFactory.removeReadyTimers();
    for (KeyedTimerData<String> keyedTimerData : readyTimers) {
        final long currentTimeStamp = keyedTimerData.getTimerData().getTimestamp().getMillis();
        assertTrue(lastTimestamp <= currentTimeStamp);
        lastTimestamp = currentTimeStamp;
    }
    assertEquals(6, readyTimers.size());
    assertEquals(4, timerInternalsFactory.getEventTimeBuffer().size());
    // watermark 10 comes, so all timers will be evicted in order.
    timerInternalsFactory.setInputWatermark(new Instant(10));
    readyTimers = timerInternalsFactory.removeReadyTimers();
    for (KeyedTimerData<String> keyedTimerData : readyTimers) {
        final long currentTimeStamp = keyedTimerData.getTimerData().getTimestamp().getMillis();
        assertTrue(lastTimestamp <= currentTimeStamp);
        lastTimestamp = currentTimeStamp;
    }
    assertEquals(4, readyTimers.size());
    assertEquals(0, timerInternalsFactory.getEventTimeBuffer().size());
    store.close();
}

Example 10 with ByteArray

use of org.apache.beam.runners.samza.runtime.SamzaStoreStateInternals.ByteArray in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testAllTimersAreFiredWithReload.

@Test
public void testAllTimersAreFiredWithReload() {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    pipelineOptions.setEventTimerBufferSize(2);
    final KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey("testKey");
    // timers in store now are timestamped from 0 - 2.
    for (int i = 0; i < 3; i++) {
        timerInternals.setTimer(nameSpace, "timer" + i, "", new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
    }
    // total number of event time timers to fire equals to the number of timers in store
    Collection<KeyedTimerData<String>> readyTimers;
    timerInternalsFactory.setInputWatermark(new Instant(3));
    readyTimers = timerInternalsFactory.removeReadyTimers();
    // buffer should reload from store and all timers are supposed to be fired.
    assertEquals(3, readyTimers.size());
    store.close();
}