Examples with SamzaPipelineOptions org.apache.beam.runners.samza.SamzaPipelineOptions used on opensource projects

Example 1 with SamzaPipelineOptions

use of org.apache.beam.runners.samza.SamzaPipelineOptions in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testBufferRefilledAfterRestoreToNonFullState.

@Test
public void testBufferRefilledAfterRestoreToNonFullState() {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    pipelineOptions.setEventTimerBufferSize(5);
    final KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey("testKey");
    // timer in store now is timestamped 6.
    for (int i = 0; i < 6; i++) {
        timerInternals.setTimer(nameSpace, "timer" + i, "", new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
    }
    // total number of event time timers to fire equals to the number of timers in store
    Collection<KeyedTimerData<String>> readyTimers;
    timerInternalsFactory.setInputWatermark(new Instant(4));
    readyTimers = timerInternalsFactory.removeReadyTimers();
    assertEquals(5, readyTimers.size());
    // reloaded timer5
    assertEquals(1, timerInternalsFactory.getEventTimeBuffer().size());
    for (int i = 6; i < 13; i++) {
        timerInternals.setTimer(nameSpace, "timer" + i, "", new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
    }
    // timers should go into buffer not state
    assertEquals(5, timerInternalsFactory.getEventTimeBuffer().size());
    // watermark 10 comes,6 timers will be evicted in order and 2 still in buffer.
    timerInternalsFactory.setInputWatermark(new Instant(10));
    readyTimers = timerInternalsFactory.removeReadyTimers();
    long lastTimestamp = 0;
    for (KeyedTimerData<String> keyedTimerData : readyTimers) {
        final long currentTimeStamp = keyedTimerData.getTimerData().getTimestamp().getMillis();
        assertTrue(lastTimestamp <= currentTimeStamp);
        lastTimestamp = currentTimeStamp;
    }
    assertEquals(6, readyTimers.size());
    assertEquals(2, timerInternalsFactory.getEventTimeBuffer().size());
    store.close();
}

Example 2 with SamzaPipelineOptions

use of org.apache.beam.runners.samza.SamzaPipelineOptions in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testAllTimersAreFiredInOrder.

/**
 * Test the total number of event time timers reloaded into memory is aligned with the number of
 * the event time timers written to the store. Moreover, event time timers reloaded into memory is
 * maintained in order.
 */
@Test
public void testAllTimersAreFiredInOrder() {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    pipelineOptions.setEventTimerBufferSize(5);
    final KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey("testKey");
    // timers in store now are timestamped from 0 - 7.
    for (int i = 0; i < 8; i++) {
        timerInternals.setTimer(nameSpace, "timer" + i, "", new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
    }
    // fire the first 2 timers.
    // timers in memory now are timestamped from 2 - 4;
    // timers in store now are timestamped from 2 - 7.
    Collection<KeyedTimerData<String>> readyTimers;
    timerInternalsFactory.setInputWatermark(new Instant(1));
    long lastTimestamp = 0;
    readyTimers = timerInternalsFactory.removeReadyTimers();
    for (KeyedTimerData<String> keyedTimerData : readyTimers) {
        final long currentTimeStamp = keyedTimerData.getTimerData().getTimestamp().getMillis();
        assertTrue(lastTimestamp <= currentTimeStamp);
        lastTimestamp = currentTimeStamp;
    }
    assertEquals(2, readyTimers.size());
    // the total number of timers to fire is 18.
    for (int i = 8; i < 20; i++) {
        timerInternals.setTimer(nameSpace, "timer" + i, "", new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
    }
    timerInternalsFactory.setInputWatermark(new Instant(20));
    lastTimestamp = 0;
    readyTimers = timerInternalsFactory.removeReadyTimers();
    for (KeyedTimerData<String> keyedTimerData : readyTimers) {
        final long currentTimeStamp = keyedTimerData.getTimerData().getTimestamp().getMillis();
        assertTrue(lastTimestamp <= currentTimeStamp);
        lastTimestamp = currentTimeStamp;
    }
    assertEquals(18, readyTimers.size());
    store.close();
}

Example 3 with SamzaPipelineOptions

use of org.apache.beam.runners.samza.SamzaPipelineOptions in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testRestoreEventBufferSize.

@Test
public void testRestoreEventBufferSize() throws Exception {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final String key = "testKey";
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey(key);
    final TimerInternals.TimerData timer1 = TimerInternals.TimerData.of("timer1", nameSpace, new Instant(10), new Instant(10), TimeDomain.EVENT_TIME);
    timerInternals.setTimer(timer1);
    store.close();
    // restore by creating a new instance
    store = createStore();
    final SamzaTimerInternalsFactory<String> restoredFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    assertEquals(1, restoredFactory.getEventTimeBuffer().size());
    restoredFactory.setInputWatermark(new Instant(150));
    Collection<KeyedTimerData<String>> readyTimers = restoredFactory.removeReadyTimers();
    assertEquals(1, readyTimers.size());
    // Timer 1 should be evicted from buffer
    assertTrue(restoredFactory.getEventTimeBuffer().isEmpty());
    final TimerInternals restoredTimerInternals = restoredFactory.timerInternalsForKey(key);
    final TimerInternals.TimerData timer2 = TimerInternals.TimerData.of("timer2", nameSpace, new Instant(200), new Instant(200), TimeDomain.EVENT_TIME);
    restoredTimerInternals.setTimer(timer2);
    // Timer 2 should be added to the Event buffer
    assertEquals(1, restoredFactory.getEventTimeBuffer().size());
    // Timer 2 should not be ready
    readyTimers = restoredFactory.removeReadyTimers();
    assertEquals(0, readyTimers.size());
    restoredFactory.setInputWatermark(new Instant(250));
    // Timer 2 should be ready
    readyTimers = restoredFactory.removeReadyTimers();
    assertEquals(1, readyTimers.size());
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    StringUtf8Coder.of().encode(key, baos);
    byte[] keyBytes = baos.toByteArray();
    assertEquals(readyTimers, Arrays.asList(new KeyedTimerData<>(keyBytes, key, timer2)));
    store.close();
}

Example 4 with SamzaPipelineOptions

use of org.apache.beam.runners.samza.SamzaPipelineOptions in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testProcessingTimeTimers.

@Test
public void testProcessingTimeTimers() throws IOException {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    TestTimerRegistry timerRegistry = new TestTimerRegistry();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(timerRegistry, "timer", pipelineOptions, store);
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey("testKey");
    final TimerInternals.TimerData timer1 = TimerInternals.TimerData.of("timer1", nameSpace, new Instant(10), new Instant(10), TimeDomain.PROCESSING_TIME);
    timerInternals.setTimer(timer1);
    final TimerInternals.TimerData timer2 = TimerInternals.TimerData.of("timer2", nameSpace, new Instant(100), new Instant(100), TimeDomain.PROCESSING_TIME);
    timerInternals.setTimer(timer2);
    final TimerInternals.TimerData timer3 = TimerInternals.TimerData.of("timer3", "timerFamilyId3", nameSpace, new Instant(100), new Instant(100), TimeDomain.PROCESSING_TIME);
    timerInternals.setTimer(timer3);
    assertEquals(3, timerRegistry.timers.size());
    store.close();
    // restore by creating a new instance
    store = createStore();
    TestTimerRegistry restoredRegistry = new TestTimerRegistry();
    final SamzaTimerInternalsFactory<String> restoredFactory = createTimerInternalsFactory(restoredRegistry, "timer", pipelineOptions, store);
    assertEquals(3, restoredRegistry.timers.size());
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    StringUtf8Coder.of().encode("testKey", baos);
    final byte[] keyBytes = baos.toByteArray();
    restoredFactory.removeProcessingTimer(new KeyedTimerData<>(keyBytes, "testKey", timer1));
    restoredFactory.removeProcessingTimer(new KeyedTimerData<>(keyBytes, "testKey", timer2));
    restoredFactory.removeProcessingTimer(new KeyedTimerData<>(keyBytes, "testKey", timer3));
    store.close();
}

Example 5 with SamzaPipelineOptions

use of org.apache.beam.runners.samza.SamzaPipelineOptions in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testRestore.

@Test
public void testRestore() throws Exception {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final String key = "testKey";
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey(key);
    final TimerInternals.TimerData timer1 = TimerInternals.TimerData.of("timer1", nameSpace, new Instant(10), new Instant(10), TimeDomain.EVENT_TIME);
    timerInternals.setTimer(timer1);
    final TimerInternals.TimerData timer2 = TimerInternals.TimerData.of("timer2", nameSpace, new Instant(100), new Instant(100), TimeDomain.EVENT_TIME);
    timerInternals.setTimer(timer2);
    store.close();
    // restore by creating a new instance
    store = createStore();
    final SamzaTimerInternalsFactory<String> restoredFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    restoredFactory.setInputWatermark(new Instant(150));
    Collection<KeyedTimerData<String>> readyTimers = restoredFactory.removeReadyTimers();
    assertEquals(2, readyTimers.size());
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    StringUtf8Coder.of().encode(key, baos);
    byte[] keyBytes = baos.toByteArray();
    assertEquals(readyTimers, Arrays.asList(new KeyedTimerData<>(keyBytes, key, timer1), new KeyedTimerData<>(keyBytes, key, timer2)));
    store.close();
}