Examples with SamzaPipelineOptions org.apache.beam.runners.samza.SamzaPipelineOptions used on opensource projects

Example 6 with SamzaPipelineOptions

use of org.apache.beam.runners.samza.SamzaPipelineOptions in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testBufferSizeNotExceedingPipelineOptionValue.

@Test
public void testBufferSizeNotExceedingPipelineOptionValue() {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    pipelineOptions.setEventTimerBufferSize(2);
    final KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey("testKey");
    // timers in store are then timestamped from 0 - 4.
    for (int i = 0; i < 5; i++) {
        timerInternals.setTimer(nameSpace, "timer" + i, "", new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
    }
    // only two timers are supposed to be in event time buffer
    assertEquals(2, timerInternalsFactory.getEventTimeBuffer().size());
    store.close();
}

Example 7 with SamzaPipelineOptions

use of org.apache.beam.runners.samza.SamzaPipelineOptions in project beam by apache.

the class SamzaTimerInternalsFactoryTest method testMaxExpiredEventTimersProcessAtOnce.

private void testMaxExpiredEventTimersProcessAtOnce(int totalNumberOfTimersInStore, int totalNumberOfExpiredTimers, int maxExpiredTimersToProcessOnce, int expectedExpiredTimersToProcess) {
    final SamzaPipelineOptions pipelineOptions = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    pipelineOptions.setMaxReadyTimersToProcessOnce(maxExpiredTimersToProcessOnce);
    final KeyValueStore<ByteArray, StateValue<?>> store = createStore();
    final SamzaTimerInternalsFactory<String> timerInternalsFactory = createTimerInternalsFactory(null, "timer", pipelineOptions, store);
    final StateNamespace nameSpace = StateNamespaces.global();
    final TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey("testKey");
    TimerInternals.TimerData timer;
    for (int i = 0; i < totalNumberOfTimersInStore; i++) {
        timer = TimerInternals.TimerData.of("timer" + i, nameSpace, new Instant(i), new Instant(i), TimeDomain.EVENT_TIME);
        timerInternals.setTimer(timer);
    }
    // Set the timestamp of the input watermark to be the value of totalNumberOfExpiredTimers
    // so that totalNumberOfExpiredTimers timers are expected be expired with respect to this
    // watermark.
    final Instant inputWatermark = new Instant(totalNumberOfExpiredTimers);
    timerInternalsFactory.setInputWatermark(inputWatermark);
    final Collection<KeyedTimerData<String>> readyTimers = timerInternalsFactory.removeReadyTimers();
    assertEquals(expectedExpiredTimersToProcess, readyTimers.size());
    store.close();
}

Example 8 with SamzaPipelineOptions

use of org.apache.beam.runners.samza.SamzaPipelineOptions in project beam by apache.

the class ConfigGeneratorTest method testDuplicateStateIdConfig.

@Test
public void testDuplicateStateIdConfig() {
    SamzaPipelineOptions options = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    options.setJobName("TestStoreConfig");
    options.setRunner(SamzaRunner.class);
    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply(Create.empty(TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.strings()))).apply(ParDo.of(new DoFn<KV<String, String>, KV<String, String>>() {

        private static final String testState = "testState";

        @StateId(testState)
        private final StateSpec<ValueState<Integer>> state = StateSpecs.value();

        @ProcessElement
        public void processElement(ProcessContext context, @StateId(testState) ValueState<Integer> state) {
            context.output(context.element());
        }
    })).apply(ParDo.of(new DoFn<KV<String, String>, Void>() {

        private static final String testState = "testState";

        @StateId(testState)
        private final StateSpec<ValueState<Integer>> state = StateSpecs.value();

        @ProcessElement
        public void processElement(ProcessContext context, @StateId(testState) ValueState<Integer> state) {
        }
    }));
    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    assertThrows(IllegalStateException.class, () -> SamzaPipelineTranslator.createConfig(pipeline, options, idMap, configBuilder));
}

Example 9 with SamzaPipelineOptions

use of org.apache.beam.runners.samza.SamzaPipelineOptions in project beam by apache.

the class ConfigGeneratorTest method testSamzaStandAloneExecutionEnvironmentConfig.

@Test
public void testSamzaStandAloneExecutionEnvironmentConfig() {
    SamzaPipelineOptions options = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    options.setJobName("TestEnvConfig");
    options.setRunner(SamzaRunner.class);
    options.setSamzaExecutionEnvironment(SamzaExecutionEnvironment.STANDALONE);
    options.setConfigOverride(ImmutableMap.<String, String>builder().put(ZkConfig.ZK_CONNECT, "localhost:2181").build());
    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply(Create.of(1, 2, 3)).apply(Sum.integersGlobally());
    pipeline.replaceAll(SamzaTransformOverrides.getDefaultOverrides());
    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPipelineTranslator.createConfig(pipeline, options, idMap, configBuilder);
    try {
        Config config = configBuilder.build();
        assertEquals(config.get(APP_RUNNER_CLASS), LocalApplicationRunner.class.getName());
        assertEquals(config.get(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY), ZkJobCoordinatorFactory.class.getName());
    } catch (IllegalArgumentException e) {
        throw new AssertionError(String.format("Failed to validate correct configs for %s samza execution environment", SamzaExecutionEnvironment.STANDALONE), e);
    }
}

Example 10 with SamzaPipelineOptions

use of org.apache.beam.runners.samza.SamzaPipelineOptions in project beam by apache.

the class ConfigGeneratorTest method testSamzaLocalExecutionEnvironmentConfig.

@Test
public void testSamzaLocalExecutionEnvironmentConfig() {
    SamzaPipelineOptions options = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    options.setJobName("TestEnvConfig");
    options.setRunner(SamzaRunner.class);
    options.setSamzaExecutionEnvironment(SamzaExecutionEnvironment.LOCAL);
    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply(Create.of(1, 2, 3)).apply(Sum.integersGlobally());
    pipeline.replaceAll(SamzaTransformOverrides.getDefaultOverrides());
    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPipelineTranslator.createConfig(pipeline, options, idMap, configBuilder);
    final Config config = configBuilder.build();
    assertTrue(Maps.difference(config, ConfigBuilder.localRunConfig()).entriesOnlyOnRight().isEmpty());
}