Examples with MapConfig org.apache.samza.config.MapConfig used on opensource projects

Example 36 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestKinesisConfig method testGetKinesisStreams.

@Test
public void testGetKinesisStreams() {
    Map<String, String> kv = new HashMap<>();
    kv.put("systems.kinesis.streams.kinesis-stream1.prop1", "value1");
    kv.put("systems.kinesis.streams.kinesis-stream1.prop2", "value2");
    kv.put("systems.kinesis.streams.kinesis-stream2.prop1", "value3");
    Config config = new MapConfig(kv);
    KinesisConfig kConfig = new KinesisConfig(config);
    Set<String> streams = kConfig.getKinesisStreams("kinesis");
    assertEquals(2, streams.size());
}

Example 37 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestKinesisConfig method testKclConfigs.

@Test
public void testKclConfigs() {
    Map<String, String> kv = new HashMap<>();
    String system = "kinesis";
    String stream = "kinesis-stream";
    String systemConfigPrefix = String.format("systems.%s.", system);
    // region config is required for setting kcl config.
    kv.put(systemConfigPrefix + "aws.region", "us-east-1");
    // Kcl Configs
    kv.put(systemConfigPrefix + "aws.kcl.TableName", "sample-table");
    kv.put(systemConfigPrefix + "aws.kcl.MaxRecords", "100");
    kv.put(systemConfigPrefix + "aws.kcl.CallProcessRecordsEvenForEmptyRecordList", "true");
    kv.put(systemConfigPrefix + "aws.kcl.InitialPositionInStream", "TRIM_HORIZON");
    // override one of the Kcl configs for kinesis-stream1
    kv.put(systemConfigPrefix + "streams.kinesis-stream1.aws.kcl.InitialPositionInStream", "LATEST");
    Config config = new MapConfig(kv);
    KinesisConfig kConfig = new KinesisConfig(config);
    KinesisClientLibConfiguration kclConfig = kConfig.getKinesisClientLibConfig(system, stream, "sample-app");
    assertEquals("sample-table", kclConfig.getTableName());
    assertEquals(100, kclConfig.getMaxRecords());
    assertTrue(kclConfig.shouldCallProcessRecordsEvenForEmptyRecordList());
    assertEquals(InitialPositionInStream.TRIM_HORIZON, kclConfig.getInitialPositionInStream());
    // verify if the overriden config is applied for kinesis-stream1
    kclConfig = kConfig.getKinesisClientLibConfig(system, "kinesis-stream1", "sample-app");
    assertEquals(InitialPositionInStream.LATEST, kclConfig.getInitialPositionInStream());
}

Example 38 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestLocalApplicationRunner method testRunIdForStream.

/**
 * For app.mode=STREAM ensure that the run.id generation utils --
 * DistributedLock, ClusterMembership and MetadataStore are NOT created.
 * Also ensure that metadataStore.put is NOT invoked
 */
@Test
public void testRunIdForStream() throws Exception {
    final Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_MODE, "STREAM");
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    cfgs.put(JobConfig.JOB_NAME, "test-task-job");
    cfgs.put(JobConfig.JOB_ID, "jobId");
    config = new MapConfig(cfgs);
    mockApp = new LegacyTaskApplication(IdentityStreamTask.class.getName());
    prepareTestForRunId();
    runner.run();
    verify(coordinationUtils, Mockito.times(0)).getLock(CoordinationConstants.RUNID_LOCK_ID);
    verify(coordinationUtils, Mockito.times(0)).getClusterMembership();
    verify(clusterMembership, Mockito.times(0)).getNumberOfProcessors();
    verify(metadataStore, Mockito.times(0)).put(eq(CoordinationConstants.RUNID_STORE_KEY), any(byte[].class));
    verify(metadataStore, Mockito.times(0)).flush();
}

Example 39 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestLocalApplicationRunner method testKill.

@Test
public void testKill() throws Exception {
    Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    config = new MapConfig(cfgs);
    ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
    mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
    prepareTest();
    // return the jobConfigs from the planner
    doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        return null;
    }).when(sp).start();
    doAnswer(new Answer() {

        private int count = 0;

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            if (++count == 1) {
                ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
                listener.afterStop();
                return null;
            }
            return null;
        }
    }).when(sp).stop();
    ExternalContext externalContext = mock(ExternalContext.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class));
    doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    runner.run(externalContext);
    runner.kill();
    verify(coordinatorStreamStore).init();
    verify(coordinatorStreamStore, atLeastOnce()).close();
    assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish);
}

Example 40 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestLocalApplicationRunner method testRunStreamTaskWithoutExternalContext.

@Test
public void testRunStreamTaskWithoutExternalContext() throws Exception {
    final Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    cfgs.put(ApplicationConfig.APP_NAME, "test-app");
    cfgs.put(ApplicationConfig.APP_ID, "test-appId");
    config = new MapConfig(cfgs);
    mockApp = new LegacyTaskApplication(IdentityStreamTask.class.getName());
    prepareTest();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore metadataStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        listener.afterStop();
        return null;
    }).when(sp).start();
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.empty()), any(CoordinatorStreamStore.class));
    doReturn(metadataStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    doReturn(ApplicationStatus.SuccessfulFinish).when(runner).status();
    runner.run();
    verify(metadataStore).init();
    verify(metadataStore).close();
    assertEquals(ApplicationStatus.SuccessfulFinish, runner.status());
}