Examples with ExternalContext org.apache.samza.context.ExternalContext used on opensource projects

Example 6 with ExternalContext

use of org.apache.samza.context.ExternalContext in project samza by apache.

the class SamzaSqlApplication method describe.

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
    try {
        // TODO: Introduce an API to return a dsl string containing one or more sql statements.
        List<String> dslStmts = SamzaSqlDslConverter.fetchSqlFromConfig(appDescriptor.getConfig());
        Map<Integer, TranslatorContext> translatorContextMap = new HashMap<>();
        // 1. Get Calcite plan
        List<String> inputSystemStreams = new LinkedList<>();
        List<String> outputSystemStreams = new LinkedList<>();
        Collection<RelRoot> relRoots = SamzaSqlApplicationConfig.populateSystemStreamsAndGetRelRoots(dslStmts, appDescriptor.getConfig(), inputSystemStreams, outputSystemStreams);
        // 2. Populate configs
        SamzaSqlApplicationConfig sqlConfig = new SamzaSqlApplicationConfig(appDescriptor.getConfig(), inputSystemStreams, outputSystemStreams);
        // 3. Translate Calcite plan to Samza stream operators
        QueryTranslator queryTranslator = new QueryTranslator(appDescriptor, sqlConfig);
        SamzaSqlExecutionContext executionContext = new SamzaSqlExecutionContext(sqlConfig);
        // QueryId implies the index of the query in multiple query statements scenario. It should always start with 0.
        int queryId = 0;
        for (RelRoot relRoot : relRoots) {
            LOG.info("Translating relRoot {} to samza stream graph with queryId {}", relRoot, queryId);
            TranslatorContext translatorContext = new TranslatorContext(appDescriptor, relRoot, executionContext);
            translatorContextMap.put(queryId, translatorContext);
            queryTranslator.translate(relRoot, sqlConfig.getOutputSystemStreams().get(queryId), translatorContext, queryId);
            queryId++;
        }
        // 4. Set all translator contexts
        /*
       * TODO When serialization of ApplicationDescriptor is actually needed, then something will need to be updated here,
       * since translatorContext is not Serializable. Currently, a new ApplicationDescriptor instance is created in each
       * container, so it does not need to be serialized. Therefore, the translatorContext is recreated in each container
       * and does not need to be serialized.
       */
        appDescriptor.withApplicationTaskContextFactory(new ApplicationTaskContextFactory<SamzaSqlApplicationContext>() {

            @Override
            public SamzaSqlApplicationContext create(ExternalContext externalContext, JobContext jobContext, ContainerContext containerContext, TaskContext taskContext, ApplicationContainerContext applicationContainerContext) {
                return new SamzaSqlApplicationContext(translatorContextMap);
            }
        });
    } catch (RuntimeException e) {
        LOG.error("SamzaSqlApplication threw exception.", e);
        throw e;
    }
}

Example 7 with ExternalContext

use of org.apache.samza.context.ExternalContext in project samza by apache.

the class TestLocalApplicationRunner method testRunCompleteWithouCoordinatorStreamStore.

@Test
public void testRunCompleteWithouCoordinatorStreamStore() throws Exception {
    Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    config = new MapConfig(cfgs);
    ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
    mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
    prepareTest();
    // return the jobConfigs from the planner
    doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();
    StreamProcessor sp = mock(StreamProcessor.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        listener.afterStop();
        return null;
    }).when(sp).start();
    ExternalContext externalContext = mock(ExternalContext.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), eq(null));
    doReturn(null).when(runner).createCoordinatorStreamStore(any(Config.class));
    runner.run(externalContext);
    runner.waitForFinish();
    assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish);
}

Example 8 with ExternalContext

use of org.apache.samza.context.ExternalContext in project samza by apache.

the class TestLocalApplicationRunner method testRunComplete.

@Test
public void testRunComplete() throws Exception {
    Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    config = new MapConfig(cfgs);
    ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
    mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
    prepareTest();
    // return the jobConfigs from the planner
    doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        listener.afterStop();
        return null;
    }).when(sp).start();
    ExternalContext externalContext = mock(ExternalContext.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class));
    doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    runner.run(externalContext);
    runner.waitForFinish();
    verify(coordinatorStreamStore).init();
    verify(coordinatorStreamStore).close();
    assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish);
}

Example 9 with ExternalContext

use of org.apache.samza.context.ExternalContext in project beam by apache.

the class SamzaRunner method runSamzaApp.

private ApplicationRunner runSamzaApp(StreamApplication app, Config config) {
    final ApplicationRunner runner = ApplicationRunners.getApplicationRunner(app, config);
    ExternalContext externalContext = null;
    if (listener != null) {
        externalContext = listener.onStart();
    }
    runner.run(externalContext);
    if (listener != null && options.getSamzaExecutionEnvironment() == SamzaExecutionEnvironment.YARN) {
        listener.onSubmit();
    }
    return runner;
}