Examples with LegacyTaskApplication org.apache.samza.application.LegacyTaskApplication used on opensource projects

Example 1 with LegacyTaskApplication

use of org.apache.samza.application.LegacyTaskApplication in project samza by apache.

the class TestLocalApplicationRunner method testRunIdForStream.

/**
 * For app.mode=STREAM ensure that the run.id generation utils --
 * DistributedLock, ClusterMembership and MetadataStore are NOT created.
 * Also ensure that metadataStore.put is NOT invoked
 */
@Test
public void testRunIdForStream() throws Exception {
    final Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_MODE, "STREAM");
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    cfgs.put(JobConfig.JOB_NAME, "test-task-job");
    cfgs.put(JobConfig.JOB_ID, "jobId");
    config = new MapConfig(cfgs);
    mockApp = new LegacyTaskApplication(IdentityStreamTask.class.getName());
    prepareTestForRunId();
    runner.run();
    verify(coordinationUtils, Mockito.times(0)).getLock(CoordinationConstants.RUNID_LOCK_ID);
    verify(coordinationUtils, Mockito.times(0)).getClusterMembership();
    verify(clusterMembership, Mockito.times(0)).getNumberOfProcessors();
    verify(metadataStore, Mockito.times(0)).put(eq(CoordinationConstants.RUNID_STORE_KEY), any(byte[].class));
    verify(metadataStore, Mockito.times(0)).flush();
}

Example 2 with LegacyTaskApplication

use of org.apache.samza.application.LegacyTaskApplication in project samza by apache.

the class TestLocalApplicationRunner method testRunStreamTaskWithoutExternalContext.

@Test
public void testRunStreamTaskWithoutExternalContext() throws Exception {
    final Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    cfgs.put(ApplicationConfig.APP_NAME, "test-app");
    cfgs.put(ApplicationConfig.APP_ID, "test-appId");
    config = new MapConfig(cfgs);
    mockApp = new LegacyTaskApplication(IdentityStreamTask.class.getName());
    prepareTest();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore metadataStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        listener.afterStop();
        return null;
    }).when(sp).start();
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.empty()), any(CoordinatorStreamStore.class));
    doReturn(metadataStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    doReturn(ApplicationStatus.SuccessfulFinish).when(runner).status();
    runner.run();
    verify(metadataStore).init();
    verify(metadataStore).close();
    assertEquals(ApplicationStatus.SuccessfulFinish, runner.status());
}

Example 3 with LegacyTaskApplication

use of org.apache.samza.application.LegacyTaskApplication in project samza by apache.

the class TestLocalApplicationRunner method testRunStreamTask.

@Test
public void testRunStreamTask() throws Exception {
    final Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    cfgs.put(ApplicationConfig.APP_NAME, "test-app");
    cfgs.put(ApplicationConfig.APP_ID, "test-appId");
    config = new MapConfig(cfgs);
    mockApp = new LegacyTaskApplication(IdentityStreamTask.class.getName());
    prepareTest();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore metadataStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        listener.afterStop();
        return null;
    }).when(sp).start();
    ExternalContext externalContext = mock(ExternalContext.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class));
    doReturn(metadataStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    doReturn(ApplicationStatus.SuccessfulFinish).when(runner).status();
    runner.run(externalContext);
    verify(metadataStore).init();
    verify(metadataStore).close();
    assertEquals(ApplicationStatus.SuccessfulFinish, runner.status());
}

Example 4 with LegacyTaskApplication

use of org.apache.samza.application.LegacyTaskApplication in project samza by apache.

the class ExecutionPlanner method createJobGraph.

/**
 * Creates the physical graph from {@link ApplicationDescriptorImpl}
 */
/* package private */
JobGraph createJobGraph(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) {
    JobGraph jobGraph = new JobGraph(config, appDesc);
    // Source streams contain both input and intermediate streams.
    Set<StreamSpec> sourceStreams = getStreamSpecs(appDesc.getInputStreamIds(), streamConfig);
    // Sink streams contain both output and intermediate streams.
    Set<StreamSpec> sinkStreams = getStreamSpecs(appDesc.getOutputStreamIds(), streamConfig);
    Set<StreamSpec> intermediateStreams = Sets.intersection(sourceStreams, sinkStreams);
    Set<StreamSpec> inputStreams = Sets.difference(sourceStreams, intermediateStreams);
    Set<StreamSpec> outputStreams = Sets.difference(sinkStreams, intermediateStreams);
    Set<TableDescriptor> tables = appDesc.getTableDescriptors();
    // Generate job.id and job.name configs from app.id and app.name if defined
    MapConfig generatedJobConfigs = JobPlanner.generateSingleJobConfig(config);
    String jobName = generatedJobConfigs.get(JobConfig.JOB_NAME);
    String jobId = generatedJobConfigs.get(JobConfig.JOB_ID, "1");
    // For this phase, we have a single job node for the whole DAG
    JobNode node = jobGraph.getOrCreateJobNode(jobName, jobId);
    // Add input streams
    inputStreams.forEach(spec -> jobGraph.addInputStream(spec, node));
    // Add output streams
    outputStreams.forEach(spec -> jobGraph.addOutputStream(spec, node));
    // Add intermediate streams
    intermediateStreams.forEach(spec -> jobGraph.addIntermediateStream(spec, node, node));
    // Add tables
    for (TableDescriptor table : tables) {
        jobGraph.addTable(table, node);
        // Add side-input streams (if any)
        if (table instanceof LocalTableDescriptor) {
            LocalTableDescriptor localTable = (LocalTableDescriptor) table;
            Iterable<String> sideInputs = ListUtils.emptyIfNull(localTable.getSideInputs());
            for (String sideInput : sideInputs) {
                jobGraph.addSideInputStream(getStreamSpec(sideInput, streamConfig));
            }
        }
    }
    if (!LegacyTaskApplication.class.isAssignableFrom(appDesc.getAppClass())) {
        // skip the validation when input streamIds are empty. This is only possible for LegacyTaskApplication
        jobGraph.validate();
    }
    return jobGraph;
}

Example 5 with LegacyTaskApplication

use of org.apache.samza.application.LegacyTaskApplication in project samza by apache.

the class TestRunner method initializeInMemoryInputStream.

/**
 * Creates an in memory stream with {@link InMemorySystemFactory} and feeds its partition with stream of messages
 * @param partitionData key of the map represents partitionId and value represents messages in the partition
 * @param descriptor describes a stream to initialize with the in memory system
 */
private <StreamMessageType> void initializeInMemoryInputStream(InMemoryInputDescriptor<?> descriptor, Map<Integer, Iterable<StreamMessageType>> partitionData) {
    String systemName = descriptor.getSystemName();
    String streamName = (String) descriptor.getPhysicalName().orElse(descriptor.getStreamId());
    if (this.app instanceof LegacyTaskApplication) {
        // for legacy applications that only specify task.class.
        if (configs.containsKey(TaskConfig.INPUT_STREAMS)) {
            configs.put(TaskConfig.INPUT_STREAMS, configs.get(TaskConfig.INPUT_STREAMS).concat("," + systemName + "." + streamName));
        } else {
            configs.put(TaskConfig.INPUT_STREAMS, systemName + "." + streamName);
        }
    }
    InMemorySystemDescriptor imsd = (InMemorySystemDescriptor) descriptor.getSystemDescriptor();
    imsd.withInMemoryScope(this.inMemoryScope);
    addConfig(descriptor.toConfig());
    addConfig(descriptor.getSystemDescriptor().toConfig());
    addSerdeConfigs(descriptor);
    StreamSpec spec = new StreamSpec(descriptor.getStreamId(), streamName, systemName, partitionData.size());
    SystemFactory factory = new InMemorySystemFactory();
    Config config = new MapConfig(descriptor.toConfig(), descriptor.getSystemDescriptor().toConfig());
    factory.getAdmin(systemName, config).createStream(spec);
    InMemorySystemProducer producer = (InMemorySystemProducer) factory.getProducer(systemName, config, null);
    SystemStream sysStream = new SystemStream(systemName, streamName);
    partitionData.forEach((partitionId, partition) -> {
        partition.forEach(e -> {
            Object key = e instanceof KV ? ((KV) e).getKey() : null;
            Object value = e instanceof KV ? ((KV) e).getValue() : e;
            if (value instanceof IncomingMessageEnvelope) {
                producer.send((IncomingMessageEnvelope) value);
            } else {
                producer.send(systemName, new OutgoingMessageEnvelope(sysStream, Integer.valueOf(partitionId), key, value));
            }
        });
        producer.send(systemName, new OutgoingMessageEnvelope(sysStream, Integer.valueOf(partitionId), null, new EndOfStreamMessage(null)));
    });
}