Examples with TaskApplicationDescriptorImpl org.apache.samza.application.descriptors.TaskApplicationDescriptorImpl used on opensource projects

Example 1 with TaskApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.TaskApplicationDescriptorImpl in project samza by apache.

the class TestTaskFactoryUtil method testGetTaskFactoryWithTaskAppDescriptor.

// test getTaskFactory with TaskApplicationDescriptor
@Test
public void testGetTaskFactoryWithTaskAppDescriptor() {
    TaskApplicationDescriptorImpl mockTaskApp = mock(TaskApplicationDescriptorImpl.class);
    TaskFactory mockTaskFactory = mock(TaskFactory.class);
    when(mockTaskApp.getTaskFactory()).thenReturn(mockTaskFactory);
    TaskFactory taskFactory = TaskFactoryUtil.getTaskFactory(mockTaskApp);
    assertEquals(mockTaskFactory, taskFactory);
}

Example 2 with TaskApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.TaskApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testCreateJobGraphForLegacyTaskApplication.

@Test
public void testCreateJobGraphForLegacyTaskApplication() {
    TaskApplicationDescriptorImpl taskAppDesc = mock(TaskApplicationDescriptorImpl.class);
    when(taskAppDesc.getInputDescriptors()).thenReturn(new HashMap<>());
    when(taskAppDesc.getOutputDescriptors()).thenReturn(new HashMap<>());
    when(taskAppDesc.getTableDescriptors()).thenReturn(new HashSet<>());
    when(taskAppDesc.getSystemDescriptors()).thenReturn(new HashSet<>());
    when(taskAppDesc.getIntermediateBroadcastStreamIds()).thenReturn(new HashSet<>());
    doReturn(LegacyTaskApplication.class).when(taskAppDesc).getAppClass();
    Map<String, String> systemStreamConfigs = new HashMap<>();
    inputDescriptors.forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig()));
    outputDescriptors.forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig()));
    systemDescriptors.forEach(sd -> systemStreamConfigs.putAll(sd.toConfig()));
    ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
    JobGraph jobGraph = planner.createJobGraph(taskAppDesc);
    assertEquals(1, jobGraph.getJobNodes().size());
    JobNode jobNode = jobGraph.getJobNodes().get(0);
    assertEquals("test-app", jobNode.getJobName());
    assertEquals("test-app-1", jobNode.getJobNameAndId());
    assertEquals(0, jobNode.getInEdges().size());
    assertEquals(0, jobNode.getOutEdges().size());
    assertEquals(0, jobNode.getTables().size());
    assertEquals(config, jobNode.getConfig());
}

Example 3 with TaskApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.TaskApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testCreateJobGraphForTaskApplication.

@Test
public void testCreateJobGraphForTaskApplication() {
    TaskApplicationDescriptorImpl taskAppDesc = mock(TaskApplicationDescriptorImpl.class);
    // add interemediate streams
    String intermediateStream1 = "intermediate-stream1";
    String intermediateBroadcast = "intermediate-broadcast1";
    // intermediate stream1, not broadcast
    GenericInputDescriptor<KV<Object, Object>> intermediateInput1 = system1Descriptor.getInputDescriptor(intermediateStream1, new KVSerde<>(new NoOpSerde(), new NoOpSerde()));
    GenericOutputDescriptor<KV<Object, Object>> intermediateOutput1 = system1Descriptor.getOutputDescriptor(intermediateStream1, new KVSerde<>(new NoOpSerde(), new NoOpSerde()));
    // intermediate stream2, broadcast
    GenericInputDescriptor<KV<Object, Object>> intermediateBroacastInput1 = system1Descriptor.getInputDescriptor(intermediateBroadcast, new KVSerde<>(new NoOpSerde<>(), new NoOpSerde<>()));
    GenericOutputDescriptor<KV<Object, Object>> intermediateBroacastOutput1 = system1Descriptor.getOutputDescriptor(intermediateBroadcast, new KVSerde<>(new NoOpSerde<>(), new NoOpSerde<>()));
    inputDescriptors.put(intermediateStream1, intermediateInput1);
    outputDescriptors.put(intermediateStream1, intermediateOutput1);
    inputDescriptors.put(intermediateBroadcast, intermediateBroacastInput1);
    outputDescriptors.put(intermediateBroadcast, intermediateBroacastOutput1);
    Set<String> broadcastStreams = new HashSet<>();
    broadcastStreams.add(intermediateBroadcast);
    when(taskAppDesc.getInputDescriptors()).thenReturn(inputDescriptors);
    when(taskAppDesc.getInputStreamIds()).thenReturn(inputDescriptors.keySet());
    when(taskAppDesc.getOutputDescriptors()).thenReturn(outputDescriptors);
    when(taskAppDesc.getOutputStreamIds()).thenReturn(outputDescriptors.keySet());
    when(taskAppDesc.getTableDescriptors()).thenReturn(Collections.emptySet());
    when(taskAppDesc.getSystemDescriptors()).thenReturn(systemDescriptors);
    when(taskAppDesc.getIntermediateBroadcastStreamIds()).thenReturn(broadcastStreams);
    doReturn(MockTaskApplication.class).when(taskAppDesc).getAppClass();
    Map<String, String> systemStreamConfigs = new HashMap<>();
    inputDescriptors.forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig()));
    outputDescriptors.forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig()));
    systemDescriptors.forEach(sd -> systemStreamConfigs.putAll(sd.toConfig()));
    ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
    JobGraph jobGraph = planner.createJobGraph(taskAppDesc);
    assertEquals(1, jobGraph.getJobNodes().size());
    assertTrue(jobGraph.getInputStreams().stream().map(edge -> edge.getName()).filter(streamId -> inputDescriptors.containsKey(streamId)).collect(Collectors.toList()).isEmpty());
    Set<String> intermediateStreams = new HashSet<>(inputDescriptors.keySet());
    jobGraph.getInputStreams().forEach(edge -> {
        if (intermediateStreams.contains(edge.getStreamSpec().getId())) {
            intermediateStreams.remove(edge.getStreamSpec().getId());
        }
    });
    assertEquals(new HashSet<>(Arrays.asList(intermediateStream1, intermediateBroadcast)), intermediateStreams);
}

Example 4 with TaskApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.TaskApplicationDescriptorImpl in project samza by apache.

the class TestJobNodeConfigurationGenerator method testGenerateJobConfigWithLegacyTaskApplication.

@Test
public void testGenerateJobConfigWithLegacyTaskApplication() {
    TaskApplicationDescriptorImpl taskAppDesc = new TaskApplicationDescriptorImpl(getLegacyTaskApplication(), mockConfig);
    configureJobNode(taskAppDesc);
    Map<String, String> originConfig = new HashMap<>(mockConfig);
    // create the JobGraphConfigureGenerator and generate the jobConfig for the jobNode
    JobNodeConfigurationGenerator configureGenerator = new JobNodeConfigurationGenerator();
    JobConfig jobConfig = configureGenerator.generateJobConfig(mockJobNode, "");
    // jobConfig should be exactly the same as original config
    Map<String, String> generatedConfig = new HashMap<>(jobConfig);
    assertEquals(originConfig, generatedConfig);
}

Example 5 with TaskApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.TaskApplicationDescriptorImpl in project samza by apache.

the class TestJobNodeConfigurationGenerator method testGenerateJobConfigWithTaskApplication.

@Test
public void testGenerateJobConfigWithTaskApplication() {
    // set the application to TaskApplication, which still wire up all input/output/intermediate streams
    TaskApplicationDescriptorImpl taskAppDesc = new TaskApplicationDescriptorImpl(getTaskApplication(), mockConfig);
    configureJobNode(taskAppDesc);
    // create the JobGraphConfigureGenerator and generate the jobConfig for the jobNode
    JobNodeConfigurationGenerator configureGenerator = new JobNodeConfigurationGenerator();
    JobConfig jobConfig = configureGenerator.generateJobConfig(mockJobNode, "testJobGraphJson");
    // Verify the results
    Config expectedJobConfig = getExpectedJobConfig(mockConfig, mockJobNode.getInEdges());
    validateJobConfig(expectedJobConfig, jobConfig);
    Map<String, Serde> deserializedSerdes = validateAndGetDeserializedSerdes(jobConfig, 2);
    validateStreamConfigures(jobConfig, deserializedSerdes);
}