Search in sources :

Example 1 with MockLocalTableDescriptor

use of org.apache.samza.table.descriptors.TestLocalTableDescriptor.MockLocalTableDescriptor in project samza by apache.

the class TestJobNodeConfigurationGenerator method testStreamApplicationWithTableAndSideInput.

@Test
public void testStreamApplicationWithTableAndSideInput() {
    mockStreamAppDesc = new StreamApplicationDescriptorImpl(getRepartitionJoinStreamApplication(), mockConfig);
    // add table to the RepartitionJoinStreamApplication
    GenericInputDescriptor<KV<String, Object>> sideInput1 = inputSystemDescriptor.getInputDescriptor("sideInput1", defaultSerde);
    BaseTableDescriptor mockTableDescriptor = new MockLocalTableDescriptor("testTable", defaultSerde).withSideInputs(Arrays.asList(sideInput1.getStreamId())).withSideInputsProcessor(mock(SideInputsProcessor.class, withSettings().serializable())).withConfig("mock.table.provider.config", "mock.config.value");
    // add side input and terminate at table in the appplication
    mockStreamAppDesc.getInputStream(sideInput1).sendTo(mockStreamAppDesc.getTable(mockTableDescriptor));
    StreamEdge sideInputEdge = new StreamEdge(new StreamSpec(sideInput1.getStreamId(), "sideInput1", inputSystemDescriptor.getSystemName()), false, false, mockConfig);
    // need to put the sideInput related stream configuration to the original config
    // TODO: this is confusing since part of the system and stream related configuration is generated outside the JobGraphConfigureGenerator
    // It would be nice if all system and stream related configuration is generated in one place and only intermediate stream
    // configuration is generated by JobGraphConfigureGenerator
    Map<String, String> configs = new HashMap<>(mockConfig);
    configs.putAll(sideInputEdge.generateConfig());
    mockConfig = spy(new MapConfig(configs));
    configureJobNode(mockStreamAppDesc);
    // create the JobGraphConfigureGenerator and generate the jobConfig for the jobNode
    JobNodeConfigurationGenerator configureGenerator = new JobNodeConfigurationGenerator();
    JobConfig jobConfig = configureGenerator.generateJobConfig(mockJobNode, "testJobGraphJson");
    Config expectedJobConfig = getExpectedJobConfig(mockConfig, mockJobNode.getInEdges());
    validateJobConfig(expectedJobConfig, jobConfig);
    Map<String, Serde> deserializedSerdes = validateAndGetDeserializedSerdes(jobConfig, 5);
    validateTableConfigure(jobConfig, deserializedSerdes, mockTableDescriptor);
}
Also used : Serde(org.apache.samza.serializers.Serde) StringSerde(org.apache.samza.serializers.StringSerde) SerializableSerde(org.apache.samza.serializers.SerializableSerde) TimestampedValueSerde(org.apache.samza.operators.impl.store.TimestampedValueSerde) StreamSpec(org.apache.samza.system.StreamSpec) HashMap(java.util.HashMap) JobConfig(org.apache.samza.config.JobConfig) MapConfig(org.apache.samza.config.MapConfig) SerializerConfig(org.apache.samza.config.SerializerConfig) TaskConfig(org.apache.samza.config.TaskConfig) Config(org.apache.samza.config.Config) KV(org.apache.samza.operators.KV) MockLocalTableDescriptor(org.apache.samza.table.descriptors.TestLocalTableDescriptor.MockLocalTableDescriptor) BaseTableDescriptor(org.apache.samza.table.descriptors.BaseTableDescriptor) JobConfig(org.apache.samza.config.JobConfig) StreamApplicationDescriptorImpl(org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl) MapConfig(org.apache.samza.config.MapConfig) Test(org.junit.Test)

Example 2 with MockLocalTableDescriptor

use of org.apache.samza.table.descriptors.TestLocalTableDescriptor.MockLocalTableDescriptor in project samza by apache.

the class TestJobNodeConfigurationGenerator method testTaskApplicationWithTableAndSideInput.

@Test
public void testTaskApplicationWithTableAndSideInput() {
    // add table to the RepartitionJoinStreamApplication
    GenericInputDescriptor<KV<String, Object>> sideInput1 = inputSystemDescriptor.getInputDescriptor("sideInput1", defaultSerde);
    BaseTableDescriptor mockTableDescriptor = new MockLocalTableDescriptor("testTable", defaultSerde).withSideInputs(Arrays.asList(sideInput1.getStreamId())).withSideInputsProcessor(mock(SideInputsProcessor.class, withSettings().serializable())).withConfig("mock.table.provider.config", "mock.config.value");
    StreamEdge sideInputEdge = new StreamEdge(new StreamSpec(sideInput1.getStreamId(), "sideInput1", inputSystemDescriptor.getSystemName()), false, false, mockConfig);
    // need to put the sideInput related stream configuration to the original config
    // TODO: this is confusing since part of the system and stream related configuration is generated outside the JobGraphConfigureGenerator
    // It would be nice if all system and stream related configuration is generated in one place and only intermediate stream
    // configuration is generated by JobGraphConfigureGenerator
    Map<String, String> configs = new HashMap<>(mockConfig);
    configs.putAll(sideInputEdge.generateConfig());
    mockConfig = spy(new MapConfig(configs));
    // set the application to TaskApplication, which still wire up all input/output/intermediate streams
    TaskApplicationDescriptorImpl taskAppDesc = new TaskApplicationDescriptorImpl(getTaskApplication(), mockConfig);
    // add table to the task application
    taskAppDesc.withTable(mockTableDescriptor);
    taskAppDesc.withInputStream(inputSystemDescriptor.getInputDescriptor("sideInput1", defaultSerde));
    configureJobNode(taskAppDesc);
    // create the JobGraphConfigureGenerator and generate the jobConfig for the jobNode
    JobNodeConfigurationGenerator configureGenerator = new JobNodeConfigurationGenerator();
    JobConfig jobConfig = configureGenerator.generateJobConfig(mockJobNode, "testJobGraphJson");
    // Verify the results
    Config expectedJobConfig = getExpectedJobConfig(mockConfig, mockJobNode.getInEdges());
    validateJobConfig(expectedJobConfig, jobConfig);
    Map<String, Serde> deserializedSerdes = validateAndGetDeserializedSerdes(jobConfig, 2);
    validateStreamConfigures(jobConfig, deserializedSerdes);
    validateTableConfigure(jobConfig, deserializedSerdes, mockTableDescriptor);
}
Also used : Serde(org.apache.samza.serializers.Serde) StringSerde(org.apache.samza.serializers.StringSerde) SerializableSerde(org.apache.samza.serializers.SerializableSerde) TimestampedValueSerde(org.apache.samza.operators.impl.store.TimestampedValueSerde) StreamSpec(org.apache.samza.system.StreamSpec) HashMap(java.util.HashMap) JobConfig(org.apache.samza.config.JobConfig) MapConfig(org.apache.samza.config.MapConfig) SerializerConfig(org.apache.samza.config.SerializerConfig) TaskConfig(org.apache.samza.config.TaskConfig) Config(org.apache.samza.config.Config) TaskApplicationDescriptorImpl(org.apache.samza.application.descriptors.TaskApplicationDescriptorImpl) KV(org.apache.samza.operators.KV) MockLocalTableDescriptor(org.apache.samza.table.descriptors.TestLocalTableDescriptor.MockLocalTableDescriptor) BaseTableDescriptor(org.apache.samza.table.descriptors.BaseTableDescriptor) JobConfig(org.apache.samza.config.JobConfig) MapConfig(org.apache.samza.config.MapConfig) Test(org.junit.Test)

Aggregations

HashMap (java.util.HashMap)2 Config (org.apache.samza.config.Config)2 JobConfig (org.apache.samza.config.JobConfig)2 MapConfig (org.apache.samza.config.MapConfig)2 SerializerConfig (org.apache.samza.config.SerializerConfig)2 TaskConfig (org.apache.samza.config.TaskConfig)2 KV (org.apache.samza.operators.KV)2 TimestampedValueSerde (org.apache.samza.operators.impl.store.TimestampedValueSerde)2 Serde (org.apache.samza.serializers.Serde)2 SerializableSerde (org.apache.samza.serializers.SerializableSerde)2 StringSerde (org.apache.samza.serializers.StringSerde)2 StreamSpec (org.apache.samza.system.StreamSpec)2 BaseTableDescriptor (org.apache.samza.table.descriptors.BaseTableDescriptor)2 MockLocalTableDescriptor (org.apache.samza.table.descriptors.TestLocalTableDescriptor.MockLocalTableDescriptor)2 Test (org.junit.Test)2 StreamApplicationDescriptorImpl (org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl)1 TaskApplicationDescriptorImpl (org.apache.samza.application.descriptors.TaskApplicationDescriptorImpl)1