Examples with GenericSystemDescriptor org.apache.samza.system.descriptors.GenericSystemDescriptor used on opensource projects

Example 6 with GenericSystemDescriptor

use of org.apache.samza.system.descriptors.GenericSystemDescriptor in project samza by apache.

the class TestStreamApplicationDescriptorImpl method testMultipleGetInputStreams.

@Test
public void testMultipleGetInputStreams() {
    String streamId1 = "test-stream-1";
    String streamId2 = "test-stream-2";
    GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
    GenericInputDescriptor isd1 = sd.getInputDescriptor(streamId1, mock(Serde.class));
    GenericInputDescriptor isd2 = sd.getInputDescriptor(streamId2, mock(Serde.class));
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
        appDesc.getInputStream(isd1);
        appDesc.getInputStream(isd2);
    }, getConfig());
    InputOperatorSpec inputOpSpec1 = streamAppDesc.getInputOperators().get(streamId1);
    InputOperatorSpec inputOpSpec2 = streamAppDesc.getInputOperators().get(streamId2);
    assertEquals(2, streamAppDesc.getInputOperators().size());
    assertEquals(streamId1, inputOpSpec1.getStreamId());
    assertEquals(streamId2, inputOpSpec2.getStreamId());
    assertEquals(2, streamAppDesc.getInputDescriptors().size());
    assertEquals(isd1, streamAppDesc.getInputDescriptors().get(streamId1));
    assertEquals(isd2, streamAppDesc.getInputDescriptors().get(streamId2));
}

Example 7 with GenericSystemDescriptor

use of org.apache.samza.system.descriptors.GenericSystemDescriptor in project samza by apache.

the class TestStreamApplicationDescriptorImpl method testGetIntermediateStreamWithDefaultSystemDescriptor.

@Test
public void testGetIntermediateStreamWithDefaultSystemDescriptor() {
    Config mockConfig = getConfig();
    String streamId = "streamId";
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
    }, mockConfig);
    GenericSystemDescriptor sd = new GenericSystemDescriptor("mock-system", "mock-system-factory");
    streamAppDesc.withDefaultSystem(sd);
    IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl = streamAppDesc.getIntermediateStream(streamId, mock(Serde.class), false);
    assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec());
    assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream());
    assertEquals(streamId, intermediateStreamImpl.getStreamId());
}

Example 8 with GenericSystemDescriptor

use of org.apache.samza.system.descriptors.GenericSystemDescriptor in project samza by apache.

the class TestStreamApplicationDescriptorImpl method testMultipleSystemDescriptorForSameSystemName.

@Test
public void testMultipleSystemDescriptorForSameSystemName() {
    GenericSystemDescriptor sd1 = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
    GenericSystemDescriptor sd2 = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
    GenericInputDescriptor isd1 = sd1.getInputDescriptor("test-stream-1", mock(Serde.class));
    GenericInputDescriptor isd2 = sd2.getInputDescriptor("test-stream-2", mock(Serde.class));
    GenericOutputDescriptor osd1 = sd2.getOutputDescriptor("test-stream-3", mock(Serde.class));
    new StreamApplicationDescriptorImpl(appDesc -> {
        appDesc.getInputStream(isd1);
        try {
            appDesc.getInputStream(isd2);
            fail("Adding input stream with the same system name but different SystemDescriptor should have failed");
        } catch (IllegalStateException e) {
        }
        try {
            appDesc.getOutputStream(osd1);
            fail("adding output stream with the same system name but different SystemDescriptor should have failed");
        } catch (IllegalStateException e) {
        }
    }, getConfig());
    new StreamApplicationDescriptorImpl(appDesc -> {
        appDesc.withDefaultSystem(sd2);
        try {
            appDesc.getInputStream(isd1);
            fail("Adding input stream with the same system name as the default system but different SystemDescriptor should have failed");
        } catch (IllegalStateException e) {
        }
    }, getConfig());
}

Example 9 with GenericSystemDescriptor

use of org.apache.samza.system.descriptors.GenericSystemDescriptor in project samza by apache.

the class TestJobGraphJsonGenerator method setUp.

@Before
public void setUp() {
    input1Spec = new StreamSpec("input1", "input1", "input-system");
    input2Spec = new StreamSpec("input2", "input2", "input-system");
    outputSpec = new StreamSpec("output", "output", "output-system");
    repartitionSpec = new StreamSpec("jobName-jobId-partition_by-p1", "partition_by-p1", "intermediate-system");
    defaultSerde = KVSerde.of(new StringSerde(), new JsonSerdeV2<>());
    inputSystemDescriptor = new GenericSystemDescriptor("input-system", "mockSystemFactoryClassName");
    outputSystemDescriptor = new GenericSystemDescriptor("output-system", "mockSystemFactoryClassName");
    intermediateSystemDescriptor = new GenericSystemDescriptor("intermediate-system", "mockSystemFactoryClassName");
    input1Descriptor = inputSystemDescriptor.getInputDescriptor("input1", defaultSerde);
    input2Descriptor = inputSystemDescriptor.getInputDescriptor("input2", defaultSerde);
    outputDescriptor = outputSystemDescriptor.getOutputDescriptor("output", defaultSerde);
    table1Descriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table1", defaultSerde);
    table2Descriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table2", defaultSerde);
    Map<String, String> configs = new HashMap<>();
    configs.put(JobConfig.JOB_NAME, "jobName");
    configs.put(JobConfig.JOB_ID, "jobId");
    mockConfig = spy(new MapConfig(configs));
    mockJobNode = mock(JobNode.class);
    StreamEdge input1Edge = new StreamEdge(input1Spec, false, false, mockConfig);
    StreamEdge input2Edge = new StreamEdge(input2Spec, false, false, mockConfig);
    StreamEdge outputEdge = new StreamEdge(outputSpec, false, false, mockConfig);
    StreamEdge repartitionEdge = new StreamEdge(repartitionSpec, true, false, mockConfig);
    Map<String, StreamEdge> inputEdges = new HashMap<>();
    inputEdges.put(input1Descriptor.getStreamId(), input1Edge);
    inputEdges.put(input2Descriptor.getStreamId(), input2Edge);
    inputEdges.put(repartitionSpec.getId(), repartitionEdge);
    Map<String, StreamEdge> outputEdges = new HashMap<>();
    outputEdges.put(outputDescriptor.getStreamId(), outputEdge);
    outputEdges.put(repartitionSpec.getId(), repartitionEdge);
    when(mockJobNode.getInEdges()).thenReturn(inputEdges);
    when(mockJobNode.getOutEdges()).thenReturn(outputEdges);
    when(mockJobNode.getConfig()).thenReturn(mockConfig);
    when(mockJobNode.getJobName()).thenReturn("jobName");
    when(mockJobNode.getJobId()).thenReturn("jobId");
    when(mockJobNode.getJobNameAndId()).thenReturn(JobNode.createJobNameAndId("jobName", "jobId"));
    Map<String, TableDescriptor> tables = new HashMap<>();
    tables.put(table1Descriptor.getTableId(), table1Descriptor);
    tables.put(table2Descriptor.getTableId(), table2Descriptor);
    when(mockJobNode.getTables()).thenReturn(tables);
}

Example 10 with GenericSystemDescriptor

use of org.apache.samza.system.descriptors.GenericSystemDescriptor in project samza by apache.

the class TestExecutionPlanner method setup.

@Before
public void setup() {
    Map<String, String> configMap = new HashMap<>();
    configMap.put(JobConfig.JOB_NAME, "test-app");
    configMap.put(JobConfig.JOB_DEFAULT_SYSTEM, DEFAULT_SYSTEM);
    StreamTestUtils.addStreamConfigs(configMap, "input1", "system1", "input1");
    StreamTestUtils.addStreamConfigs(configMap, "input2", "system2", "input2");
    StreamTestUtils.addStreamConfigs(configMap, "input3", "system2", "input3");
    StreamTestUtils.addStreamConfigs(configMap, "input4", "system1", "input4");
    StreamTestUtils.addStreamConfigs(configMap, "output1", "system1", "output1");
    StreamTestUtils.addStreamConfigs(configMap, "output2", "system2", "output2");
    config = new MapConfig(configMap);
    input1Spec = new StreamSpec("input1", "input1", "system1");
    input2Spec = new StreamSpec("input2", "input2", "system2");
    input3Spec = new StreamSpec("input3", "input3", "system2");
    output1Spec = new StreamSpec("output1", "output1", "system1");
    output2Spec = new StreamSpec("output2", "output2", "system2");
    KVSerde<Object, Object> kvSerde = new KVSerde<>(new NoOpSerde(), new NoOpSerde());
    String mockSystemFactoryClass = "factory.class.name";
    system1Descriptor = new GenericSystemDescriptor("system1", mockSystemFactoryClass);
    system2Descriptor = new GenericSystemDescriptor("system2", mockSystemFactoryClass);
    input1Descriptor = system1Descriptor.getInputDescriptor("input1", kvSerde);
    input2Descriptor = system2Descriptor.getInputDescriptor("input2", kvSerde);
    input3Descriptor = system2Descriptor.getInputDescriptor("input3", kvSerde);
    input4Descriptor = system1Descriptor.getInputDescriptor("input4", kvSerde);
    output1Descriptor = system1Descriptor.getOutputDescriptor("output1", kvSerde);
    output2Descriptor = system2Descriptor.getOutputDescriptor("output2", kvSerde);
    // clean and set up sets and maps of descriptors
    systemDescriptors.clear();
    inputDescriptors.clear();
    outputDescriptors.clear();
    tableDescriptors.clear();
    systemDescriptors.add(system1Descriptor);
    systemDescriptors.add(system2Descriptor);
    inputDescriptors.put(input1Descriptor.getStreamId(), input1Descriptor);
    inputDescriptors.put(input2Descriptor.getStreamId(), input2Descriptor);
    inputDescriptors.put(input3Descriptor.getStreamId(), input3Descriptor);
    inputDescriptors.put(input4Descriptor.getStreamId(), input4Descriptor);
    outputDescriptors.put(output1Descriptor.getStreamId(), output1Descriptor);
    outputDescriptors.put(output2Descriptor.getStreamId(), output2Descriptor);
    // set up external partition count
    Map<String, Integer> system1Map = new HashMap<>();
    system1Map.put("input1", 64);
    system1Map.put("output1", 8);
    system1Map.put("input4", IntermediateStreamManager.MAX_INFERRED_PARTITIONS * 2);
    Map<String, Integer> system2Map = new HashMap<>();
    system2Map.put("input2", 16);
    system2Map.put("input3", 32);
    system2Map.put("output2", 16);
    SystemAdmin systemAdmin1 = createSystemAdmin(system1Map);
    SystemAdmin systemAdmin2 = createSystemAdmin(system2Map);
    systemAdmins = mock(SystemAdmins.class);
    when(systemAdmins.getSystemAdmin("system1")).thenReturn(systemAdmin1);
    when(systemAdmins.getSystemAdmin("system2")).thenReturn(systemAdmin2);
    streamManager = new StreamManager(systemAdmins);
}