Examples with OperatorSpecGraph org.apache.samza.operators.OperatorSpecGraph used on opensource projects

Example 11 with OperatorSpecGraph

use of org.apache.samza.operators.OperatorSpecGraph in project samza by apache.

the class TestQueryTranslator method testTranslateStreamTableLeftJoin.

@Test
public void testTranslateStreamTableLeftJoin() {
    Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 10);
    String sql = "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)" + " select p.name as profileName, pv.pageKey" + " from testavro.PAGEVIEW as pv" + " left join testavro.PROFILE.`$table` as p" + " on p.id = pv.profileId";
    config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
    Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
    List<String> sqlStmts = fetchSqlFromConfig(config);
    List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
    SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream).collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> {
    }, samzaConfig);
    QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
    translator.translate(queryInfo.get(0), streamAppDesc, 0);
    OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph();
    StreamConfig streamConfig = new StreamConfig(samzaConfig);
    String input1StreamId = specGraph.getInputOperators().keySet().stream().findFirst().get();
    String input1System = streamConfig.getSystem(input1StreamId);
    String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId);
    String input2StreamId = specGraph.getInputOperators().keySet().stream().skip(1).findFirst().get();
    String input2System = streamConfig.getSystem(input2StreamId);
    String input2PhysicalName = streamConfig.getPhysicalName(input2StreamId);
    String input3StreamId = specGraph.getInputOperators().keySet().stream().skip(2).findFirst().get();
    String input3System = streamConfig.getSystem(input3StreamId);
    String input3PhysicalName = streamConfig.getPhysicalName(input3StreamId);
    String input4StreamId = specGraph.getInputOperators().keySet().stream().skip(3).findFirst().get();
    String input4System = streamConfig.getSystem(input4StreamId);
    String input4PhysicalName = streamConfig.getPhysicalName(input4StreamId);
    String output1StreamId = specGraph.getOutputStreams().keySet().stream().findFirst().get();
    String output1System = streamConfig.getSystem(output1StreamId);
    String output1PhysicalName = streamConfig.getPhysicalName(output1StreamId);
    String output2StreamId = specGraph.getOutputStreams().keySet().stream().skip(1).findFirst().get();
    String output2System = streamConfig.getSystem(output2StreamId);
    String output2PhysicalName = streamConfig.getPhysicalName(output2StreamId);
    String output3StreamId = specGraph.getOutputStreams().keySet().stream().skip(2).findFirst().get();
    String output3System = streamConfig.getSystem(output3StreamId);
    String output3PhysicalName = streamConfig.getPhysicalName(output3StreamId);
    Assert.assertEquals(3, specGraph.getOutputStreams().size());
    Assert.assertEquals("kafka", output1System);
    Assert.assertEquals("sql-job-1-partition_by-table_sql_0_join_2", output1PhysicalName);
    Assert.assertEquals("kafka", output2System);
    Assert.assertEquals("sql-job-1-partition_by-stream_sql_0_join_2", output2PhysicalName);
    Assert.assertEquals("testavro", output3System);
    Assert.assertEquals("enrichedPageViewTopic", output3PhysicalName);
    Assert.assertEquals(4, specGraph.getInputOperators().size());
    Assert.assertEquals("testavro", input1System);
    Assert.assertEquals("PAGEVIEW", input1PhysicalName);
    Assert.assertEquals("testavro", input2System);
    Assert.assertEquals("PROFILE", input2PhysicalName);
    Assert.assertEquals("kafka", input3System);
    Assert.assertEquals("sql-job-1-partition_by-table_sql_0_join_2", input3PhysicalName);
    Assert.assertEquals("kafka", input4System);
    Assert.assertEquals("sql-job-1-partition_by-stream_sql_0_join_2", input4PhysicalName);
}

Example 12 with OperatorSpecGraph

use of org.apache.samza.operators.OperatorSpecGraph in project samza by apache.

the class TestQueryTranslator method testTranslateStreamTableRightJoin.

@Test
public void testTranslateStreamTableRightJoin() {
    Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 10);
    String sql = "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)" + " select p.name as profileName, pv.pageKey" + " from testavro.PROFILE.`$table` as p" + " right join testavro.PAGEVIEW as pv" + " on p.id = pv.profileId";
    config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
    Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
    List<String> sqlStmts = fetchSqlFromConfig(config);
    List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
    SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream).collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> {
    }, samzaConfig);
    QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
    translator.translate(queryInfo.get(0), streamAppDesc, 0);
    OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph();
    StreamConfig streamConfig = new StreamConfig(samzaConfig);
    String input1StreamId = specGraph.getInputOperators().keySet().stream().findFirst().get();
    String input1System = streamConfig.getSystem(input1StreamId);
    String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId);
    String input2StreamId = specGraph.getInputOperators().keySet().stream().skip(1).findFirst().get();
    String input2System = streamConfig.getSystem(input2StreamId);
    String input2PhysicalName = streamConfig.getPhysicalName(input2StreamId);
    String input3StreamId = specGraph.getInputOperators().keySet().stream().skip(2).findFirst().get();
    String input3System = streamConfig.getSystem(input3StreamId);
    String input3PhysicalName = streamConfig.getPhysicalName(input3StreamId);
    String input4StreamId = specGraph.getInputOperators().keySet().stream().skip(3).findFirst().get();
    String input4System = streamConfig.getSystem(input4StreamId);
    String input4PhysicalName = streamConfig.getPhysicalName(input4StreamId);
    String output1StreamId = specGraph.getOutputStreams().keySet().stream().findFirst().get();
    String output1System = streamConfig.getSystem(output1StreamId);
    String output1PhysicalName = streamConfig.getPhysicalName(output1StreamId);
    String output2StreamId = specGraph.getOutputStreams().keySet().stream().skip(1).findFirst().get();
    String output2System = streamConfig.getSystem(output2StreamId);
    String output2PhysicalName = streamConfig.getPhysicalName(output2StreamId);
    String output3StreamId = specGraph.getOutputStreams().keySet().stream().skip(2).findFirst().get();
    String output3System = streamConfig.getSystem(output3StreamId);
    String output3PhysicalName = streamConfig.getPhysicalName(output3StreamId);
    Assert.assertEquals(3, specGraph.getOutputStreams().size());
    Assert.assertEquals("kafka", output1System);
    Assert.assertEquals("sql-job-1-partition_by-table_sql_0_join_2", output1PhysicalName);
    Assert.assertEquals("kafka", output2System);
    Assert.assertEquals("sql-job-1-partition_by-stream_sql_0_join_2", output2PhysicalName);
    Assert.assertEquals("testavro", output3System);
    Assert.assertEquals("enrichedPageViewTopic", output3PhysicalName);
    Assert.assertEquals(4, specGraph.getInputOperators().size());
    Assert.assertEquals("testavro", input1System);
    Assert.assertEquals("PROFILE", input1PhysicalName);
    Assert.assertEquals("testavro", input2System);
    Assert.assertEquals("PAGEVIEW", input2PhysicalName);
    Assert.assertEquals("kafka", input3System);
    Assert.assertEquals("sql-job-1-partition_by-table_sql_0_join_2", input3PhysicalName);
    Assert.assertEquals("kafka", input4System);
    Assert.assertEquals("sql-job-1-partition_by-stream_sql_0_join_2", input4PhysicalName);
}

Example 13 with OperatorSpecGraph

use of org.apache.samza.operators.OperatorSpecGraph in project samza by apache.

the class TestQueryTranslator method testTranslateFanIn.

@Test
public void testTranslateFanIn() {
    Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
    String sql1 = "Insert into testavro.simpleOutputTopic select * from testavro.SIMPLE2";
    String sql2 = "Insert into testavro.simpleOutputTopic select * from testavro.SIMPLE1";
    List<String> sqlStmts = Arrays.asList(sql1, sql2);
    config.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
    List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
    SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream).collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
    StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(streamApp -> {
    }, samzaConfig);
    QueryTranslator translator = new QueryTranslator(appDesc, samzaSqlApplicationConfig);
    translator.translate(queryInfo.get(0), appDesc, 0);
    translator.translate(queryInfo.get(1), appDesc, 1);
    OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph();
    StreamConfig streamConfig = new StreamConfig(samzaConfig);
    String inputStreamId1 = specGraph.getInputOperators().keySet().stream().findFirst().get();
    String inputSystem1 = streamConfig.getSystem(inputStreamId1);
    String inputPhysicalName1 = streamConfig.getPhysicalName(inputStreamId1);
    String inputStreamId2 = specGraph.getInputOperators().keySet().stream().skip(1).findFirst().get();
    String inputSystem2 = streamConfig.getSystem(inputStreamId2);
    String inputPhysicalName2 = streamConfig.getPhysicalName(inputStreamId2);
    String outputStreamId = specGraph.getOutputStreams().keySet().stream().findFirst().get();
    String outputSystem = streamConfig.getSystem(outputStreamId);
    String outputPhysicalName = streamConfig.getPhysicalName(outputStreamId);
    Assert.assertEquals(1, specGraph.getOutputStreams().size());
    Assert.assertEquals("testavro", outputSystem);
    Assert.assertEquals("simpleOutputTopic", outputPhysicalName);
    Assert.assertEquals(2, specGraph.getInputOperators().size());
    Assert.assertEquals("testavro", inputSystem1);
    Assert.assertEquals("SIMPLE2", inputPhysicalName1);
    Assert.assertEquals("testavro", inputSystem2);
    Assert.assertEquals("SIMPLE1", inputPhysicalName2);
}

Example 14 with OperatorSpecGraph

use of org.apache.samza.operators.OperatorSpecGraph in project samza by apache.

the class TestTaskFactoryUtil method testGetTaskFactoryWithStreamAppDescriptor.

// test getTaskFactory with StreamApplicationDescriptor
@Test
public void testGetTaskFactoryWithStreamAppDescriptor() {
    StreamApplicationDescriptorImpl mockStreamApp = mock(StreamApplicationDescriptorImpl.class);
    OperatorSpecGraph mockSpecGraph = mock(OperatorSpecGraph.class);
    when(mockStreamApp.getOperatorSpecGraph()).thenReturn(mockSpecGraph);
    TaskFactory streamTaskFactory = TaskFactoryUtil.getTaskFactory(mockStreamApp);
    assertTrue(streamTaskFactory instanceof StreamOperatorTaskFactory);
    AsyncStreamTask streamTask = ((StreamOperatorTaskFactory) streamTaskFactory).createInstance();
    assertTrue(streamTask instanceof StreamOperatorTask);
    verify(mockSpecGraph).clone();
}

Example 15 with OperatorSpecGraph

use of org.apache.samza.operators.OperatorSpecGraph in project samza by apache.

the class TestQueryTranslator method testTranslateComplex.

@Test
public void testTranslateComplex() {
    Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
    config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, "Insert into testavro.outputTopic(string_value) select Flatten(array_values) from testavro.COMPLEX1");
    Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
    List<String> sqlStmts = fetchSqlFromConfig(config);
    List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
    SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream).collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> {
    }, samzaConfig);
    QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
    translator.translate(queryInfo.get(0), streamAppDesc, 0);
    OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph();
    StreamConfig streamConfig = new StreamConfig(samzaConfig);
    String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get();
    String inputSystem = streamConfig.getSystem(inputStreamId);
    String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId);
    String outputStreamId = specGraph.getOutputStreams().keySet().stream().findFirst().get();
    String outputSystem = streamConfig.getSystem(outputStreamId);
    String outputPhysicalName = streamConfig.getPhysicalName(outputStreamId);
    Assert.assertEquals(1, specGraph.getOutputStreams().size());
    Assert.assertEquals("testavro", outputSystem);
    Assert.assertEquals("outputTopic", outputPhysicalName);
    Assert.assertEquals(1, specGraph.getInputOperators().size());
    Assert.assertEquals("testavro", inputSystem);
    Assert.assertEquals("COMPLEX1", inputPhysicalName);
}