Search in sources :

Example 6 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlEndToEnd method testEndToEndFanOut.

@Ignore
@Test
public void testEndToEndFanOut() throws SamzaSqlValidatorException {
    int numMessages = 20;
    TestAvroSystemFactory.messages.clear();
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
    String sql1 = "Insert into testavro.SIMPLE2 select * from testavro.SIMPLE1";
    String sql2 = "Insert into testavro.SIMPLE3 select * from testavro.SIMPLE1";
    List<String> sqlStmts = Arrays.asList(sql1, sql2);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
    runApplication(config);
    List<Integer> outMessages = TestAvroSystemFactory.messages.stream().map(x -> Integer.valueOf(((GenericRecord) x.getMessage()).get("id").toString())).sorted().collect(Collectors.toList());
    Assert.assertEquals(numMessages * 2, outMessages.size());
    Set<Integer> outMessagesSet = new HashSet<>(outMessages);
    Assert.assertEquals(numMessages, outMessagesSet.size());
    Assert.assertTrue(IntStream.range(0, numMessages).boxed().collect(Collectors.toList()).equals(new ArrayList<>(outMessagesSet)));
}
Also used : SamzaSqlValidator(org.apache.samza.sql.planner.SamzaSqlValidator) MapConfig(org.apache.samza.config.MapConfig) SamzaSqlTestConfig(org.apache.samza.sql.util.SamzaSqlTestConfig) SamzaSqlApplicationConfig(org.apache.samza.sql.runner.SamzaSqlApplicationConfig) Config(org.apache.samza.config.Config) ArrayList(java.util.ArrayList) MapConfig(org.apache.samza.config.MapConfig) HashSet(java.util.HashSet) Ignore(org.junit.Ignore) Test(org.junit.Test)

Example 7 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlRemoteTable method testJoinEndToEndWithFilterHelper.

void testJoinEndToEndWithFilterHelper(boolean enableOptimizer) throws SamzaSqlValidatorException {
    int numMessages = 20;
    TestAvroSystemFactory.messages.clear();
    RemoteStoreIOResolverTestFactory.records.clear();
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
    populateProfileTable(staticConfigs, numMessages);
    String sql = "Insert into testavro.enrichedPageViewTopic " + "select pv.pageKey as __key__, pv.pageKey as pageKey, coalesce(null, 'N/A') as companyName," + "       p.name as profileName, p.address as profileAddress " + "from testRemoteStore.Profile.`$table` as p " + "join testavro.PAGEVIEW as pv " + " on p.__key__ = pv.profileId" + " where p.name = 'Mike' and pv.profileId = 1";
    List<String> sqlStmts = Arrays.asList(sql);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_ENABLE_PLAN_OPTIMIZER, Boolean.toString(enableOptimizer));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
    runApplication(config);
    List<String> outMessages = TestAvroSystemFactory.messages.stream().map(x -> ((GenericRecord) x.getMessage()).get("pageKey").toString() + "," + (((GenericRecord) x.getMessage()).get("profileName") == null ? "null" : ((GenericRecord) x.getMessage()).get("profileName").toString())).collect(Collectors.toList());
    Assert.assertEquals(1, outMessages.size());
    Assert.assertEquals(outMessages.get(0), "home,Mike");
}
Also used : Arrays(java.util.Arrays) GenericRecord(org.apache.avro.generic.GenericRecord) TestAvroSystemFactory(org.apache.samza.sql.system.TestAvroSystemFactory) SamzaSqlTestConfig(org.apache.samza.sql.util.SamzaSqlTestConfig) SamzaSqlValidatorException(org.apache.samza.sql.planner.SamzaSqlValidatorException) SamzaSqlApplicationConfig(org.apache.samza.sql.runner.SamzaSqlApplicationConfig) RemoteStoreIOResolverTestFactory(org.apache.samza.sql.util.RemoteStoreIOResolverTestFactory) HashMap(java.util.HashMap) SamzaSqlValidator(org.apache.samza.sql.planner.SamzaSqlValidator) Test(org.junit.Test) JsonUtil(org.apache.samza.sql.util.JsonUtil) Collectors(java.util.stream.Collectors) SamzaException(org.apache.samza.SamzaException) List(java.util.List) Ignore(org.junit.Ignore) Map(java.util.Map) Config(org.apache.samza.config.Config) Assert(org.junit.Assert) MapConfig(org.apache.samza.config.MapConfig) SamzaSqlValidator(org.apache.samza.sql.planner.SamzaSqlValidator) SamzaSqlTestConfig(org.apache.samza.sql.util.SamzaSqlTestConfig) SamzaSqlApplicationConfig(org.apache.samza.sql.runner.SamzaSqlApplicationConfig) Config(org.apache.samza.config.Config) MapConfig(org.apache.samza.config.MapConfig) MapConfig(org.apache.samza.config.MapConfig) GenericRecord(org.apache.avro.generic.GenericRecord)

Example 8 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlRemoteTable method testSameJoinTargetSinkEndToEndRightOuterJoin.

@Test
public void testSameJoinTargetSinkEndToEndRightOuterJoin() throws SamzaSqlValidatorException {
    int numMessages = 21;
    TestAvroSystemFactory.messages.clear();
    RemoteStoreIOResolverTestFactory.records.clear();
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(new HashMap<>(), numMessages, true);
    populateProfileTable(staticConfigs, numMessages);
    // The below query reads messages from a stream and deletes the corresponding records from the table.
    // Since the stream has alternate messages with null foreign key, only half of the messages will have
    // successful joins and hence only half of the records in the table will be deleted. Although join is
    // redundant here, keeping it just for testing purpose.
    String sql = "Insert into testRemoteStore.Profile.`$table` " + "select p.__key__ as __key__, 'DELETE' as __op__ " + "from testRemoteStore.Profile.`$table` as p " + "join testavro.PAGEVIEW as pv " + " on p.__key__ = pv.profileId ";
    List<String> sqlStmts = Arrays.asList(sql);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
    runApplication(config);
    Assert.assertEquals((numMessages + 1) / 2, RemoteStoreIOResolverTestFactory.records.size());
}
Also used : SamzaSqlValidator(org.apache.samza.sql.planner.SamzaSqlValidator) SamzaSqlTestConfig(org.apache.samza.sql.util.SamzaSqlTestConfig) SamzaSqlApplicationConfig(org.apache.samza.sql.runner.SamzaSqlApplicationConfig) Config(org.apache.samza.config.Config) MapConfig(org.apache.samza.config.MapConfig) MapConfig(org.apache.samza.config.MapConfig) Test(org.junit.Test)

Example 9 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlRemoteTable method testDeleteOpValidation.

@Test
public void testDeleteOpValidation() throws SamzaSqlValidatorException {
    int numMessages = 1;
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(new HashMap<>(), numMessages, true);
    String sql = "Insert into testRemoteStore.Profile.`$table` " + "select p.__key__ as __key__, 'DELETE' as __op__ " + "from testRemoteStore.Profile.`$table` as p " + "join testavro.PAGEVIEW as pv " + " on p.__key__ = pv.profileId ";
    List<String> sqlStmts = Arrays.asList(sql);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
}
Also used : SamzaSqlValidator(org.apache.samza.sql.planner.SamzaSqlValidator) SamzaSqlTestConfig(org.apache.samza.sql.util.SamzaSqlTestConfig) SamzaSqlApplicationConfig(org.apache.samza.sql.runner.SamzaSqlApplicationConfig) Config(org.apache.samza.config.Config) MapConfig(org.apache.samza.config.MapConfig) MapConfig(org.apache.samza.config.MapConfig) Test(org.junit.Test)

Example 10 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlRemoteTable method testUnsupportedOpValidation.

@Test(expected = SamzaSqlValidatorException.class)
public void testUnsupportedOpValidation() throws SamzaSqlValidatorException {
    int numMessages = 1;
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(new HashMap<>(), numMessages, true);
    String sql = "Insert into testRemoteStore.Profile.`$table` " + "select p.__key__ as __key__, 'UPDATE' as __op__ " + "from testRemoteStore.Profile.`$table` as p " + "join testavro.PAGEVIEW as pv " + " on p.__key__ = pv.profileId";
    List<String> sqlStmts = Arrays.asList(sql);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
}
Also used : SamzaSqlValidator(org.apache.samza.sql.planner.SamzaSqlValidator) SamzaSqlTestConfig(org.apache.samza.sql.util.SamzaSqlTestConfig) SamzaSqlApplicationConfig(org.apache.samza.sql.runner.SamzaSqlApplicationConfig) Config(org.apache.samza.config.Config) MapConfig(org.apache.samza.config.MapConfig) MapConfig(org.apache.samza.config.MapConfig) Test(org.junit.Test)

Aggregations

Config (org.apache.samza.config.Config)55 MapConfig (org.apache.samza.config.MapConfig)55 SamzaSqlValidator (org.apache.samza.sql.planner.SamzaSqlValidator)55 SamzaSqlApplicationConfig (org.apache.samza.sql.runner.SamzaSqlApplicationConfig)55 SamzaSqlTestConfig (org.apache.samza.sql.util.SamzaSqlTestConfig)55 Test (org.junit.Test)55 Ignore (org.junit.Ignore)28 ArrayList (java.util.ArrayList)27 HashMap (java.util.HashMap)25 Arrays (java.util.Arrays)24 List (java.util.List)24 Map (java.util.Map)24 Collectors (java.util.stream.Collectors)24 GenericRecord (org.apache.avro.generic.GenericRecord)24 SamzaSqlValidatorException (org.apache.samza.sql.planner.SamzaSqlValidatorException)24 TestAvroSystemFactory (org.apache.samza.sql.system.TestAvroSystemFactory)24 JsonUtil (org.apache.samza.sql.util.JsonUtil)24 Assert (org.junit.Assert)24 OutgoingMessageEnvelope (org.apache.samza.system.OutgoingMessageEnvelope)23 HashSet (java.util.HashSet)22