Examples with SamzaSqlValidator org.apache.samza.sql.planner.SamzaSqlValidator used on opensource projects

Example 46 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlEndToEnd method testEndToEndStreamTableNestedJoinWithCompositeKey.

@Test
public void testEndToEndStreamTableNestedJoinWithCompositeKey() throws Exception {
    int numMessages = 20;
    TestAvroSystemFactory.messages.clear();
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
    String sql = "Insert into testavro.enrichedPageViewTopic " + "select pv.pageKey as __key__, pv.pageKey as pageKey, c.name as companyName, p.name as profileName," + "       p.address as profileAddress " + "from testavro.PAGEVIEW as pv " + "join testavro.PROFILE.`$table` as p " + " on p.id = pv.profileId " + " join testavro.COMPANY.`$table` as c " + " on p.companyId = c.id AND c.id = pv.profileId";
    List<String> sqlStmts = Arrays.asList(sql);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
    runApplication(config);
    List<String> outMessages = TestAvroSystemFactory.messages.stream().map(x -> ((GenericRecord) x.getMessage()).get("pageKey").toString() + "," + ((GenericRecord) x.getMessage()).get("profileName").toString() + "," + ((GenericRecord) x.getMessage()).get("companyName").toString()).collect(Collectors.toList());
    Assert.assertEquals(TestAvroSystemFactory.COMPANIES.length, outMessages.size());
    List<String> expectedOutMessages = TestAvroSystemFactory.getPageKeyProfileCompanyNameJoin(TestAvroSystemFactory.COMPANIES.length);
    Assert.assertEquals(expectedOutMessages, outMessages);
}

Example 47 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlEndToEnd method testEndToEndUdf.

@Test
public void testEndToEndUdf() throws Exception {
    int numMessages = 20;
    TestAvroSystemFactory.messages.clear();
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
    String sql1 = "Insert into testavro.outputTopic(id, bool_value, long_value) " + "select id, NOT(id = 5) as bool_value, MYTest(id) as long_value from testavro.SIMPLE1;;";
    List<String> sqlStmts = Collections.singletonList(sql1);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
    runApplication(config);
    LOG.info("output Messages " + TestAvroSystemFactory.messages);
    List<Integer> outMessages = TestAvroSystemFactory.messages.stream().map(x -> Integer.valueOf(((GenericRecord) x.getMessage()).get("long_value").toString())).sorted().collect(Collectors.toList());
    Assert.assertEquals(outMessages.size(), numMessages);
    MyTestUdf udf = new MyTestUdf();
    Assert.assertTrue(IntStream.range(0, numMessages).map(udf::execute).boxed().collect(Collectors.toList()).equals(outMessages));
}

Example 48 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlEndToEnd method testEndToEnd.

@Test
public void testEndToEnd() throws SamzaSqlValidatorException {
    int numMessages = 20;
    TestAvroSystemFactory.messages.clear();
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
    String sql = "Insert into testavro.simpleOutputTopic select * from testavro.SIMPLE1";
    List<String> sqlStmts = Arrays.asList(sql);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
    runApplication(config);
    List<Integer> outMessages = TestAvroSystemFactory.messages.stream().map(x -> Integer.valueOf(((GenericRecord) x.getMessage()).get("id").toString())).sorted().collect(Collectors.toList());
    Assert.assertEquals(numMessages, outMessages.size());
    Assert.assertTrue(IntStream.range(0, numMessages).boxed().collect(Collectors.toList()).equals(outMessages));
}

Example 49 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlEndToEnd method testEndToEndFlatten.

@Test
public void testEndToEndFlatten() {
    int numMessages = 20;
    TestAvroSystemFactory.messages.clear();
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
    String sql1 = "Insert into testavro.outputTopic(string_value, id, bool_value, bytes_value, fixed_value, float_value0, array_values) " + " select Flatten(array_values) as string_value, id, NOT(id = 5) as bool_value, bytes_value, fixed_value, float_value0, array_values" + " from testavro.COMPLEX1";
    List<String> sqlStmts = Collections.singletonList(sql1);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
    runApplication(config);
    List<OutgoingMessageEnvelope> outMessages = new ArrayList<>(TestAvroSystemFactory.messages);
    // Test invariant for each input Row with rank i will contain a column array_values with i elements $\sum_1^n{i}$.
    int expectedMessages = (numMessages * (numMessages - 1)) / 2;
    // Assert.assertEquals(outMessages.size(), actualList.size());
    Assert.assertEquals(expectedMessages, outMessages.size());
    // check that values are actually not null and within the expected range
    Optional<GenericRecord> nullValueRecord = outMessages.stream().map(x -> (GenericRecord) x.getMessage()).filter(x -> x.get("string_value") == null).findFirst();
    // The String value column is result of dot product thus must be present in the Array column
    Optional<GenericRecord> missingValue = outMessages.stream().map(x -> (GenericRecord) x.getMessage()).filter(x -> {
        String value = (String) x.get("string_value");
        List<Object> arrayValues = (List<Object>) x.get("array_values");
        if (arrayValues == null) {
            return true;
        }
        Optional<Object> notThere = arrayValues.stream().filter(v -> v.toString().equalsIgnoreCase(value)).findAny();
        return !notThere.isPresent();
    }).findFirst();
    Assert.assertFalse("Null value " + nullValueRecord.orElse(null), nullValueRecord.isPresent());
    Assert.assertFalse("Absent Value " + missingValue.orElse(null), missingValue.isPresent());
}

Example 50 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlEndToEnd method testEndToEndMultiSqlStmtsWithSameSystemStreamAsInputAndOutput.

@Test
public void testEndToEndMultiSqlStmtsWithSameSystemStreamAsInputAndOutput() throws SamzaSqlValidatorException {
    int numMessages = 20;
    TestAvroSystemFactory.messages.clear();
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
    String sql1 = "Insert into testavro.SIMPLE1 select * from testavro.SIMPLE2";
    String sql2 = "Insert into testavro.simpleOutputTopic select * from testavro.SIMPLE1";
    List<String> sqlStmts = Arrays.asList(sql1, sql2);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
    runApplication(config);
    List<Integer> outMessages = TestAvroSystemFactory.messages.stream().map(x -> Integer.valueOf(((GenericRecord) x.getMessage()).get("id").toString())).sorted().collect(Collectors.toList());
    Assert.assertEquals(numMessages * 2, outMessages.size());
    Set<Integer> outMessagesSet = new HashSet<>(outMessages);
    Assert.assertEquals(numMessages, outMessagesSet.size());
    Assert.assertTrue(IntStream.range(0, numMessages).boxed().collect(Collectors.toList()).equals(new ArrayList<>(outMessagesSet)));
}