Examples with SamzaSqlValidator org.apache.samza.sql.planner.SamzaSqlValidator used on opensource projects

Example 16 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlEndToEnd method testEndToEndWithProjection.

@Test
public void testEndToEndWithProjection() throws Exception {
    int numMessages = 20;
    TestAvroSystemFactory.messages.clear();
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
    String sql1 = "Insert into testavro.outputTopic(id, bool_value, long_value) " + " select id, NOT(id = 5) as bool_value, TIMESTAMPDIFF(HOUR, CURRENT_TIMESTAMP(), LOCALTIMESTAMP()) + MONTH(CURRENT_DATE()) as long_value from testavro.SIMPLE1";
    List<String> sqlStmts = Arrays.asList(sql1);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
    runApplication(config);
    List<Integer> outMessages = TestAvroSystemFactory.messages.stream().map(x -> Integer.valueOf(((GenericRecord) x.getMessage()).get("id").toString())).sorted().collect(Collectors.toList());
    Assert.assertEquals(numMessages, outMessages.size());
    Assert.assertEquals(IntStream.range(0, numMessages).boxed().collect(Collectors.toList()), outMessages);
}

Example 17 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlEndToEnd method testEndToEndStreamTableInnerJoin.

@Test
public void testEndToEndStreamTableInnerJoin() throws Exception {
    int numMessages = 20;
    TestAvroSystemFactory.messages.clear();
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
    String sql = "Insert into testavro.enrichedPageViewTopic " + "select pv.pageKey as __key__, pv.pageKey as pageKey, coalesce(null, 'N/A') as companyName," + "       p.name as profileName, p.address as profileAddress " + "from testavro.PROFILE.`$table` as p " + "join testavro.PAGEVIEW as pv " + " on p.id = pv.profileId";
    List<String> sqlStmts = Arrays.asList(sql);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
    runApplication(config);
    List<String> outMessages = TestAvroSystemFactory.messages.stream().map(x -> ((GenericRecord) x.getMessage()).get("pageKey").toString() + "," + (((GenericRecord) x.getMessage()).get("profileName") == null ? "null" : ((GenericRecord) x.getMessage()).get("profileName").toString())).collect(Collectors.toList());
    Assert.assertEquals(numMessages, outMessages.size());
    List<String> expectedOutMessages = TestAvroSystemFactory.getPageKeyProfileNameJoin(numMessages);
    Assert.assertEquals(expectedOutMessages, outMessages);
}

Example 18 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlEndToEnd method testEndToEndNestedRecord.

@Ignore
@Test
public void testEndToEndNestedRecord() throws SamzaSqlValidatorException {
    int numMessages = 10;
    TestAvroSystemFactory.messages.clear();
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
    String sql1 = "Insert into testavro.outputTopic" + " select `phoneNumbers`[0].`kind`" + " from testavro.PROFILE as p";
    List<String> sqlStmts = Collections.singletonList(sql1);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
    runApplication(config);
    List<OutgoingMessageEnvelope> outMessages = new ArrayList<>(TestAvroSystemFactory.messages);
    Assert.assertEquals(numMessages, outMessages.size());
}

Example 19 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlEndToEnd method testEndToEndFlattenWithUdf.

@Test
public void testEndToEndFlattenWithUdf() throws Exception {
    int numMessages = 20;
    TestAvroSystemFactory.messages.clear();
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
    String sql1 = "Insert into testavro.outputTopic(id, bool_value) select Flatten(MyTestArray(id)) as id, NOT(id = 5) as bool_value" + " from testavro.SIMPLE1";
    List<String> sqlStmts = Collections.singletonList(sql1);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
    runApplication(config);
    List<OutgoingMessageEnvelope> outMessages = new ArrayList<>(TestAvroSystemFactory.messages);
    // Test invariant for each input Row with rank i will contain a column array_values with i elements $\sum_1^n{i}$.
    int expectedMessages = (numMessages * (numMessages - 1)) / 2;
    // Flatten de-normalizes the data. So there is separate record for each entry in the array.
    Assert.assertEquals(expectedMessages, outMessages.size());
    // check that values are actually not null and within the expected range
    Optional<GenericRecord> nullValueRecord = outMessages.stream().map(x -> (GenericRecord) x.getMessage()).filter(x -> x.get("id") == null).findFirst();
    Assert.assertFalse("Null value " + nullValueRecord.orElse(null), nullValueRecord.isPresent());
// TODO this is failing for now and that is because of udf weak type system, fixing it will be beyond this work.
/* // The String value column is result of dot product thus must be present in the Array column
    Optional<GenericRecord> missingValue = outMessages.stream().map(x -> (GenericRecord) x.getMessage()).filter(x -> {
      String value = (String) x.get("string_value");
      List<Object> arrayValues = (List<Object>) x.get("array_values");
      if (arrayValues == null) {
        return true;
      }
      Optional<Object> notThere = arrayValues.stream().filter(v -> v.toString().equalsIgnoreCase(value)).findAny();
      return !notThere.isPresent();
    }).findFirst();
    Assert.assertFalse("Absent Value " + missingValue.orElse(null), missingValue.isPresent());
    */
}

Example 20 with SamzaSqlValidator

use of org.apache.samza.sql.planner.SamzaSqlValidator in project samza by apache.

the class TestSamzaSqlEndToEnd method testEndToEndWithBooleanCheck.

@Test
public void testEndToEndWithBooleanCheck() throws Exception {
    int numMessages = 20;
    TestAvroSystemFactory.messages.clear();
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
    String sql1 = "Insert into testavro.outputTopic" + " select * from testavro.COMPLEX1 where bool_value IS TRUE";
    List<String> sqlStmts = Arrays.asList(sql1);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config config = new MapConfig(staticConfigs);
    new SamzaSqlValidator(config).validate(sqlStmts);
    runApplication(config);
    List<OutgoingMessageEnvelope> outMessages = new ArrayList<>(TestAvroSystemFactory.messages);
    Assert.assertEquals(numMessages / 2, outMessages.size());
}