Examples with CSVMessageFormat org.apache.apex.malhar.sql.table.CSVMessageFormat used on opensource projects

Example 6 with CSVMessageFormat

use of org.apache.apex.malhar.sql.table.CSVMessageFormat in project apex-malhar by apache.

the class SerDeTest method testJoin.

@Test
public void testJoin() throws IOException, ClassNotFoundException {
    LogicalPlan dag = new LogicalPlan();
    String schemaIn0 = "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[" + "{\"name\":\"RowTime\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss Z\"}}," + "{\"name\":\"id\",\"type\":\"Integer\"}," + "{\"name\":\"Product\",\"type\":\"String\"}," + "{\"name\":\"units\",\"type\":\"Integer\"}]}";
    String schemaIn1 = "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[" + "{\"name\":\"id\",\"type\":\"Integer\"}," + "{\"name\":\"Category\",\"type\":\"String\"}]}";
    String schemaOut = "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[" + "{\"name\":\"RowTime1\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss Z\"}}," + "{\"name\":\"RowTime2\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss Z\"}}," + "{\"name\":\"Product\",\"type\":\"String\"}," + "{\"name\":\"Category\",\"type\":\"String\"}]}";
    String sql = "INSERT INTO SALES SELECT STREAM A.ROWTIME, FLOOR(A.ROWTIME TO DAY), " + "APEXCONCAT('OILPAINT', SUBSTRING(A.PRODUCT, 6, 7)), B.CATEGORY " + "FROM ORDERS AS A " + "JOIN CATEGORY AS B ON A.id = B.id " + "WHERE A.id > 3 AND A.PRODUCT LIKE 'paint%'";
    SQLExecEnvironment.getEnvironment().registerTable("ORDERS", new KafkaEndpoint("localhost:9092", "testdata0", new CSVMessageFormat(schemaIn0))).registerTable("CATEGORY", new KafkaEndpoint("localhost:9092", "testdata1", new CSVMessageFormat(schemaIn1))).registerTable("SALES", new KafkaEndpoint("localhost:9092", "testresult", new CSVMessageFormat(schemaOut))).registerFunction("APEXCONCAT", FileEndpointTest.class, "apex_concat_str").executeSQL(dag, sql);
    dag.validate();
}

Example 7 with CSVMessageFormat

use of org.apache.apex.malhar.sql.table.CSVMessageFormat in project apex-malhar by apache.

the class SerDeTest method testJoinFilter.

@Test
public void testJoinFilter() throws IOException, ClassNotFoundException {
    LogicalPlan dag = new LogicalPlan();
    String schemaIn0 = "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[" + "{\"name\":\"RowTime\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss Z\"}}," + "{\"name\":\"id\",\"type\":\"Integer\"}," + "{\"name\":\"Product\",\"type\":\"String\"}," + "{\"name\":\"units\",\"type\":\"Integer\"}]}";
    String schemaIn1 = "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[" + "{\"name\":\"id\",\"type\":\"Integer\"}," + "{\"name\":\"Category\",\"type\":\"String\"}]}";
    String schemaOut = "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[" + "{\"name\":\"RowTime1\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss Z\"}}," + "{\"name\":\"RowTime2\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss Z\"}}," + "{\"name\":\"Product\",\"type\":\"String\"}," + "{\"name\":\"Category\",\"type\":\"String\"}]}";
    String sql = "INSERT INTO SALES SELECT STREAM A.ROWTIME, FLOOR(A.ROWTIME TO DAY), " + "APEXCONCAT('OILPAINT', SUBSTRING(A.PRODUCT, 6, 7)), B.CATEGORY " + "FROM ORDERS AS A JOIN CATEGORY AS B ON A.id = B.id AND A.id > 3" + "WHERE A.PRODUCT LIKE 'paint%'";
    SQLExecEnvironment.getEnvironment().registerTable("ORDERS", new KafkaEndpoint("localhost:9092", "testdata0", new CSVMessageFormat(schemaIn0))).registerTable("CATEGORY", new KafkaEndpoint("localhost:9092", "testdata1", new CSVMessageFormat(schemaIn1))).registerTable("SALES", new KafkaEndpoint("localhost:9092", "testresult", new CSVMessageFormat(schemaOut))).registerFunction("APEXCONCAT", FileEndpointTest.class, "apex_concat_str").executeSQL(dag, sql);
    dag.validate();
}

Example 8 with CSVMessageFormat

use of org.apache.apex.malhar.sql.table.CSVMessageFormat in project apex-malhar by apache.

the class SQLApplicationWithAPI method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    // Source definition
    String schemaInName = conf.get("csvSchemaInName");
    String schemaIn = conf.get("csvSchemaIn");
    String sourceFile = conf.get("sourceFile");
    SQLExecEnvironment.getEnvironment().registerTable(schemaInName, new FileEndpoint(sourceFile, new CSVMessageFormat(schemaIn))).executeSQL(dag, conf.get("sql"));
}