Examples with BeamSqlEnv org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv used on opensource projects

Example 16 with BeamSqlEnv

use of org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv in project beam by apache.

the class BigQueryReadWriteIT method testSQLWriteAndRead_withExport.

@Test
public void testSQLWriteAndRead_withExport() {
    BeamSqlEnv sqlEnv = BeamSqlEnv.inMemory(new BigQueryTableProvider());
    String createTableStatement = "CREATE EXTERNAL TABLE TEST( \n" + "   c_bigint BIGINT, \n" + "   c_tinyint TINYINT, \n" + "   c_smallint SMALLINT, \n" + "   c_integer INTEGER, \n" + "   c_float FLOAT, \n" + "   c_double DOUBLE, \n" + "   c_boolean BOOLEAN, \n" + "   c_timestamp TIMESTAMP, \n" + "   c_varchar VARCHAR, \n " + "   c_char CHAR, \n" + "   c_arr ARRAY<VARCHAR> \n" + ") \n" + "TYPE 'bigquery' \n" + "LOCATION '" + bigQueryTestingTypes.tableSpec() + "' \n" + "TBLPROPERTIES " + "'{ " + METHOD_PROPERTY + ": \"" + Method.EXPORT.toString() + "\" }'";
    sqlEnv.executeDdl(createTableStatement);
    String insertStatement = "INSERT INTO TEST VALUES (" + "9223372036854775807, " + "127, " + "32767, " + "2147483647, " + "1.0, " + "1.0, " + "TRUE, " + "TIMESTAMP '2018-05-28 20:17:40.123', " + "'varchar', " + "'char', " + "ARRAY['123', '456']" + ")";
    sqlEnv.parseQuery(insertStatement);
    BeamSqlRelUtils.toPCollection(pipeline, sqlEnv.parseQuery(insertStatement));
    pipeline.run().waitUntilFinish(Duration.standardMinutes(5));
    String selectTableStatement = "SELECT * FROM TEST";
    PCollection<Row> output = BeamSqlRelUtils.toPCollection(readPipeline, sqlEnv.parseQuery(selectTableStatement));
    PAssert.that(output).containsInAnyOrder(row(SOURCE_SCHEMA_TWO, 9223372036854775807L, (byte) 127, (short) 32767, 2147483647, (float) 1.0, 1.0, true, parseTimestampWithUTCTimeZone("2018-05-28 20:17:40.123"), "varchar", "char", Arrays.asList("123", "456")));
    PipelineResult.State state = readPipeline.run().waitUntilFinish(Duration.standardMinutes(5));
    assertThat(state, equalTo(State.DONE));
}

Example 17 with BeamSqlEnv

use of org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv in project beam by apache.

the class BeamDDLTest method testParseCreateExternalTable_withoutTableComment.

@Test
public void testParseCreateExternalTable_withoutTableComment() throws Exception {
    TestTableProvider tableProvider = new TestTableProvider();
    BeamSqlEnv env = BeamSqlEnv.withTableProvider(tableProvider);
    JSONObject properties = new JSONObject();
    JSONArray hello = new JSONArray();
    hello.add("james");
    hello.add("bond");
    properties.put("hello", hello);
    env.executeDdl("CREATE EXTERNAL TABLE person (\n" + "id int COMMENT 'id', \n" + "name varchar COMMENT 'name') \n" + "TYPE 'text' \n" + "LOCATION '/home/admin/person'\n" + "TBLPROPERTIES '{\"hello\": [\"james\", \"bond\"]}'");
    assertEquals(mockTable("person", "text", null, properties), tableProvider.getTables().get("person"));
}

Example 18 with BeamSqlEnv

use of org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv in project beam by apache.

the class BeamDDLTest method testParseCreateExternalTable_withoutTblProperties.

@Test
public void testParseCreateExternalTable_withoutTblProperties() throws Exception {
    TestTableProvider tableProvider = new TestTableProvider();
    BeamSqlEnv env = BeamSqlEnv.withTableProvider(tableProvider);
    env.executeDdl("CREATE EXTERNAL TABLE person (\n" + "id int COMMENT 'id', \n" + "name varchar COMMENT 'name') \n" + "TYPE 'text' \n" + "COMMENT 'person table' \n" + "LOCATION '/home/admin/person'\n");
    assertEquals(mockTable("person", "text", "person table", new JSONObject()), tableProvider.getTables().get("person"));
}

Example 19 with BeamSqlEnv

use of org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv in project beam by apache.

the class BeamDDLTest method testParseCreateExternalTable_WithComplexFields.

@Test
public void testParseCreateExternalTable_WithComplexFields() {
    TestTableProvider tableProvider = new TestTableProvider();
    BeamSqlEnv env = BeamSqlEnv.withTableProvider(tableProvider);
    env.executeDdl("CREATE EXTERNAL TABLE PersonDetails" + " ( personInfo MAP<VARCHAR, ROW<field_1 INTEGER,field_2 VARCHAR>> , " + " additionalInfo ROW<field_0 TIMESTAMP,field_1 INTEGER,field_2 TINYINT> )" + " TYPE 'text'" + " LOCATION '/home/admin/person'");
    assertNotNull(tableProvider.getTables().get("PersonDetails"));
}

Example 20 with BeamSqlEnv

use of org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv in project beam by apache.

the class BeamComplexTypeTest method testRowConstructor.

@Test
public void testRowConstructor() {
    BeamSqlEnv sqlEnv = BeamSqlEnv.inMemory(readOnlyTableProvider);
    PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, sqlEnv.parseQuery("SELECT ROW(1, ROW(2, 3), 'str', ROW('str2', 'str3'))"));
    Schema intRow = Schema.builder().addInt32Field("field2").addInt32Field("field3").build();
    Schema strRow = Schema.builder().addStringField("field5").addStringField("field6").build();
    Schema innerRow = Schema.builder().addInt32Field("field1").addRowField("intRow", intRow).addStringField("field4").addRowField("strRow", strRow).build();
    PAssert.that(stream).containsInAnyOrder(Row.withSchema(Schema.builder().addRowField("row", innerRow).build()).addValues(Row.withSchema(innerRow).addValues(1, Row.withSchema(intRow).addValues(2, 3).build(), "str", Row.withSchema(strRow).addValues("str2", "str3").build()).build()).build());
    pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}