Examples with InMemoryMetaStore org.apache.beam.sdk.extensions.sql.meta.store.InMemoryMetaStore used on opensource projects

Example 1 with InMemoryMetaStore

use of org.apache.beam.sdk.extensions.sql.meta.store.InMemoryMetaStore in project beam by apache.

the class BigtableTableFlatTest method testCreatesFlatSchemaCorrectly.

@Test
public void testCreatesFlatSchemaCorrectly() {
    final String tableId = "flatTableSchema";
    InMemoryMetaStore metaStore = new InMemoryMetaStore();
    metaStore.registerProvider(new BigtableTableProvider());
    BeamSqlCli cli = new BeamSqlCli().metaStore(metaStore);
    cli.execute(createFlatTableString(tableId, location(tableId)));
    Table table = metaStore.getTables().get(tableId);
    assertNotNull(table);
    assertEquals(TEST_FLAT_SCHEMA, table.getSchema());
    JSONObject properties = table.getProperties();
    assertTrue(properties.containsKey(COLUMNS_MAPPING));
    assertEquals(columnsMappingString(), properties.getString(COLUMNS_MAPPING));
}

Example 2 with InMemoryMetaStore

use of org.apache.beam.sdk.extensions.sql.meta.store.InMemoryMetaStore in project beam by apache.

the class BeamSqlCliTest method testExecute_createTableWithPrefixArrayField.

@Test
public void testExecute_createTableWithPrefixArrayField() throws Exception {
    InMemoryMetaStore metaStore = new InMemoryMetaStore();
    metaStore.registerProvider(new TextTableProvider());
    BeamSqlCli cli = new BeamSqlCli().metaStore(metaStore);
    cli.execute("CREATE EXTERNAL TABLE person (\n" + "id int COMMENT 'id', \n" + "name varchar COMMENT 'name', \n" + "age int COMMENT 'age', \n" + "tags ARRAY<VARCHAR>, \n" + "matrix ARRAY<ARRAY<INTEGER>> \n" + ") \n" + "TYPE 'text' \n" + "COMMENT '' LOCATION '/home/admin/orders'");
    Table table = metaStore.getTables().get("person");
    assertNotNull(table);
    assertEquals(Stream.of(Field.of("id", INTEGER).withDescription("id").withNullable(true), Field.of("name", VARCHAR).withDescription("name").withNullable(true), Field.of("age", INTEGER).withDescription("age").withNullable(true), Field.of("tags", Schema.FieldType.array(VARCHAR)).withNullable(true), Field.of("matrix", Schema.FieldType.array(Schema.FieldType.array(INTEGER))).withNullable(true)).collect(toSchema()), table.getSchema());
}

Example 3 with InMemoryMetaStore

use of org.apache.beam.sdk.extensions.sql.meta.store.InMemoryMetaStore in project beam by apache.

the class BeamSqlCliTest method testExecute_createTableWithPrefixMapField.

@Test
public void testExecute_createTableWithPrefixMapField() throws Exception {
    InMemoryMetaStore metaStore = new InMemoryMetaStore();
    metaStore.registerProvider(new TextTableProvider());
    BeamSqlCli cli = new BeamSqlCli().metaStore(metaStore);
    cli.execute("CREATE EXTERNAL TABLE person (\n" + "id int COMMENT 'id', \n" + "name varchar COMMENT 'name', \n" + "age int COMMENT 'age', \n" + "tags MAP<VARCHAR, VARCHAR>, \n" + "nestedMap MAP<INTEGER, MAP<VARCHAR, INTEGER>> \n" + ") \n" + "TYPE 'text' \n" + "COMMENT '' LOCATION '/home/admin/orders'");
    Table table = metaStore.getTables().get("person");
    assertNotNull(table);
    assertEquals(Stream.of(Field.of("id", INTEGER).withDescription("id").withNullable(true), Field.of("name", VARCHAR).withDescription("name").withNullable(true), Field.of("age", INTEGER).withDescription("age").withNullable(true), Field.of("tags", Schema.FieldType.map(VARCHAR, VARCHAR)).withNullable(true), Field.of("nestedMap", Schema.FieldType.map(INTEGER, Schema.FieldType.map(VARCHAR, INTEGER))).withNullable(true)).collect(toSchema()), table.getSchema());
}

Example 4 with InMemoryMetaStore

use of org.apache.beam.sdk.extensions.sql.meta.store.InMemoryMetaStore in project beam by apache.

the class BeamSqlCliTest method testExecute_createTableWithRowField.

@Test
public void testExecute_createTableWithRowField() throws Exception {
    InMemoryMetaStore metaStore = new InMemoryMetaStore();
    metaStore.registerProvider(new TextTableProvider());
    BeamSqlCli cli = new BeamSqlCli().metaStore(metaStore);
    cli.execute("CREATE EXTERNAL TABLE person (\n" + "id int COMMENT 'id', \n" + "name varchar COMMENT 'name', \n" + "age int COMMENT 'age', \n" + "address ROW ( \n" + "  street VARCHAR, \n" + "  country VARCHAR \n" + "  ), \n" + "addressAngular ROW< \n" + "  street VARCHAR, \n" + "  country VARCHAR \n" + "  >, \n" + "isRobot BOOLEAN" + ") \n" + "TYPE 'text' \n" + "COMMENT '' LOCATION '/home/admin/orders'");
    Table table = metaStore.getTables().get("person");
    assertNotNull(table);
    assertEquals(Stream.of(Field.of("id", INTEGER).withDescription("id").withNullable(true), Field.of("name", VARCHAR).withDescription("name").withNullable(true), Field.of("age", INTEGER).withDescription("age").withNullable(true), Field.of("address", Schema.FieldType.row(Schema.builder().addNullableField("street", VARCHAR).addNullableField("country", VARCHAR).build())).withNullable(true), Field.of("addressAngular", Schema.FieldType.row(Schema.builder().addNullableField("street", VARCHAR).addNullableField("country", VARCHAR).build())).withNullable(true), Field.of("isRobot", BOOLEAN).withNullable(true)).collect(toSchema()), table.getSchema());
}

Example 5 with InMemoryMetaStore

use of org.apache.beam.sdk.extensions.sql.meta.store.InMemoryMetaStore in project beam by apache.

the class BeamSqlCliTest method test_time_types.

@Test
public void test_time_types() throws Exception {
    InMemoryMetaStore metaStore = new InMemoryMetaStore();
    TestTableProvider testTableProvider = new TestTableProvider();
    metaStore.registerProvider(testTableProvider);
    BeamSqlCli cli = new BeamSqlCli().metaStore(metaStore);
    cli.execute("CREATE EXTERNAL TABLE test_table (\n" + "f_date DATE, \n" + "f_time TIME, \n" + "f_ts TIMESTAMP" + ") \n" + "TYPE 'test'");
    cli.execute("INSERT INTO test_table VALUES (" + "DATE '2018-11-01', " + "TIME '15:23:59', " + "TIMESTAMP '2018-07-01 21:26:07.123' )");
    Table table = metaStore.getTables().get("test_table");
    assertNotNull(table);
    TestTableProvider.TableWithRows tableWithRows = testTableProvider.tables().get(table.getName());
    assertEquals(1, tableWithRows.getRows().size());
    Row row = tableWithRows.getRows().get(0);
    assertEquals(3, row.getFieldCount());
    // test DATE field
    assertEquals("2018-11-01", row.getLogicalTypeValue("f_date", LocalDate.class).toString());
    // test TIME field
    assertEquals("15:23:59", row.getLogicalTypeValue("f_time", LocalTime.class).toString());
    // test TIMESTAMP field
    assertEquals(parseTimestampWithUTCTimeZone("2018-07-01 21:26:07.123"), row.getDateTime("f_ts"));
}