Examples with Table org.apache.beam.sdk.extensions.sql.meta.Table used on opensource projects

Example 56 with Table

use of org.apache.beam.sdk.extensions.sql.meta.Table in project beam by apache.

the class BigQueryRowCountIT method testNonEmptyTable.

@Test
public void testNonEmptyTable() {
    BigQueryTableProvider provider = new BigQueryTableProvider();
    Table table = getTable("testTable", bigQuery.tableSpec());
    pipeline.apply(Create.of(new TableRow().set("id", 1).set("name", "name1"), new TableRow().set("id", 2).set("name", "name2"), new TableRow().set("id", 3).set("name", "name3")).withCoder(TableRowJsonCoder.of())).apply(BigQueryIO.writeTableRows().to(bigQuery.tableSpec()).withSchema(new TableSchema().setFields(ImmutableList.of(new TableFieldSchema().setName("id").setType("INTEGER"), new TableFieldSchema().setName("name").setType("STRING")))).withoutValidation());
    pipeline.run().waitUntilFinish();
    BeamSqlTable sqlTable = provider.buildBeamSqlTable(table);
    BeamTableStatistics size1 = sqlTable.getTableStatistics(TestPipeline.testingPipelineOptions());
    assertNotNull(size1);
    assertEquals(3d, size1.getRowCount(), 0.1);
}

Example 57 with Table

use of org.apache.beam.sdk.extensions.sql.meta.Table in project beam by apache.

the class BigQueryRowCountIT method testFakeTable.

@Test
public void testFakeTable() {
    BigQueryTableProvider provider = new BigQueryTableProvider();
    Table table = getTable("fakeTable", "project:dataset.table");
    BeamSqlTable sqlTable = provider.buildBeamSqlTable(table);
    BeamTableStatistics size = sqlTable.getTableStatistics(TestPipeline.testingPipelineOptions());
    assertTrue(size.isUnknown());
}

Example 58 with Table

use of org.apache.beam.sdk.extensions.sql.meta.Table in project beam by apache.

the class BigtableTableCreationFailuresTest method testCreateWithoutKeyFails.

@Test
public void testCreateWithoutKeyFails() {
    String createTable = "CREATE EXTERNAL TABLE fail(something VARCHAR) \n" + "TYPE bigtable \n" + "LOCATION '" + location() + "'";
    cli.execute(createTable);
    Table table = metaStore.getTables().get("fail");
    IllegalStateException e = assertThrows(IllegalStateException.class, () -> tableProvider.buildBeamSqlTable(table));
    checkMessage(e.getMessage(), "Schema has to contain 'key' field");
}

Example 59 with Table

use of org.apache.beam.sdk.extensions.sql.meta.Table in project beam by apache.

the class BigtableTableCreationFailuresTest method testShouldFailOnIncorrectLocation.

@Test
public void testShouldFailOnIncorrectLocation() {
    String createTable = "CREATE EXTERNAL TABLE fail(key VARCHAR, q BIGINT) \n" + "TYPE bigtable \n" + "LOCATION 'googleapis.com/incorrect/projects/fakeProject/instances/fakeInstance/tables/beamTable' \n" + "TBLPROPERTIES '{\"columnsMapping\": \"f:q\"}'";
    cli.execute(createTable);
    Table table = metaStore.getTables().get("fail");
    InvalidTableException e = assertThrows(InvalidTableException.class, () -> tableProvider.buildBeamSqlTable(table));
    checkMessage(e.getMessage(), "Bigtable location must be in the following format:");
}

Example 60 with Table

use of org.apache.beam.sdk.extensions.sql.meta.Table in project beam by apache.

the class IOPushDownRuleTest method buildUp.

@Before
public void buildUp() {
    TestTableProvider tableProvider = new TestTableProvider();
    Table table = getTable("TEST", PushDownOptions.PROJECT);
    tableProvider.createTable(table);
    tableProvider.addRows(table.getName(), row(BASIC_SCHEMA, 100, 1, "one", 100), row(BASIC_SCHEMA, 200, 2, "two", 200));
    sqlEnv = BeamSqlEnv.builder(tableProvider).setPipelineOptions(PipelineOptionsFactory.create()).setRuleSets(ImmutableList.of(RuleSets.ofList(defaultRules))).build();
}