Examples with BeamTableStatistics org.apache.beam.sdk.extensions.sql.impl.BeamTableStatistics used on opensource projects

Example 6 with BeamTableStatistics

use of org.apache.beam.sdk.extensions.sql.impl.BeamTableStatistics in project beam by apache.

the class BeamKafkaTableStatisticsTest method testAllLate.

@Test
public void testAllLate() {
    KafkaTestTable table = testTable(3);
    table.addRecord(createKafkaTestRecord("1", 132, 1000L));
    for (int i = 0; i < 98; i++) {
        table.addRecord(createKafkaTestRecord("1", i, 500L));
    }
    table.setNumberOfRecordsForRate(200);
    BeamTableStatistics stats = table.getTableStatistics(null);
    Assert.assertTrue(stats.isUnknown());
}

Example 7 with BeamTableStatistics

use of org.apache.beam.sdk.extensions.sql.impl.BeamTableStatistics in project beam by apache.

the class BeamKafkaTableStatisticsTest method testEmptyPartitionsRate.

@Test
public void testEmptyPartitionsRate() {
    KafkaTestTable table = testTable(3);
    BeamTableStatistics stats = table.getTableStatistics(null);
    Assert.assertTrue(stats.isUnknown());
}

Example 8 with BeamTableStatistics

use of org.apache.beam.sdk.extensions.sql.impl.BeamTableStatistics in project beam by apache.

the class BeamKafkaTableStatisticsTest method testLateRecords.

@Test
public void testLateRecords() {
    KafkaTestTable table = testTable(3);
    table.addRecord(createKafkaTestRecord("1", 132, 1000L));
    for (int i = 0; i < 98; i++) {
        table.addRecord(createKafkaTestRecord("1", i, 500L));
    }
    table.addRecord(createKafkaTestRecord("1", 133, 2000L));
    table.setNumberOfRecordsForRate(200);
    BeamTableStatistics stats = table.getTableStatistics(null);
    Assert.assertEquals(1d, stats.getRate(), 0.001);
}

Example 9 with BeamTableStatistics

use of org.apache.beam.sdk.extensions.sql.impl.BeamTableStatistics in project beam by apache.

the class BigQueryRowCountIT method testNonEmptyTable.

@Test
public void testNonEmptyTable() {
    BigQueryTableProvider provider = new BigQueryTableProvider();
    Table table = getTable("testTable", bigQuery.tableSpec());
    pipeline.apply(Create.of(new TableRow().set("id", 1).set("name", "name1"), new TableRow().set("id", 2).set("name", "name2"), new TableRow().set("id", 3).set("name", "name3")).withCoder(TableRowJsonCoder.of())).apply(BigQueryIO.writeTableRows().to(bigQuery.tableSpec()).withSchema(new TableSchema().setFields(ImmutableList.of(new TableFieldSchema().setName("id").setType("INTEGER"), new TableFieldSchema().setName("name").setType("STRING")))).withoutValidation());
    pipeline.run().waitUntilFinish();
    BeamSqlTable sqlTable = provider.buildBeamSqlTable(table);
    BeamTableStatistics size1 = sqlTable.getTableStatistics(TestPipeline.testingPipelineOptions());
    assertNotNull(size1);
    assertEquals(3d, size1.getRowCount(), 0.1);
}

Example 10 with BeamTableStatistics

use of org.apache.beam.sdk.extensions.sql.impl.BeamTableStatistics in project beam by apache.

the class BigQueryRowCountIT method testFakeTable.

@Test
public void testFakeTable() {
    BigQueryTableProvider provider = new BigQueryTableProvider();
    Table table = getTable("fakeTable", "project:dataset.table");
    BeamSqlTable sqlTable = provider.buildBeamSqlTable(table);
    BeamTableStatistics size = sqlTable.getTableStatistics(TestPipeline.testingPipelineOptions());
    assertTrue(size.isUnknown());
}