Examples with TableProvider org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider used on opensource projects

Example 1 with TableProvider

use of org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider in project beam by apache.

the class BeamCalciteSchema method getSubSchema.

@Override
public Schema getSubSchema(String name) {
    if (!subSchemas.containsKey(name)) {
        TableProvider subProvider = tableProvider.getSubProvider(name);
        BeamCalciteSchema subSchema = subProvider == null ? null : new BeamCalciteSchema(connection, subProvider);
        subSchemas.put(name, subSchema);
    }
    return subSchemas.get(name);
}

Example 2 with TableProvider

use of org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider in project beam by apache.

the class KafkaTableProviderIT method testFake.

@Test
public void testFake() throws InterruptedException {
    pipeline.getOptions().as(DirectOptions.class).setBlockOnRun(false);
    String createTableString = String.format("CREATE EXTERNAL TABLE kafka_table(\n" + "f_long BIGINT NOT NULL, \n" + "f_int INTEGER NOT NULL, \n" + "f_string VARCHAR NOT NULL \n" + ") \n" + "TYPE 'kafka' \n" + "LOCATION '%s'\n" + "TBLPROPERTIES '%s'", buildLocation(), objectsProvider.getKafkaPropertiesString());
    TableProvider tb = new KafkaTableProvider();
    BeamSqlEnv env = BeamSqlEnv.inMemory(tb);
    env.executeDdl(createTableString);
    PCollection<Row> queryOutput = BeamSqlRelUtils.toPCollection(pipeline, env.parseQuery("SELECT * FROM kafka_table"));
    queryOutput.apply(ParDo.of(new FakeKvPair())).setCoder(KvCoder.of(StringUtf8Coder.of(), RowCoder.of(TEST_TABLE_SCHEMA))).apply("waitForSuccess", ParDo.of(new StreamAssertEqual(ImmutableSet.of(generateRow(0), generateRow(1), generateRow(2)))));
    queryOutput.apply(logRecords(""));
    pipeline.run();
    TimeUnit.SECONDS.sleep(4);
    produceSomeRecords(3);
    for (int i = 0; i < 200; i++) {
        if (FLAG.getOrDefault(pipeline.getOptions().getOptionsId(), false)) {
            return;
        }
        TimeUnit.MILLISECONDS.sleep(90);
    }
    Assert.fail();
}

Example 3 with TableProvider

use of org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider in project beam by apache.

the class BeamSqlMultipleSchemasTest method testJoinWithExtraSchema.

@Test
public void testJoinWithExtraSchema() {
    PCollection<Row> inputMain = pipeline.apply("mainInput", create(row(1, "pcollection_1"), row(2, "pcollection_2")));
    PCollection<Row> inputExtra = pipeline.apply("extraInput", create(row(1, "_extra_table_1"), row(2, "_extra_table_2")));
    TableProvider extraInputProvider = extraTableProvider("extraTable", inputExtra);
    PCollection<Row> result = inputMain.apply(SqlTransform.query("SELECT extra.f_int, main.f_string || extra.f_string AS f_string \n" + "FROM extraSchema.extraTable AS extra \n" + "   INNER JOIN \n" + " PCOLLECTION AS main \n" + "   ON main.f_int = extra.f_int").withTableProvider("extraSchema", extraInputProvider));
    PAssert.that(result).containsInAnyOrder(row(1, "pcollection_1_extra_table_1"), row(2, "pcollection_2_extra_table_2"));
    pipeline.run();
}

Example 4 with TableProvider

use of org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider in project beam by apache.

the class BeamSqlMultipleSchemasTest method testJoinQualifiedMainWithExtraSchema.

@Test
public void testJoinQualifiedMainWithExtraSchema() {
    PCollection<Row> inputMain = pipeline.apply("mainInput", create(row(1, "pcollection_1"), row(2, "pcollection_2")));
    PCollection<Row> inputExtra = pipeline.apply("extraInput", create(row(1, "_extra_table_1"), row(2, "_extra_table_2")));
    TableProvider extraInputProvider = extraTableProvider("extraTable", inputExtra);
    PCollection<Row> result = inputMain.apply(SqlTransform.query("SELECT extra.f_int, main.f_string || extra.f_string AS f_string \n" + "FROM extraSchema.extraTable AS extra \n" + "   INNER JOIN \n" + " beam.PCOLLECTION AS main \n" + "   ON main.f_int = extra.f_int").withTableProvider("extraSchema", extraInputProvider));
    PAssert.that(result).containsInAnyOrder(row(1, "pcollection_1_extra_table_1"), row(2, "pcollection_2_extra_table_2"));
    pipeline.run();
}

Example 5 with TableProvider

use of org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider in project beam by apache.

the class BeamSqlMultipleSchemasTest method testSetDefaultUnqualifiedSchemaAndJoin.

@Test
public void testSetDefaultUnqualifiedSchemaAndJoin() {
    PCollection<Row> inputMain = pipeline.apply("mainInput", create(row(1, "pcollection_1"), row(2, "pcollection_2")));
    PCollection<Row> inputExtra = pipeline.apply("extraInput", create(row(1, "_extra_table_1"), row(2, "_extra_table_2")));
    TableProvider extraInputProvider = extraTableProvider("extraTable", inputExtra);
    PCollection<Row> result = inputMain.apply(SqlTransform.query("SELECT extra.f_int, main.f_string || extra.f_string AS f_string \n" + "FROM extraTable AS extra \n" + "   INNER JOIN \n" + " beam.PCOLLECTION AS main \n" + "   ON main.f_int = extra.f_int").withDefaultTableProvider("extraSchema", extraInputProvider));
    PAssert.that(result).containsInAnyOrder(row(1, "pcollection_1_extra_table_1"), row(2, "pcollection_2_extra_table_2"));
    pipeline.run();
}