Examples with StreamTableEnvironment org.apache.flink.table.api.bridge.java.StreamTableEnvironment used on opensource projects

Example 41 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class HiveTableSourceITCase method testStreamPartitionReadByPartitionName.

@Test(timeout = 120000)
public void testStreamPartitionReadByPartitionName() throws Exception {
    final String catalogName = "hive";
    final String dbName = "source_db";
    final String tblName = "stream_partition_name_test";
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.enableCheckpointing(100);
    StreamTableEnvironment tEnv = HiveTestUtils.createTableEnvInStreamingMode(env, SqlDialect.HIVE);
    tEnv.registerCatalog(catalogName, hiveCatalog);
    tEnv.useCatalog(catalogName);
    tEnv.executeSql("CREATE TABLE source_db.stream_partition_name_test (x int, y string, z int)" + " PARTITIONED BY (" + " pt_year int, pt_mon string, pt_day string) TBLPROPERTIES(" + "'streaming-source.enable'='true'," + "'streaming-source.monitor-interval'='1s'," + "'streaming-source.consume-start-offset'='pt_year=2019/pt_month=09/pt_day=02'" + ")");
    HiveTestUtils.createTextTableInserter(hiveCatalog, dbName, tblName).addRow(new Object[] { 0, "a", 11 }).commit("pt_year='2019',pt_mon='09',pt_day='01'");
    HiveTestUtils.createTextTableInserter(hiveCatalog, dbName, tblName).addRow(new Object[] { 1, "b", 12 }).commit("pt_year='2020',pt_mon='09',pt_day='03'");
    TableResult result = tEnv.executeSql("select * from hive.source_db.stream_partition_name_test");
    CloseableIterator<Row> iter = result.collect();
    Assert.assertEquals(Row.of(1, "b", "12", "2020", "09", "03").toString(), fetchRows(iter, 1).get(0));
    for (int i = 2; i < 6; i++) {
        try {
            Thread.sleep(1_000);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        HiveTestUtils.createTextTableInserter(hiveCatalog, dbName, tblName).addRow(new Object[] { i, "new_add", 11 + i }).addRow(new Object[] { i, "new_add_1", 11 + i }).commit("pt_year='2020',pt_mon='10',pt_day='0" + i + "'");
        Assert.assertEquals(Arrays.asList(Row.of(i, "new_add", 11 + i, "2020", "10", "0" + i).toString(), Row.of(i, "new_add_1", 11 + i, "2020", "10", "0" + i).toString()), fetchRows(iter, 2));
    }
    result.getJobClient().get().cancel();
}

Example 42 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class HiveTestUtils method createTableEnvInStreamingMode.

public static StreamTableEnvironment createTableEnvInStreamingMode(StreamExecutionEnvironment env, SqlDialect dialect) {
    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
    tableEnv.getConfig().getConfiguration().setInteger(TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM.key(), 1);
    tableEnv.getConfig().setSqlDialect(dialect);
    return tableEnv;
}

Example 43 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class JdbcDynamicTableSinkITCase method testAppend.

@Test
public void testAppend() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().enableObjectReuse();
    env.getConfig().setParallelism(1);
    StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
    Table t = tEnv.fromDataStream(get4TupleDataStream(env), $("id"), $("num"), $("text"), $("ts"));
    tEnv.registerTable("T", t);
    tEnv.executeSql("CREATE TABLE upsertSink (" + "  id INT," + "  num BIGINT," + "  ts TIMESTAMP(3)" + ") WITH (" + "  'connector'='jdbc'," + "  'url'='" + DB_URL + "'," + "  'table-name'='" + OUTPUT_TABLE2 + "'" + ")");
    tEnv.executeSql("INSERT INTO upsertSink SELECT id, num, ts FROM T WHERE id IN (2, 10, 20)").await();
    check(new Row[] { Row.of(2, 2, Timestamp.valueOf("1970-01-01 00:00:00.002")), Row.of(10, 4, Timestamp.valueOf("1970-01-01 00:00:00.01")), Row.of(20, 6, Timestamp.valueOf("1970-01-01 00:00:00.02")) }, DB_URL, OUTPUT_TABLE2, new String[] { "id", "num", "ts" });
}

Example 44 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class JdbcDynamicTableSinkITCase method testReal.

@Test
public void testReal() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().enableObjectReuse();
    StreamTableEnvironment tEnv = StreamTableEnvironment.create(env, EnvironmentSettings.inStreamingMode());
    tEnv.executeSql("CREATE TABLE upsertSink (" + "  real_data float" + ") WITH (" + "  'connector'='jdbc'," + "  'url'='" + DB_URL + "'," + "  'table-name'='" + OUTPUT_TABLE4 + "'" + ")");
    tEnv.executeSql("INSERT INTO upsertSink SELECT CAST(1.0 as FLOAT)").await();
    check(new Row[] { Row.of(1.0f) }, DB_URL, "REAL_TABLE", new String[] { "real_data" });
}

Example 45 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class HiveTableSinkITCase method testStreamingWrite.

private void testStreamingWrite(boolean part, boolean useMr, String format, Consumer<String> pathConsumer) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    env.enableCheckpointing(100);
    StreamTableEnvironment tEnv = HiveTestUtils.createTableEnvInStreamingMode(env);
    tEnv.registerCatalog(hiveCatalog.getName(), hiveCatalog);
    tEnv.useCatalog(hiveCatalog.getName());
    tEnv.getConfig().setSqlDialect(SqlDialect.HIVE);
    if (useMr) {
        tEnv.getConfig().getConfiguration().set(HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_WRITER, true);
    } else {
        tEnv.getConfig().getConfiguration().set(HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_WRITER, false);
    }
    try {
        tEnv.executeSql("create database db1");
        tEnv.useDatabase("db1");
        // prepare source
        List<Row> data = Arrays.asList(Row.of(1, "a", "b", "2020-05-03", "7"), Row.of(2, "p", "q", "2020-05-03", "8"), Row.of(3, "x", "y", "2020-05-03", "9"), Row.of(4, "x", "y", "2020-05-03", "10"), Row.of(5, "x", "y", "2020-05-03", "11"));
        DataStream<Row> stream = env.addSource(new FiniteTestSource<>(data), new RowTypeInfo(Types.INT, Types.STRING, Types.STRING, Types.STRING, Types.STRING));
        tEnv.createTemporaryView("my_table", stream, $("a"), $("b"), $("c"), $("d"), $("e"));
        // DDL
        tEnv.executeSql("create external table sink_table (a int,b string,c string" + (part ? "" : ",d string,e string") + ") " + (part ? "partitioned by (d string,e string) " : "") + " stored as " + format + " TBLPROPERTIES (" + "'" + PARTITION_TIME_EXTRACTOR_TIMESTAMP_PATTERN.key() + "'='$d $e:00:00'," + "'" + SINK_PARTITION_COMMIT_DELAY.key() + "'='1h'," + "'" + SINK_PARTITION_COMMIT_POLICY_KIND.key() + "'='metastore,success-file'," + "'" + SINK_PARTITION_COMMIT_SUCCESS_FILE_NAME.key() + "'='_MY_SUCCESS'" + ")");
        // hive dialect only works with hive tables at the moment, switch to default dialect
        tEnv.getConfig().setSqlDialect(SqlDialect.DEFAULT);
        tEnv.sqlQuery("select * from my_table").executeInsert("sink_table").await();
        assertBatch("db1.sink_table", Arrays.asList("+I[1, a, b, 2020-05-03, 7]", "+I[1, a, b, 2020-05-03, 7]", "+I[2, p, q, 2020-05-03, 8]", "+I[2, p, q, 2020-05-03, 8]", "+I[3, x, y, 2020-05-03, 9]", "+I[3, x, y, 2020-05-03, 9]", "+I[4, x, y, 2020-05-03, 10]", "+I[4, x, y, 2020-05-03, 10]", "+I[5, x, y, 2020-05-03, 11]", "+I[5, x, y, 2020-05-03, 11]"));
        pathConsumer.accept(URI.create(hiveCatalog.getHiveTable(ObjectPath.fromString("db1.sink_table")).getSd().getLocation()).getPath());
    } finally {
        tEnv.executeSql("drop database db1 cascade");
    }
}