Examples with TableResult org.apache.flink.table.api.TableResult used on opensource projects

Example 11 with TableResult

use of org.apache.flink.table.api.TableResult in project flink by apache.

the class JdbcDynamicTableSinkITCase method testBatchSink.

@Test
public void testBatchSink() throws Exception {
    TableEnvironment tEnv = TableEnvironment.create(EnvironmentSettings.inBatchMode());
    tEnv.executeSql("CREATE TABLE USER_RESULT(" + "NAME VARCHAR," + "SCORE BIGINT" + ") WITH ( " + "'connector' = 'jdbc'," + "'url'='" + DB_URL + "'," + "'table-name' = '" + OUTPUT_TABLE3 + "'," + "'sink.buffer-flush.max-rows' = '2'," + "'sink.buffer-flush.interval' = '300ms'," + "'sink.max-retries' = '4'" + ")");
    TableResult tableResult = tEnv.executeSql("INSERT INTO USER_RESULT\n" + "SELECT user_name, score " + "FROM (VALUES (1, 'Bob'), (22, 'Tom'), (42, 'Kim'), " + "(42, 'Kim'), (1, 'Bob')) " + "AS UserCountTable(score, user_name)");
    tableResult.await();
    check(new Row[] { Row.of("Bob", 1), Row.of("Tom", 22), Row.of("Kim", 42), Row.of("Kim", 42), Row.of("Bob", 1) }, DB_URL, OUTPUT_TABLE3, new String[] { "NAME", "SCORE" });
}

Example 12 with TableResult

use of org.apache.flink.table.api.TableResult in project flink by apache.

the class KafkaTableTestUtils method collectRows.

public static List<Row> collectRows(Table table, int expectedSize) throws Exception {
    final TableResult result = table.execute();
    final List<Row> collectedRows = new ArrayList<>();
    try (CloseableIterator<Row> iterator = result.collect()) {
        while (collectedRows.size() < expectedSize && iterator.hasNext()) {
            collectedRows.add(iterator.next());
        }
    }
    result.getJobClient().ifPresent(jc -> {
        try {
            jc.cancel().get(5, TimeUnit.SECONDS);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    });
    return collectedRows;
}

Example 13 with TableResult

use of org.apache.flink.table.api.TableResult in project flink by apache.

the class KafkaChangelogTableITCase method testKafkaDebeziumChangelogSource.

@Test
public void testKafkaDebeziumChangelogSource() throws Exception {
    final String topic = "changelog_topic";
    createTestTopic(topic, 1, 1);
    // enables MiniBatch processing to verify MiniBatch + FLIP-95, see FLINK-18769
    Configuration tableConf = tEnv.getConfig().getConfiguration();
    tableConf.setString("table.exec.mini-batch.enabled", "true");
    tableConf.setString("table.exec.mini-batch.allow-latency", "1s");
    tableConf.setString("table.exec.mini-batch.size", "5000");
    tableConf.setString("table.optimizer.agg-phase-strategy", "TWO_PHASE");
    // ---------- Write the Debezium json into Kafka -------------------
    List<String> lines = readLines("debezium-data-schema-exclude.txt");
    try {
        writeRecordsToKafka(topic, lines);
    } catch (Exception e) {
        throw new Exception("Failed to write debezium data to Kafka.", e);
    }
    // ---------- Produce an event time stream into Kafka -------------------
    String bootstraps = getBootstrapServers();
    String sourceDDL = String.format("CREATE TABLE debezium_source (" + // test format metadata
    " origin_ts TIMESTAMP(3) METADATA FROM 'value.ingestion-timestamp' VIRTUAL," + // unused
    " origin_table STRING METADATA FROM 'value.source.table' VIRTUAL," + " id INT NOT NULL," + " name STRING," + " description STRING," + " weight DECIMAL(10,3)," + // test connector metadata
    " origin_topic STRING METADATA FROM 'topic' VIRTUAL," + // unused
    " origin_partition STRING METADATA FROM 'partition' VIRTUAL" + ") WITH (" + " 'connector' = 'kafka'," + " 'topic' = '%s'," + " 'properties.bootstrap.servers' = '%s'," + " 'scan.startup.mode' = 'earliest-offset'," + " 'value.format' = 'debezium-json'" + ")", topic, bootstraps);
    String sinkDDL = "CREATE TABLE sink (" + " origin_topic STRING," + " origin_table STRING," + " name STRING," + " weightSum DECIMAL(10,3)," + " PRIMARY KEY (name) NOT ENFORCED" + ") WITH (" + " 'connector' = 'values'," + " 'sink-insert-only' = 'false'" + ")";
    tEnv.executeSql(sourceDDL);
    tEnv.executeSql(sinkDDL);
    TableResult tableResult = tEnv.executeSql("INSERT INTO sink " + "SELECT FIRST_VALUE(origin_topic), FIRST_VALUE(origin_table), name, SUM(weight) " + "FROM debezium_source GROUP BY name");
    /*
         * Debezium captures change data on the `products` table:
         *
         * <pre>
         * CREATE TABLE products (
         *  id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
         *  name VARCHAR(255),
         *  description VARCHAR(512),
         *  weight FLOAT
         * );
         * ALTER TABLE products AUTO_INCREMENT = 101;
         *
         * INSERT INTO products
         * VALUES (default,"scooter","Small 2-wheel scooter",3.14),
         *        (default,"car battery","12V car battery",8.1),
         *        (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8),
         *        (default,"hammer","12oz carpenter's hammer",0.75),
         *        (default,"hammer","14oz carpenter's hammer",0.875),
         *        (default,"hammer","16oz carpenter's hammer",1.0),
         *        (default,"rocks","box of assorted rocks",5.3),
         *        (default,"jacket","water resistent black wind breaker",0.1),
         *        (default,"spare tire","24 inch spare tire",22.2);
         * UPDATE products SET description='18oz carpenter hammer' WHERE id=106;
         * UPDATE products SET weight='5.1' WHERE id=107;
         * INSERT INTO products VALUES (default,"jacket","water resistent white wind breaker",0.2);
         * INSERT INTO products VALUES (default,"scooter","Big 2-wheel scooter ",5.18);
         * UPDATE products SET description='new water resistent white wind breaker', weight='0.5' WHERE id=110;
         * UPDATE products SET weight='5.17' WHERE id=111;
         * DELETE FROM products WHERE id=111;
         *
         * > SELECT * FROM products;
         * +-----+--------------------+---------------------------------------------------------+--------+
         * | id  | name               | description                                             | weight |
         * +-----+--------------------+---------------------------------------------------------+--------+
         * | 101 | scooter            | Small 2-wheel scooter                                   |   3.14 |
         * | 102 | car battery        | 12V car battery                                         |    8.1 |
         * | 103 | 12-pack drill bits | 12-pack of drill bits with sizes ranging from #40 to #3 |    0.8 |
         * | 104 | hammer             | 12oz carpenter's hammer                                 |   0.75 |
         * | 105 | hammer             | 14oz carpenter's hammer                                 |  0.875 |
         * | 106 | hammer             | 18oz carpenter hammer                                   |      1 |
         * | 107 | rocks              | box of assorted rocks                                   |    5.1 |
         * | 108 | jacket             | water resistent black wind breaker                      |    0.1 |
         * | 109 | spare tire         | 24 inch spare tire                                      |   22.2 |
         * | 110 | jacket             | new water resistent white wind breaker                  |    0.5 |
         * +-----+--------------------+---------------------------------------------------------+--------+
         * </pre>
         */
    List<String> expected = Arrays.asList("+I[changelog_topic, products, scooter, 3.140]", "+I[changelog_topic, products, car battery, 8.100]", "+I[changelog_topic, products, 12-pack drill bits, 0.800]", "+I[changelog_topic, products, hammer, 2.625]", "+I[changelog_topic, products, rocks, 5.100]", "+I[changelog_topic, products, jacket, 0.600]", "+I[changelog_topic, products, spare tire, 22.200]");
    waitingExpectedResults("sink", expected, Duration.ofSeconds(10));
    // ------------- cleanup -------------------
    // stop the job
    tableResult.getJobClient().get().cancel().get();
    deleteTestTopic(topic);
}

Example 14 with TableResult

use of org.apache.flink.table.api.TableResult in project flink by apache.

the class KafkaTableITCase method testPerPartitionWatermarkKafka.

@Test
public void testPerPartitionWatermarkKafka() throws Exception {
    // we always use a different topic name for each parameterized topic,
    // in order to make sure the topic can be created.
    final String topic = "per_partition_watermark_topic_" + format;
    createTestTopic(topic, 4, 1);
    // ---------- Produce an event time stream into Kafka -------------------
    String groupId = getStandardProps().getProperty("group.id");
    String bootstraps = getBootstrapServers();
    final String createTable = String.format("CREATE TABLE kafka (\n" + "  `partition_id` INT,\n" + "  `name` STRING,\n" + "  `timestamp` TIMESTAMP(3),\n" + "  WATERMARK FOR `timestamp` AS `timestamp`\n" + ") WITH (\n" + "  'connector' = 'kafka',\n" + "  'topic' = '%s',\n" + "  'properties.bootstrap.servers' = '%s',\n" + "  'properties.group.id' = '%s',\n" + "  'scan.startup.mode' = 'earliest-offset',\n" + "  'sink.partitioner' = '%s',\n" + "  'format' = '%s'\n" + ")", topic, bootstraps, groupId, TestPartitioner.class.getName(), format);
    tEnv.executeSql(createTable);
    // make every partition have more than one record
    String initialValues = "INSERT INTO kafka\n" + "VALUES\n" + " (0, 'partition-0-name-0', TIMESTAMP '2020-03-08 13:12:11.123'),\n" + " (0, 'partition-0-name-1', TIMESTAMP '2020-03-08 14:12:12.223'),\n" + " (0, 'partition-0-name-2', TIMESTAMP '2020-03-08 15:12:13.323'),\n" + " (1, 'partition-1-name-0', TIMESTAMP '2020-03-09 13:13:11.123'),\n" + " (1, 'partition-1-name-1', TIMESTAMP '2020-03-09 15:13:11.133'),\n" + " (1, 'partition-1-name-2', TIMESTAMP '2020-03-09 16:13:11.143'),\n" + " (2, 'partition-2-name-0', TIMESTAMP '2020-03-10 13:12:14.123'),\n" + " (2, 'partition-2-name-1', TIMESTAMP '2020-03-10 14:12:14.123'),\n" + " (2, 'partition-2-name-2', TIMESTAMP '2020-03-10 14:13:14.123'),\n" + " (2, 'partition-2-name-3', TIMESTAMP '2020-03-10 14:14:14.123'),\n" + " (2, 'partition-2-name-4', TIMESTAMP '2020-03-10 14:15:14.123'),\n" + " (2, 'partition-2-name-5', TIMESTAMP '2020-03-10 14:16:14.123'),\n" + " (3, 'partition-3-name-0', TIMESTAMP '2020-03-11 17:12:11.123'),\n" + " (3, 'partition-3-name-1', TIMESTAMP '2020-03-11 18:12:11.123')";
    tEnv.executeSql(initialValues).await();
    // ---------- Consume stream from Kafka -------------------
    env.setParallelism(1);
    String createSink = "CREATE TABLE MySink(\n" + "  id INT,\n" + "  name STRING,\n" + "  ts TIMESTAMP(3),\n" + "  WATERMARK FOR ts as ts\n" + ") WITH (\n" + "  'connector' = 'values',\n" + "  'sink.drop-late-event' = 'true'\n" + ")";
    tEnv.executeSql(createSink);
    TableResult tableResult = tEnv.executeSql("INSERT INTO MySink SELECT * FROM kafka");
    final List<String> expected = Arrays.asList("+I[0, partition-0-name-0, 2020-03-08T13:12:11.123]", "+I[0, partition-0-name-1, 2020-03-08T14:12:12.223]", "+I[0, partition-0-name-2, 2020-03-08T15:12:13.323]", "+I[1, partition-1-name-0, 2020-03-09T13:13:11.123]", "+I[1, partition-1-name-1, 2020-03-09T15:13:11.133]", "+I[1, partition-1-name-2, 2020-03-09T16:13:11.143]", "+I[2, partition-2-name-0, 2020-03-10T13:12:14.123]", "+I[2, partition-2-name-1, 2020-03-10T14:12:14.123]", "+I[2, partition-2-name-2, 2020-03-10T14:13:14.123]", "+I[2, partition-2-name-3, 2020-03-10T14:14:14.123]", "+I[2, partition-2-name-4, 2020-03-10T14:15:14.123]", "+I[2, partition-2-name-5, 2020-03-10T14:16:14.123]", "+I[3, partition-3-name-0, 2020-03-11T17:12:11.123]", "+I[3, partition-3-name-1, 2020-03-11T18:12:11.123]");
    KafkaTableTestUtils.waitingExpectedResults("MySink", expected, Duration.ofSeconds(5));
    // ------------- cleanup -------------------
    tableResult.getJobClient().ifPresent(JobClient::cancel);
    deleteTestTopic(topic);
}

Example 15 with TableResult

use of org.apache.flink.table.api.TableResult in project flink by apache.

the class JsonBatchFileSystemITCase method bigDataTest.

@Test
public void bigDataTest() throws IOException {
    int numRecords = 1000;
    File dir = generateTestData(numRecords);
    env().setParallelism(1);
    String sql = String.format("CREATE TABLE bigdata_source ( " + "	id INT, " + "	content STRING" + ") PARTITIONED by (id) WITH (" + "	'connector' = 'filesystem'," + "	'path' = '%s'," + "	'format' = 'json'" + ")", dir);
    tEnv().executeSql(sql);
    TableResult result = tEnv().executeSql("select * from bigdata_source");
    List<String> elements = new ArrayList<>();
    result.collect().forEachRemaining(r -> elements.add((String) r.getField(1)));
    Assert.assertEquals(numRecords, elements.size());
    elements.sort(String::compareTo);
    List<String> expected = new ArrayList<>();
    for (int i = 0; i < numRecords; i++) {
        expected.add(String.valueOf(i));
    }
    expected.sort(String::compareTo);
    Assert.assertEquals(expected, elements);
}