Examples with StreamTableEnvironment org.apache.flink.table.api.bridge.java.StreamTableEnvironment used on opensource projects

Example 36 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class DataStreamJavaITCase method testFromAndToChangelogStreamEventTime.

@Test
public void testFromAndToChangelogStreamEventTime() throws Exception {
    final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
    final DataStream<Tuple3<Long, Integer, String>> dataStream = getWatermarkedDataStream();
    final DataStream<Row> changelogStream = dataStream.map(t -> Row.ofKind(RowKind.INSERT, t.f1, t.f2)).returns(Types.ROW(Types.INT, Types.STRING));
    // derive physical columns and add a rowtime
    final Table table = tableEnv.fromChangelogStream(changelogStream, Schema.newBuilder().columnByMetadata("rowtime", TIMESTAMP_LTZ(3)).columnByExpression("computed", $("f1").upperCase()).watermark("rowtime", sourceWatermark()).build());
    tableEnv.createTemporaryView("t", table);
    // access and reorder columns
    final Table reordered = tableEnv.sqlQuery("SELECT computed, rowtime, f0 FROM t");
    // write out the rowtime column with fully declared schema
    final DataStream<Row> result = tableEnv.toChangelogStream(reordered, Schema.newBuilder().column("f1", STRING()).columnByMetadata("rowtime", TIMESTAMP_LTZ(3)).columnByExpression("ignored", $("f1").upperCase()).column("f0", INT()).build());
    // test event time window and field access
    testResult(result.keyBy(k -> k.getField("f1")).window(TumblingEventTimeWindows.of(Time.milliseconds(5))).<Row>apply((key, window, input, out) -> {
        int sum = 0;
        for (Row row : input) {
            sum += row.<Integer>getFieldAs("f0");
        }
        out.collect(Row.of(key, sum));
    }).returns(Types.ROW(Types.STRING, Types.INT)), Row.of("A", 47), Row.of("C", 1000), Row.of("C", 1000));
}

Example 37 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class DataStreamJavaITCase method testFromAndToChangelogStreamUpsert.

@Test
public void testFromAndToChangelogStreamUpsert() throws Exception {
    final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
    final List<Either<Row, Row>> inputOrOutput = Arrays.asList(input(RowKind.INSERT, "bob", 0), output(RowKind.INSERT, "bob", 0), // --
    input(RowKind.UPDATE_AFTER, "bob", 1), output(RowKind.UPDATE_AFTER, "bob", 1), // --
    input(RowKind.INSERT, "alice", 1), output(RowKind.INSERT, "alice", 1), // no impact
    input(RowKind.INSERT, "alice", 1), // --
    input(RowKind.UPDATE_AFTER, "alice", 2), output(RowKind.UPDATE_AFTER, "alice", 2), // --
    input(RowKind.UPDATE_AFTER, "alice", 100), output(RowKind.UPDATE_AFTER, "alice", 100));
    final DataStream<Row> changelogStream = env.fromElements(getInput(inputOrOutput));
    tableEnv.createTemporaryView("t", tableEnv.fromChangelogStream(changelogStream, Schema.newBuilder().primaryKey("f0").build(), ChangelogMode.upsert()));
    final Table result = tableEnv.sqlQuery("SELECT f0, SUM(f1) FROM t GROUP BY f0");
    testResult(tableEnv.toChangelogStream(result, Schema.newBuilder().primaryKey("f0").build(), ChangelogMode.upsert()), getOutput(inputOrOutput));
}

Example 38 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class DataStreamJavaITCase method testAttachAsDataStream.

@Test
public void testAttachAsDataStream() throws Exception {
    final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
    final String input1DataId = TestValuesTableFactory.registerData(Arrays.asList(Row.of(1, "a"), Row.of(2, "b")));
    tableEnv.createTemporaryTable("InputTable1", TableDescriptor.forConnector("values").option("data-id", input1DataId).schema(Schema.newBuilder().column("i", INT()).column("s", STRING()).build()).build());
    tableEnv.createTemporaryTable("OutputTable1", TableDescriptor.forConnector("values").schema(Schema.newBuilder().column("i", INT()).column("s", STRING()).build()).build());
    tableEnv.createTemporaryView("InputTable2", env.fromElements(1, 2, 3));
    tableEnv.createTemporaryTable("OutputTable2", TableDescriptor.forConnector("values").schema(Schema.newBuilder().column("i", INT()).build()).build());
    tableEnv.createStatementSet().addInsert("OutputTable1", tableEnv.from("InputTable1")).addInsert("OutputTable2", tableEnv.from("InputTable2")).attachAsDataStream();
    // submits all source-to-sink pipelines
    testResult(env.fromElements(3, 4, 5), 3, 4, 5);
    assertThat(TestValuesTableFactory.getResults("OutputTable1"), containsInAnyOrder("+I[1, a]", "+I[2, b]"));
    assertThat(TestValuesTableFactory.getResults("OutputTable2"), containsInAnyOrder("+I[1]", "+I[2]", "+I[3]"));
}

Example 39 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class DataStreamJavaITCase method testTableStreamConversionBatch.

@Test
public void testTableStreamConversionBatch() throws Exception {
    env.setRuntimeMode(RuntimeExecutionMode.BATCH);
    DataStreamSource<Row> streamSource = env.fromElements(Row.of("Alice"), Row.of("alice"), Row.of("lily"), Row.of("Bob"), Row.of("lily"), Row.of("lily"));
    StreamTableEnvironment tableEnvironment = StreamTableEnvironment.create(env);
    Table sourceTable = tableEnvironment.fromDataStream(streamSource).as("word");
    tableEnvironment.createTemporaryView("tmp_table", sourceTable);
    Table resultTable = tableEnvironment.sqlQuery("select UPPER(word) as word from tmp_table");
    SingleOutputStreamOperator<Tuple2<String, Integer>> resultStream = tableEnvironment.toDataStream(resultTable).map(row -> (String) row.getField("word")).returns(TypeInformation.of(String.class)).map(s -> new Tuple2<>(s, 1)).returns(TypeInformation.of(new TypeHint<Tuple2<String, Integer>>() {
    })).keyBy(tuple -> tuple.f0).sum(1);
    testResult(resultStream, new Tuple2<>("ALICE", 2), new Tuple2<>("BOB", 1), new Tuple2<>("LILY", 3));
}

Example 40 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class DataStreamJavaITCase method testToDataStreamCustomEventTime.

@Test
public void testToDataStreamCustomEventTime() throws Exception {
    final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
    final TableConfig tableConfig = tableEnv.getConfig();
    // session time zone should not have an impact on the conversion
    final ZoneId originalZone = tableConfig.getLocalTimeZone();
    tableConfig.setLocalTimeZone(ZoneId.of("Europe/Berlin"));
    final LocalDateTime localDateTime1 = LocalDateTime.parse("1970-01-01T00:00:00.000");
    final LocalDateTime localDateTime2 = LocalDateTime.parse("1970-01-01T01:00:00.000");
    final DataStream<Tuple2<LocalDateTime, String>> dataStream = env.fromElements(new Tuple2<>(localDateTime1, "alice"), new Tuple2<>(localDateTime2, "bob"));
    final Table table = tableEnv.fromDataStream(dataStream, Schema.newBuilder().column("f0", "TIMESTAMP(3)").column("f1", "STRING").watermark("f0", "SOURCE_WATERMARK()").build());
    testSchema(table, new ResolvedSchema(Arrays.asList(Column.physical("f0", TIMESTAMP(3)), Column.physical("f1", STRING())), Collections.singletonList(WatermarkSpec.of("f0", ResolvedExpressionMock.of(TIMESTAMP(3), "`SOURCE_WATERMARK`()"))), null));
    final DataStream<Long> rowtimeStream = tableEnv.toDataStream(table).process(new ProcessFunction<Row, Long>() {

        @Override
        public void processElement(Row value, Context ctx, Collector<Long> out) {
            out.collect(ctx.timestamp());
        }
    });
    testResult(rowtimeStream, localDateTime1.atOffset(ZoneOffset.UTC).toInstant().toEpochMilli(), localDateTime2.atOffset(ZoneOffset.UTC).toInstant().toEpochMilli());
    tableConfig.setLocalTimeZone(originalZone);
}