Examples with StreamTableEnvironment org.apache.flink.table.api.bridge.java.StreamTableEnvironment used on opensource projects

Example 31 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class DataStreamJavaITCase method testFromChangelogStreamUpsert.

@Test
public void testFromChangelogStreamUpsert() {
    final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
    final List<Either<Row, Row>> inputOrOutput = Arrays.asList(input(RowKind.INSERT, "bob", 0), output(RowKind.INSERT, "bob", 0), // --
    input(RowKind.UPDATE_AFTER, "bob", 1), output(RowKind.UPDATE_BEFORE, "bob", 0), output(RowKind.UPDATE_AFTER, "bob", 1), // --
    input(RowKind.INSERT, "alice", 1), output(RowKind.INSERT, "alice", 1), // no impact
    input(RowKind.INSERT, "alice", 1), // --
    input(RowKind.UPDATE_AFTER, "alice", 2), output(RowKind.UPDATE_BEFORE, "alice", 1), output(RowKind.UPDATE_AFTER, "alice", 2), // --
    input(RowKind.UPDATE_AFTER, "alice", 100), output(RowKind.UPDATE_BEFORE, "alice", 2), output(RowKind.UPDATE_AFTER, "alice", 100));
    final DataStream<Row> changelogStream = env.fromElements(getInput(inputOrOutput));
    tableEnv.createTemporaryView("t", tableEnv.fromChangelogStream(changelogStream, Schema.newBuilder().primaryKey("f0").build(), ChangelogMode.upsert()));
    final Table result = tableEnv.sqlQuery("SELECT f0, SUM(f1) FROM t GROUP BY f0");
    testResult(result.execute(), getOutput(inputOrOutput));
}

Example 32 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class DataStreamJavaITCase method testFromAndToDataStreamWithRaw.

@Test
public void testFromAndToDataStreamWithRaw() throws Exception {
    final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
    final List<Tuple2<DayOfWeek, ZoneOffset>> rawRecords = Arrays.asList(Tuple2.of(DayOfWeek.MONDAY, ZoneOffset.UTC), Tuple2.of(DayOfWeek.FRIDAY, ZoneOffset.ofHours(5)));
    final DataStream<Tuple2<DayOfWeek, ZoneOffset>> dataStream = env.fromCollection(rawRecords);
    // verify incoming type information
    assertThat(dataStream.getType(), instanceOf(TupleTypeInfo.class));
    final TupleTypeInfo<?> tupleInfo = (TupleTypeInfo<?>) dataStream.getType();
    assertThat(tupleInfo.getFieldTypes()[0], instanceOf(EnumTypeInfo.class));
    assertThat(tupleInfo.getFieldTypes()[1], instanceOf(GenericTypeInfo.class));
    final Table table = tableEnv.fromDataStream(dataStream);
    // verify schema conversion
    final List<DataType> columnDataTypes = table.getResolvedSchema().getColumnDataTypes();
    assertThat(columnDataTypes.get(0).getLogicalType(), instanceOf(RawType.class));
    assertThat(columnDataTypes.get(1).getLogicalType(), instanceOf(RawType.class));
    // test reverse operation
    testResult(table.execute(), Row.of(DayOfWeek.MONDAY, ZoneOffset.UTC), Row.of(DayOfWeek.FRIDAY, ZoneOffset.ofHours(5)));
    testResult(tableEnv.toDataStream(table, DataTypes.of(dataStream.getType())), rawRecords.toArray(new Tuple2[0]));
}

Example 33 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class DataStreamJavaITCase method testFromAndToDataStreamEventTime.

@Test
public void testFromAndToDataStreamEventTime() throws Exception {
    final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
    final DataStream<Tuple3<Long, Integer, String>> dataStream = getWatermarkedDataStream();
    final Table table = tableEnv.fromDataStream(dataStream, Schema.newBuilder().columnByMetadata("rowtime", "TIMESTAMP_LTZ(3)").watermark("rowtime", "SOURCE_WATERMARK()").build());
    testSchema(table, new ResolvedSchema(Arrays.asList(Column.physical("f0", BIGINT().notNull()), Column.physical("f1", INT().notNull()), Column.physical("f2", STRING()), Column.metadata("rowtime", TIMESTAMP_LTZ(3), null, false)), Collections.singletonList(WatermarkSpec.of("rowtime", ResolvedExpressionMock.of(TIMESTAMP_LTZ(3), "`SOURCE_WATERMARK`()"))), null));
    tableEnv.createTemporaryView("t", table);
    final TableResult result = tableEnv.executeSql("SELECT f2, SUM(f1) FROM t GROUP BY f2, TUMBLE(rowtime, INTERVAL '0.005' SECOND)");
    testResult(result, Row.of("a", 47), Row.of("c", 1000), Row.of("c", 1000));
    testResult(tableEnv.toDataStream(table).keyBy(k -> k.getField("f2")).window(TumblingEventTimeWindows.of(Time.milliseconds(5))).<Row>apply((key, window, input, out) -> {
        int sum = 0;
        for (Row row : input) {
            sum += row.<Integer>getFieldAs("f1");
        }
        out.collect(Row.of(key, sum));
    }).returns(Types.ROW(Types.STRING, Types.INT)), Row.of("a", 47), Row.of("c", 1000), Row.of("c", 1000));
}

Example 34 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class DataStreamJavaITCase method getComplexUnifiedPipeline.

// --------------------------------------------------------------------------------------------
// Helper methods
// --------------------------------------------------------------------------------------------
private Table getComplexUnifiedPipeline(StreamExecutionEnvironment env) {
    final DataStream<String> allowedNamesStream = env.fromElements("Bob", "Alice");
    final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
    tableEnv.createTemporaryView("AllowedNamesTable", tableEnv.fromDataStream(allowedNamesStream).as("allowedName"));
    final Table nameCountTable = tableEnv.sqlQuery("SELECT name, COUNT(*) AS c " + "FROM (VALUES ('Bob'), ('Alice'), ('Greg'), ('Bob')) AS NameTable(name) " + "WHERE name IN (SELECT allowedName FROM AllowedNamesTable)" + "GROUP BY name");
    final DataStream<Row> nameCountStream = tableEnv.toChangelogStream(nameCountTable);
    final DataStream<Tuple2<String, Long>> updatesPerNameStream = nameCountStream.keyBy(r -> r.<String>getFieldAs("name")).process(new KeyedProcessFunction<String, Row, Tuple2<String, Long>>() {

        ValueState<Long> count;

        @Override
        public void open(Configuration parameters) {
            count = getRuntimeContext().getState(new ValueStateDescriptor<>("count", Long.class));
        }

        @Override
        public void processElement(Row r, Context ctx, Collector<Tuple2<String, Long>> out) throws IOException {
            Long currentCount = count.value();
            if (currentCount == null) {
                currentCount = 0L;
            }
            final long updatedCount = currentCount + 1;
            count.update(updatedCount);
            out.collect(Tuple2.of(ctx.getCurrentKey(), updatedCount));
        }
    });
    tableEnv.createTemporaryView("UpdatesPerName", updatesPerNameStream);
    return tableEnv.sqlQuery("SELECT DISTINCT f0, f1 FROM UpdatesPerName");
}

Example 35 with StreamTableEnvironment

use of org.apache.flink.table.api.bridge.java.StreamTableEnvironment in project flink by apache.

the class DataStreamJavaITCase method testToDataStreamAtomic.

@Test
public void testToDataStreamAtomic() throws Exception {
    final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
    final Table table = tableEnv.fromValues(1, 2, 3, 4, 5);
    testResult(tableEnv.toDataStream(table, Integer.class), 1, 2, 3, 4, 5);
}