Examples with STRING org.apache.flink.table.api.DataTypes.STRING used on opensource projects

Example 1 with STRING

use of org.apache.flink.table.api.DataTypes.STRING in project flink by apache.

the class DataStreamJavaITCase method getComplexUnifiedPipeline.

// --------------------------------------------------------------------------------------------
// Helper methods
// --------------------------------------------------------------------------------------------
private Table getComplexUnifiedPipeline(StreamExecutionEnvironment env) {
    final DataStream<String> allowedNamesStream = env.fromElements("Bob", "Alice");
    final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
    tableEnv.createTemporaryView("AllowedNamesTable", tableEnv.fromDataStream(allowedNamesStream).as("allowedName"));
    final Table nameCountTable = tableEnv.sqlQuery("SELECT name, COUNT(*) AS c " + "FROM (VALUES ('Bob'), ('Alice'), ('Greg'), ('Bob')) AS NameTable(name) " + "WHERE name IN (SELECT allowedName FROM AllowedNamesTable)" + "GROUP BY name");
    final DataStream<Row> nameCountStream = tableEnv.toChangelogStream(nameCountTable);
    final DataStream<Tuple2<String, Long>> updatesPerNameStream = nameCountStream.keyBy(r -> r.<String>getFieldAs("name")).process(new KeyedProcessFunction<String, Row, Tuple2<String, Long>>() {

        ValueState<Long> count;

        @Override
        public void open(Configuration parameters) {
            count = getRuntimeContext().getState(new ValueStateDescriptor<>("count", Long.class));
        }

        @Override
        public void processElement(Row r, Context ctx, Collector<Tuple2<String, Long>> out) throws IOException {
            Long currentCount = count.value();
            if (currentCount == null) {
                currentCount = 0L;
            }
            final long updatedCount = currentCount + 1;
            count.update(updatedCount);
            out.collect(Tuple2.of(ctx.getCurrentKey(), updatedCount));
        }
    });
    tableEnv.createTemporaryView("UpdatesPerName", updatesPerNameStream);
    return tableEnv.sqlQuery("SELECT DISTINCT f0, f1 FROM UpdatesPerName");
}

Example 2 with STRING

use of org.apache.flink.table.api.DataTypes.STRING in project flink by apache.

the class DataStreamJavaITCase method testFromAndToChangelogStreamEventTime.

@Test
public void testFromAndToChangelogStreamEventTime() throws Exception {
    final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
    final DataStream<Tuple3<Long, Integer, String>> dataStream = getWatermarkedDataStream();
    final DataStream<Row> changelogStream = dataStream.map(t -> Row.ofKind(RowKind.INSERT, t.f1, t.f2)).returns(Types.ROW(Types.INT, Types.STRING));
    // derive physical columns and add a rowtime
    final Table table = tableEnv.fromChangelogStream(changelogStream, Schema.newBuilder().columnByMetadata("rowtime", TIMESTAMP_LTZ(3)).columnByExpression("computed", $("f1").upperCase()).watermark("rowtime", sourceWatermark()).build());
    tableEnv.createTemporaryView("t", table);
    // access and reorder columns
    final Table reordered = tableEnv.sqlQuery("SELECT computed, rowtime, f0 FROM t");
    // write out the rowtime column with fully declared schema
    final DataStream<Row> result = tableEnv.toChangelogStream(reordered, Schema.newBuilder().column("f1", STRING()).columnByMetadata("rowtime", TIMESTAMP_LTZ(3)).columnByExpression("ignored", $("f1").upperCase()).column("f0", INT()).build());
    // test event time window and field access
    testResult(result.keyBy(k -> k.getField("f1")).window(TumblingEventTimeWindows.of(Time.milliseconds(5))).<Row>apply((key, window, input, out) -> {
        int sum = 0;
        for (Row row : input) {
            sum += row.<Integer>getFieldAs("f0");
        }
        out.collect(Row.of(key, sum));
    }).returns(Types.ROW(Types.STRING, Types.INT)), Row.of("A", 47), Row.of("C", 1000), Row.of("C", 1000));
}

Example 3 with STRING

use of org.apache.flink.table.api.DataTypes.STRING in project flink by apache.

the class DataStreamJavaITCase method testTableStreamConversionBatch.

@Test
public void testTableStreamConversionBatch() throws Exception {
    env.setRuntimeMode(RuntimeExecutionMode.BATCH);
    DataStreamSource<Row> streamSource = env.fromElements(Row.of("Alice"), Row.of("alice"), Row.of("lily"), Row.of("Bob"), Row.of("lily"), Row.of("lily"));
    StreamTableEnvironment tableEnvironment = StreamTableEnvironment.create(env);
    Table sourceTable = tableEnvironment.fromDataStream(streamSource).as("word");
    tableEnvironment.createTemporaryView("tmp_table", sourceTable);
    Table resultTable = tableEnvironment.sqlQuery("select UPPER(word) as word from tmp_table");
    SingleOutputStreamOperator<Tuple2<String, Integer>> resultStream = tableEnvironment.toDataStream(resultTable).map(row -> (String) row.getField("word")).returns(TypeInformation.of(String.class)).map(s -> new Tuple2<>(s, 1)).returns(TypeInformation.of(new TypeHint<Tuple2<String, Integer>>() {
    })).keyBy(tuple -> tuple.f0).sum(1);
    testResult(resultStream, new Tuple2<>("ALICE", 2), new Tuple2<>("BOB", 1), new Tuple2<>("LILY", 3));
}

Example 4 with STRING

use of org.apache.flink.table.api.DataTypes.STRING in project flink by apache.

the class OggJsonSerDeSchemaTest method testDeserializationWithMetadata.

public void testDeserializationWithMetadata(String resourceFile) throws Exception {
    // we only read the first line for keeping the test simple
    final String firstLine = readLines(resourceFile).get(0);
    final List<ReadableMetadata> requestedMetadata = Arrays.asList(ReadableMetadata.values());
    final DataType producedDataTypes = DataTypeUtils.appendRowFields(PHYSICAL_DATA_TYPE, requestedMetadata.stream().map(m -> DataTypes.FIELD(m.key, m.dataType)).collect(Collectors.toList()));
    final OggJsonDeserializationSchema deserializationSchema = new OggJsonDeserializationSchema(PHYSICAL_DATA_TYPE, requestedMetadata, InternalTypeInfo.of(producedDataTypes.getLogicalType()), false, TimestampFormat.ISO_8601);
    final SimpleCollector collector = new SimpleCollector();
    deserializationSchema.deserialize(firstLine.getBytes(StandardCharsets.UTF_8), collector);
    assertEquals(1, collector.list.size());
    Consumer<RowData> consumer = row -> {
        assertEquals(101, row.getInt(0));
        assertEquals("scooter", row.getString(1).toString());
        assertEquals("Small 2-wheel scooter", row.getString(2).toString());
        assertEquals(3.140000104904175, row.getFloat(3), 1e-15);
        assertEquals("OGG.TBL_TEST", row.getString(4).toString());
        assertEquals("id", row.getArray(5).getString(0).toString());
        assertEquals(1589377175766L, row.getTimestamp(6, 6).getMillisecond());
        assertEquals(1589384406000L, row.getTimestamp(7, 6).getMillisecond());
    };
    consumer.accept(collector.list.get(0));
}

Example 5 with STRING

use of org.apache.flink.table.api.DataTypes.STRING in project flink by apache.

the class MaxwellJsonSerDerTest method testDeserializationWithMetadata.

@Test
public void testDeserializationWithMetadata() throws Exception {
    // we only read the first line for keeping the test simple
    final String firstLine = readLines("maxwell-data.txt").get(0);
    final List<ReadableMetadata> requestedMetadata = Arrays.asList(ReadableMetadata.values());
    final DataType producedDataType = DataTypeUtils.appendRowFields(PHYSICAL_DATA_TYPE, requestedMetadata.stream().map(m -> DataTypes.FIELD(m.key, m.dataType)).collect(Collectors.toList()));
    final MaxwellJsonDeserializationSchema deserializationSchema = new MaxwellJsonDeserializationSchema(PHYSICAL_DATA_TYPE, requestedMetadata, InternalTypeInfo.of(producedDataType.getLogicalType()), false, TimestampFormat.ISO_8601);
    final SimpleCollector collector = new SimpleCollector();
    deserializationSchema.deserialize(firstLine.getBytes(StandardCharsets.UTF_8), collector);
    assertEquals(1, collector.list.size());
    Consumer<RowData> consumer = row -> {
        assertThat(row.getInt(0), equalTo(101));
        assertThat(row.getString(1).toString(), equalTo("scooter"));
        assertThat(row.getString(2).toString(), equalTo("Small 2-wheel scooter"));
        assertThat(row.getFloat(3), equalTo(3.14f));
        assertThat(row.getString(4).toString(), equalTo("test"));
        assertThat(row.getString(5).toString(), equalTo("product"));
        assertThat(row.getArray(6).getString(0).toString(), equalTo("id"));
        assertThat(row.getTimestamp(7, 3).getMillisecond(), equalTo(1596684883000L));
    };
    consumer.accept(collector.list.get(0));
}