Examples with RowData org.apache.flink.table.data.RowData used on opensource projects

Example 31 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class HiveSourceITCase method testRegularRead.

@Test
public void testRegularRead() throws Exception {
    // test non-partitioned table
    ObjectPath tablePath = new ObjectPath("default", "tbl1");
    Map<String, String> tableOptions = new HashMap<>();
    tableOptions.put(CONNECTOR.key(), IDENTIFIER);
    hiveCatalog.createTable(tablePath, new CatalogTableImpl(TableSchema.builder().field("i", DataTypes.INT()).build(), tableOptions, null), false);
    HiveTestUtils.createTextTableInserter(hiveCatalog, tablePath.getDatabaseName(), tablePath.getObjectName()).addRow(new Object[] { 1 }).addRow(new Object[] { 2 }).commit();
    StreamExecutionEnvironment streamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
    streamEnv.setParallelism(1);
    HiveSource<RowData> hiveSource = new HiveSourceBuilder(new JobConf(hiveCatalog.getHiveConf()), new Configuration(), HiveShimLoader.getHiveVersion(), tablePath.getDatabaseName(), tablePath.getObjectName(), Collections.emptyMap()).buildWithDefaultBulkFormat();
    List<RowData> results = CollectionUtil.iteratorToList(streamEnv.fromSource(hiveSource, WatermarkStrategy.noWatermarks(), "HiveSource-tbl1").executeAndCollect());
    assertEquals(2, results.size());
    assertEquals(1, results.get(0).getInt(0));
    assertEquals(2, results.get(1).getInt(0));
    hiveCatalog.dropTable(tablePath, false);
    // test partitioned table
    tablePath = new ObjectPath("default", "tbl2");
    hiveCatalog.createTable(tablePath, new CatalogTableImpl(TableSchema.builder().field("i", DataTypes.INT()).field("p", DataTypes.STRING()).build(), Collections.singletonList("p"), tableOptions, null), false);
    HiveTestUtils.createTextTableInserter(hiveCatalog, tablePath.getDatabaseName(), tablePath.getObjectName()).addRow(new Object[] { 1 }).addRow(new Object[] { 2 }).commit("p='a'");
    hiveSource = new HiveSourceBuilder(new JobConf(hiveCatalog.getHiveConf()), new Configuration(), HiveShimLoader.getHiveVersion(), tablePath.getDatabaseName(), tablePath.getObjectName(), Collections.emptyMap()).setLimit(1L).buildWithDefaultBulkFormat();
    results = CollectionUtil.iteratorToList(streamEnv.fromSource(hiveSource, WatermarkStrategy.noWatermarks(), "HiveSource-tbl2").executeAndCollect());
    assertEquals(1, results.size());
    assertEquals(1, results.get(0).getInt(0));
    assertEquals("a", results.get(0).getString(1).toString());
    HiveTestUtils.createTextTableInserter(hiveCatalog, tablePath.getDatabaseName(), tablePath.getObjectName()).addRow(new Object[] { 3 }).commit("p='b'");
    LinkedHashMap<String, String> spec = new LinkedHashMap<>();
    spec.put("p", "b");
    hiveSource = new HiveSourceBuilder(new JobConf(hiveCatalog.getHiveConf()), new Configuration(), null, tablePath.getDatabaseName(), tablePath.getObjectName(), Collections.emptyMap()).setPartitions(Collections.singletonList(HiveTablePartition.ofPartition(hiveCatalog.getHiveConf(), hiveCatalog.getHiveVersion(), tablePath.getDatabaseName(), tablePath.getObjectName(), spec))).buildWithDefaultBulkFormat();
    results = CollectionUtil.iteratorToList(streamEnv.fromSource(hiveSource, WatermarkStrategy.noWatermarks(), "HiveSource-tbl2").executeAndCollect());
    assertEquals(1, results.size());
    assertEquals(3, results.get(0).getInt(0));
    assertEquals("b", results.get(0).getString(1).toString());
    hiveCatalog.dropTable(tablePath, false);
}

Example 32 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class KinesisDynamicTableFactoryTest method testGoodTableSource.

// --------------------------------------------------------------------------------------------
// Positive tests
// --------------------------------------------------------------------------------------------
@Test
public void testGoodTableSource() {
    ResolvedSchema sourceSchema = defaultSourceSchema();
    Map<String, String> sourceOptions = defaultTableOptions().build();
    // Construct actual DynamicTableSource using FactoryUtil
    KinesisDynamicSource actualSource = (KinesisDynamicSource) createTableSource(sourceSchema, sourceOptions);
    // Construct expected DynamicTableSink using factory under test
    KinesisDynamicSource expectedSource = new KinesisDynamicSource(sourceSchema.toPhysicalRowDataType(), STREAM_NAME, defaultConsumerProperties(), new TestFormatFactory.DecodingFormatMock(",", true));
    // verify that the constructed DynamicTableSink is as expected
    assertEquals(expectedSource, actualSource);
    // verify that the copy of the constructed DynamicTableSink is as expected
    assertEquals(expectedSource, actualSource.copy());
    // verify produced sink
    ScanTableSource.ScanRuntimeProvider functionProvider = actualSource.getScanRuntimeProvider(ScanRuntimeProviderContext.INSTANCE);
    SourceFunction<RowData> sourceFunction = as(functionProvider, SourceFunctionProvider.class).createSourceFunction();
    assertThat(sourceFunction, instanceOf(FlinkKinesisConsumer.class));
}

Example 33 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class ChangelogCsvDeserializer method deserialize.

@Override
public RowData deserialize(byte[] message) {
    // parse the columns including a changelog flag
    final String[] columns = new String(message).split(Pattern.quote(columnDelimiter));
    final RowKind kind = RowKind.valueOf(columns[0]);
    final Row row = new Row(kind, parsingTypes.size());
    for (int i = 0; i < parsingTypes.size(); i++) {
        row.setField(i, parse(parsingTypes.get(i).getTypeRoot(), columns[i + 1]));
    }
    // convert to internal data structure
    return (RowData) converter.toInternal(row);
}

Example 34 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class CanalJsonSerDeSchemaTest method runTest.

public void runTest(List<String> lines, CanalJsonDeserializationSchema deserializationSchema) throws Exception {
    SimpleCollector collector = new SimpleCollector();
    for (String line : lines) {
        deserializationSchema.deserialize(line.getBytes(StandardCharsets.UTF_8), collector);
    }
    // Canal captures change data (`canal-data.txt`) on the `product` table:
    // 
    // CREATE TABLE product (
    // id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
    // name VARCHAR(255),
    // description VARCHAR(512),
    // weight FLOAT
    // );
    // ALTER TABLE product AUTO_INCREMENT = 101;
    // 
    // INSERT INTO product
    // VALUES (default,"scooter","Small 2-wheel scooter",3.14),
    // (default,"car battery","12V car battery",8.1),
    // (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40
    // to #3",0.8),
    // (default,"hammer","12oz carpenter's hammer",0.75),
    // (default,"hammer","14oz carpenter's hammer",0.875),
    // (default,"hammer","16oz carpenter's hammer",1.0),
    // (default,"rocks","box of assorted rocks",5.3),
    // (default,"jacket","water resistent black wind breaker",0.1),
    // (default,"spare tire","24 inch spare tire",22.2);
    // UPDATE product SET description='18oz carpenter hammer' WHERE id=106;
    // UPDATE product SET weight='5.1' WHERE id=107;
    // INSERT INTO product VALUES (default,"jacket","water resistent white wind breaker",0.2);
    // INSERT INTO product VALUES (default,"scooter","Big 2-wheel scooter ",5.18);
    // UPDATE product SET description='new water resistent white wind breaker', weight='0.5'
    // WHERE id=110;
    // UPDATE product SET weight='5.17' WHERE id=111;
    // DELETE FROM product WHERE id=111;
    // UPDATE product SET weight='5.17' WHERE id=102 or id = 101;
    // DELETE FROM product WHERE id=102 or id = 103;
    List<String> expected = Arrays.asList("+I(101,scooter,Small 2-wheel scooter,3.14)", "+I(102,car battery,12V car battery,8.1)", "+I(103,12-pack drill bits,12-pack of drill bits with sizes ranging from #40 to #3,0.8)", "+I(104,hammer,12oz carpenter's hammer,0.75)", "+I(105,hammer,14oz carpenter's hammer,0.875)", "+I(106,hammer,null,1.0)", "+I(107,rocks,box of assorted rocks,5.3)", "+I(108,jacket,water resistent black wind breaker,0.1)", "+I(109,spare tire,24 inch spare tire,22.2)", "-U(106,hammer,null,1.0)", "+U(106,hammer,18oz carpenter hammer,1.0)", "-U(107,rocks,box of assorted rocks,5.3)", "+U(107,rocks,box of assorted rocks,5.1)", "+I(110,jacket,water resistent white wind breaker,0.2)", "+I(111,scooter,Big 2-wheel scooter ,5.18)", "-U(110,jacket,water resistent white wind breaker,0.2)", "+U(110,jacket,new water resistent white wind breaker,0.5)", "-U(111,scooter,Big 2-wheel scooter ,5.18)", "+U(111,scooter,Big 2-wheel scooter ,5.17)", "-D(111,scooter,Big 2-wheel scooter ,5.17)", "-U(101,scooter,Small 2-wheel scooter,3.14)", "+U(101,scooter,Small 2-wheel scooter,5.17)", "-U(102,car battery,12V car battery,8.1)", "+U(102,car battery,12V car battery,5.17)", "-D(102,car battery,12V car battery,5.17)", "-D(103,12-pack drill bits,12-pack of drill bits with sizes ranging from #40 to #3,0.8)");
    List<String> actual = collector.list.stream().map(Object::toString).collect(Collectors.toList());
    assertEquals(expected, actual);
    // test Serialization
    CanalJsonSerializationSchema serializationSchema = new CanalJsonSerializationSchema((RowType) PHYSICAL_DATA_TYPE.getLogicalType(), TimestampFormat.ISO_8601, JsonFormatOptions.MapNullKeyMode.LITERAL, "null", true);
    serializationSchema.open(null);
    List<String> result = new ArrayList<>();
    for (RowData rowData : collector.list) {
        result.add(new String(serializationSchema.serialize(rowData), StandardCharsets.UTF_8));
    }
    List<String> expectedResult = Arrays.asList("{\"data\":[{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":3.14}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":8.1}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":103,\"name\":\"12-pack drill bits\",\"description\":\"12-pack of drill bits with sizes ranging from #40 to #3\",\"weight\":0.8}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":104,\"name\":\"hammer\",\"description\":\"12oz carpenter's hammer\",\"weight\":0.75}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":105,\"name\":\"hammer\",\"description\":\"14oz carpenter's hammer\",\"weight\":0.875}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":106,\"name\":\"hammer\",\"description\":null,\"weight\":1.0}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.3}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":108,\"name\":\"jacket\",\"description\":\"water resistent black wind breaker\",\"weight\":0.1}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":109,\"name\":\"spare tire\",\"description\":\"24 inch spare tire\",\"weight\":22.2}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":106,\"name\":\"hammer\",\"description\":null,\"weight\":1.0}],\"type\":\"DELETE\"}", "{\"data\":[{\"id\":106,\"name\":\"hammer\",\"description\":\"18oz carpenter hammer\",\"weight\":1.0}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.3}],\"type\":\"DELETE\"}", "{\"data\":[{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.1}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":110,\"name\":\"jacket\",\"description\":\"water resistent white wind breaker\",\"weight\":0.2}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.18}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":110,\"name\":\"jacket\",\"description\":\"water resistent white wind breaker\",\"weight\":0.2}],\"type\":\"DELETE\"}", "{\"data\":[{\"id\":110,\"name\":\"jacket\",\"description\":\"new water resistent white wind breaker\",\"weight\":0.5}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.18}],\"type\":\"DELETE\"}", "{\"data\":[{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.17}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.17}],\"type\":\"DELETE\"}", "{\"data\":[{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":3.14}],\"type\":\"DELETE\"}", "{\"data\":[{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":5.17}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":8.1}],\"type\":\"DELETE\"}", "{\"data\":[{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":5.17}],\"type\":\"INSERT\"}", "{\"data\":[{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":5.17}],\"type\":\"DELETE\"}", "{\"data\":[{\"id\":103,\"name\":\"12-pack drill bits\",\"description\":\"12-pack of drill bits with sizes ranging from #40 to #3\",\"weight\":0.8}],\"type\":\"DELETE\"}");
    assertEquals(expectedResult, result);
}

Example 35 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class DebeziumJsonSerDeSchemaTest method testSerializationDeserialization.

private void testSerializationDeserialization(String resourceFile, boolean schemaInclude) throws Exception {
    List<String> lines = readLines(resourceFile);
    DebeziumJsonDeserializationSchema deserializationSchema = new DebeziumJsonDeserializationSchema(PHYSICAL_DATA_TYPE, Collections.emptyList(), InternalTypeInfo.of(PHYSICAL_DATA_TYPE.getLogicalType()), schemaInclude, false, TimestampFormat.ISO_8601);
    SimpleCollector collector = new SimpleCollector();
    for (String line : lines) {
        deserializationSchema.deserialize(line.getBytes(StandardCharsets.UTF_8), collector);
    }
    // Debezium captures change data (`debezium-data-schema-include.txt`) on the `product`
    // table:
    // 
    // CREATE TABLE product (
    // id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
    // name VARCHAR(255),
    // description VARCHAR(512),
    // weight FLOAT
    // );
    // ALTER TABLE product AUTO_INCREMENT = 101;
    // 
    // INSERT INTO product
    // VALUES (default,"scooter","Small 2-wheel scooter",3.14),
    // (default,"car battery","12V car battery",8.1),
    // (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40
    // to #3",0.8),
    // (default,"hammer","12oz carpenter's hammer",0.75),
    // (default,"hammer","14oz carpenter's hammer",0.875),
    // (default,"hammer","16oz carpenter's hammer",1.0),
    // (default,"rocks","box of assorted rocks",5.3),
    // (default,"jacket","water resistent black wind breaker",0.1),
    // (default,"spare tire","24 inch spare tire",22.2);
    // UPDATE product SET description='18oz carpenter hammer' WHERE id=106;
    // UPDATE product SET weight='5.1' WHERE id=107;
    // INSERT INTO product VALUES (default,"jacket","water resistent white wind breaker",0.2);
    // INSERT INTO product VALUES (default,"scooter","Big 2-wheel scooter ",5.18);
    // UPDATE product SET description='new water resistent white wind breaker', weight='0.5'
    // WHERE id=110;
    // UPDATE product SET weight='5.17' WHERE id=111;
    // DELETE FROM product WHERE id=111;
    List<String> expected = Arrays.asList("+I(101,scooter,Small 2-wheel scooter,3.14)", "+I(102,car battery,12V car battery,8.1)", "+I(103,12-pack drill bits,12-pack of drill bits with sizes ranging from #40 to #3,0.8)", "+I(104,hammer,12oz carpenter's hammer,0.75)", "+I(105,hammer,14oz carpenter's hammer,0.875)", "+I(106,hammer,16oz carpenter's hammer,1.0)", "+I(107,rocks,box of assorted rocks,5.3)", "+I(108,jacket,water resistent black wind breaker,0.1)", "+I(109,spare tire,24 inch spare tire,22.2)", "-U(106,hammer,16oz carpenter's hammer,1.0)", "+U(106,hammer,18oz carpenter hammer,1.0)", "-U(107,rocks,box of assorted rocks,5.3)", "+U(107,rocks,box of assorted rocks,5.1)", "+I(110,jacket,water resistent white wind breaker,0.2)", "+I(111,scooter,Big 2-wheel scooter ,5.18)", "-U(110,jacket,water resistent white wind breaker,0.2)", "+U(110,jacket,new water resistent white wind breaker,0.5)", "-U(111,scooter,Big 2-wheel scooter ,5.18)", "+U(111,scooter,Big 2-wheel scooter ,5.17)", "-D(111,scooter,Big 2-wheel scooter ,5.17)");
    List<String> actual = collector.list.stream().map(Object::toString).collect(Collectors.toList());
    assertEquals(expected, actual);
    DebeziumJsonSerializationSchema serializationSchema = new DebeziumJsonSerializationSchema((RowType) PHYSICAL_DATA_TYPE.getLogicalType(), TimestampFormat.SQL, JsonFormatOptions.MapNullKeyMode.LITERAL, "null", true);
    serializationSchema.open(null);
    actual = new ArrayList<>();
    for (RowData rowData : collector.list) {
        actual.add(new String(serializationSchema.serialize(rowData), StandardCharsets.UTF_8));
    }
    expected = Arrays.asList("{\"before\":null,\"after\":{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":3.14},\"op\":\"c\"}", "{\"before\":null,\"after\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":8.1},\"op\":\"c\"}", "{\"before\":null,\"after\":{\"id\":103,\"name\":\"12-pack drill bits\",\"description\":\"12-pack of drill bits with sizes ranging from #40 to #3\",\"weight\":0.8},\"op\":\"c\"}", "{\"before\":null,\"after\":{\"id\":104,\"name\":\"hammer\",\"description\":\"12oz carpenter's hammer\",\"weight\":0.75},\"op\":\"c\"}", "{\"before\":null,\"after\":{\"id\":105,\"name\":\"hammer\",\"description\":\"14oz carpenter's hammer\",\"weight\":0.875},\"op\":\"c\"}", "{\"before\":null,\"after\":{\"id\":106,\"name\":\"hammer\",\"description\":\"16oz carpenter's hammer\",\"weight\":1.0},\"op\":\"c\"}", "{\"before\":null,\"after\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.3},\"op\":\"c\"}", "{\"before\":null,\"after\":{\"id\":108,\"name\":\"jacket\",\"description\":\"water resistent black wind breaker\",\"weight\":0.1},\"op\":\"c\"}", "{\"before\":null,\"after\":{\"id\":109,\"name\":\"spare tire\",\"description\":\"24 inch spare tire\",\"weight\":22.2},\"op\":\"c\"}", "{\"before\":{\"id\":106,\"name\":\"hammer\",\"description\":\"16oz carpenter's hammer\",\"weight\":1.0},\"after\":null,\"op\":\"d\"}", "{\"before\":null,\"after\":{\"id\":106,\"name\":\"hammer\",\"description\":\"18oz carpenter hammer\",\"weight\":1.0},\"op\":\"c\"}", "{\"before\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.3},\"after\":null,\"op\":\"d\"}", "{\"before\":null,\"after\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.1},\"op\":\"c\"}", "{\"before\":null,\"after\":{\"id\":110,\"name\":\"jacket\",\"description\":\"water resistent white wind breaker\",\"weight\":0.2},\"op\":\"c\"}", "{\"before\":null,\"after\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.18},\"op\":\"c\"}", "{\"before\":{\"id\":110,\"name\":\"jacket\",\"description\":\"water resistent white wind breaker\",\"weight\":0.2},\"after\":null,\"op\":\"d\"}", "{\"before\":null,\"after\":{\"id\":110,\"name\":\"jacket\",\"description\":\"new water resistent white wind breaker\",\"weight\":0.5},\"op\":\"c\"}", "{\"before\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.18},\"after\":null,\"op\":\"d\"}", "{\"before\":null,\"after\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.17},\"op\":\"c\"}", "{\"before\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.17},\"after\":null,\"op\":\"d\"}");
    assertEquals(expected, actual);
}