Examples with GenericRowData org.apache.flink.table.data.GenericRowData used on opensource projects

Example 6 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class CsvToRowDataConverters method createRowConverter.

public CsvToRowDataConverter createRowConverter(RowType rowType, boolean isTopLevel) {
    final CsvToRowDataConverter[] fieldConverters = rowType.getFields().stream().map(RowType.RowField::getType).map(this::createNullableConverter).toArray(CsvToRowDataConverter[]::new);
    final String[] fieldNames = rowType.getFieldNames().toArray(new String[0]);
    final int arity = fieldNames.length;
    return jsonNode -> {
        int nodeSize = jsonNode.size();
        if (nodeSize != 0) {
            validateArity(arity, nodeSize, ignoreParseErrors);
        } else {
            return null;
        }
        GenericRowData row = new GenericRowData(arity);
        for (int i = 0; i < arity; i++) {
            JsonNode field;
            // Jackson only supports mapping by name in the first level
            if (isTopLevel) {
                field = jsonNode.get(fieldNames[i]);
            } else {
                field = jsonNode.get(i);
            }
            try {
                if (field == null) {
                    row.setField(i, null);
                } else {
                    row.setField(i, fieldConverters[i].convert(field));
                }
            } catch (Throwable t) {
                throw new RuntimeException(String.format("Fail to deserialize at field: %s.", fieldNames[i]), t);
            }
        }
        return row;
    };
}

Example 7 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class HiveLookupJoinITCase method testPartitionFetcherAndReader.

@Test
public void testPartitionFetcherAndReader() throws Exception {
    // constructs test data using dynamic partition
    TableEnvironment batchEnv = HiveTestUtils.createTableEnvInBatchMode(SqlDialect.HIVE);
    batchEnv.registerCatalog(hiveCatalog.getName(), hiveCatalog);
    batchEnv.useCatalog(hiveCatalog.getName());
    batchEnv.executeSql("insert overwrite partition_table values " + "(1,'a',08,2019,'08','01')," + "(1,'a',10,2020,'08','31')," + "(2,'a',21,2020,'08','31')," + "(2,'b',22,2020,'08','31')," + "(3,'c',33,2020,'09','31')").await();
    FileSystemLookupFunction<HiveTablePartition> lookupFunction = getLookupFunction("partition_table");
    lookupFunction.open(null);
    PartitionFetcher<HiveTablePartition> fetcher = lookupFunction.getPartitionFetcher();
    PartitionFetcher.Context<HiveTablePartition> context = lookupFunction.getFetcherContext();
    List<HiveTablePartition> partitions = fetcher.fetch(context);
    // fetch latest partition by partition-name
    assertEquals(1, partitions.size());
    PartitionReader<HiveTablePartition, RowData> reader = lookupFunction.getPartitionReader();
    reader.open(partitions);
    List<RowData> res = new ArrayList<>();
    ObjectIdentifier tableIdentifier = ObjectIdentifier.of(hiveCatalog.getName(), "default", "partition_table");
    CatalogTable catalogTable = (CatalogTable) hiveCatalog.getTable(tableIdentifier.toObjectPath());
    GenericRowData reuse = new GenericRowData(catalogTable.getSchema().getFieldCount());
    TypeSerializer<RowData> serializer = InternalSerializers.create(catalogTable.getSchema().toRowDataType().getLogicalType());
    RowData row;
    while ((row = reader.read(reuse)) != null) {
        res.add(serializer.copy(row));
    }
    res.sort(Comparator.comparingInt(o -> o.getInt(0)));
    assertEquals("[+I(3,c,33,2020,09,31)]", res.toString());
}

Example 8 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class MaxwellJsonDeserializationSchema method emitRow.

private void emitRow(GenericRowData rootRow, GenericRowData physicalRow, Collector<RowData> out) {
    // shortcut in case no output projection is required
    if (!hasMetadata) {
        out.collect(physicalRow);
        return;
    }
    final int metadataArity = metadataConverters.length;
    final GenericRowData producedRow = new GenericRowData(physicalRow.getRowKind(), fieldCount + metadataArity);
    for (int physicalPos = 0; physicalPos < fieldCount; physicalPos++) {
        producedRow.setField(physicalPos, physicalRow.getField(physicalPos));
    }
    for (int metadataPos = 0; metadataPos < metadataArity; metadataPos++) {
        producedRow.setField(fieldCount + metadataPos, metadataConverters[metadataPos].convert(rootRow));
    }
    out.collect(producedRow);
}

Example 9 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class JsonRowDataSerDeSchemaTest method testSerializationMapNullKey.

@Test
public void testSerializationMapNullKey() throws Exception {
    RowType rowType = (RowType) ROW(FIELD("nestedMap", MAP(STRING(), MAP(STRING(), INT())))).getLogicalType();
    // test data
    // use LinkedHashMap to make sure entries order
    Map<StringData, Integer> map = new LinkedHashMap<>();
    map.put(StringData.fromString("no-null key"), 1);
    map.put(StringData.fromString(null), 2);
    GenericMapData mapData = new GenericMapData(map);
    Map<StringData, GenericMapData> nestedMap = new LinkedHashMap<>();
    nestedMap.put(StringData.fromString("no-null key"), mapData);
    nestedMap.put(StringData.fromString(null), mapData);
    GenericMapData nestedMapData = new GenericMapData(nestedMap);
    GenericRowData rowData = new GenericRowData(1);
    rowData.setField(0, nestedMapData);
    JsonRowDataSerializationSchema serializationSchema1 = new JsonRowDataSerializationSchema(rowType, TimestampFormat.SQL, JsonFormatOptions.MapNullKeyMode.FAIL, "null", true);
    // expect message for serializationSchema1
    String errorMessage1 = "JSON format doesn't support to serialize map data with null keys." + " You can drop null key entries or encode null in literals by specifying map-null-key.mode option.";
    JsonRowDataSerializationSchema serializationSchema2 = new JsonRowDataSerializationSchema(rowType, TimestampFormat.SQL, JsonFormatOptions.MapNullKeyMode.DROP, "null", true);
    // expect result for serializationSchema2
    String expectResult2 = "{\"nestedMap\":{\"no-null key\":{\"no-null key\":1}}}";
    JsonRowDataSerializationSchema serializationSchema3 = new JsonRowDataSerializationSchema(rowType, TimestampFormat.SQL, JsonFormatOptions.MapNullKeyMode.LITERAL, "nullKey", true);
    // expect result for serializationSchema3
    String expectResult3 = "{\"nestedMap\":{\"no-null key\":{\"no-null key\":1,\"nullKey\":2},\"nullKey\":{\"no-null key\":1,\"nullKey\":2}}}";
    try {
        // throw exception when mapNullKey Mode is fail
        serializationSchema1.serialize(rowData);
        Assert.fail("expecting exception message: " + errorMessage1);
    } catch (Throwable t) {
        assertThat(t, FlinkMatchers.containsMessage(errorMessage1));
    }
    // mapNullKey Mode is drop
    byte[] actual2 = serializationSchema2.serialize(rowData);
    assertEquals(expectResult2, new String(actual2));
    // mapNullKey Mode is literal
    byte[] actual3 = serializationSchema3.serialize(rowData);
    assertEquals(expectResult3, new String(actual3));
}

Example 10 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class RowDataVectorizer method convert.

/**
 * Converting ArrayData to RowData for calling {@link RowDataVectorizer#setColumn(int,
 * ColumnVector, LogicalType, RowData, int)} recursively with array.
 *
 * @param arrayData input ArrayData.
 * @param arrayFieldType LogicalType of input ArrayData.
 * @return RowData.
 */
private static RowData convert(ArrayData arrayData, LogicalType arrayFieldType) {
    GenericRowData rowData = new GenericRowData(arrayData.size());
    ArrayData.ElementGetter elementGetter = ArrayData.createElementGetter(arrayFieldType);
    for (int i = 0; i < arrayData.size(); i++) {
        rowData.setField(i, elementGetter.getElementOrNull(arrayData, i));
    }
    return rowData;
}