Examples with GenericRowData org.apache.flink.table.data.GenericRowData used on opensource projects

Example 21 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class CanalJsonDeserializationSchema method deserialize.

@Override
public void deserialize(@Nullable byte[] message, Collector<RowData> out) throws IOException {
    if (message == null || message.length == 0) {
        return;
    }
    try {
        final JsonNode root = jsonDeserializer.deserializeToJsonNode(message);
        if (database != null) {
            if (!databasePattern.matcher(root.get(ReadableMetadata.DATABASE.key).asText()).matches()) {
                return;
            }
        }
        if (table != null) {
            if (!tablePattern.matcher(root.get(ReadableMetadata.TABLE.key).asText()).matches()) {
                return;
            }
        }
        final GenericRowData row = (GenericRowData) jsonDeserializer.convertToRowData(root);
        // "type" field
        String type = row.getString(2).toString();
        if (OP_INSERT.equals(type)) {
            // "data" field is an array of row, contains inserted rows
            ArrayData data = row.getArray(0);
            for (int i = 0; i < data.size(); i++) {
                GenericRowData insert = (GenericRowData) data.getRow(i, fieldCount);
                insert.setRowKind(RowKind.INSERT);
                emitRow(row, insert, out);
            }
        } else if (OP_UPDATE.equals(type)) {
            // "data" field is an array of row, contains new rows
            ArrayData data = row.getArray(0);
            // "old" field is an array of row, contains old values
            ArrayData old = row.getArray(1);
            for (int i = 0; i < data.size(); i++) {
                // the underlying JSON deserialization schema always produce GenericRowData.
                GenericRowData after = (GenericRowData) data.getRow(i, fieldCount);
                GenericRowData before = (GenericRowData) old.getRow(i, fieldCount);
                final JsonNode oldField = root.get(FIELD_OLD);
                for (int f = 0; f < fieldCount; f++) {
                    if (before.isNullAt(f) && oldField.findValue(fieldNames.get(f)) == null) {
                        // fields in "old" (before) means the fields are changed
                        // fields not in "old" (before) means the fields are not changed
                        // so we just copy the not changed fields into before
                        before.setField(f, after.getField(f));
                    }
                }
                before.setRowKind(RowKind.UPDATE_BEFORE);
                after.setRowKind(RowKind.UPDATE_AFTER);
                emitRow(row, before, out);
                emitRow(row, after, out);
            }
        } else if (OP_DELETE.equals(type)) {
            // "data" field is an array of row, contains deleted rows
            ArrayData data = row.getArray(0);
            for (int i = 0; i < data.size(); i++) {
                GenericRowData insert = (GenericRowData) data.getRow(i, fieldCount);
                insert.setRowKind(RowKind.DELETE);
                emitRow(row, insert, out);
            }
        } else if (OP_CREATE.equals(type)) {
            // this is a DDL change event, and we should skip it.
            return;
        } else {
            if (!ignoreParseErrors) {
                throw new IOException(format("Unknown \"type\" value \"%s\". The Canal JSON message is '%s'", type, new String(message)));
            }
        }
    } catch (Throwable t) {
        // a big try catch to protect the processing.
        if (!ignoreParseErrors) {
            throw new IOException(format("Corrupt Canal JSON message '%s'.", new String(message)), t);
        }
    }
}

Example 22 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class CanalJsonDeserializationSchema method emitRow.

private void emitRow(GenericRowData rootRow, GenericRowData physicalRow, Collector<RowData> out) {
    // shortcut in case no output projection is required
    if (!hasMetadata) {
        out.collect(physicalRow);
        return;
    }
    final int physicalArity = physicalRow.getArity();
    final int metadataArity = metadataConverters.length;
    final GenericRowData producedRow = new GenericRowData(physicalRow.getRowKind(), physicalArity + metadataArity);
    for (int physicalPos = 0; physicalPos < physicalArity; physicalPos++) {
        producedRow.setField(physicalPos, physicalRow.getField(physicalPos));
    }
    for (int metadataPos = 0; metadataPos < metadataArity; metadataPos++) {
        producedRow.setField(physicalArity + metadataPos, metadataConverters[metadataPos].convert(rootRow));
    }
    out.collect(producedRow);
}

Example 23 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class DebeziumJsonDeserializationSchema method emitRow.

private void emitRow(GenericRowData rootRow, GenericRowData physicalRow, Collector<RowData> out) {
    // shortcut in case no output projection is required
    if (!hasMetadata) {
        out.collect(physicalRow);
        return;
    }
    final int physicalArity = physicalRow.getArity();
    final int metadataArity = metadataConverters.length;
    final GenericRowData producedRow = new GenericRowData(physicalRow.getRowKind(), physicalArity + metadataArity);
    for (int physicalPos = 0; physicalPos < physicalArity; physicalPos++) {
        producedRow.setField(physicalPos, physicalRow.getField(physicalPos));
    }
    for (int metadataPos = 0; metadataPos < metadataArity; metadataPos++) {
        producedRow.setField(physicalArity + metadataPos, metadataConverters[metadataPos].convert(rootRow));
    }
    out.collect(producedRow);
}

Example 24 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class InternalDataUtils method toGenericRow.

static GenericRowData toGenericRow(RowData rowData, LogicalType logicalType) {
    final List<LogicalType> fieldTypes = LogicalTypeChecks.getFieldTypes(logicalType);
    final GenericRowData row = new GenericRowData(fieldTypes.size());
    row.setRowKind(rowData.getRowKind());
    for (int i = 0; i < fieldTypes.size(); i++) {
        if (rowData.isNullAt(i)) {
            row.setField(i, null);
        } else {
            LogicalType fieldType = fieldTypes.get(i);
            RowData.FieldGetter fieldGetter = RowData.createFieldGetter(fieldType, i);
            row.setField(i, toGenericInternalData(fieldGetter.getFieldOrNull(rowData), fieldType));
        }
    }
    return row;
}

Example 25 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class RowDataAssert method asGeneric.

public RowDataAssert asGeneric(LogicalType logicalType) {
    GenericRowData actual = InternalDataUtils.toGenericRow(this.actual, logicalType);
    return new RowDataAssert(actual).usingComparator((x, y) -> {
        // Avoid converting actual again
        x = x == actual ? x : InternalDataUtils.toGenericRow(x, logicalType);
        y = y == actual ? y : InternalDataUtils.toGenericRow(y, logicalType);
        if (Objects.equals(x, y)) {
            return 0;
        }
        return Objects.hashCode(x) < Objects.hashCode(y) ? -1 : 1;
    });
}