Examples with GenericRowData org.apache.flink.table.data.GenericRowData used on opensource projects

Example 11 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class FileInfoExtractorBulkFormat method wrapReader.

private Reader<RowData> wrapReader(Reader<RowData> superReader, FileSourceSplit split) {
    // Fill the metadata + partition columns row
    final GenericRowData fileInfoRowData = new GenericRowData(metadataColumnsFunctions.size() + partitionColumnTypes.size());
    int fileInfoRowIndex = 0;
    for (; fileInfoRowIndex < metadataColumnsFunctions.size(); fileInfoRowIndex++) {
        fileInfoRowData.setField(fileInfoRowIndex, metadataColumnsFunctions.get(fileInfoRowIndex).getValue(split));
    }
    if (!partitionColumnTypes.isEmpty()) {
        final LinkedHashMap<String, String> partitionSpec = PartitionPathUtils.extractPartitionSpecFromPath(split.path());
        for (int partitionFieldIndex = 0; fileInfoRowIndex < fileInfoRowData.getArity(); fileInfoRowIndex++, partitionFieldIndex++) {
            final String fieldName = partitionColumnTypes.get(partitionFieldIndex).getKey();
            final DataType fieldType = partitionColumnTypes.get(partitionFieldIndex).getValue();
            if (!partitionSpec.containsKey(fieldName)) {
                throw new RuntimeException("Cannot find the partition value from path for partition: " + fieldName);
            }
            String valueStr = partitionSpec.get(fieldName);
            valueStr = valueStr.equals(defaultPartName) ? null : valueStr;
            fileInfoRowData.setField(fileInfoRowIndex, PartitionPathUtils.convertStringToInternalValue(valueStr, fieldType));
        }
    }
    // This row is going to be reused for every record
    final EnrichedRowData producedRowData = new EnrichedRowData(fileInfoRowData, this.extendedRowIndexMapping);
    return RecordMapperWrapperRecordIterator.wrapReader(superReader, physicalRowData -> {
        producedRowData.replaceMutableRow(physicalRowData);
        return producedRowData;
    });
}

Example 12 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class RowDataKinesisDeserializationSchema method deserialize.

@Override
public RowData deserialize(byte[] recordValue, String partitionKey, String seqNum, long approxArrivalTimestamp, String stream, String shardId) throws IOException {
    RowData physicalRow = physicalDeserializer.deserialize(recordValue);
    GenericRowData metadataRow = new GenericRowData(requestedMetadataFields.size());
    for (int i = 0; i < metadataRow.getArity(); i++) {
        Metadata metadataField = requestedMetadataFields.get(i);
        if (metadataField == Metadata.Timestamp) {
            metadataRow.setField(i, TimestampData.fromEpochMillis(approxArrivalTimestamp));
        } else if (metadataField == Metadata.SequenceNumber) {
            metadataRow.setField(i, StringData.fromString(seqNum));
        } else if (metadataField == Metadata.ShardId) {
            metadataRow.setField(i, StringData.fromString(shardId));
        } else {
            String msg = String.format("Unsupported metadata key %s", metadataField);
            // should never happen
            throw new RuntimeException(msg);
        }
    }
    return new JoinedRowData(physicalRow.getRowKind(), physicalRow, metadataRow);
}

Example 13 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class OrcBulkRowDataWriterTest method readList.

/**
 * Read ListColumnVector with specify schema {@literal array<struct<_col2_col0:string>>}.
 */
private static ArrayData readList(ListColumnVector listVector, int row) {
    int offset = (int) listVector.offsets[row];
    StructColumnVector structChild = (StructColumnVector) listVector.child;
    BytesColumnVector valueChild = (BytesColumnVector) structChild.fields[0];
    StringData value1 = readStringData(valueChild, offset);
    GenericRowData arrayValue1 = new GenericRowData(1);
    arrayValue1.setField(0, value1);
    StringData value2 = readStringData(valueChild, offset + 1);
    GenericRowData arrayValue2 = new GenericRowData(1);
    arrayValue2.setField(0, (value2));
    return new GenericArrayData(new Object[] { arrayValue1, arrayValue2 });
}

Example 14 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class RowDataSerializer method deserialize.

@Override
public RowData deserialize(DataInputView source) throws IOException {
    // read bitmask
    readIntoMask(source, mask);
    GenericRowData row = new GenericRowData(fieldSerializers.length);
    row.setRowKind(readKindFromMask(mask));
    for (int i = 0; i < row.getArity(); i++) {
        if (mask[i + ROW_KIND_OFFSET]) {
            row.setField(i, null);
        } else {
            row.setField(i, fieldSerializers[i].deserialize(source));
        }
    }
    return row;
}

Example 15 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class EmbeddedPythonScalarFunctionOperator method open.

@SuppressWarnings("unchecked")
@Override
public void open() throws Exception {
    isOneArg = udfInputOffsets.length == 1;
    isOneFieldResult = udfOutputType.getFieldCount() == 1;
    super.open();
    rowDataWrapper = new StreamRecordRowDataWrappingCollector(output);
    reuseResultRowData = new GenericRowData(udfOutputType.getFieldCount());
    RowType userDefinedFunctionInputType = new RowType(Arrays.stream(udfInputOffsets).mapToObj(i -> inputType.getFields().get(i)).collect(Collectors.toList()));
    userDefinedFunctionInputConverters = userDefinedFunctionInputType.getFields().stream().map(RowType.RowField::getType).map(PythonTypeUtils::toDataConverter).toArray(PythonTypeUtils.DataConverter[]::new);
    userDefinedFunctionInputArgs = new Object[udfInputOffsets.length];
    userDefinedFunctionOutputConverters = udfOutputType.getFields().stream().map(RowType.RowField::getType).map(PythonTypeUtils::toDataConverter).toArray(PythonTypeUtils.DataConverter[]::new);
    if (forwardedFieldGeneratedProjection != null) {
        forwardedFieldProjection = forwardedFieldGeneratedProjection.newInstance(Thread.currentThread().getContextClassLoader());
    }
}