Examples with StructVector org.apache.arrow.vector.complex.StructVector used on opensource projects

Example 1 with StructVector

use of org.apache.arrow.vector.complex.StructVector in project flink by apache.

the class ArrowUtils method createArrowFieldWriterForArray.

private static ArrowFieldWriter<ArrayData> createArrowFieldWriterForArray(ValueVector vector, LogicalType fieldType) {
    if (vector instanceof TinyIntVector) {
        return TinyIntWriter.forArray((TinyIntVector) vector);
    } else if (vector instanceof SmallIntVector) {
        return SmallIntWriter.forArray((SmallIntVector) vector);
    } else if (vector instanceof IntVector) {
        return IntWriter.forArray((IntVector) vector);
    } else if (vector instanceof BigIntVector) {
        return BigIntWriter.forArray((BigIntVector) vector);
    } else if (vector instanceof BitVector) {
        return BooleanWriter.forArray((BitVector) vector);
    } else if (vector instanceof Float4Vector) {
        return FloatWriter.forArray((Float4Vector) vector);
    } else if (vector instanceof Float8Vector) {
        return DoubleWriter.forArray((Float8Vector) vector);
    } else if (vector instanceof VarCharVector) {
        return VarCharWriter.forArray((VarCharVector) vector);
    } else if (vector instanceof VarBinaryVector) {
        return VarBinaryWriter.forArray((VarBinaryVector) vector);
    } else if (vector instanceof DecimalVector) {
        DecimalVector decimalVector = (DecimalVector) vector;
        return DecimalWriter.forArray(decimalVector, getPrecision(decimalVector), decimalVector.getScale());
    } else if (vector instanceof DateDayVector) {
        return DateWriter.forArray((DateDayVector) vector);
    } else if (vector instanceof TimeSecVector || vector instanceof TimeMilliVector || vector instanceof TimeMicroVector || vector instanceof TimeNanoVector) {
        return TimeWriter.forArray(vector);
    } else if (vector instanceof TimeStampVector && ((ArrowType.Timestamp) vector.getField().getType()).getTimezone() == null) {
        int precision;
        if (fieldType instanceof LocalZonedTimestampType) {
            precision = ((LocalZonedTimestampType) fieldType).getPrecision();
        } else {
            precision = ((TimestampType) fieldType).getPrecision();
        }
        return TimestampWriter.forArray(vector, precision);
    } else if (vector instanceof ListVector) {
        ListVector listVector = (ListVector) vector;
        LogicalType elementType = ((ArrayType) fieldType).getElementType();
        return ArrayWriter.forArray(listVector, createArrowFieldWriterForArray(listVector.getDataVector(), elementType));
    } else if (vector instanceof StructVector) {
        RowType rowType = (RowType) fieldType;
        ArrowFieldWriter<RowData>[] fieldsWriters = new ArrowFieldWriter[rowType.getFieldCount()];
        for (int i = 0; i < fieldsWriters.length; i++) {
            fieldsWriters[i] = createArrowFieldWriterForRow(((StructVector) vector).getVectorById(i), rowType.getTypeAt(i));
        }
        return RowWriter.forArray((StructVector) vector, fieldsWriters);
    } else {
        throw new UnsupportedOperationException(String.format("Unsupported type %s.", fieldType));
    }
}

Example 2 with StructVector

use of org.apache.arrow.vector.complex.StructVector in project flink by apache.

the class RowWriter method doWrite.

@Override
public void doWrite(T in, int ordinal) {
    RowData row;
    if (isNullAt(in, ordinal)) {
        row = nullRow;
        ((StructVector) getValueVector()).setNull(getCount());
    } else {
        row = readRow(in, ordinal);
        ((StructVector) getValueVector()).setIndexDefined(getCount());
    }
    for (int i = 0; i < fieldsWriters.length; i++) {
        fieldsWriters[i].write(row, i);
    }
}

Example 3 with StructVector

use of org.apache.arrow.vector.complex.StructVector in project textdb by TextDB.

the class NltkSentimentOperator method convertListOfSpans.

// For now we're only allowing List<Span>. This can (and should) be generalized in the future.
private void convertListOfSpans(ImmutableList<Span> spansList, ListVector listVector, int index, String name) {
    if (index == 0) {
        if (innerIndexMap.containsKey(name))
            innerIndexMap.replace(name, 0);
        else
            innerIndexMap.put(name, 0);
    }
    int innerIndex = innerIndexMap.get(name);
    int size = spansList.size();
    StructVector subElementsVector = (StructVector) listVector.getDataVector();
    listVector.startNewValue(index);
    VarCharVector attributeNameVector = (VarCharVector) subElementsVector.getVectorById(0);
    IntVector startVector = (IntVector) subElementsVector.getVectorById(1);
    IntVector endVector = (IntVector) subElementsVector.getVectorById(2);
    VarCharVector keyVector = (VarCharVector) subElementsVector.getVectorById(3);
    VarCharVector valueVector = (VarCharVector) subElementsVector.getVectorById(4);
    IntVector tokenOffsetVector = (IntVector) subElementsVector.getVectorById(5);
    for (int i = 0; i < size; i++) {
        if (spansList.get(i) == null) {
            subElementsVector.setNull(innerIndex);
        } else {
            subElementsVector.setIndexDefined(innerIndex);
            Span span = spansList.get(i);
            // For all the fields of the struct
            if (span.getAttributeName() != null)
                attributeNameVector.setSafe(innerIndex, span.getAttributeName().getBytes(StandardCharsets.UTF_8));
            startVector.setSafe(innerIndex, span.getStart());
            endVector.setSafe(innerIndex, span.getEnd());
            if (span.getKey() != null)
                keyVector.setSafe(innerIndex, span.getKey().getBytes(StandardCharsets.UTF_8));
            if (span.getValue() != null)
                valueVector.setSafe(innerIndex, span.getValue().getBytes(StandardCharsets.UTF_8));
            tokenOffsetVector.setSafe(innerIndex, span.getTokenOffset());
        }
        innerIndex++;
    }
    innerIndexMap.replace(name, innerIndex);
    listVector.endValue(index, size);
}

Example 4 with StructVector

use of org.apache.arrow.vector.complex.StructVector in project textdb by TextDB.

the class NltkSentimentOperator method vectorizeTupleToPython.

private void vectorizeTupleToPython(Tuple tuple, int index, VectorSchemaRoot schemaRoot) {
    for (Attribute a : tuple.getSchema().getAttributes()) {
        String name = a.getName();
        // When it is null, skip it.
        if (tuple.getField(name).getValue() == null)
            continue;
        switch(a.getType()) {
            case INTEGER:
                ((IntVector) schemaRoot.getVector(name)).setSafe(index, (int) tuple.getField(name).getValue());
                break;
            case DOUBLE:
                ((Float8Vector) schemaRoot.getVector(name)).setSafe(index, (double) tuple.getField(name).getValue());
                break;
            case BOOLEAN:
            // break;
            case TEXT:
            case STRING:
            case _ID_TYPE:
                ((VarCharVector) schemaRoot.getVector(name)).setSafe(index, tuple.getField(name).getValue().toString().getBytes(StandardCharsets.UTF_8));
                break;
            case DATE:
                ((DateDayVector) schemaRoot.getVector(name)).setSafe(index, (int) ((LocalDate) tuple.getField(name).getValue()).toEpochDay());
                break;
            case DATETIME:
                StructVector dateTimeStructs = ((StructVector) schemaRoot.getVector(name));
                if (tuple.getField(name).getValue() != null) {
                    dateTimeStructs.setIndexDefined(index);
                    DateDayVector subVectorDay = (DateDayVector) dateTimeStructs.getVectorById(0);
                    TimeSecVector subVectorTime = (TimeSecVector) dateTimeStructs.getVectorById(1);
                    LocalDateTime value = (LocalDateTime) tuple.getField(name).getValue();
                    subVectorDay.setSafe(index, (int) value.toLocalDate().toEpochDay());
                    subVectorTime.setSafe(index, value.toLocalTime().toSecondOfDay());
                } else
                    dateTimeStructs.setNull(index);
                break;
            case LIST:
                // For now only supporting span.
                if (((ImmutableList) tuple.getField(name).getValue()).get(0).getClass() != Span.class) {
                    throw (new DataflowException("Unsupported Element Type for List Field!"));
                } else {
                    ListVector listVector = (ListVector) schemaRoot.getVector(name);
                    ImmutableList<Span> spansList = (ImmutableList<Span>) tuple.getField(name).getValue();
                    convertListOfSpans(spansList, listVector, index, name);
                }
                break;
            default:
                break;
        }
    }
}

Example 5 with StructVector

use of org.apache.arrow.vector.complex.StructVector in project flink by apache.

the class ArrowUtils method createColumnVector.

public static ColumnVector createColumnVector(ValueVector vector, LogicalType fieldType) {
    if (vector instanceof TinyIntVector) {
        return new ArrowTinyIntColumnVector((TinyIntVector) vector);
    } else if (vector instanceof SmallIntVector) {
        return new ArrowSmallIntColumnVector((SmallIntVector) vector);
    } else if (vector instanceof IntVector) {
        return new ArrowIntColumnVector((IntVector) vector);
    } else if (vector instanceof BigIntVector) {
        return new ArrowBigIntColumnVector((BigIntVector) vector);
    } else if (vector instanceof BitVector) {
        return new ArrowBooleanColumnVector((BitVector) vector);
    } else if (vector instanceof Float4Vector) {
        return new ArrowFloatColumnVector((Float4Vector) vector);
    } else if (vector instanceof Float8Vector) {
        return new ArrowDoubleColumnVector((Float8Vector) vector);
    } else if (vector instanceof VarCharVector) {
        return new ArrowVarCharColumnVector((VarCharVector) vector);
    } else if (vector instanceof VarBinaryVector) {
        return new ArrowVarBinaryColumnVector((VarBinaryVector) vector);
    } else if (vector instanceof DecimalVector) {
        return new ArrowDecimalColumnVector((DecimalVector) vector);
    } else if (vector instanceof DateDayVector) {
        return new ArrowDateColumnVector((DateDayVector) vector);
    } else if (vector instanceof TimeSecVector || vector instanceof TimeMilliVector || vector instanceof TimeMicroVector || vector instanceof TimeNanoVector) {
        return new ArrowTimeColumnVector(vector);
    } else if (vector instanceof TimeStampVector && ((ArrowType.Timestamp) vector.getField().getType()).getTimezone() == null) {
        return new ArrowTimestampColumnVector(vector);
    } else if (vector instanceof ListVector) {
        ListVector listVector = (ListVector) vector;
        return new ArrowArrayColumnVector(listVector, createColumnVector(listVector.getDataVector(), ((ArrayType) fieldType).getElementType()));
    } else if (vector instanceof StructVector) {
        StructVector structVector = (StructVector) vector;
        ColumnVector[] fieldColumns = new ColumnVector[structVector.size()];
        for (int i = 0; i < fieldColumns.length; ++i) {
            fieldColumns[i] = createColumnVector(structVector.getVectorById(i), ((RowType) fieldType).getTypeAt(i));
        }
        return new ArrowRowColumnVector(structVector, fieldColumns);
    } else {
        throw new UnsupportedOperationException(String.format("Unsupported type %s.", fieldType));
    }
}