use of org.apache.arrow.vector.complex.ListVector in project flink by apache.
the class ArrowUtils method createArrowFieldWriterForRow.
private static ArrowFieldWriter<RowData> createArrowFieldWriterForRow(ValueVector vector, LogicalType fieldType) {
if (vector instanceof TinyIntVector) {
return TinyIntWriter.forRow((TinyIntVector) vector);
} else if (vector instanceof SmallIntVector) {
return SmallIntWriter.forRow((SmallIntVector) vector);
} else if (vector instanceof IntVector) {
return IntWriter.forRow((IntVector) vector);
} else if (vector instanceof BigIntVector) {
return BigIntWriter.forRow((BigIntVector) vector);
} else if (vector instanceof BitVector) {
return BooleanWriter.forRow((BitVector) vector);
} else if (vector instanceof Float4Vector) {
return FloatWriter.forRow((Float4Vector) vector);
} else if (vector instanceof Float8Vector) {
return DoubleWriter.forRow((Float8Vector) vector);
} else if (vector instanceof VarCharVector) {
return VarCharWriter.forRow((VarCharVector) vector);
} else if (vector instanceof VarBinaryVector) {
return VarBinaryWriter.forRow((VarBinaryVector) vector);
} else if (vector instanceof DecimalVector) {
DecimalVector decimalVector = (DecimalVector) vector;
return DecimalWriter.forRow(decimalVector, getPrecision(decimalVector), decimalVector.getScale());
} else if (vector instanceof DateDayVector) {
return DateWriter.forRow((DateDayVector) vector);
} else if (vector instanceof TimeSecVector || vector instanceof TimeMilliVector || vector instanceof TimeMicroVector || vector instanceof TimeNanoVector) {
return TimeWriter.forRow(vector);
} else if (vector instanceof TimeStampVector && ((ArrowType.Timestamp) vector.getField().getType()).getTimezone() == null) {
int precision;
if (fieldType instanceof LocalZonedTimestampType) {
precision = ((LocalZonedTimestampType) fieldType).getPrecision();
} else {
precision = ((TimestampType) fieldType).getPrecision();
}
return TimestampWriter.forRow(vector, precision);
} else if (vector instanceof ListVector) {
ListVector listVector = (ListVector) vector;
LogicalType elementType = ((ArrayType) fieldType).getElementType();
return ArrayWriter.forRow(listVector, createArrowFieldWriterForArray(listVector.getDataVector(), elementType));
} else if (vector instanceof StructVector) {
RowType rowType = (RowType) fieldType;
ArrowFieldWriter<RowData>[] fieldsWriters = new ArrowFieldWriter[rowType.getFieldCount()];
for (int i = 0; i < fieldsWriters.length; i++) {
fieldsWriters[i] = createArrowFieldWriterForRow(((StructVector) vector).getVectorById(i), rowType.getTypeAt(i));
}
return RowWriter.forRow((StructVector) vector, fieldsWriters);
} else {
throw new UnsupportedOperationException(String.format("Unsupported type %s.", fieldType));
}
}
use of org.apache.arrow.vector.complex.ListVector in project beam by apache.
the class ArrowConversionTest method rowIterator.
@Test
public void rowIterator() {
org.apache.arrow.vector.types.pojo.Schema schema = new org.apache.arrow.vector.types.pojo.Schema(asList(field("int32", new ArrowType.Int(32, true)), field("float64", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), field("string", new ArrowType.Utf8()), field("timestampMicroUTC", new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC")), field("timestampMilliUTC", new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), field("int32_list", new ArrowType.List(), field("int32s", new ArrowType.Int(32, true))), field("boolean", new ArrowType.Bool()), field("fixed_size_binary", new ArrowType.FixedSizeBinary(3))));
Schema beamSchema = ArrowConversion.ArrowSchemaTranslator.toBeamSchema(schema);
VectorSchemaRoot expectedSchemaRoot = VectorSchemaRoot.create(schema, allocator);
expectedSchemaRoot.allocateNew();
expectedSchemaRoot.setRowCount(16);
IntVector intVector = (IntVector) expectedSchemaRoot.getFieldVectors().get(0);
Float8Vector floatVector = (Float8Vector) expectedSchemaRoot.getFieldVectors().get(1);
VarCharVector strVector = (VarCharVector) expectedSchemaRoot.getFieldVectors().get(2);
TimeStampMicroTZVector timestampMicroUtcVector = (TimeStampMicroTZVector) expectedSchemaRoot.getFieldVectors().get(3);
TimeStampMilliTZVector timeStampMilliTZVector = (TimeStampMilliTZVector) expectedSchemaRoot.getFieldVectors().get(4);
ListVector int32ListVector = (ListVector) expectedSchemaRoot.getFieldVectors().get(5);
IntVector int32ListElementVector = int32ListVector.<IntVector>addOrGetVector(new org.apache.arrow.vector.types.pojo.FieldType(false, new ArrowType.Int(32, true), null)).getVector();
BitVector boolVector = (BitVector) expectedSchemaRoot.getFieldVectors().get(6);
FixedSizeBinaryVector fixedSizeBinaryVector = (FixedSizeBinaryVector) expectedSchemaRoot.getFieldVectors().get(7);
ArrayList<Row> expectedRows = new ArrayList<>();
for (int i = 0; i < 16; i++) {
DateTime dt = new DateTime(2019, 1, i + 1, i, i, i, DateTimeZone.UTC);
expectedRows.add(Row.withSchema(beamSchema).addValues(i, i + .1 * i, "" + i, dt, dt, ImmutableList.of(i), (i % 2) != 0, new byte[] { (byte) i, (byte) (i + 1), (byte) (i + 2) }).build());
intVector.set(i, i);
floatVector.set(i, i + .1 * i);
strVector.set(i, new Text("" + i));
timestampMicroUtcVector.set(i, dt.getMillis() * 1000);
timeStampMilliTZVector.set(i, dt.getMillis());
int32ListVector.startNewValue(i);
int32ListElementVector.set(i, i);
int32ListVector.endValue(i, 1);
boolVector.set(i, i % 2);
fixedSizeBinaryVector.set(i, new byte[] { (byte) i, (byte) (i + 1), (byte) (i + 2) });
}
assertThat(ImmutableList.copyOf(ArrowConversion.rowsFromRecordBatch(beamSchema, expectedSchemaRoot)), IsIterableContainingInOrder.contains(expectedRows.stream().map((row) -> equalTo(row)).collect(ImmutableList.toImmutableList())));
expectedSchemaRoot.close();
}
use of org.apache.arrow.vector.complex.ListVector in project carbondata by apache.
the class ArrowWriter method createFieldWriter.
private static ArrowFieldWriter createFieldWriter(ValueVector valueVector) {
if (valueVector instanceof BitVector) {
return new BooleanWriter((BitVector) valueVector);
} else if (valueVector instanceof TinyIntVector) {
return new ByteWriter((TinyIntVector) valueVector);
} else if (valueVector instanceof SmallIntVector) {
return new ShortWriter((SmallIntVector) valueVector);
} else if (valueVector instanceof IntVector) {
return new IntWriter((IntVector) valueVector);
} else if (valueVector instanceof BigIntVector) {
return new LongWriter((BigIntVector) valueVector);
} else if (valueVector instanceof DecimalVector) {
DecimalVector decimalVector = (DecimalVector) valueVector;
final Field field = decimalVector.getField();
ArrowType.Decimal c = (ArrowType.Decimal) field.getType();
return new DecimalWriter((DecimalVector) valueVector, c.getPrecision(), c.getScale());
} else if (valueVector instanceof VarCharVector) {
return new StringWriter((VarCharVector) valueVector);
} else if (valueVector instanceof Float4Vector) {
return new FloatWriter((Float4Vector) valueVector);
} else if (valueVector instanceof Float8Vector) {
return new DoubleWriter((Float8Vector) valueVector);
} else if (valueVector instanceof ListVector) {
ArrowFieldWriter elementVector = createFieldWriter(((ListVector) valueVector).getDataVector());
return new ArrayWriter((ListVector) valueVector, elementVector);
} else if (valueVector instanceof StructVector) {
StructVector s = (StructVector) valueVector;
List<ArrowFieldWriter> arrowFieldWriters = new ArrayList<>();
for (int i = 0; i < s.size(); i++) {
arrowFieldWriters.add(createFieldWriter(s.getChildByOrdinal(i)));
}
return new StructWriter(s, arrowFieldWriters.toArray(new ArrowFieldWriter[arrowFieldWriters.size()]));
} else if (valueVector instanceof VarBinaryVector) {
return new BinaryWriter((VarBinaryVector) valueVector);
} else if (valueVector instanceof DateDayVector) {
return new DateWriter((DateDayVector) valueVector);
} else if (valueVector instanceof TimeStampMicroTZVector) {
return new TimeStampWriter((TimeStampMicroTZVector) valueVector);
} else {
throw new UnsupportedOperationException("Invalid data type");
}
}
Aggregations