Examples with ValueVector org.apache.drill.exec.vector.ValueVector used on opensource projects

Example 1 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class VectorAccessibleSerializable method readVectors.

@SuppressWarnings("resource")
private void readVectors(InputStream input, RecordBatchDef batchDef) throws IOException {
    final VectorContainer container = new VectorContainer();
    final List<ValueVector> vectorList = Lists.newArrayList();
    final List<SerializedField> fieldList = batchDef.getFieldList();
    for (SerializedField metaData : fieldList) {
        final int dataLength = metaData.getBufferLength();
        final MaterializedField field = MaterializedField.create(metaData);
        final DrillBuf buf = allocator.read(dataLength, input);
        final ValueVector vector = TypeHelper.getNewVector(field, allocator);
        vector.load(metaData, buf);
        // Vector now owns the buffer
        buf.release();
        vectorList.add(vector);
    }
    container.addCollection(vectorList);
    container.buildSchema(svMode);
    container.setRecordCount(recordCount);
    va = container;
}

Example 2 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class ParquetPartitionDescriptor method populatePartitionVectors.

@Override
public void populatePartitionVectors(ValueVector[] vectors, List<PartitionLocation> partitions, BitSet partitionColumnBitSet, Map<Integer, String> fieldNameMap) {
    int record = 0;
    for (PartitionLocation partitionLocation : partitions) {
        for (int partitionColumnIndex : BitSets.toIter(partitionColumnBitSet)) {
            SchemaPath column = SchemaPath.getSimplePath(fieldNameMap.get(partitionColumnIndex));
            ((ParquetGroupScan) scanRel.getGroupScan()).populatePruningVector(vectors[partitionColumnIndex], record, column, partitionLocation.getEntirePartitionLocation());
        }
        record++;
    }
    for (ValueVector v : vectors) {
        if (v == null) {
            continue;
        }
        v.getMutator().setValueCount(partitions.size());
    }
}

Example 3 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class FieldIdUtil method getFieldIdIfMatches.

public static TypedFieldId getFieldIdIfMatches(ValueVector vector, TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
    if (vector instanceof RepeatedMapVector && seg != null && seg.isArray() && !seg.isLastPath()) {
        if (addToBreadCrumb) {
            addToBreadCrumb = false;
            builder.remainder(seg);
        }
        // skip the first array segment as there is no corresponding child vector.
        seg = seg.getChild();
        // segment.
        if (seg.isArray()) {
            return null;
        }
    }
    if (seg == null) {
        if (addToBreadCrumb) {
            builder.intermediateType(vector.getField().getType());
        }
        return builder.finalType(vector.getField().getType()).build();
    }
    if (seg.isArray()) {
        if (seg.isLastPath()) {
            MajorType type;
            if (vector instanceof AbstractContainerVector) {
                type = ((AbstractContainerVector) vector).getLastPathType();
            } else if (vector instanceof ListVector) {
                type = ((ListVector) vector).getDataVector().getField().getType();
                builder.listVector();
            } else {
                throw new UnsupportedOperationException("FieldIdUtil does not support vector of type " + vector.getField().getType());
            }
            //
            builder.withIndex().finalType(type);
            // only set remainder when it's the only array segment.
            if (addToBreadCrumb) {
                addToBreadCrumb = false;
                builder.remainder(seg);
            }
            return builder.build();
        } else {
            if (addToBreadCrumb) {
                addToBreadCrumb = false;
                builder.remainder(seg);
            }
        }
    } else {
        if (vector instanceof ListVector) {
            return null;
        }
    }
    ValueVector v;
    if (vector instanceof AbstractContainerVector) {
        VectorWithOrdinal vord = ((AbstractContainerVector) vector).getChildVectorWithOrdinal(seg.isArray() ? null : seg.getNameSegment().getPath());
        if (vord == null) {
            return null;
        }
        v = vord.vector;
        if (addToBreadCrumb) {
            builder.intermediateType(v.getField().getType());
            builder.addId(vord.ordinal);
        }
    } else if (vector instanceof ListVector) {
        v = ((ListVector) vector).getDataVector();
    } else {
        throw new UnsupportedOperationException("FieldIdUtil does not support vector of type " + vector.getField().getType());
    }
    if (v instanceof AbstractContainerVector) {
        // we're looking for a multi path.
        AbstractContainerVector c = (AbstractContainerVector) v;
        return getFieldIdIfMatches(c, builder, addToBreadCrumb, seg.getChild());
    } else if (v instanceof ListVector) {
        ListVector list = (ListVector) v;
        return getFieldIdIfMatches(list, builder, addToBreadCrumb, seg.getChild());
    } else if (v instanceof UnionVector) {
        return getFieldIdIfMatchesUnion((UnionVector) v, builder, addToBreadCrumb, seg.getChild());
    } else {
        if (seg.isNamed()) {
            if (addToBreadCrumb) {
                builder.intermediateType(v.getField().getType());
            }
            builder.finalType(v.getField().getType());
        } else {
            builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build());
        }
        if (seg.isLastPath()) {
            return builder.build();
        } else {
            PathSegment child = seg.getChild();
            if (child.isLastPath() && child.isArray()) {
                if (addToBreadCrumb) {
                    builder.remainder(child);
                }
                builder.withIndex();
                builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build());
                return builder.build();
            } else {
                logger.warn("You tried to request a complex type inside a scalar object or path or type is wrong.");
                return null;
            }
        }
    }
}

Example 4 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class TestOptiqPlans method testFilterPlan.

@Test
public void testFilterPlan() throws Exception {
    final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
    try (final Drillbit bit1 = new Drillbit(config, serviceSet);
        final DrillClient client = new DrillClient(config, serviceSet.getCoordinator())) {
        bit1.run();
        client.connect();
        final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Resources.toString(Resources.getResource("physical_filter.json"), Charsets.UTF_8));
        final RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
        for (final QueryDataBatch b : results) {
            System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
            loader.load(b.getHeader().getDef(), b.getData());
            for (final VectorWrapper<?> vw : loader) {
                System.out.println(vw.getValueVector().getField().getPath());
                final ValueVector vv = vw.getValueVector();
                for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
                    final Object o = vv.getAccessor().getObject(i);
                    System.out.println(o);
                }
            }
            loader.clear();
            b.release();
        }
        client.close();
    }
}

Example 5 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class TestOptiqPlans method testOrderVarbinary.

@Test
public void testOrderVarbinary() throws Exception {
    final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
    try (final Drillbit bit1 = new Drillbit(config, serviceSet);
        final DrillClient client = new DrillClient(config, serviceSet.getCoordinator())) {
        bit1.run();
        client.connect();
        final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Resources.toString(Resources.getResource("physical_order_varbinary.json"), Charsets.UTF_8));
        final RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
        for (final QueryDataBatch b : results) {
            System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
            loader.load(b.getHeader().getDef(), b.getData());
            for (final VectorWrapper vw : loader) {
                System.out.println(vw.getValueVector().getField().getPath());
                final ValueVector vv = vw.getValueVector();
                for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
                    final Object o = vv.getAccessor().getObject(i);
                    if (vv instanceof VarBinaryVector) {
                        final VarBinaryVector.Accessor x = ((VarBinaryVector) vv).getAccessor();
                        final VarBinaryHolder vbh = new VarBinaryHolder();
                        x.get(i, vbh);
                        System.out.printf("%d..%d", vbh.start, vbh.end);
                        System.out.println("[" + new String((byte[]) vv.getAccessor().getObject(i)) + "]");
                    } else {
                        System.out.println(vv.getAccessor().getObject(i));
                    }
                }
            }
            loader.clear();
            b.release();
        }
        client.close();
    }
}