Examples with ValueVector org.apache.drill.exec.vector.ValueVector used on opensource projects

Example 86 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class UnionAllRecordBatch method setValueCount.

private void setValueCount(int count) {
    for (ValueVector v : allocationVectors) {
        ValueVector.Mutator m = v.getMutator();
        m.setValueCount(count);
    }
}

Example 87 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class MockRecordReader method next.

@Override
public int next() {
    if (recordsRead >= this.config.getRecords()) {
        return 0;
    }
    final int recordSetSize = Math.min(batchRecordCount, this.config.getRecords() - recordsRead);
    recordsRead += recordSetSize;
    for (final ValueVector v : valueVectors) {
        final ValueVector.Mutator m = v.getMutator();
        m.generateTestData(recordSetSize);
    }
    return recordSetSize;
}

Example 88 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class DrillTestWrapper method addToCombinedVectorResults.

/**
   * Add to result vectors and compare batch schema against expected schema while iterating batches.
   * @param batches
   * @param  expectedSchema: the expected schema the batches should contain. Through SchemaChangeException
   *                       if encounter different batch schema.
   * @return
   * @throws SchemaChangeException
   * @throws UnsupportedEncodingException
   */
public static Map<String, List<Object>> addToCombinedVectorResults(Iterable<VectorAccessible> batches, BatchSchema expectedSchema) throws SchemaChangeException, UnsupportedEncodingException {
    // TODO - this does not handle schema changes
    Map<String, List<Object>> combinedVectors = new TreeMap<>();
    long totalRecords = 0;
    BatchSchema schema = null;
    for (VectorAccessible loader : batches) {
        if (expectedSchema != null) {
            if (!expectedSchema.equals(loader.getSchema())) {
                throw new SchemaChangeException(String.format("Batch schema does not match expected schema\n" + "Actual schema: %s.  Expected schema : %s", loader.getSchema(), expectedSchema));
            }
        }
        // SchemaChangeException, so check/clean throws clause above.
        if (schema == null) {
            schema = loader.getSchema();
            for (MaterializedField mf : schema) {
                combinedVectors.put(SchemaPath.getSimplePath(mf.getPath()).toExpr(), new ArrayList<Object>());
            }
        } else {
            // TODO - actually handle schema changes, this is just to get access to the SelectionVectorMode
            // of the current batch, the check for a null schema is used to only mutate the schema once
            // need to add new vectors and null fill for previous batches? distinction between null and non-existence important?
            schema = loader.getSchema();
        }
        logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords);
        totalRecords += loader.getRecordCount();
        for (VectorWrapper<?> w : loader) {
            String field = SchemaPath.getSimplePath(w.getField().getPath()).toExpr();
            ValueVector[] vectors;
            if (w.isHyper()) {
                vectors = w.getValueVectors();
            } else {
                vectors = new ValueVector[] { w.getValueVector() };
            }
            SelectionVector2 sv2 = null;
            SelectionVector4 sv4 = null;
            switch(schema.getSelectionVectorMode()) {
                case TWO_BYTE:
                    sv2 = loader.getSelectionVector2();
                    break;
                case FOUR_BYTE:
                    sv4 = loader.getSelectionVector4();
                    break;
            }
            if (sv4 != null) {
                for (int j = 0; j < sv4.getCount(); j++) {
                    int complexIndex = sv4.get(j);
                    int batchIndex = complexIndex >> 16;
                    int recordIndexInBatch = complexIndex & 65535;
                    Object obj = vectors[batchIndex].getAccessor().getObject(recordIndexInBatch);
                    if (obj != null) {
                        if (obj instanceof Text) {
                            obj = obj.toString();
                        }
                    }
                    combinedVectors.get(field).add(obj);
                }
            } else {
                for (ValueVector vv : vectors) {
                    for (int j = 0; j < loader.getRecordCount(); j++) {
                        int index;
                        if (sv2 != null) {
                            index = sv2.getIndex(j);
                        } else {
                            index = j;
                        }
                        Object obj = vv.getAccessor().getObject(index);
                        if (obj != null) {
                            if (obj instanceof Text) {
                                obj = obj.toString();
                            }
                        }
                        combinedVectors.get(field).add(obj);
                    }
                }
            }
        }
    }
    return combinedVectors;
}

Example 89 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class DrillTestWrapper method addToHyperVectorMap.

private Map<String, HyperVectorValueIterator> addToHyperVectorMap(final List<QueryDataBatch> records, final RecordBatchLoader loader) throws SchemaChangeException, UnsupportedEncodingException {
    // TODO - this does not handle schema changes
    Map<String, HyperVectorValueIterator> combinedVectors = new TreeMap<>();
    long totalRecords = 0;
    QueryDataBatch batch;
    int size = records.size();
    for (int i = 0; i < size; i++) {
        batch = records.get(i);
        loader.load(batch.getHeader().getDef(), batch.getData());
        logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords);
        totalRecords += loader.getRecordCount();
        for (VectorWrapper<?> w : loader) {
            String field = SchemaPath.getSimplePath(w.getField().getPath()).toExpr();
            if (!combinedVectors.containsKey(field)) {
                MaterializedField mf = w.getField();
                ValueVector[] vvList = (ValueVector[]) Array.newInstance(mf.getValueClass(), 1);
                vvList[0] = w.getValueVector();
                combinedVectors.put(field, new HyperVectorValueIterator(mf, new HyperVectorWrapper<>(mf, vvList)));
            } else {
                combinedVectors.get(field).getHyperVector().addVector(w.getValueVector());
            }
        }
    }
    for (HyperVectorValueIterator hvi : combinedVectors.values()) {
        hvi.determineTotalSize();
    }
    return combinedVectors;
}

Example 90 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class MergingRecordBatch method buildSchema.

@Override
public void buildSchema() throws SchemaChangeException {
    // find frag provider that has data to use to build schema, and put in tempBatchHolder for later use
    tempBatchHolder = new RawFragmentBatch[fragProviders.length];
    int i = 0;
    try {
        while (true) {
            if (i >= fragProviders.length) {
                state = BatchState.DONE;
                return;
            }
            final RawFragmentBatch batch = getNext(i);
            if (batch == null) {
                if (!context.shouldContinue()) {
                    state = BatchState.STOP;
                } else {
                    state = BatchState.DONE;
                }
                break;
            }
            if (batch.getHeader().getDef().getFieldCount() == 0) {
                i++;
                continue;
            }
            tempBatchHolder[i] = batch;
            for (final SerializedField field : batch.getHeader().getDef().getFieldList()) {
                @SuppressWarnings("resource") final ValueVector v = outgoingContainer.addOrGet(MaterializedField.create(field));
                v.allocateNew();
            }
            break;
        }
    } catch (final IOException e) {
        throw new DrillRuntimeException(e);
    }
    outgoingContainer = VectorContainer.canonicalize(outgoingContainer);
    outgoingContainer.buildSchema(SelectionVectorMode.NONE);
}