Examples with ValueVector org.apache.drill.exec.vector.ValueVector used on opensource projects

Example 76 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class OrderedPartitionRecordBatch method buildTable.

private void buildTable() throws SchemaChangeException, ClassTransformationException, IOException {
    // Get all samples from distributed map
    @SuppressWarnings("resource") SortRecordBatchBuilder containerBuilder = new SortRecordBatchBuilder(context.getAllocator());
    final VectorContainer allSamplesContainer = new VectorContainer();
    final VectorContainer candidatePartitionTable = new VectorContainer();
    CachedVectorContainer wrap = null;
    try {
        for (CachedVectorContainer w : mmap.get(mapKey)) {
            containerBuilder.add(w.get());
        }
        containerBuilder.build(context, allSamplesContainer);
        List<Ordering> orderDefs = Lists.newArrayList();
        int i = 0;
        for (Ordering od : popConfig.getOrderings()) {
            SchemaPath sp = SchemaPath.getSimplePath("f" + i++);
            orderDefs.add(new Ordering(od.getDirection(), new FieldReference(sp)));
        }
        // sort the data incoming samples.
        @SuppressWarnings("resource") SelectionVector4 newSv4 = containerBuilder.getSv4();
        Sorter sorter = SortBatch.createNewSorter(context, orderDefs, allSamplesContainer);
        sorter.setup(context, newSv4, allSamplesContainer);
        sorter.sort(newSv4, allSamplesContainer);
        // Copy every Nth record from the samples into a candidate partition table, where N = totalSampledRecords/partitions
        // Attempt to push this to the distributed map. Only the first candidate to get pushed will be used.
        SampleCopier copier = null;
        List<ValueVector> localAllocationVectors = Lists.newArrayList();
        copier = getCopier(newSv4, allSamplesContainer, candidatePartitionTable, orderDefs, localAllocationVectors);
        int allocationSize = 50;
        while (true) {
            for (ValueVector vv : localAllocationVectors) {
                AllocationHelper.allocate(vv, samplingFactor * partitions, allocationSize);
            }
            int skipRecords = containerBuilder.getSv4().getTotalCount() / partitions;
            if (copier.copyRecords(skipRecords, skipRecords, partitions - 1)) {
                assert copier.getOutputRecords() == partitions - 1 : String.format("output records: %d partitions: %d", copier.getOutputRecords(), partitions);
                for (VectorWrapper<?> vw : candidatePartitionTable) {
                    vw.getValueVector().getMutator().setValueCount(copier.getOutputRecords());
                }
                break;
            } else {
                candidatePartitionTable.zeroVectors();
                allocationSize *= 2;
            }
        }
        candidatePartitionTable.setRecordCount(copier.getOutputRecords());
        @SuppressWarnings("resource") WritableBatch batch = WritableBatch.getBatchNoHVWrap(candidatePartitionTable.getRecordCount(), candidatePartitionTable, false);
        wrap = new CachedVectorContainer(batch, context.getDrillbitContext().getAllocator());
        tableMap.putIfAbsent(mapKey + "final", wrap, 1, TimeUnit.MINUTES);
    } finally {
        candidatePartitionTable.clear();
        allSamplesContainer.clear();
        containerBuilder.clear();
        containerBuilder.close();
        if (wrap != null) {
            wrap.clear();
        }
    }
}

Example 77 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class NoFrameSupportTemplate method allocateInternal.

private void allocateInternal() {
    for (VectorWrapper<?> w : container) {
        ValueVector vv = internal.addOrGet(w.getField());
        vv.allocateNew();
    }
}

Example 78 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class WindowFrameRecordBatch method doWork.

private void doWork() throws DrillException {
    final WindowDataBatch current = batches.get(0);
    final int recordCount = current.getRecordCount();
    logger.trace("WindowFramer.doWork() START, num batches {}, current batch has {} rows", batches.size(), recordCount);
    // allocate outgoing vectors
    for (VectorWrapper<?> w : container) {
        w.getValueVector().allocateNew();
    }
    for (WindowFramer framer : framers) {
        framer.doWork();
    }
    // transfer "non aggregated" vectors
    for (VectorWrapper<?> vw : current) {
        ValueVector v = container.addOrGet(vw.getField());
        TransferPair tp = vw.getValueVector().makeTransferPair(v);
        tp.transfer();
    }
    container.setRecordCount(recordCount);
    for (VectorWrapper<?> v : container) {
        v.getValueVector().getMutator().setValueCount(recordCount);
    }
    // we can safely free the current batch
    current.clear();
    batches.remove(0);
    logger.trace("doWork() END");
}

Example 79 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class RemovingRecordBatch method getGenerated4Copier.

public static Copier getGenerated4Copier(RecordBatch batch, FragmentContext context, BufferAllocator allocator, VectorContainer container, RecordBatch outgoing, SchemaChangeCallBack callBack) throws SchemaChangeException {
    for (VectorWrapper<?> vv : batch) {
        @SuppressWarnings("resource") ValueVector v = vv.getValueVectors()[0];
        v.makeTransferPair(container.addOrGet(v.getField(), callBack));
    }
    try {
        final CodeGenerator<Copier> cg = CodeGenerator.get(Copier.TEMPLATE_DEFINITION4, context.getFunctionRegistry(), context.getOptions());
        CopyUtil.generateCopies(cg.getRoot(), batch, true);
        cg.plainJavaCapable(true);
        // Uncomment out this line to debug the generated code.
        //      cg.saveCodeForDebugging(true);
        Copier copier = context.getImplementationClass(cg);
        copier.setupRemover(context, batch, outgoing);
        return copier;
    } catch (ClassTransformationException | IOException e) {
        throw new SchemaChangeException("Failure while attempting to load generated class", e);
    }
}

Example 80 with ValueVector

use of org.apache.drill.exec.vector.ValueVector in project drill by apache.

the class UnionAllRecordBatch method setValueCount.

private void setValueCount(int count) {
    for (ValueVector v : allocationVectors) {
        ValueVector.Mutator m = v.getMutator();
        m.setValueCount(count);
    }
}