Examples with SelectionVector2 org.apache.drill.exec.record.selection.SelectionVector2 used on opensource projects

Example 1 with SelectionVector2

use of org.apache.drill.exec.record.selection.SelectionVector2 in project drill by apache.

the class VectorAccessibleSerializable method readSv2.

private void readSv2(InputStream input) throws IOException {
    if (sv2 != null) {
        sv2.clear();
    }
    final int dataLength = recordCount * SelectionVector2.RECORD_SIZE;
    svMode = BatchSchema.SelectionVectorMode.TWO_BYTE;
    @SuppressWarnings("resource") DrillBuf buf = allocator.read(dataLength, input);
    sv2 = new SelectionVector2(allocator, buf, recordCount);
    // SV2 now owns the buffer
    buf.release();
}

Example 2 with SelectionVector2

use of org.apache.drill.exec.record.selection.SelectionVector2 in project drill by apache.

the class PriorityQueueTemplate method add.

@SuppressWarnings("resource")
@Override
public void add(FragmentContext context, RecordBatchData batch) throws SchemaChangeException {
    Stopwatch watch = Stopwatch.createStarted();
    if (hyperBatch == null) {
        hyperBatch = new ExpandableHyperContainer(batch.getContainer());
    } else {
        hyperBatch.addBatch(batch.getContainer());
    }
    // may not need to do this every time
    doSetup(context, hyperBatch, null);
    int count = 0;
    SelectionVector2 sv2 = null;
    if (hasSv2) {
        sv2 = batch.getSv2();
    }
    for (; queueSize < limit && count < batch.getRecordCount(); count++) {
        heapSv4.set(queueSize, batchCount, hasSv2 ? sv2.getIndex(count) : count);
        queueSize++;
        siftUp();
    }
    for (; count < batch.getRecordCount(); count++) {
        heapSv4.set(limit, batchCount, hasSv2 ? sv2.getIndex(count) : count);
        if (compare(limit, 0) < 0) {
            swap(limit, 0);
            siftDown();
        }
    }
    batchCount++;
    if (hasSv2) {
        sv2.clear();
    }
    logger.debug("Took {} us to add {} records", watch.elapsed(TimeUnit.MICROSECONDS), count);
}

Example 3 with SelectionVector2

use of org.apache.drill.exec.record.selection.SelectionVector2 in project drill by apache.

the class ExternalSortBatch method processBatch.

/**
   * Process the converted incoming batch by adding it to the in-memory store
   * of data, or spilling data to disk when necessary.
   */
@SuppressWarnings("resource")
private void processBatch() {
    if (incoming.getRecordCount() == 0) {
        return;
    }
    // Determine actual sizes of the incoming batch before taking
    // ownership. Allows us to figure out if we need to spill first,
    // to avoid overflowing memory simply due to ownership transfer.
    RecordBatchSizer sizer = analyzeIncomingBatch();
    if (isSpillNeeded(sizer.actualSize())) {
        spillFromMemory();
    }
    // Sanity check. We should now be below the buffer memory maximum.
    long startMem = allocator.getAllocatedMemory();
    if (startMem > bufferMemoryPool) {
        logger.error("ERROR: Failed to spill above buffer limit. Buffer pool = {}, memory = {}", bufferMemoryPool, startMem);
    }
    // Convert the incoming batch to the agreed-upon schema.
    // No converted batch means we got an empty input batch.
    // Converting the batch transfers memory ownership to our
    // allocator. This gives a round-about way to learn the batch
    // size: check the before and after memory levels, then use
    // the difference as the batch size, in bytes.
    VectorContainer convertedBatch = convertBatch();
    if (convertedBatch == null) {
        return;
    }
    SelectionVector2 sv2;
    try {
        sv2 = makeSelectionVector();
    } catch (Exception e) {
        convertedBatch.clear();
        throw e;
    }
    // Compute batch size, including allocation of an sv2.
    long endMem = allocator.getAllocatedMemory();
    long batchSize = endMem - startMem;
    int count = sv2.getCount();
    inputRecordCount += count;
    inputBatchCount++;
    totalInputBytes += sizer.actualSize();
    if (minimumBufferSpace == 0) {
        minimumBufferSpace = endMem;
    } else {
        minimumBufferSpace = Math.min(minimumBufferSpace, endMem);
    }
    stats.setLongStat(Metric.MIN_BUFFER, minimumBufferSpace);
    // Update the size based on the actual record count, not
    // the effective count as given by the selection vector
    // (which may exclude some records due to filtering.)
    updateMemoryEstimates(batchSize, sizer);
    // Sort the incoming batch using either the original selection vector,
    // or a new one created here.
    SingleBatchSorter sorter;
    sorter = opCodeGen.getSorter(convertedBatch);
    try {
        sorter.setup(context, sv2, convertedBatch);
    } catch (SchemaChangeException e) {
        convertedBatch.clear();
        throw UserException.unsupportedError(e).message("Unexpected schema change.").build(logger);
    }
    try {
        sorter.sort(sv2);
    } catch (SchemaChangeException e) {
        convertedBatch.clear();
        throw UserException.unsupportedError(e).message("Unexpected schema change.").build(logger);
    }
    RecordBatchData rbd = new RecordBatchData(convertedBatch, allocator);
    try {
        rbd.setSv2(sv2);
        bufferedBatches.add(new BatchGroup.InputBatch(rbd.getContainer(), rbd.getSv2(), oContext, sizer.netSize()));
        if (peakNumBatches < bufferedBatches.size()) {
            peakNumBatches = bufferedBatches.size();
            stats.setLongStat(Metric.PEAK_BATCHES_IN_MEMORY, peakNumBatches);
        }
    } catch (Throwable t) {
        rbd.clear();
        throw t;
    }
}

Example 4 with SelectionVector2

use of org.apache.drill.exec.record.selection.SelectionVector2 in project drill by apache.

the class ExternalSortBatch method newSV2.

private SelectionVector2 newSV2() throws OutOfMemoryException, InterruptedException {
    @SuppressWarnings("resource") SelectionVector2 sv2 = new SelectionVector2(oAllocator);
    if (!sv2.allocateNewSafe(incoming.getRecordCount())) {
        try {
            @SuppressWarnings("resource") final BatchGroup merged = mergeAndSpill(batchGroups);
            if (merged != null) {
                spilledBatchGroups.add(merged);
            } else {
                throw UserException.memoryError("Unable to allocate sv2 for %d records, and not enough batchGroups to spill.", incoming.getRecordCount()).addContext("batchGroups.size", batchGroups.size()).addContext("spilledBatchGroups.size", spilledBatchGroups.size()).addContext("allocated memory", oAllocator.getAllocatedMemory()).addContext("allocator limit", oAllocator.getLimit()).build(logger);
            }
        } catch (SchemaChangeException e) {
            throw new RuntimeException(e);
        }
        int waitTime = 1;
        while (true) {
            try {
                Thread.sleep(waitTime * 1000);
            } catch (final InterruptedException e) {
                if (!context.shouldContinue()) {
                    throw e;
                }
            }
            waitTime *= 2;
            if (sv2.allocateNewSafe(incoming.getRecordCount())) {
                break;
            }
            if (waitTime >= 32) {
                throw new OutOfMemoryException("Unable to allocate sv2 buffer after repeated attempts");
            }
        }
    }
    for (int i = 0; i < incoming.getRecordCount(); i++) {
        sv2.setIndex(i, (char) i);
    }
    sv2.setRecordCount(incoming.getRecordCount());
    return sv2;
}

Example 5 with SelectionVector2

use of org.apache.drill.exec.record.selection.SelectionVector2 in project drill by apache.

the class ExternalSortBatch method newSV2.

/**
   * Allocate and initialize the selection vector used as the sort index.
   * Assumes that memory is available for the vector since memory management
   * ensured space is available.
   *
   * @return a new, populated selection vector 2
   */
private SelectionVector2 newSV2() {
    SelectionVector2 sv2 = new SelectionVector2(allocator);
    if (!sv2.allocateNewSafe(incoming.getRecordCount())) {
        throw UserException.resourceError(new OutOfMemoryException("Unable to allocate sv2 buffer")).build(logger);
    }
    for (int i = 0; i < incoming.getRecordCount(); i++) {
        sv2.setIndex(i, (char) i);
    }
    sv2.setRecordCount(incoming.getRecordCount());
    return sv2;
}