Examples with OutOfMemoryException org.apache.drill.exec.exception.OutOfMemoryException used on opensource projects

Example 26 with OutOfMemoryException

use of org.apache.drill.exec.exception.OutOfMemoryException in project drill by apache.

the class RepeatedMapVector method allocateNew.

@Override
public void allocateNew(int groupCount, int innerValueCount) {
    clear();
    try {
        offsets.allocateNew(groupCount + 1);
        for (ValueVector v : getChildren()) {
            AllocationHelper.allocatePrecomputedChildCount(v, groupCount, 50, innerValueCount);
        }
    } catch (OutOfMemoryException e) {
        clear();
        throw e;
    }
    offsets.zeroVector();
    mutator.reset();
}

Example 27 with OutOfMemoryException

use of org.apache.drill.exec.exception.OutOfMemoryException in project drill by axbaretto.

the class ExternalSortBatch method newSV2.

private SelectionVector2 newSV2() throws OutOfMemoryException, InterruptedException {
    @SuppressWarnings("resource") SelectionVector2 sv2 = new SelectionVector2(oAllocator);
    if (!sv2.allocateNewSafe(incoming.getRecordCount())) {
        try {
            @SuppressWarnings("resource") final BatchGroup merged = mergeAndSpill(batchGroups);
            if (merged != null) {
                spilledBatchGroups.add(merged);
            } else {
                throw UserException.memoryError("Unable to allocate sv2 for %d records, and not enough batchGroups to spill.", incoming.getRecordCount()).addContext("batchGroups.size", batchGroups.size()).addContext("spilledBatchGroups.size", spilledBatchGroups.size()).addContext("allocated memory", oAllocator.getAllocatedMemory()).addContext("allocator limit", oAllocator.getLimit()).build(logger);
            }
        } catch (SchemaChangeException e) {
            throw new RuntimeException(e);
        }
        int waitTime = 1;
        while (true) {
            try {
                Thread.sleep(waitTime * 1000);
            } catch (final InterruptedException e) {
                if (!context.getExecutorState().shouldContinue()) {
                    throw e;
                }
            }
            waitTime *= 2;
            if (sv2.allocateNewSafe(incoming.getRecordCount())) {
                break;
            }
            if (waitTime >= 32) {
                throw new OutOfMemoryException("Unable to allocate sv2 buffer after repeated attempts");
            }
        }
    }
    for (int i = 0; i < incoming.getRecordCount(); i++) {
        sv2.setIndex(i, (char) i);
    }
    sv2.setRecordCount(incoming.getRecordCount());
    return sv2;
}

Example 28 with OutOfMemoryException

use of org.apache.drill.exec.exception.OutOfMemoryException in project drill by axbaretto.

the class TestAllocators method testAllocators.

@Test
public void testAllocators() throws Exception {
    // Setup a drillbit (initializes a root allocator)
    final DrillConfig config = DrillConfig.create(TEST_CONFIGURATIONS);
    try (final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
        final Drillbit bit = new Drillbit(config, serviceSet)) {
        ;
        bit.run();
        final DrillbitContext bitContext = bit.getContext();
        FunctionImplementationRegistry functionRegistry = bitContext.getFunctionImplementationRegistry();
        StoragePluginRegistry storageRegistry = new StoragePluginRegistryImpl(bitContext);
        // Create a few Fragment Contexts
        BitControl.PlanFragment.Builder pfBuilder1 = BitControl.PlanFragment.newBuilder();
        pfBuilder1.setMemInitial(1500000);
        BitControl.PlanFragment pf1 = pfBuilder1.build();
        BitControl.PlanFragment.Builder pfBuilder2 = BitControl.PlanFragment.newBuilder();
        pfBuilder2.setMemInitial(500000);
        BitControl.PlanFragment pf2 = pfBuilder1.build();
        FragmentContextImpl fragmentContext1 = new FragmentContextImpl(bitContext, pf1, null, functionRegistry);
        FragmentContextImpl fragmentContext2 = new FragmentContextImpl(bitContext, pf2, null, functionRegistry);
        // Get a few physical operators. Easiest way is to read a physical plan.
        PhysicalPlanReader planReader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(bitContext, storageRegistry);
        PhysicalPlan plan = planReader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile(planFile), Charsets.UTF_8));
        List<PhysicalOperator> physicalOperators = plan.getSortedOperators();
        Iterator<PhysicalOperator> physicalOperatorIterator = physicalOperators.iterator();
        PhysicalOperator physicalOperator1 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator2 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator3 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator4 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator5 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator6 = physicalOperatorIterator.next();
        // Create some bogus Operator profile defs and stats to create operator contexts
        OpProfileDef def;
        OperatorStats stats;
        // Use some bogus operator type to create a new operator context.
        def = new OpProfileDef(physicalOperator1.getOperatorId(), UserBitShared.CoreOperatorType.MOCK_SUB_SCAN_VALUE, OperatorUtilities.getChildCount(physicalOperator1));
        stats = fragmentContext1.getStats().newOperatorStats(def, fragmentContext1.getAllocator());
        // Add a couple of Operator Contexts
        // Initial allocation = 1000000 bytes for all operators
        OperatorContext oContext11 = fragmentContext1.newOperatorContext(physicalOperator1);
        DrillBuf b11 = oContext11.getAllocator().buffer(1000000);
        OperatorContext oContext12 = fragmentContext1.newOperatorContext(physicalOperator2, stats);
        DrillBuf b12 = oContext12.getAllocator().buffer(500000);
        OperatorContext oContext21 = fragmentContext1.newOperatorContext(physicalOperator3);
        def = new OpProfileDef(physicalOperator4.getOperatorId(), UserBitShared.CoreOperatorType.TEXT_WRITER_VALUE, OperatorUtilities.getChildCount(physicalOperator4));
        stats = fragmentContext2.getStats().newOperatorStats(def, fragmentContext2.getAllocator());
        OperatorContext oContext22 = fragmentContext2.newOperatorContext(physicalOperator4, stats);
        DrillBuf b22 = oContext22.getAllocator().buffer(2000000);
        // New Fragment begins
        BitControl.PlanFragment.Builder pfBuilder3 = BitControl.PlanFragment.newBuilder();
        pfBuilder3.setMemInitial(1000000);
        BitControl.PlanFragment pf3 = pfBuilder3.build();
        FragmentContextImpl fragmentContext3 = new FragmentContextImpl(bitContext, pf3, null, functionRegistry);
        // New fragment starts an operator that allocates an amount within the limit
        def = new OpProfileDef(physicalOperator5.getOperatorId(), UserBitShared.CoreOperatorType.UNION_VALUE, OperatorUtilities.getChildCount(physicalOperator5));
        stats = fragmentContext3.getStats().newOperatorStats(def, fragmentContext3.getAllocator());
        OperatorContext oContext31 = fragmentContext3.newOperatorContext(physicalOperator5, stats);
        DrillBuf b31a = oContext31.getAllocator().buffer(200000);
        // Previously running operator completes
        b22.release();
        ((AutoCloseable) oContext22).close();
        // Fragment 3 asks for more and fails
        boolean outOfMem = false;
        try {
            oContext31.getAllocator().buffer(44000000);
            fail("Fragment 3 should fail to allocate buffer");
        } catch (OutOfMemoryException e) {
            // Expected.
            outOfMem = true;
        }
        assertTrue(outOfMem);
        // Operator is Exempt from Fragment limits. Fragment 3 asks for more and succeeds
        OperatorContext oContext32 = fragmentContext3.newOperatorContext(physicalOperator6);
        try {
            DrillBuf b32 = oContext32.getAllocator().buffer(4400000);
            b32.release();
        } catch (OutOfMemoryException e) {
            fail("Fragment 3 failed to allocate buffer");
        } finally {
            closeOp(oContext32);
        }
        b11.release();
        closeOp(oContext11);
        b12.release();
        closeOp(oContext12);
        closeOp(oContext21);
        b31a.release();
        closeOp(oContext31);
        fragmentContext1.close();
        fragmentContext2.close();
        fragmentContext3.close();
    }
}

Example 29 with OutOfMemoryException

use of org.apache.drill.exec.exception.OutOfMemoryException in project drill by axbaretto.

the class IndirectRowSet method makeSv2.

private static SelectionVector2 makeSv2(BufferAllocator allocator, VectorContainer container, Set<Integer> skipIndices) {
    int rowCount = container.getRecordCount() - skipIndices.size();
    SelectionVector2 sv2 = new SelectionVector2(allocator);
    if (!sv2.allocateNewSafe(rowCount)) {
        throw new OutOfMemoryException("Unable to allocate sv2 buffer");
    }
    for (int srcIndex = 0, destIndex = 0; srcIndex < container.getRecordCount(); srcIndex++) {
        if (skipIndices.contains(srcIndex)) {
            continue;
        }
        sv2.setIndex(destIndex, (char) srcIndex);
        destIndex++;
    }
    sv2.setRecordCount(rowCount);
    container.buildSchema(SelectionVectorMode.TWO_BYTE);
    return sv2;
}

Example 30 with OutOfMemoryException

use of org.apache.drill.exec.exception.OutOfMemoryException in project drill by axbaretto.

the class HashTableTemplate method resizeAndRehashIfNeeded.

// Resize the hash table if needed by creating a new one with double the number of buckets.
// For each entry in the old hash table, re-hash it to the new table and update the metadata
// in the new table.. the metadata consists of the startIndices, links and hashValues.
// Note that the keys stored in the BatchHolders are not moved around.
private void resizeAndRehashIfNeeded() {
    if (numEntries < threshold) {
        return;
    }
    if (EXTRA_DEBUG) {
        logger.debug("Hash table numEntries = {}, threshold = {}; resizing the table...", numEntries, threshold);
    }
    // future attempts to resize will return immediately.
    if (tableSize == MAXIMUM_CAPACITY) {
        threshold = Integer.MAX_VALUE;
        return;
    }
    int newTableSize = 2 * tableSize;
    newTableSize = roundUpToPowerOf2(newTableSize);
    // the new hash-values (to replace the existing ones - inside rehash() ), then OOM
    if (4 * /* sizeof(int) */
    (newTableSize + 2 * HashTable.BATCH_SIZE) >= allocator.getLimit() - allocator.getAllocatedMemory()) {
        throw new OutOfMemoryException("Resize Hash Table");
    }
    tableSize = newTableSize;
    if (tableSize > MAXIMUM_CAPACITY) {
        tableSize = MAXIMUM_CAPACITY;
    }
    long t0 = System.currentTimeMillis();
    // set the new threshold based on the new table size and load factor
    threshold = (int) Math.ceil(tableSize * htConfig.getLoadFactor());
    IntVector newStartIndices = allocMetadataVector(tableSize, EMPTY_SLOT);
    for (int i = 0; i < batchHolders.size(); i++) {
        BatchHolder bh = batchHolders.get(i);
        int batchStartIdx = i * BATCH_SIZE;
        bh.rehash(tableSize, newStartIndices, batchStartIdx);
    }
    startIndices.clear();
    startIndices = newStartIndices;
    if (EXTRA_DEBUG) {
        logger.debug("After resizing and rehashing, dumping the hash table...");
        logger.debug("Number of buckets = {}.", startIndices.getAccessor().getValueCount());
        for (int i = 0; i < startIndices.getAccessor().getValueCount(); i++) {
            logger.debug("Bucket: {}, startIdx[ {} ] = {}.", i, i, startIndices.getAccessor().get(i));
            int idx = startIndices.getAccessor().get(i);
            BatchHolder bh = batchHolders.get((idx >>> 16) & BATCH_MASK);
            bh.dump(idx);
        }
    }
    resizingTime += System.currentTimeMillis() - t0;
    numResizing++;
}