Examples with OutOfMemoryException org.apache.drill.exec.exception.OutOfMemoryException used on opensource projects

Example 21 with OutOfMemoryException

use of org.apache.drill.exec.exception.OutOfMemoryException in project drill by apache.

the class AbstractRepeatedMapVector method allocateNew.

public void allocateNew(int groupCount, int innerValueCount) {
    clear();
    try {
        allocateOffsetsNew(groupCount);
        for (ValueVector v : getChildren()) {
            AllocationHelper.allocatePrecomputedChildCount(v, groupCount, 50, innerValueCount);
        }
    } catch (OutOfMemoryException e) {
        clear();
        throw e;
    }
    getMutator().reset();
}

Example 22 with OutOfMemoryException

use of org.apache.drill.exec.exception.OutOfMemoryException in project drill by apache.

the class SaslDecryptionHandler method decode.

public void decode(ChannelHandlerContext ctx, ByteBuf msg, List<Object> out) throws IOException {
    if (!ctx.channel().isOpen()) {
        logger.trace("Channel closed before decoding the message of {} bytes", msg.readableBytes());
        msg.skipBytes(msg.readableBytes());
        return;
    }
    try {
        if (logger.isTraceEnabled()) {
            logger.trace("Trying to decrypt the encrypted message of size: {} with maxWrappedSize", msg.readableBytes());
        }
        // All the encrypted blocks are prefixed with it's length in network byte order (or BigEndian format). Netty's
        // default Byte order of ByteBuf is Little Endian, so we cannot just do msg.getInt() as that will read the 4
        // octets in little endian format.
        // 
        // We will read the length of one complete encrypted chunk and decode that.
        msg.getBytes(msg.readerIndex(), lengthOctets.array(), 0, RpcConstants.LENGTH_FIELD_LENGTH);
        final int wrappedMsgLength = lengthOctets.getInt(0);
        msg.skipBytes(RpcConstants.LENGTH_FIELD_LENGTH);
        // Since lengthBasedFrameDecoder will ensure we have enough bytes it's good to have this check here.
        assert (msg.readableBytes() == wrappedMsgLength);
        // should close the connection since it represents a security attack.
        if (wrappedMsgLength > maxWrappedSize) {
            throw new RpcException(String.format("Received encoded buffer size: %d is larger than negotiated " + "maxWrappedSize: %d. Closing the connection as this is unexpected.", wrappedMsgLength, maxWrappedSize));
        }
        final byte[] wrappedMsg = encodedMsg;
        // Copy the wrappedMsgLength of bytes into the byte array
        msg.getBytes(msg.readerIndex(), wrappedMsg, 0, wrappedMsgLength);
        // }
        // SASL library always copies the origMsg internally to a new byte array
        // and return another new byte array after decrypting the message. The memory for this
        // will be Garbage collected by JVM since SASL Library releases it's reference after
        // returning the byte array.
        final byte[] decodedMsg = saslCodec.unwrap(wrappedMsg, 0, wrappedMsgLength);
        if (logger.isTraceEnabled()) {
            logger.trace("Successfully decrypted incoming message. Length after decryption: {}", decodedMsg.length);
        }
        // Update the msg reader index since we have decrypted this chunk
        msg.skipBytes(wrappedMsgLength);
        // Allocate a new Bytebuf to copy the decrypted chunk.
        final ByteBuf decodedMsgBuf = ctx.alloc().buffer(decodedMsg.length);
        decodedMsgBuf.writeBytes(decodedMsg);
        // Add the decrypted chunk to output buffer for next handler to take care of it.
        out.add(decodedMsgBuf);
    } catch (OutOfMemoryException e) {
        logger.warn("Failure allocating buffer on incoming stream due to memory limits.");
        msg.resetReaderIndex();
        outOfMemoryHandler.handle();
    } catch (IOException e) {
        logger.error("Something went wrong while unwrapping the message: {} with MaxEncodeSize: {} and " + "error: {}", msg, maxWrappedSize, e.getMessage());
        throw e;
    }
}

Example 23 with OutOfMemoryException

use of org.apache.drill.exec.exception.OutOfMemoryException in project drill by apache.

the class UnorderedReceiverBatch method next.

@Override
public IterOutcome next() {
    batchLoader.resetRecordCount();
    stats.startProcessing();
    try {
        RawFragmentBatch batch;
        try {
            stats.startWait();
            batch = getNextBatch();
            // skip over empty batches. we do this since these are basically control messages.
            while (batch != null && batch.getHeader().getDef().getRecordCount() == 0 && (!first || batch.getHeader().getDef().getFieldCount() == 0)) {
                batch = getNextBatch();
            }
        } finally {
            stats.stopWait();
        }
        first = false;
        if (batch == null) {
            lastOutcome = IterOutcome.NONE;
            batchLoader.zero();
            context.getExecutorState().checkContinue();
            return lastOutcome;
        }
        if (context.getAllocator().isOverLimit()) {
            context.requestMemory(this);
            if (context.getAllocator().isOverLimit()) {
                throw new OutOfMemoryException("Allocator over limit");
            }
        }
        RecordBatchDef rbd = batch.getHeader().getDef();
        boolean schemaChanged = batchLoader.load(rbd, batch.getBody());
        stats.addLongStat(Metric.BYTES_RECEIVED, batch.getByteCount());
        batch.release();
        if (schemaChanged) {
            this.schema = batchLoader.getSchema();
            stats.batchReceived(0, rbd.getRecordCount(), true);
            lastOutcome = IterOutcome.OK_NEW_SCHEMA;
        } else {
            stats.batchReceived(0, rbd.getRecordCount(), false);
            lastOutcome = IterOutcome.OK;
        }
        return lastOutcome;
    } finally {
        stats.stopProcessing();
    }
}

Example 24 with OutOfMemoryException

use of org.apache.drill.exec.exception.OutOfMemoryException in project drill by apache.

the class HashAggTemplate method checkGroupAndAggrValues.

// Check if a group is present in the hash table; if not, insert it in the hash table.
// The htIdxHolder contains the index of the group in the hash table container; this same
// index is also used for the aggregation values maintained by the hash aggregate.
private void checkGroupAndAggrValues(int incomingRowIdx) {
    assert incomingRowIdx >= 0;
    assert !earlyOutput;
    // The hash code is computed once, then its lower bits are used to determine the
    // partition to use, and the higher bits determine the location in the hash table.
    int hashCode;
    try {
        // htables[0].updateBatches();
        hashCode = htables[0].getBuildHashCode(incomingRowIdx);
    } catch (SchemaChangeException e) {
        throw new UnsupportedOperationException("Unexpected schema change", e);
    }
    // right shift hash code for secondary (or tertiary...) spilling
    for (int i = 0; i < spilledState.getCycle(); i++) {
        hashCode >>>= spilledState.getBitsInMask();
    }
    int currentPartition = hashCode & spilledState.getPartitionMask();
    hashCode >>>= spilledState.getBitsInMask();
    HashTable.PutStatus putStatus = null;
    long allocatedBeforeHTput = allocator.getAllocatedMemory();
    String tryingTo = phase.is1st() ? "early return" : "spill";
    // Proactive spill - in case there is no reserve memory - spill and retry putting later
    if (reserveValueBatchMemory == 0 && canSpill) {
        logger.trace("Reserved memory runs short, trying to {} a partition and retry Hash Table put() again.", tryingTo);
        // spill to free some memory
        doSpill(currentPartition);
        retrySameIndex = true;
        // to retry this put()
        return;
    }
    // ==========================================
    try {
        putStatus = htables[currentPartition].put(incomingRowIdx, htIdxHolder, hashCode, getTargetBatchCount());
    } catch (RetryAfterSpillException re) {
        if (!canSpill) {
            throw new OutOfMemoryException(getOOMErrorMsg("Can not spill"));
        }
        logger.trace("HT put failed with an OOM, trying to {} a partition and retry Hash Table put() again.", tryingTo);
        // for debugging - in case there's a leak
        long memDiff = allocator.getAllocatedMemory() - allocatedBeforeHTput;
        if (memDiff > 0) {
            logger.warn("Leak: HashTable put() OOM left behind {} bytes allocated", memDiff);
        }
        // spill to free some memory
        doSpill(currentPartition);
        retrySameIndex = true;
        // to retry this put()
        return;
    } catch (OutOfMemoryException exc) {
        throw new OutOfMemoryException(getOOMErrorMsg("HT was: " + allocatedBeforeHTput), exc);
    } catch (SchemaChangeException e) {
        throw new UnsupportedOperationException("Unexpected schema change", e);
    }
    long allocatedBeforeAggCol = allocator.getAllocatedMemory();
    boolean needToCheckIfSpillIsNeeded = allocatedBeforeAggCol > allocatedBeforeHTput;
    // 
    if (putStatus == HashTable.PutStatus.NEW_BATCH_ADDED) {
        try {
            // try to preempt an OOM by using the reserve
            useReservedValuesMemory();
            // allocate a new (internal) values batch
            addBatchHolder(currentPartition, getTargetBatchCount());
            // restore the reserve, if possible
            restoreReservedMemory();
            // A reason to check for a spill - In case restore-reserve failed
            needToCheckIfSpillIsNeeded = (0 == reserveValueBatchMemory);
            // just allocated a planned batch
            if (plannedBatches > 0) {
                plannedBatches--;
            }
            long totalAddedMem = allocator.getAllocatedMemory() - allocatedBeforeHTput;
            long aggValuesAddedMem = allocator.getAllocatedMemory() - allocatedBeforeAggCol;
            logger.trace("MEMORY CHECK AGG: allocated now {}, added {}, total (with HT) added {}", allocator.getAllocatedMemory(), aggValuesAddedMem, totalAddedMem);
            // resize the batch estimates if needed (e.g., varchars may take more memory than estimated)
            if (totalAddedMem > estMaxBatchSize) {
                logger.trace("Adjusting Batch size estimate from {} to {}", estMaxBatchSize, totalAddedMem);
                estMaxBatchSize = totalAddedMem;
                needToCheckIfSpillIsNeeded = true;
            }
            if (aggValuesAddedMem > estValuesBatchSize) {
                logger.trace("Adjusting Values Batch size from {} to {}", estValuesBatchSize, aggValuesAddedMem);
                estValuesBatchSize = aggValuesAddedMem;
                needToCheckIfSpillIsNeeded = true;
            }
        } catch (OutOfMemoryException exc) {
            throw new OutOfMemoryException(getOOMErrorMsg("AGGR"), exc);
        }
    } else if (putStatus == HashTable.PutStatus.KEY_ADDED_LAST) {
        // If a batch just became full (i.e. another batch would be allocated soon) -- then need to
        // check (later, see below) if the memory limits are too close, and if so -- then spill !
        // planning to allocate one more batch
        plannedBatches++;
        needToCheckIfSpillIsNeeded = true;
    }
    // =================================================================
    // Locate the matching aggregate columns and perform the aggregation
    // =================================================================
    int currentIdx = htIdxHolder.value;
    BatchHolder bh = batchHolders[currentPartition].get((currentIdx >>> 16) & BATCH_MASK);
    int idxWithinBatch = currentIdx & BATCH_MASK;
    if (bh.updateAggrValues(incomingRowIdx, idxWithinBatch)) {
        numGroupedRecords++;
    }
    // ===================================================================================
    if (needToCheckIfSpillIsNeeded && canSpill && useMemoryPrediction) {
        spillIfNeeded(currentPartition);
    }
}

Example 25 with OutOfMemoryException

use of org.apache.drill.exec.exception.OutOfMemoryException in project drill by apache.

the class ExternalSortBatch method innerNext.

@SuppressWarnings("resource")
@Override
public IterOutcome innerNext() {
    if (schema != null) {
        if (spillCount == 0) {
            return (getSelectionVector4().next()) ? IterOutcome.OK : IterOutcome.NONE;
        } else {
            Stopwatch w = Stopwatch.createStarted();
            int count = copier.next(targetRecordCount);
            if (count > 0) {
                long t = w.elapsed(TimeUnit.MICROSECONDS);
                logger.debug("Took {} us to merge {} records", t, count);
                container.setRecordCount(count);
                return IterOutcome.OK;
            } else {
                logger.debug("copier returned 0 records");
                return IterOutcome.NONE;
            }
        }
    }
    int totalCount = 0;
    // total number of batches received so far
    int totalBatches = 0;
    try {
        container.clear();
        outer: while (true) {
            IterOutcome upstream;
            if (first) {
                upstream = IterOutcome.OK_NEW_SCHEMA;
            } else {
                upstream = next(incoming);
            }
            if (upstream == IterOutcome.OK && sorter == null) {
                upstream = IterOutcome.OK_NEW_SCHEMA;
            }
            switch(upstream) {
                case NONE:
                    if (first) {
                        return upstream;
                    }
                    break outer;
                case NOT_YET:
                    throw new UnsupportedOperationException();
                case STOP:
                    return upstream;
                case OK_NEW_SCHEMA:
                case OK:
                    VectorContainer convertedBatch;
                    // only change in the case that the schema truly changes.  Artificial schema changes are ignored.
                    if (upstream == IterOutcome.OK_NEW_SCHEMA && !incoming.getSchema().equals(schema)) {
                        if (schema != null) {
                            if (unionTypeEnabled) {
                                this.schema = SchemaUtil.mergeSchemas(schema, incoming.getSchema());
                            } else {
                                throw SchemaChangeException.schemaChanged("Schema changes not supported in External Sort. Please enable Union type", schema, incoming.getSchema());
                            }
                        } else {
                            schema = incoming.getSchema();
                        }
                        convertedBatch = SchemaUtil.coerceContainer(incoming, schema, oContext);
                        for (BatchGroup b : batchGroups) {
                            b.setSchema(schema);
                        }
                        for (BatchGroup b : spilledBatchGroups) {
                            b.setSchema(schema);
                        }
                        this.sorter = createNewSorter(context, convertedBatch);
                    } else {
                        convertedBatch = SchemaUtil.coerceContainer(incoming, schema, oContext);
                    }
                    if (first) {
                        first = false;
                    }
                    if (convertedBatch.getRecordCount() == 0) {
                        for (VectorWrapper<?> w : convertedBatch) {
                            w.clear();
                        }
                        break;
                    }
                    SelectionVector2 sv2;
                    if (incoming.getSchema().getSelectionVectorMode() == BatchSchema.SelectionVectorMode.TWO_BYTE) {
                        sv2 = incoming.getSelectionVector2().clone();
                    } else {
                        try {
                            sv2 = newSV2();
                        } catch (InterruptedException e) {
                            return IterOutcome.STOP;
                        } catch (OutOfMemoryException e) {
                            throw new OutOfMemoryException(e);
                        }
                    }
                    int count = sv2.getCount();
                    totalCount += count;
                    totalBatches++;
                    sorter.setup(context, sv2, convertedBatch);
                    sorter.sort(sv2);
                    RecordBatchData rbd = new RecordBatchData(convertedBatch, oAllocator);
                    boolean success = false;
                    try {
                        rbd.setSv2(sv2);
                        batchGroups.add(new BatchGroup(rbd.getContainer(), rbd.getSv2(), oContext));
                        if (peakNumBatches < batchGroups.size()) {
                            peakNumBatches = batchGroups.size();
                            stats.setLongStat(Metric.PEAK_BATCHES_IN_MEMORY, peakNumBatches);
                        }
                        batchesSinceLastSpill++;
                        if (// If we haven't spilled so far, do we have enough memory for MSorter if this turns out to be the last incoming batch?
                        (spillCount == 0 && !hasMemoryForInMemorySort(totalCount)) || // If we haven't spilled so far, make sure we don't exceed the maximum number of batches SV4 can address
                        (spillCount == 0 && totalBatches > Character.MAX_VALUE) || // current memory used is more than 95% of memory usage limit of this operator
                        (oAllocator.getAllocatedMemory() > .95 * oAllocator.getLimit()) || // since the last spill exceed the defined limit
                        (batchGroups.size() > SPILL_THRESHOLD && batchesSinceLastSpill >= SPILL_BATCH_GROUP_SIZE)) {
                            if (firstSpillBatchCount == 0) {
                                firstSpillBatchCount = batchGroups.size();
                            }
                            if (spilledBatchGroups.size() > firstSpillBatchCount / 2) {
                                logger.info("Merging spills");
                                final BatchGroup merged = mergeAndSpill(spilledBatchGroups);
                                if (merged != null) {
                                    spilledBatchGroups.addFirst(merged);
                                }
                            }
                            final BatchGroup merged = mergeAndSpill(batchGroups);
                            if (merged != null) {
                                // make sure we don't add null to spilledBatchGroups
                                spilledBatchGroups.add(merged);
                                batchesSinceLastSpill = 0;
                            }
                        }
                        success = true;
                    } finally {
                        if (!success) {
                            rbd.clear();
                        }
                    }
                    break;
                case OUT_OF_MEMORY:
                    logger.debug("received OUT_OF_MEMORY, trying to spill");
                    if (batchesSinceLastSpill > 2) {
                        final BatchGroup merged = mergeAndSpill(batchGroups);
                        if (merged != null) {
                            spilledBatchGroups.add(merged);
                            batchesSinceLastSpill = 0;
                        }
                    } else {
                        logger.debug("not enough batches to spill, sending OUT_OF_MEMORY downstream");
                        return IterOutcome.OUT_OF_MEMORY;
                    }
                    break;
                default:
                    throw new UnsupportedOperationException();
            }
        }
        if (totalCount == 0) {
            return IterOutcome.NONE;
        }
        if (spillCount == 0) {
            if (builder != null) {
                builder.clear();
                builder.close();
            }
            builder = new SortRecordBatchBuilder(oAllocator);
            for (BatchGroup group : batchGroups) {
                RecordBatchData rbd = new RecordBatchData(group.getContainer(), oAllocator);
                rbd.setSv2(group.getSv2());
                builder.add(rbd);
            }
            builder.build(context, container);
            sv4 = builder.getSv4();
            mSorter = createNewMSorter();
            mSorter.setup(context, oAllocator, getSelectionVector4(), this.container);
            // For testing memory-leak purpose, inject exception after mSorter finishes setup
            injector.injectUnchecked(context.getExecutionControls(), INTERRUPTION_AFTER_SETUP);
            mSorter.sort(this.container);
            // sort may have prematurely exited due to should continue returning false.
            if (!context.shouldContinue()) {
                return IterOutcome.STOP;
            }
            // For testing memory-leak purpose, inject exception after mSorter finishes sorting
            injector.injectUnchecked(context.getExecutionControls(), INTERRUPTION_AFTER_SORT);
            sv4 = mSorter.getSV4();
            container.buildSchema(SelectionVectorMode.FOUR_BYTE);
        } else {
            // some batches were spilled
            final BatchGroup merged = mergeAndSpill(batchGroups);
            if (merged != null) {
                spilledBatchGroups.add(merged);
            }
            batchGroups.addAll(spilledBatchGroups);
            // no need to cleanup spilledBatchGroups, all it's batches are in batchGroups now
            spilledBatchGroups = null;
            logger.warn("Starting to merge. {} batch groups. Current allocated memory: {}", batchGroups.size(), oAllocator.getAllocatedMemory());
            VectorContainer hyperBatch = constructHyperBatch(batchGroups);
            createCopier(hyperBatch, batchGroups, container, false);
            int estimatedRecordSize = 0;
            for (VectorWrapper<?> w : batchGroups.get(0)) {
                try {
                    estimatedRecordSize += TypeHelper.getSize(w.getField().getType());
                } catch (UnsupportedOperationException e) {
                    estimatedRecordSize += 50;
                }
            }
            targetRecordCount = Math.min(MAX_BATCH_SIZE, Math.max(1, COPIER_BATCH_MEM_LIMIT / estimatedRecordSize));
            int count = copier.next(targetRecordCount);
            container.buildSchema(SelectionVectorMode.NONE);
            container.setRecordCount(count);
        }
        return IterOutcome.OK_NEW_SCHEMA;
    } catch (SchemaChangeException ex) {
        kill(false);
        context.fail(UserException.unsupportedError(ex).message("Sort doesn't currently support sorts with changing schemas").build(logger));
        return IterOutcome.STOP;
    } catch (ClassTransformationException | IOException ex) {
        kill(false);
        context.fail(ex);
        return IterOutcome.STOP;
    } catch (UnsupportedOperationException e) {
        throw new RuntimeException(e);
    }
}