Examples with RamAccounting io.crate.breaker.RamAccounting used on opensource projects

Example 1 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class WindowProjector method fromProjection.

public static Projector fromProjection(WindowAggProjection projection, NodeContext nodeCtx, InputFactory inputFactory, TransactionContext txnCtx, RamAccounting ramAccounting, MemoryManager memoryManager, Version minNodeVersion, Version indexVersionCreated, IntSupplier numThreads, Executor executor) {
    var windowFunctionSymbols = projection.windowFunctions();
    var numWindowFunctions = windowFunctionSymbols.size();
    assert numWindowFunctions > 0 : "WindowAggProjection must have at least 1 window function.";
    ArrayList<WindowFunction> windowFunctions = new ArrayList<>(numWindowFunctions);
    ArrayList<CollectExpression<Row, ?>> windowFuncArgsExpressions = new ArrayList<>(numWindowFunctions);
    Input[][] windowFuncArgsInputs = new Input[numWindowFunctions][];
    Boolean[] ignoreNulls = new Boolean[numWindowFunctions];
    for (int idx = 0; idx < numWindowFunctions; idx++) {
        var windowFunctionSymbol = windowFunctionSymbols.get(idx);
        InputFactory.Context<CollectExpression<Row, ?>> ctx = inputFactory.ctxForInputColumns(txnCtx);
        ctx.add(windowFunctionSymbol.arguments());
        FunctionImplementation impl = nodeCtx.functions().getQualified(windowFunctionSymbol, txnCtx.sessionSettings().searchPath());
        assert impl != null : "Function implementation not found using full qualified lookup";
        if (impl instanceof AggregationFunction) {
            var filterInputFactoryCtx = inputFactory.ctxForInputColumns(txnCtx);
            var filterSymbol = windowFunctionSymbol.filter();
            // noinspection unchecked
            Input<Boolean> filterInput = filterSymbol == null ? Literal.BOOLEAN_TRUE : (Input<Boolean>) filterInputFactoryCtx.add(filterSymbol);
            ExpressionsInput<Row, Boolean> filter = new ExpressionsInput<>(filterInput, filterInputFactoryCtx.expressions());
            windowFunctions.add(new AggregateToWindowFunctionAdapter((AggregationFunction) impl, filter, indexVersionCreated, ramAccounting, memoryManager, minNodeVersion));
        } else if (impl instanceof WindowFunction) {
            windowFunctions.add((WindowFunction) impl);
        } else {
            throw new AssertionError("Function needs to be either a window or an aggregate function");
        }
        windowFuncArgsExpressions.addAll(ctx.expressions());
        windowFuncArgsInputs[idx] = ctx.topLevelInputs().toArray(new Input[0]);
        ignoreNulls[idx] = windowFunctionSymbol.ignoreNulls();
    }
    var windowDefinition = projection.windowDefinition();
    var partitions = windowDefinition.partitions();
    Supplier<InputFactory.Context<CollectExpression<Row, ?>>> createInputFactoryContext = () -> inputFactory.ctxForInputColumns(txnCtx);
    int arrayListElementOverHead = 32;
    RowAccountingWithEstimators accounting = new RowAccountingWithEstimators(Symbols.typeView(projection.standalone()), ramAccounting, arrayListElementOverHead);
    Comparator<Object[]> cmpPartitionBy = partitions.isEmpty() ? null : createComparator(createInputFactoryContext, new OrderBy(windowDefinition.partitions()));
    Comparator<Object[]> cmpOrderBy = createComparator(createInputFactoryContext, windowDefinition.orderBy());
    int numCellsInSourceRow = projection.standalone().size();
    ComputeFrameBoundary<Object[]> computeFrameStart = createComputeStartFrameBoundary(numCellsInSourceRow, txnCtx, nodeCtx, windowDefinition, cmpOrderBy);
    ComputeFrameBoundary<Object[]> computeFrameEnd = createComputeEndFrameBoundary(numCellsInSourceRow, txnCtx, nodeCtx, windowDefinition, cmpOrderBy);
    return sourceRows -> WindowFunctionBatchIterator.of(sourceRows, accounting, computeFrameStart, computeFrameEnd, cmpPartitionBy, cmpOrderBy, numCellsInSourceRow, numThreads, executor, windowFunctions, windowFuncArgsExpressions, ignoreNulls, windowFuncArgsInputs);
}

Example 2 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class JobSetup method registerContextPhases.

private void registerContextPhases(Iterable<? extends NodeOperation> nodeOperations, Context context) {
    for (NodeOperation nodeOperation : nodeOperations) {
        // context for nodeOperations without dependencies can be built immediately (e.g. FetchPhase)
        if (nodeOperation.downstreamExecutionPhaseId() == NodeOperation.NO_DOWNSTREAM) {
            LOGGER.trace("Building context for nodeOp without downstream: {}", nodeOperation);
            createContexts(nodeOperation.executionPhase(), context);
            context.opCtx.builtNodeOperations.set(nodeOperation.executionPhase().phaseId());
        }
        if (ExecutionPhases.hasDirectResponseDownstream(nodeOperation.downstreamNodes())) {
            var executionPhase = nodeOperation.executionPhase();
            CircuitBreaker breaker = breaker();
            int ramAccountingBlockSizeInBytes = BlockBasedRamAccounting.blockSizeInBytes(breaker.getLimit());
            var ramAccounting = new BlockBasedRamAccounting(b -> breaker.addEstimateBytesAndMaybeBreak(b, executionPhase.label()), ramAccountingBlockSizeInBytes);
            Streamer<?>[] streamers = StreamerVisitor.streamersFromOutputs(executionPhase);
            SingleBucketBuilder bucketBuilder = new SingleBucketBuilder(streamers, ramAccounting);
            context.directResponseFutures.add(bucketBuilder.completionFuture().whenComplete((res, err) -> ramAccounting.close()));
            context.registerBatchConsumer(nodeOperation.downstreamExecutionPhaseId(), bucketBuilder);
        }
    }
}

Example 3 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class FetchProjector method create.

public static Projector create(FetchProjection projection, RamAccounting ramAccounting, LongSupplier getBucketsBytesThreshold, TransactionContext txnCtx, NodeContext nodeCtx, FetchOperation fetchOperation) {
    final FetchRows fetchRows = FetchRows.create(txnCtx, nodeCtx, projection.fetchSources(), projection.outputSymbols());
    EstimateCellsSize estimateRowSize = new EstimateCellsSize(projection.inputTypes());
    return (BatchIterator<Row> source) -> {
        final long maxBucketsSizeInBytes = getBucketsBytesThreshold.getAsLong();
        BatchIterator<ReaderBuckets> buckets = BatchIterators.partition(source, projection.getFetchSize(), () -> new ReaderBuckets(fetchRows, projection::getFetchSourceByReader, estimateRowSize, ramAccounting), ReaderBuckets::add, readerBuckets -> readerBuckets.ramBytesUsed() > maxBucketsSizeInBytes);
        return new AsyncFlatMapBatchIterator<>(buckets, new FetchMapper(fetchOperation, projection.nodeReaders()));
    };
}

Example 4 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class HyperLogLogDistinctAggregation method getDocValueAggregator.

@Nullable
@Override
public DocValueAggregator<?> getDocValueAggregator(List<Reference> aggregationReferences, DocTableInfo table, List<Literal<?>> optionalParams) {
    if (aggregationReferences.stream().anyMatch(x -> !x.hasDocValues())) {
        return null;
    }
    Reference reference = aggregationReferences.get(0);
    var dataType = reference.valueType();
    switch(dataType.id()) {
        case ByteType.ID:
        case ShortType.ID:
        case IntegerType.ID:
        case LongType.ID:
        case TimestampType.ID_WITH_TZ:
        case TimestampType.ID_WITHOUT_TZ:
            return new SortedNumericDocValueAggregator<>(reference.column().fqn(), (ramAccounting, memoryManager, minNodeVersion) -> {
                var state = new HllState(dataType, minNodeVersion.onOrAfter(Version.V_4_1_0));
                var precision = optionalParams.size() == 1 ? (Integer) optionalParams.get(0).value() : HyperLogLogPlusPlus.DEFAULT_PRECISION;
                return initIfNeeded(state, memoryManager, precision);
            }, (values, state) -> {
                var hash = BitMixer.mix64(values.nextValue());
                state.addHash(hash);
            });
        case DoubleType.ID:
            return new SortedNumericDocValueAggregator<>(reference.column().fqn(), (ramAccounting, memoryManager, minNodeVersion) -> {
                var state = new HllState(dataType, minNodeVersion.onOrAfter(Version.V_4_1_0));
                var precision = optionalParams.size() == 1 ? (Integer) optionalParams.get(0).value() : HyperLogLogPlusPlus.DEFAULT_PRECISION;
                return initIfNeeded(state, memoryManager, precision);
            }, (values, state) -> {
                // Murmur3Hash.Double in regular aggregation calls mix64 of doubleToLongBits(double value).
                // Equivalent of that in context of double DocValue is
                // mix64(doubleToLongBits(decoded))
                // => mix64(doubleToLongBits(NumericUtils.sortableLongToDouble(values.nextValue())))
                // => mix64(doubleToLongBits(Double.longBitsToDouble(sortableDoubleBits(values.nextValue()))))
                // => mix64(sortableDoubleBits(values.nextValue()))
                var hash = BitMixer.mix64(NumericUtils.sortableDoubleBits(values.nextValue()));
                state.addHash(hash);
            });
        case FloatType.ID:
            return new SortedNumericDocValueAggregator<>(reference.column().fqn(), (ramAccounting, memoryManager, minNodeVersion) -> {
                var state = new HllState(dataType, minNodeVersion.onOrAfter(Version.V_4_1_0));
                var precision = optionalParams.size() == 1 ? (Integer) optionalParams.get(0).value() : HyperLogLogPlusPlus.DEFAULT_PRECISION;
                return initIfNeeded(state, memoryManager, precision);
            }, (values, state) -> {
                // Murmur3Hash.Float in regular aggregation calls mix64 of doubleToLongBits(double value).
                // Equivalent of that in context of float DocValue is
                // mix64(doubleToLongBits(decoded))
                // => mix64(doubleToLongBits((double)NumericUtils.sortableIntToFloat((int)values.nextValue())))
                // => mix64(doubleToLongBits(NumericUtils.sortableIntToFloat((int)values.nextValue()))) - no need to double cast, auto widening
                var hash = BitMixer.mix64(doubleToLongBits(NumericUtils.sortableIntToFloat((int) values.nextValue())));
                state.addHash(hash);
            });
        case StringType.ID:
            var precision = optionalParams.size() == 1 ? (Integer) optionalParams.get(0).value() : HyperLogLogPlusPlus.DEFAULT_PRECISION;
            return new HllAggregator(reference.column().fqn(), dataType, precision) {

                @Override
                public void apply(RamAccounting ramAccounting, int doc, HllState state) throws IOException {
                    if (super.values.advanceExact(doc) && super.values.docValueCount() == 1) {
                        BytesRef ref = super.values.nextValue();
                        var hash = state.isAllOn4_1() ? MurmurHash3.hash64(ref.bytes, ref.offset, ref.length) : MurmurHash3.hash128(ref.bytes, ref.offset, ref.length, 0, super.hash128).h1;
                        state.addHash(hash);
                    }
                }
            };
        case IpType.ID:
            var ipPrecision = optionalParams.size() == 1 ? (Integer) optionalParams.get(0).value() : HyperLogLogPlusPlus.DEFAULT_PRECISION;
            return new HllAggregator(reference.column().fqn(), dataType, ipPrecision) {

                @Override
                public void apply(RamAccounting ramAccounting, int doc, HllState state) throws IOException {
                    if (super.values.advanceExact(doc) && super.values.docValueCount() == 1) {
                        BytesRef ref = super.values.nextValue();
                        byte[] bytes = ((String) DocValueFormat.IP.format(ref)).getBytes(StandardCharsets.UTF_8);
                        var hash = state.isAllOn4_1() ? MurmurHash3.hash64(bytes, 0, bytes.length) : MurmurHash3.hash128(bytes, 0, bytes.length, 0, super.hash128).h1;
                        state.addHash(hash);
                    }
                }
            };
        default:
            return null;
    }
}

Example 5 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class WindowBatchIteratorTest method testWindowBatchIteratorAccountsUsedMemory.

@Test
public void testWindowBatchIteratorAccountsUsedMemory() {
    RamAccounting ramAccounting = ConcurrentRamAccounting.forCircuitBreaker("test", new NoopCircuitBreaker("dummy"));
    BatchIterator<Row> iterator = WindowFunctionBatchIterator.of(TestingBatchIterators.range(0, 10), new RowAccountingWithEstimators(List.of(DataTypes.INTEGER), ramAccounting, 32), (partitionStart, partitionEnd, currentIndex, sortedRows) -> 0, (partitionStart, partitionEnd, currentIndex, sortedRows) -> currentIndex, null, null, 1, () -> 1, Runnable::run, List.of(rowNumberWindowFunction()), List.of(), new Boolean[] { null }, new Input[][] { new Input[0] });
    TestingRowConsumer consumer = new TestingRowConsumer();
    consumer.accept(iterator, null);
    // should've accounted for 10 integers of 48 bytes each (16 for the integer, 32 for the ArrayList element)
    assertThat(ramAccounting.totalBytes(), is(480L));
}