Examples with RowCellsAccountingWithEstimators io.crate.breaker.RowCellsAccountingWithEstimators used on opensource projects

Example 1 with RowCellsAccountingWithEstimators

use of io.crate.breaker.RowCellsAccountingWithEstimators in project crate by crate.

the class NestedLoopOperation method buildCrossJoinBatchIterator.

private static BatchIterator<Row> buildCrossJoinBatchIterator(BatchIterator<Row> left, BatchIterator<Row> right, CombinedRow combiner, CircuitBreaker circuitBreaker, RamAccounting ramAccounting, List<DataType<?>> leftSideColumnTypes, long estimatedRowsSizeLeft, long estimatedNumberOfRowsLeft, boolean blockNestedLoop) {
    if (blockNestedLoop) {
        IntSupplier blockSizeCalculator = new RamBlockSizeCalculator(Paging.PAGE_SIZE, circuitBreaker, estimatedRowsSizeLeft, estimatedNumberOfRowsLeft);
        var rowAccounting = new RowCellsAccountingWithEstimators(leftSideColumnTypes, ramAccounting, 0);
        return new CrossJoinBlockNLBatchIterator(left, right, combiner, blockSizeCalculator, rowAccounting);
    } else {
        return new CrossJoinNLBatchIterator<>(left, right, combiner);
    }
}

Example 2 with RowCellsAccountingWithEstimators

use of io.crate.breaker.RowCellsAccountingWithEstimators in project crate by crate.

the class SortingProjectorTest method testUsedMemoryIsAccountedFor.

@Test
public void testUsedMemoryIsAccountedFor() throws Exception {
    MemoryCircuitBreaker circuitBreaker = new MemoryCircuitBreaker(new ByteSizeValue(30, ByteSizeUnit.BYTES), 1, LogManager.getLogger(SortingProjectorTest.class));
    RowCellsAccountingWithEstimators rowAccounting = new RowCellsAccountingWithEstimators(List.of(DataTypes.LONG, DataTypes.BOOLEAN), ConcurrentRamAccounting.forCircuitBreaker("testContext", circuitBreaker), 0);
    Projector projector = createProjector(rowAccounting, 1, 0);
    consumer.accept(projector.apply(TestingBatchIterators.range(1, 11)), null);
    expectedException.expect(CircuitBreakingException.class);
    consumer.getResult();
}

Example 3 with RowCellsAccountingWithEstimators

use of io.crate.breaker.RowCellsAccountingWithEstimators in project crate by crate.

the class SortingTopNProjectorTest method testUsedMemoryIsAccountedFor.

@Test
public void testUsedMemoryIsAccountedFor() throws Exception {
    MemoryCircuitBreaker circuitBreaker = new MemoryCircuitBreaker(new ByteSizeValue(30, ByteSizeUnit.BYTES), 1, LogManager.getLogger(SortingTopNProjectorTest.class));
    RowCellsAccountingWithEstimators rowAccounting = new RowCellsAccountingWithEstimators(List.of(DataTypes.LONG, DataTypes.BOOLEAN), ConcurrentRamAccounting.forCircuitBreaker("testContext", circuitBreaker), 0);
    Projector projector = getProjector(rowAccounting, 1, 100_000, TopN.NO_OFFSET, FIRST_CELL_ORDERING);
    consumer.accept(projector.apply(TestingBatchIterators.range(1, 11)), null);
    expectedException.expect(CircuitBreakingException.class);
    consumer.getResult();
}

Example 4 with RowCellsAccountingWithEstimators

use of io.crate.breaker.RowCellsAccountingWithEstimators in project crate by crate.

the class ProjectionToProjectorVisitor method visitOrderedTopN.

@Override
public Projector visitOrderedTopN(OrderedTopNProjection projection, Context context) {
    /* OrderBy symbols are added to the rows to enable sorting on them post-collect. E.g.:
         *
         * outputs: [x]
         * orderBy: [y]
         *
         * topLevelInputs: [x, y]
         *                    /
         * orderByIndices: [1]
         */
    InputFactory.Context<CollectExpression<Row, ?>> ctx = inputFactory.ctxForInputColumns(context.txnCtx);
    ctx.add(projection.outputs());
    ctx.add(projection.orderBy());
    int numOutputs = projection.outputs().size();
    List<Input<?>> inputs = ctx.topLevelInputs();
    int[] orderByIndices = new int[inputs.size() - numOutputs];
    int idx = 0;
    for (int i = numOutputs; i < inputs.size(); i++) {
        orderByIndices[idx++] = i;
    }
    // priority queues implementation are backed by an arrayList
    int rowMemoryOverhead = 32;
    RowCellsAccountingWithEstimators rowAccounting = new RowCellsAccountingWithEstimators(Symbols.typeView(Lists2.concat(projection.outputs(), projection.orderBy())), context.ramAccounting, rowMemoryOverhead);
    if (projection.limit() > TopN.NO_LIMIT) {
        return new SortingTopNProjector(rowAccounting, inputs, ctx.expressions(), numOutputs, OrderingByPosition.arrayOrdering(orderByIndices, projection.reverseFlags(), projection.nullsFirst()), projection.limit(), projection.offset(), UNBOUNDED_COLLECTOR_THRESHOLD);
    }
    return new SortingProjector(rowAccounting, inputs, ctx.expressions(), numOutputs, OrderingByPosition.arrayOrdering(orderByIndices, projection.reverseFlags(), projection.nullsFirst()), projection.offset());
}

Example 5 with RowCellsAccountingWithEstimators

use of io.crate.breaker.RowCellsAccountingWithEstimators in project crate by crate.

the class ReservoirSampler method getSamples.

private Samples getSamples(List<Reference> columns, int maxSamples, DocTableInfo docTable, Random random, Metadata metadata, CoordinatorTxnCtx coordinatorTxnCtx, List<Streamer> streamers, List<Engine.Searcher> searchersToRelease, RamAccounting ramAccounting) {
    ramAccounting.addBytes(DataTypes.LONG.fixedSize() * maxSamples);
    Reservoir<Long> fetchIdSamples = new Reservoir<>(maxSamples, random);
    ArrayList<DocIdToRow> docIdToRowsFunctionPerReader = new ArrayList<>();
    long totalNumDocs = 0;
    long totalSizeInBytes = 0;
    for (String index : docTable.concreteOpenIndices()) {
        var indexMetadata = metadata.index(index);
        if (indexMetadata == null) {
            continue;
        }
        var indexService = indicesService.indexService(indexMetadata.getIndex());
        if (indexService == null) {
            continue;
        }
        var mapperService = indexService.mapperService();
        FieldTypeLookup fieldTypeLookup = mapperService::fullName;
        var ctx = new DocInputFactory(nodeCtx, new LuceneReferenceResolver(indexService.index().getName(), fieldTypeLookup, docTable.partitionedByColumns())).getCtx(coordinatorTxnCtx);
        ctx.add(columns);
        List<Input<?>> inputs = ctx.topLevelInputs();
        List<? extends LuceneCollectorExpression<?>> expressions = ctx.expressions();
        CollectorContext collectorContext = new CollectorContext();
        for (LuceneCollectorExpression<?> expression : expressions) {
            expression.startCollect(collectorContext);
        }
        for (IndexShard indexShard : indexService) {
            if (!indexShard.routingEntry().primary()) {
                continue;
            }
            try {
                Engine.Searcher searcher = indexShard.acquireSearcher("update-table-statistics");
                searchersToRelease.add(searcher);
                totalNumDocs += searcher.getIndexReader().numDocs();
                totalSizeInBytes += indexShard.storeStats().getSizeInBytes();
                DocIdToRow docIdToRow = new DocIdToRow(searcher, inputs, expressions);
                docIdToRowsFunctionPerReader.add(docIdToRow);
                try {
                    // We do the sampling in 2 phases. First we get the docIds;
                    // then we retrieve the column values for the sampled docIds.
                    // we do this in 2 phases because the reservoir sampling might override previously seen
                    // items and we want to avoid unnecessary disk-lookup
                    var collector = new ReservoirCollector(fetchIdSamples, searchersToRelease.size() - 1);
                    searcher.search(new MatchAllDocsQuery(), collector);
                } catch (IOException e) {
                    throw new UncheckedIOException(e);
                }
            } catch (IllegalIndexShardStateException | AlreadyClosedException ignored) {
            }
        }
    }
    var rowAccounting = new RowCellsAccountingWithEstimators(Symbols.typeView(columns), ramAccounting, 0);
    ArrayList<Row> records = new ArrayList<>();
    for (long fetchId : fetchIdSamples.samples()) {
        int readerId = FetchId.decodeReaderId(fetchId);
        DocIdToRow docIdToRow = docIdToRowsFunctionPerReader.get(readerId);
        Object[] row = docIdToRow.apply(FetchId.decodeDocId(fetchId));
        try {
            rowAccounting.accountForAndMaybeBreak(row);
        } catch (CircuitBreakingException e) {
            LOGGER.info("Stopped gathering samples for `ANALYZE` operation because circuit breaker triggered. " + "Generating statistics with {} instead of {} records", records.size(), maxSamples);
            break;
        }
        records.add(new RowN(row));
    }
    return new Samples(records, streamers, totalNumDocs, totalSizeInBytes);
}