Examples with RamAccounting io.crate.breaker.RamAccounting used on opensource projects

Example 11 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class GroupByOptimizedIterator method tryOptimizeSingleStringKey.

@Nullable
static BatchIterator<Row> tryOptimizeSingleStringKey(IndexShard indexShard, DocTableInfo table, LuceneQueryBuilder luceneQueryBuilder, FieldTypeLookup fieldTypeLookup, BigArrays bigArrays, InputFactory inputFactory, DocInputFactory docInputFactory, RoutedCollectPhase collectPhase, CollectTask collectTask) {
    Collection<? extends Projection> shardProjections = shardProjections(collectPhase.projections());
    GroupProjection groupProjection = getSingleStringKeyGroupProjection(shardProjections);
    if (groupProjection == null) {
        return null;
    }
    assert groupProjection.keys().size() == 1 : "Must have 1 key if getSingleStringKeyGroupProjection returned a projection";
    Reference keyRef = getKeyRef(collectPhase.toCollect(), groupProjection.keys().get(0));
    if (keyRef == null) {
        // group by on non-reference
        return null;
    }
    keyRef = (Reference) DocReferences.inverseSourceLookup(keyRef);
    MappedFieldType keyFieldType = fieldTypeLookup.get(keyRef.column().fqn());
    if (keyFieldType == null || !keyFieldType.hasDocValues()) {
        return null;
    }
    if (Symbols.containsColumn(collectPhase.toCollect(), DocSysColumns.SCORE) || Symbols.containsColumn(collectPhase.where(), DocSysColumns.SCORE)) {
        // to keep the optimized implementation a bit simpler
        return null;
    }
    if (hasHighCardinalityRatio(() -> indexShard.acquireSearcher("group-by-cardinality-check"), keyFieldType.name())) {
        return null;
    }
    ShardId shardId = indexShard.shardId();
    SharedShardContext sharedShardContext = collectTask.sharedShardContexts().getOrCreateContext(shardId);
    var searcher = sharedShardContext.acquireSearcher("group-by-ordinals:" + formatSource(collectPhase));
    collectTask.addSearcher(sharedShardContext.readerId(), searcher);
    final QueryShardContext queryShardContext = sharedShardContext.indexService().newQueryShardContext();
    InputFactory.Context<? extends LuceneCollectorExpression<?>> docCtx = docInputFactory.getCtx(collectTask.txnCtx());
    docCtx.add(collectPhase.toCollect().stream()::iterator);
    InputFactory.Context<CollectExpression<Row, ?>> ctxForAggregations = inputFactory.ctxForAggregations(collectTask.txnCtx());
    ctxForAggregations.add(groupProjection.values());
    final List<CollectExpression<Row, ?>> aggExpressions = ctxForAggregations.expressions();
    List<AggregationContext> aggregations = ctxForAggregations.aggregations();
    List<? extends LuceneCollectorExpression<?>> expressions = docCtx.expressions();
    RamAccounting ramAccounting = collectTask.getRamAccounting();
    CollectorContext collectorContext = new CollectorContext(sharedShardContext.readerId());
    InputRow inputRow = new InputRow(docCtx.topLevelInputs());
    LuceneQueryBuilder.Context queryContext = luceneQueryBuilder.convert(collectPhase.where(), collectTask.txnCtx(), indexShard.mapperService(), indexShard.shardId().getIndexName(), queryShardContext, table, sharedShardContext.indexService().cache());
    return getIterator(bigArrays, searcher.item(), keyRef.column().fqn(), aggregations, expressions, aggExpressions, ramAccounting, collectTask.memoryManager(), collectTask.minNodeVersion(), inputRow, queryContext.query(), collectorContext, groupProjection.mode());
}

Example 12 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class TransportFetchOperationTest method test_ram_accounting_on_non_empty_fetch_ids_and_close.

@Test
public void test_ram_accounting_on_non_empty_fetch_ids_and_close() {
    var toFetch = new IntObjectHashMap<IntContainer>();
    toFetch.put(1, new IntArrayList());
    RamAccounting ramAccounting = TransportFetchOperation.ramAccountingForIncomingResponse(RamAccounting.NO_ACCOUNTING, toFetch, true);
    assertThat(ramAccounting, instanceOf(BlockBasedRamAccounting.class));
}

Example 13 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class ReservoirSampler method getSamples.

public Samples getSamples(RelationName relationName, List<Reference> columns, int maxSamples) {
    TableInfo table;
    try {
        table = schemas.getTableInfo(relationName);
    } catch (RelationUnknown e) {
        return Samples.EMPTY;
    }
    if (!(table instanceof DocTableInfo)) {
        return Samples.EMPTY;
    }
    DocTableInfo docTable = (DocTableInfo) table;
    Random random = Randomness.get();
    Metadata metadata = clusterService.state().metadata();
    CoordinatorTxnCtx coordinatorTxnCtx = CoordinatorTxnCtx.systemTransactionContext();
    List<Streamer> streamers = Arrays.asList(Symbols.streamerArray(columns));
    List<Engine.Searcher> searchersToRelease = new ArrayList<>();
    CircuitBreaker breaker = circuitBreakerService.getBreaker(HierarchyCircuitBreakerService.QUERY);
    RamAccounting ramAccounting = new BlockBasedRamAccounting(b -> breaker.addEstimateBytesAndMaybeBreak(b, "Reservoir-sampling"), MAX_BLOCK_SIZE_IN_BYTES);
    try {
        return getSamples(columns, maxSamples, docTable, random, metadata, coordinatorTxnCtx, streamers, searchersToRelease, ramAccounting);
    } finally {
        ramAccounting.close();
        for (Engine.Searcher searcher : searchersToRelease) {
            searcher.close();
        }
    }
}

Example 14 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class SqlHttpHandler method executeSimpleRequest.

private CompletableFuture<XContentBuilder> executeSimpleRequest(Session session, String stmt, List<Object> args, boolean includeTypes) throws IOException {
    long startTimeInNs = System.nanoTime();
    session.parse(UNNAMED, stmt, emptyList());
    session.bind(UNNAMED, UNNAMED, args == null ? emptyList() : args, null);
    DescribeResult description = session.describe('P', UNNAMED);
    List<Symbol> resultFields = description.getFields();
    ResultReceiver<XContentBuilder> resultReceiver;
    if (resultFields == null) {
        resultReceiver = new RestRowCountReceiver(JsonXContent.contentBuilder(), startTimeInNs, includeTypes);
    } else {
        CircuitBreaker breaker = circuitBreakerProvider.apply(HierarchyCircuitBreakerService.QUERY);
        RamAccounting ramAccounting = new BlockBasedRamAccounting(b -> breaker.addEstimateBytesAndMaybeBreak(b, "http-result"), MAX_BLOCK_SIZE_IN_BYTES);
        resultReceiver = new RestResultSetReceiver(JsonXContent.contentBuilder(), resultFields, startTimeInNs, new RowAccountingWithEstimators(Symbols.typeView(resultFields), ramAccounting), includeTypes);
        resultReceiver.completionFuture().whenComplete((result, error) -> ramAccounting.close());
    }
    session.execute(UNNAMED, 0, resultReceiver);
    return session.sync().thenCompose(ignored -> resultReceiver.completionFuture());
}