Examples with RamAccounting io.crate.breaker.RamAccounting used on opensource projects

Example 6 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class TransportFetchOperationTest method test_no_ram_accounting_on_empty_fetch_ids_and_close.

@Test
public void test_no_ram_accounting_on_empty_fetch_ids_and_close() {
    RamAccounting ramAccounting = TransportFetchOperation.ramAccountingForIncomingResponse(RamAccounting.NO_ACCOUNTING, new IntObjectHashMap<>(), true);
    assertThat(ramAccounting, is(RamAccounting.NO_ACCOUNTING));
}

Example 7 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class PercentileAggregationTest method testSingleItemFractionsArgumentResultsInArrayResult.

@Test
public void testSingleItemFractionsArgumentResultsInArrayResult() {
    AggregationFunction impl = (AggregationFunction<?, ?>) nodeCtx.functions().getQualified(Signature.aggregate(PercentileAggregation.NAME, DataTypes.LONG.getTypeSignature(), DataTypes.DOUBLE_ARRAY.getTypeSignature(), DataTypes.DOUBLE_ARRAY.getTypeSignature()), List.of(DataTypes.LONG, DataTypes.DOUBLE_ARRAY), DataTypes.DOUBLE_ARRAY);
    RamAccounting ramAccounting = RamAccounting.NO_ACCOUNTING;
    Object state = impl.newState(ramAccounting, Version.CURRENT, Version.CURRENT, memoryManager);
    Literal<List<Double>> fractions = Literal.of(Collections.singletonList(0.95D), DataTypes.DOUBLE_ARRAY);
    impl.iterate(ramAccounting, memoryManager, state, Literal.of(10L), fractions);
    impl.iterate(ramAccounting, memoryManager, state, Literal.of(20L), fractions);
    Object result = impl.terminatePartial(ramAccounting, state);
    assertThat("result must be an array", result, instanceOf(List.class));
}

Example 8 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class CollectTaskTest method setUp.

@Before
public void setUp() throws Exception {
    super.setUp();
    localNodeId = "dummyLocalNodeId";
    collectPhase = Mockito.mock(RoutedCollectPhase.class);
    Routing routing = Mockito.mock(Routing.class);
    when(routing.containsShards(localNodeId)).thenReturn(true);
    when(collectPhase.routing()).thenReturn(routing);
    when(collectPhase.maxRowGranularity()).thenReturn(RowGranularity.DOC);
    collectOperation = mock(MapSideDataCollectOperation.class);
    Mockito.doAnswer(new Answer<>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            Runnable runnable = invocation.getArgument(0);
            runnable.run();
            return null;
        }
    }).when(collectOperation).launch(Mockito.any(), Mockito.anyString());
    consumer = new TestingRowConsumer();
    collectTask = new CollectTask(collectPhase, CoordinatorTxnCtx.systemTransactionContext(), collectOperation, RamAccounting.NO_ACCOUNTING, ramAccounting -> new OnHeapMemoryManager(ramAccounting::addBytes), consumer, mock(SharedShardContexts.class), Version.CURRENT, 4096);
}

Example 9 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class ShardingUpsertExecutor method apply.

@Override
public CompletableFuture<? extends Iterable<Row>> apply(BatchIterator<Row> batchIterator) {
    final ConcurrencyLimit nodeLimit = nodeLimits.get(localNode);
    long startTime = nodeLimit.startSample();
    var isUsedBytesOverThreshold = new IsUsedBytesOverThreshold(queryCircuitBreaker, nodeLimit);
    var reqBatchIterator = BatchIterators.partition(batchIterator, bulkSize, () -> new ShardedRequests<>(requestFactory, ramAccounting), grouper, bulkShardCreationLimiter.or(isUsedBytesOverThreshold));
    // If IO is involved the source iterator should pause when the target node reaches a concurrent job counter limit.
    // Without IO, we assume that the source iterates over in-memory structures which should be processed as
    // fast as possible to free resources.
    Predicate<ShardedRequests<ShardUpsertRequest, ShardUpsertRequest.Item>> shouldPause = this::shouldPauseOnPartitionCreation;
    if (batchIterator.hasLazyResultSet()) {
        shouldPause = shouldPause.or(this::shouldPauseOnTargetNodeJobsCounter).or(isUsedBytesOverThreshold);
    }
    BatchIteratorBackpressureExecutor<ShardedRequests<ShardUpsertRequest, ShardUpsertRequest.Item>, UpsertResults> executor = new BatchIteratorBackpressureExecutor<>(jobId, scheduler, this.executor, reqBatchIterator, this::execute, resultCollector.combiner(), resultCollector.supplier().get(), shouldPause, earlyTerminationCondition, earlyTerminationExceptionGenerator, this::getMaxLastRttInMs);
    return executor.consumeIteratorAndExecute().thenApply(upsertResults -> resultCollector.finisher().apply(upsertResults)).whenComplete((res, err) -> {
        nodeLimit.onSample(startTime, err != null);
    });
}

Example 10 with RamAccounting

use of io.crate.breaker.RamAccounting in project crate by crate.

the class NodeFetchOperation method doFetch.

private CompletableFuture<? extends IntObjectMap<StreamBucket>> doFetch(FetchTask fetchTask, IntObjectMap<IntArrayList> toFetch) throws Exception {
    HashMap<RelationName, TableFetchInfo> tableFetchInfos = getTableFetchInfos(fetchTask);
    // RamAccounting is per doFetch call instead of per FetchTask/fetchPhase
    // To be able to free up the memory count when the operation is complete
    final var ramAccounting = ConcurrentRamAccounting.forCircuitBreaker("fetch-" + fetchTask.id(), circuitBreaker);
    ArrayList<Supplier<StreamBucket>> collectors = new ArrayList<>(toFetch.size());
    for (IntObjectCursor<IntArrayList> toFetchCursor : toFetch) {
        final int readerId = toFetchCursor.key;
        final IntArrayList docIds = toFetchCursor.value;
        RelationName ident = fetchTask.tableIdent(readerId);
        final TableFetchInfo tfi = tableFetchInfos.get(ident);
        assert tfi != null : "tfi must not be null";
        var collector = tfi.createCollector(readerId, new BlockBasedRamAccounting(ramAccounting::addBytes, BlockBasedRamAccounting.MAX_BLOCK_SIZE_IN_BYTES));
        collectors.add(() -> collector.collect(docIds));
    }
    return ThreadPools.runWithAvailableThreads(executor, ThreadPools.numIdleThreads(executor, numProcessors), collectors).thenApply(buckets -> {
        var toFetchIt = toFetch.iterator();
        assert toFetch.size() == buckets.size() : "Must have a bucket per reader and they must be in the same order";
        IntObjectHashMap<StreamBucket> bucketByReader = new IntObjectHashMap<>(toFetch.size());
        for (var bucket : buckets) {
            assert toFetchIt.hasNext() : "toFetchIt must have an element if there is one in buckets";
            int readerId = toFetchIt.next().key;
            bucketByReader.put(readerId, bucket);
        }
        return bucketByReader;
    }).whenComplete((result, err) -> ramAccounting.close());
}