Examples with RowAccountingWithEstimators io.crate.breaker.RowAccountingWithEstimators used on opensource projects

Example 1 with RowAccountingWithEstimators

use of io.crate.breaker.RowAccountingWithEstimators in project crate by crate.

the class ShardCollectSource method createMultiShardScoreDocCollector.

private CompletableFuture<BatchIterator<Row>> createMultiShardScoreDocCollector(RoutedCollectPhase collectPhase, boolean supportMoveToStart, CollectTask collectTask, String localNodeId) {
    Map<String, Map<String, IntIndexedContainer>> locations = collectPhase.routing().locations();
    SharedShardContexts sharedShardContexts = collectTask.sharedShardContexts();
    Map<String, IntIndexedContainer> indexShards = locations.get(localNodeId);
    List<CompletableFuture<OrderedDocCollector>> orderedDocCollectors = new ArrayList<>();
    Metadata metadata = clusterService.state().metadata();
    for (Map.Entry<String, IntIndexedContainer> entry : indexShards.entrySet()) {
        String indexName = entry.getKey();
        Index index = metadata.index(indexName).getIndex();
        for (IntCursor shard : entry.getValue()) {
            ShardId shardId = new ShardId(index, shard.value);
            try {
                SharedShardContext context = sharedShardContexts.getOrCreateContext(shardId);
                ShardCollectorProvider shardCollectorProvider = getCollectorProviderSafe(shardId);
                orderedDocCollectors.add(shardCollectorProvider.getFutureOrderedCollector(collectPhase, context, collectTask, supportMoveToStart));
            } catch (ShardNotFoundException | IllegalIndexShardStateException e) {
                throw e;
            } catch (IndexNotFoundException e) {
                if (IndexParts.isPartitioned(indexName)) {
                    break;
                }
                throw e;
            }
        }
    }
    List<DataType<?>> columnTypes = Symbols.typeView(collectPhase.toCollect());
    OrderBy orderBy = collectPhase.orderBy();
    assert orderBy != null : "orderBy must not be null";
    return CompletableFutures.allAsList(orderedDocCollectors).thenApply(collectors -> OrderedLuceneBatchIteratorFactory.newInstance(collectors, OrderingByPosition.rowOrdering(OrderByPositionVisitor.orderByPositions(orderBy.orderBySymbols(), collectPhase.toCollect()), orderBy.reverseFlags(), orderBy.nullsFirst()), new RowAccountingWithEstimators(columnTypes, collectTask.getRamAccounting()), executor, availableThreads, supportMoveToStart));
}

Example 2 with RowAccountingWithEstimators

use of io.crate.breaker.RowAccountingWithEstimators in project crate by crate.

the class WindowProjector method fromProjection.

public static Projector fromProjection(WindowAggProjection projection, NodeContext nodeCtx, InputFactory inputFactory, TransactionContext txnCtx, RamAccounting ramAccounting, MemoryManager memoryManager, Version minNodeVersion, Version indexVersionCreated, IntSupplier numThreads, Executor executor) {
    var windowFunctionSymbols = projection.windowFunctions();
    var numWindowFunctions = windowFunctionSymbols.size();
    assert numWindowFunctions > 0 : "WindowAggProjection must have at least 1 window function.";
    ArrayList<WindowFunction> windowFunctions = new ArrayList<>(numWindowFunctions);
    ArrayList<CollectExpression<Row, ?>> windowFuncArgsExpressions = new ArrayList<>(numWindowFunctions);
    Input[][] windowFuncArgsInputs = new Input[numWindowFunctions][];
    Boolean[] ignoreNulls = new Boolean[numWindowFunctions];
    for (int idx = 0; idx < numWindowFunctions; idx++) {
        var windowFunctionSymbol = windowFunctionSymbols.get(idx);
        InputFactory.Context<CollectExpression<Row, ?>> ctx = inputFactory.ctxForInputColumns(txnCtx);
        ctx.add(windowFunctionSymbol.arguments());
        FunctionImplementation impl = nodeCtx.functions().getQualified(windowFunctionSymbol, txnCtx.sessionSettings().searchPath());
        assert impl != null : "Function implementation not found using full qualified lookup";
        if (impl instanceof AggregationFunction) {
            var filterInputFactoryCtx = inputFactory.ctxForInputColumns(txnCtx);
            var filterSymbol = windowFunctionSymbol.filter();
            // noinspection unchecked
            Input<Boolean> filterInput = filterSymbol == null ? Literal.BOOLEAN_TRUE : (Input<Boolean>) filterInputFactoryCtx.add(filterSymbol);
            ExpressionsInput<Row, Boolean> filter = new ExpressionsInput<>(filterInput, filterInputFactoryCtx.expressions());
            windowFunctions.add(new AggregateToWindowFunctionAdapter((AggregationFunction) impl, filter, indexVersionCreated, ramAccounting, memoryManager, minNodeVersion));
        } else if (impl instanceof WindowFunction) {
            windowFunctions.add((WindowFunction) impl);
        } else {
            throw new AssertionError("Function needs to be either a window or an aggregate function");
        }
        windowFuncArgsExpressions.addAll(ctx.expressions());
        windowFuncArgsInputs[idx] = ctx.topLevelInputs().toArray(new Input[0]);
        ignoreNulls[idx] = windowFunctionSymbol.ignoreNulls();
    }
    var windowDefinition = projection.windowDefinition();
    var partitions = windowDefinition.partitions();
    Supplier<InputFactory.Context<CollectExpression<Row, ?>>> createInputFactoryContext = () -> inputFactory.ctxForInputColumns(txnCtx);
    int arrayListElementOverHead = 32;
    RowAccountingWithEstimators accounting = new RowAccountingWithEstimators(Symbols.typeView(projection.standalone()), ramAccounting, arrayListElementOverHead);
    Comparator<Object[]> cmpPartitionBy = partitions.isEmpty() ? null : createComparator(createInputFactoryContext, new OrderBy(windowDefinition.partitions()));
    Comparator<Object[]> cmpOrderBy = createComparator(createInputFactoryContext, windowDefinition.orderBy());
    int numCellsInSourceRow = projection.standalone().size();
    ComputeFrameBoundary<Object[]> computeFrameStart = createComputeStartFrameBoundary(numCellsInSourceRow, txnCtx, nodeCtx, windowDefinition, cmpOrderBy);
    ComputeFrameBoundary<Object[]> computeFrameEnd = createComputeEndFrameBoundary(numCellsInSourceRow, txnCtx, nodeCtx, windowDefinition, cmpOrderBy);
    return sourceRows -> WindowFunctionBatchIterator.of(sourceRows, accounting, computeFrameStart, computeFrameEnd, cmpPartitionBy, cmpOrderBy, numCellsInSourceRow, numThreads, executor, windowFunctions, windowFuncArgsExpressions, ignoreNulls, windowFuncArgsInputs);
}

Example 3 with RowAccountingWithEstimators

use of io.crate.breaker.RowAccountingWithEstimators in project crate by crate.

the class WindowBatchIteratorTest method testWindowBatchIteratorAccountsUsedMemory.

@Test
public void testWindowBatchIteratorAccountsUsedMemory() {
    RamAccounting ramAccounting = ConcurrentRamAccounting.forCircuitBreaker("test", new NoopCircuitBreaker("dummy"));
    BatchIterator<Row> iterator = WindowFunctionBatchIterator.of(TestingBatchIterators.range(0, 10), new RowAccountingWithEstimators(List.of(DataTypes.INTEGER), ramAccounting, 32), (partitionStart, partitionEnd, currentIndex, sortedRows) -> 0, (partitionStart, partitionEnd, currentIndex, sortedRows) -> currentIndex, null, null, 1, () -> 1, Runnable::run, List.of(rowNumberWindowFunction()), List.of(), new Boolean[] { null }, new Input[][] { new Input[0] });
    TestingRowConsumer consumer = new TestingRowConsumer();
    consumer.accept(iterator, null);
    // should've accounted for 10 integers of 48 bytes each (16 for the integer, 32 for the ArrayList element)
    assertThat(ramAccounting.totalBytes(), is(480L));
}

Example 4 with RowAccountingWithEstimators

use of io.crate.breaker.RowAccountingWithEstimators in project crate by crate.

the class RamAccountingPageIteratorTest method testNoCircuitBreaking.

@Test
public void testNoCircuitBreaking() {
    PagingIterator<Integer, Row> pagingIterator = PagingIterator.create(2, true, null, () -> new RowAccountingWithEstimators(List.of(DataTypes.STRING, DataTypes.STRING, DataTypes.STRING), RamAccounting.NO_ACCOUNTING));
    assertThat(pagingIterator, instanceOf(RamAccountingPageIterator.class));
    assertThat(((RamAccountingPageIterator) pagingIterator).delegatePagingIterator, instanceOf(PassThroughPagingIterator.class));
    pagingIterator.merge(Arrays.asList(new KeyIterable<>(0, Collections.singletonList(TEST_ROWS[0])), new KeyIterable<>(1, Collections.singletonList(TEST_ROWS[1]))));
    pagingIterator.finish();
    var rows = new ArrayList<Row>();
    pagingIterator.forEachRemaining(rows::add);
    assertThat(rows.get(0), TestingHelpers.isRow("a", "b", "c"));
    assertThat(rows.get(1), TestingHelpers.isRow("d", "e", "f"));
}

Example 5 with RowAccountingWithEstimators

use of io.crate.breaker.RowAccountingWithEstimators in project crate by crate.

the class RamAccountingPageIteratorTest method testCircuitBreaking.

@Test
public void testCircuitBreaking() throws Exception {
    PagingIterator<Integer, Row> pagingIterator = PagingIterator.create(2, true, null, () -> new RowAccountingWithEstimators(List.of(DataTypes.STRING, DataTypes.STRING, DataTypes.STRING), ConcurrentRamAccounting.forCircuitBreaker("test", new MemoryCircuitBreaker(new ByteSizeValue(197, ByteSizeUnit.BYTES), 1, LogManager.getLogger(RowAccountingWithEstimatorsTest.class)))));
    expectedException.expect(CircuitBreakingException.class);
    expectedException.expectMessage("Data too large, data for field [test] would be [288/288b], which is larger than the limit of [197/197b]");
    pagingIterator.merge(Arrays.asList(new KeyIterable<>(0, Collections.singletonList(TEST_ROWS[0])), new KeyIterable<>(1, Collections.singletonList(TEST_ROWS[1]))));
}