Examples with InputCollectExpression io.crate.execution.engine.collect.InputCollectExpression used on opensource projects

Example 6 with InputCollectExpression

use of io.crate.execution.engine.collect.InputCollectExpression in project crate by crate.

the class AbstractWindowFunctionTest method assertEvaluate.

@SuppressWarnings("unchecked")
protected <T> void assertEvaluate(String functionExpression, Matcher<T> expectedValue, List<ColumnIdent> rowsColumnDescription, Object[]... inputRows) throws Throwable {
    performInputSanityChecks(inputRows);
    Symbol normalizedFunctionSymbol = sqlExpressions.normalize(sqlExpressions.asSymbol(functionExpression));
    assertThat(normalizedFunctionSymbol, instanceOf(io.crate.expression.symbol.WindowFunction.class));
    var windowFunctionSymbol = (io.crate.expression.symbol.WindowFunction) normalizedFunctionSymbol;
    ReferenceResolver<InputCollectExpression> referenceResolver = r -> new InputCollectExpression(rowsColumnDescription.indexOf(r.column()));
    var sourceSymbols = Lists2.map(rowsColumnDescription, x -> sqlExpressions.normalize(sqlExpressions.asSymbol(x.sqlFqn())));
    ensureInputRowsHaveCorrectType(sourceSymbols, inputRows);
    var argsCtx = inputFactory.ctxForRefs(txnCtx, referenceResolver);
    argsCtx.add(windowFunctionSymbol.arguments());
    FunctionImplementation impl = sqlExpressions.nodeCtx.functions().getQualified(windowFunctionSymbol, txnCtx.sessionSettings().searchPath());
    assert impl instanceof WindowFunction || impl instanceof AggregationFunction : "Got " + impl + " but expected a window function";
    WindowFunction windowFunctionImpl;
    if (impl instanceof AggregationFunction) {
        windowFunctionImpl = new AggregateToWindowFunctionAdapter((AggregationFunction) impl, new ExpressionsInput<>(Literal.BOOLEAN_TRUE, List.of()), Version.CURRENT, RamAccounting.NO_ACCOUNTING, memoryManager, Version.CURRENT);
    } else {
        windowFunctionImpl = (WindowFunction) impl;
    }
    int numCellsInSourceRows = inputRows[0].length;
    var windowDef = windowFunctionSymbol.windowDefinition();
    var partitionOrderBy = windowDef.partitions().isEmpty() ? null : new OrderBy(windowDef.partitions());
    Comparator<Object[]> cmpOrderBy = createComparator(() -> inputFactory.ctxForRefs(txnCtx, referenceResolver), windowDef.orderBy());
    InputColumns.SourceSymbols inputColSources = new InputColumns.SourceSymbols(sourceSymbols);
    var mappedWindowDef = windowDef.map(s -> InputColumns.create(s, inputColSources));
    BatchIterator<Row> iterator = WindowFunctionBatchIterator.of(InMemoryBatchIterator.of(Arrays.stream(inputRows).map(RowN::new).collect(Collectors.toList()), SENTINEL, true), new IgnoreRowAccounting(), WindowProjector.createComputeStartFrameBoundary(numCellsInSourceRows, txnCtx, sqlExpressions.nodeCtx, mappedWindowDef, cmpOrderBy), WindowProjector.createComputeEndFrameBoundary(numCellsInSourceRows, txnCtx, sqlExpressions.nodeCtx, mappedWindowDef, cmpOrderBy), createComparator(() -> inputFactory.ctxForRefs(txnCtx, referenceResolver), partitionOrderBy), cmpOrderBy, numCellsInSourceRows, () -> 1, Runnable::run, List.of(windowFunctionImpl), argsCtx.expressions(), new Boolean[] { windowFunctionSymbol.ignoreNulls() }, argsCtx.topLevelInputs().toArray(new Input[0]));
    List<Object> actualResult;
    try {
        actualResult = BatchIterators.collect(iterator, Collectors.mapping(row -> row.get(numCellsInSourceRows), Collectors.toList())).get(5, TimeUnit.SECONDS);
    } catch (ExecutionException e) {
        throw e.getCause();
    }
    assertThat((T) actualResult, expectedValue);
}

Example 7 with InputCollectExpression

use of io.crate.execution.engine.collect.InputCollectExpression in project crate by crate.

the class TableFunctionCollectSource method getIterator.

@Override
@SuppressWarnings("unchecked")
public CompletableFuture<BatchIterator<Row>> getIterator(TransactionContext txnCtx, CollectPhase collectPhase, CollectTask collectTask, boolean supportMoveToStart) {
    TableFunctionCollectPhase phase = (TableFunctionCollectPhase) collectPhase;
    TableFunctionImplementation<?> functionImplementation = phase.functionImplementation();
    RowType rowType = functionImplementation.returnType();
    // noinspection unchecked  Only literals can be passed to table functions. Anything else is invalid SQL
    List<Input<?>> inputs = (List<Input<?>>) (List<?>) phase.functionArguments();
    List<Input<?>> topLevelInputs = new ArrayList<>(phase.toCollect().size());
    List<String> columns = rowType.fieldNames();
    InputFactory.Context<InputCollectExpression> ctx = inputFactory.ctxForRefs(txnCtx, ref -> {
        for (int i = 0; i < columns.size(); i++) {
            String column = columns.get(i);
            if (ref.column().isTopLevel() && ref.column().name().equals(column)) {
                return new InputCollectExpression(i);
            }
        }
        throw new IllegalStateException("Column `" + ref + "` not found in " + functionImplementation.signature().getName().displayName());
    });
    for (Symbol symbol : phase.toCollect()) {
        topLevelInputs.add(ctx.add(symbol));
    }
    var inputRow = new ValueAndInputRow<>(topLevelInputs, ctx.expressions());
    Input<Boolean> condition = (Input<Boolean>) ctx.add(phase.where());
    Iterable<Row> rows = () -> StreamSupport.stream(functionImplementation.evaluate(txnCtx, nodeCtx, inputs.toArray(new Input[0])).spliterator(), false).map(inputRow).filter(InputCondition.asPredicate(condition)).iterator();
    return CompletableFuture.completedFuture(InMemoryBatchIterator.of(rows, SentinelRow.SENTINEL, functionImplementation.hasLazyResultSet()));
}

Example 8 with InputCollectExpression

use of io.crate.execution.engine.collect.InputCollectExpression in project crate by crate.

the class IndexWriterProjectorTest method testIndexWriter.

@Test
public void testIndexWriter() throws Throwable {
    execute("create table bulk_import (id int primary key, name string) with (number_of_replicas=0)");
    ensureGreen();
    InputCollectExpression sourceInput = new InputCollectExpression(1);
    List<CollectExpression<Row, ?>> collectExpressions = Collections.<CollectExpression<Row, ?>>singletonList(sourceInput);
    RelationName bulkImportIdent = new RelationName(sqlExecutor.getCurrentSchema(), "bulk_import");
    ClusterState state = clusterService().state();
    Settings tableSettings = TableSettingsResolver.get(state.getMetadata(), bulkImportIdent, false);
    ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class);
    IndexWriterProjector writerProjector = new IndexWriterProjector(clusterService(), new NodeLimits(new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), new NoopCircuitBreaker("dummy"), RamAccounting.NO_ACCOUNTING, threadPool.scheduler(), threadPool.executor(ThreadPool.Names.SEARCH), CoordinatorTxnCtx.systemTransactionContext(), new NodeContext(internalCluster().getInstance(Functions.class)), Settings.EMPTY, IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.get(tableSettings), NumberOfReplicas.fromSettings(tableSettings, state.getNodes().getSize()), internalCluster().getInstance(TransportCreatePartitionsAction.class), internalCluster().getInstance(TransportShardUpsertAction.class)::execute, IndexNameResolver.forTable(bulkImportIdent), new Reference(new ReferenceIdent(bulkImportIdent, DocSysColumns.RAW), RowGranularity.DOC, DataTypes.STRING, 0, null), Collections.singletonList(ID_IDENT), Collections.<Symbol>singletonList(new InputColumn(0)), null, null, sourceInput, collectExpressions, 20, null, null, false, false, UUID.randomUUID(), UpsertResultContext.forRowCount(), false);
    BatchIterator rowsIterator = InMemoryBatchIterator.of(IntStream.range(0, 100).mapToObj(i -> new RowN(new Object[] { i, "{\"id\": " + i + ", \"name\": \"Arthur\"}" })).collect(Collectors.toList()), SENTINEL, true);
    TestingRowConsumer consumer = new TestingRowConsumer();
    consumer.accept(writerProjector.apply(rowsIterator), null);
    Bucket objects = consumer.getBucket();
    assertThat(objects, contains(isRow(100L)));
    execute("refresh table bulk_import");
    execute("select count(*) from bulk_import");
    assertThat(response.rowCount(), is(1L));
    assertThat(response.rows()[0][0], is(100L));
}

Example 9 with InputCollectExpression

use of io.crate.execution.engine.collect.InputCollectExpression in project crate by crate.

the class IndexWriterProjectorUnitTest method testNullPKValue.

@Test
public void testNullPKValue() throws Throwable {
    InputCollectExpression sourceInput = new InputCollectExpression(0);
    List<CollectExpression<Row, ?>> collectExpressions = Collections.<CollectExpression<Row, ?>>singletonList(sourceInput);
    TransportCreatePartitionsAction transportCreatePartitionsAction = mock(TransportCreatePartitionsAction.class);
    IndexWriterProjector indexWriter = new IndexWriterProjector(clusterService, new NodeLimits(new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), new NoopCircuitBreaker("dummy"), RamAccounting.NO_ACCOUNTING, scheduler, executor, CoordinatorTxnCtx.systemTransactionContext(), createNodeContext(), Settings.EMPTY, 5, 1, transportCreatePartitionsAction, (request, listener) -> {
    }, IndexNameResolver.forTable(BULK_IMPORT_IDENT), RAW_SOURCE_REFERENCE, Collections.singletonList(ID_IDENT), Collections.<Symbol>singletonList(new InputColumn(1)), null, null, sourceInput, collectExpressions, 20, null, null, false, false, UUID.randomUUID(), UpsertResultContext.forRowCount(), false);
    RowN rowN = new RowN(new Object[] { new BytesRef("{\"y\": \"x\"}"), null });
    BatchIterator<Row> batchIterator = InMemoryBatchIterator.of(Collections.singletonList(rowN), SENTINEL, true);
    batchIterator = indexWriter.apply(batchIterator);
    TestingRowConsumer testingBatchConsumer = new TestingRowConsumer();
    testingBatchConsumer.accept(batchIterator, null);
    List<Object[]> result = testingBatchConsumer.getResult();
    // Zero affected rows as a NULL as a PK value will result in an exception.
    // It must never bubble up as other rows might already have been written.
    assertThat(result.get(0)[0], is(0L));
}