Search in sources :

Example 16 with CollectExpression

use of io.crate.execution.engine.collect.CollectExpression in project crate by crate.

the class InsertFromValues method createRowsByShardGrouper.

private GroupRowsByShard<ShardUpsertRequest, ShardUpsertRequest.Item> createRowsByShardGrouper(Symbol[] assignmentSources, ArrayList<Input<?>> insertInputs, Supplier<String> indexNameResolver, InputFactory.Context<CollectExpression<Row, ?>> collectContext, PlannerContext plannerContext, ClusterService clusterService) {
    InputRow insertValues = new InputRow(insertInputs);
    Function<String, ShardUpsertRequest.Item> itemFactory = id -> new ShardUpsertRequest.Item(id, assignmentSources, insertValues.materialize(), null, null, null);
    var rowShardResolver = new RowShardResolver(plannerContext.transactionContext(), plannerContext.nodeContext(), writerProjection.primaryKeys(), writerProjection.ids(), writerProjection.clusteredByIdent(), writerProjection.clusteredBy());
    return new GroupRowsByShard<>(clusterService, rowShardResolver, new TypeGuessEstimateRowSize(), indexNameResolver, collectContext.expressions(), itemFactory, true);
}
Also used : GeneratedColumns(io.crate.execution.dml.upsert.GeneratedColumns) IndexParts(io.crate.metadata.IndexParts) INDEX_CLOSED_BLOCK(org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_CLOSED_BLOCK) Arrays(java.util.Arrays) TransportShardUpsertAction(io.crate.execution.dml.upsert.TransportShardUpsertAction) ShardIterator(org.elasticsearch.cluster.routing.ShardIterator) ShardedRequests(io.crate.execution.engine.indexing.ShardedRequests) TableFunctionRelation(io.crate.analyze.relations.TableFunctionRelation) NodeLimits(io.crate.execution.jobs.NodeLimits) TransportCreatePartitionsAction(org.elasticsearch.action.admin.indices.create.TransportCreatePartitionsAction) RetryListener(io.crate.execution.support.RetryListener) DependencyCarrier(io.crate.planner.DependencyCarrier) ClusterState(org.elasticsearch.cluster.ClusterState) RowN(io.crate.data.RowN) SymbolEvaluator(io.crate.analyze.SymbolEvaluator) TableStats(io.crate.statistics.TableStats) ClusterBlock(org.elasticsearch.cluster.block.ClusterBlock) ColumnIndexWriterProjection(io.crate.execution.dsl.projection.ColumnIndexWriterProjection) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) IntArrayList(com.carrotsearch.hppc.IntArrayList) IndexNotFoundException(org.elasticsearch.index.IndexNotFoundException) Map(java.util.Map) TypeGuessEstimateRowSize(io.crate.breaker.TypeGuessEstimateRowSize) ConcurrencyLimit(io.crate.concurrent.limits.ConcurrencyLimit) SelectSymbol(io.crate.expression.symbol.SelectSymbol) GroupRowsByShard(io.crate.execution.engine.indexing.GroupRowsByShard) DocTableInfo(io.crate.metadata.doc.DocTableInfo) Collection(java.util.Collection) InMemoryBatchIterator(io.crate.data.InMemoryBatchIterator) Set(java.util.Set) UUID(java.util.UUID) InputRow(io.crate.expression.InputRow) ShardRequest(io.crate.execution.dml.ShardRequest) ExecutionPlan(io.crate.planner.ExecutionPlan) List(java.util.List) OrderBy(io.crate.analyze.OrderBy) Row(io.crate.data.Row) Symbol(io.crate.expression.symbol.Symbol) RowShardResolver(io.crate.execution.engine.collect.RowShardResolver) Assignments(io.crate.expression.symbol.Assignments) Row1(io.crate.data.Row1) ShardRouting(org.elasticsearch.cluster.routing.ShardRouting) Input(io.crate.data.Input) SENTINEL(io.crate.data.SentinelRow.SENTINEL) ClusterService(org.elasticsearch.cluster.service.ClusterService) CollectExpression(io.crate.execution.engine.collect.CollectExpression) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Operation(io.crate.metadata.table.Operation) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) Supplier(java.util.function.Supplier) InsertSourceFromCells(io.crate.execution.dml.upsert.InsertSourceFromCells) ArrayList(java.util.ArrayList) BackoffPolicy(org.elasticsearch.action.bulk.BackoffPolicy) Metadata(org.elasticsearch.cluster.metadata.Metadata) ClusterBlockException(org.elasticsearch.cluster.block.ClusterBlockException) ShardLocation(io.crate.execution.engine.indexing.ShardLocation) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) StreamSupport(java.util.stream.StreamSupport) ColumnValidationException(io.crate.exceptions.ColumnValidationException) Nullable(javax.annotation.Nullable) FutureActionListener(io.crate.action.FutureActionListener) ProjectionBuilder(io.crate.execution.dsl.projection.builder.ProjectionBuilder) BULK_REQUEST_TIMEOUT_SETTING(io.crate.execution.engine.indexing.ShardingUpsertExecutor.BULK_REQUEST_TIMEOUT_SETTING) Iterator(java.util.Iterator) Reference(io.crate.metadata.Reference) DataType(io.crate.types.DataType) AcknowledgedResponse(org.elasticsearch.action.support.master.AcknowledgedResponse) RamAccounting(io.crate.breaker.RamAccounting) Consumer(java.util.function.Consumer) RowConsumer(io.crate.data.RowConsumer) ShardResponse(io.crate.execution.dml.ShardResponse) ShardUpsertRequest(io.crate.execution.dml.upsert.ShardUpsertRequest) CollectionBucket(io.crate.data.CollectionBucket) TableFunctionImplementation(io.crate.metadata.tablefunctions.TableFunctionImplementation) IndexNameResolver(io.crate.execution.engine.indexing.IndexNameResolver) NotSerializableExceptionWrapper(org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper) AbstractTableRelation(io.crate.analyze.relations.AbstractTableRelation) PlannerContext(io.crate.planner.PlannerContext) InputColumns(io.crate.execution.dsl.projection.builder.InputColumns) SQLExceptions(io.crate.exceptions.SQLExceptions) InputFactory(io.crate.expression.InputFactory) CreatePartitionsRequest(org.elasticsearch.action.admin.indices.create.CreatePartitionsRequest) ActionListener(org.elasticsearch.action.ActionListener) InputRow(io.crate.expression.InputRow) TypeGuessEstimateRowSize(io.crate.breaker.TypeGuessEstimateRowSize) GroupRowsByShard(io.crate.execution.engine.indexing.GroupRowsByShard) RowShardResolver(io.crate.execution.engine.collect.RowShardResolver)

Example 17 with CollectExpression

use of io.crate.execution.engine.collect.CollectExpression in project crate by crate.

the class IndexWriterProjectorTest method testIndexWriter.

@Test
public void testIndexWriter() throws Throwable {
    execute("create table bulk_import (id int primary key, name string) with (number_of_replicas=0)");
    ensureGreen();
    InputCollectExpression sourceInput = new InputCollectExpression(1);
    List<CollectExpression<Row, ?>> collectExpressions = Collections.<CollectExpression<Row, ?>>singletonList(sourceInput);
    RelationName bulkImportIdent = new RelationName(sqlExecutor.getCurrentSchema(), "bulk_import");
    ClusterState state = clusterService().state();
    Settings tableSettings = TableSettingsResolver.get(state.getMetadata(), bulkImportIdent, false);
    ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class);
    IndexWriterProjector writerProjector = new IndexWriterProjector(clusterService(), new NodeLimits(new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), new NoopCircuitBreaker("dummy"), RamAccounting.NO_ACCOUNTING, threadPool.scheduler(), threadPool.executor(ThreadPool.Names.SEARCH), CoordinatorTxnCtx.systemTransactionContext(), new NodeContext(internalCluster().getInstance(Functions.class)), Settings.EMPTY, IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.get(tableSettings), NumberOfReplicas.fromSettings(tableSettings, state.getNodes().getSize()), internalCluster().getInstance(TransportCreatePartitionsAction.class), internalCluster().getInstance(TransportShardUpsertAction.class)::execute, IndexNameResolver.forTable(bulkImportIdent), new Reference(new ReferenceIdent(bulkImportIdent, DocSysColumns.RAW), RowGranularity.DOC, DataTypes.STRING, 0, null), Collections.singletonList(ID_IDENT), Collections.<Symbol>singletonList(new InputColumn(0)), null, null, sourceInput, collectExpressions, 20, null, null, false, false, UUID.randomUUID(), UpsertResultContext.forRowCount(), false);
    BatchIterator rowsIterator = InMemoryBatchIterator.of(IntStream.range(0, 100).mapToObj(i -> new RowN(new Object[] { i, "{\"id\": " + i + ", \"name\": \"Arthur\"}" })).collect(Collectors.toList()), SENTINEL, true);
    TestingRowConsumer consumer = new TestingRowConsumer();
    consumer.accept(writerProjector.apply(rowsIterator), null);
    Bucket objects = consumer.getBucket();
    assertThat(objects, contains(isRow(100L)));
    execute("refresh table bulk_import");
    execute("select count(*) from bulk_import");
    assertThat(response.rowCount(), is(1L));
    assertThat(response.rows()[0][0], is(100L));
}
Also used : TransportCreatePartitionsAction(org.elasticsearch.action.admin.indices.create.TransportCreatePartitionsAction) ClusterState(org.elasticsearch.cluster.ClusterState) ClusterSettings(org.elasticsearch.common.settings.ClusterSettings) NodeContext(io.crate.metadata.NodeContext) Reference(io.crate.metadata.Reference) ThreadPool(org.elasticsearch.threadpool.ThreadPool) BatchIterator(io.crate.data.BatchIterator) InMemoryBatchIterator(io.crate.data.InMemoryBatchIterator) CollectExpression(io.crate.execution.engine.collect.CollectExpression) InputCollectExpression(io.crate.execution.engine.collect.InputCollectExpression) ReferenceIdent(io.crate.metadata.ReferenceIdent) RowN(io.crate.data.RowN) InputCollectExpression(io.crate.execution.engine.collect.InputCollectExpression) Bucket(io.crate.data.Bucket) InputColumn(io.crate.expression.symbol.InputColumn) NodeLimits(io.crate.execution.jobs.NodeLimits) RelationName(io.crate.metadata.RelationName) NoopCircuitBreaker(org.elasticsearch.common.breaker.NoopCircuitBreaker) Settings(org.elasticsearch.common.settings.Settings) ClusterSettings(org.elasticsearch.common.settings.ClusterSettings) TestingRowConsumer(io.crate.testing.TestingRowConsumer) Test(org.junit.Test)

Example 18 with CollectExpression

use of io.crate.execution.engine.collect.CollectExpression in project crate by crate.

the class IndexWriterProjectorUnitTest method testNullPKValue.

@Test
public void testNullPKValue() throws Throwable {
    InputCollectExpression sourceInput = new InputCollectExpression(0);
    List<CollectExpression<Row, ?>> collectExpressions = Collections.<CollectExpression<Row, ?>>singletonList(sourceInput);
    TransportCreatePartitionsAction transportCreatePartitionsAction = mock(TransportCreatePartitionsAction.class);
    IndexWriterProjector indexWriter = new IndexWriterProjector(clusterService, new NodeLimits(new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), new NoopCircuitBreaker("dummy"), RamAccounting.NO_ACCOUNTING, scheduler, executor, CoordinatorTxnCtx.systemTransactionContext(), createNodeContext(), Settings.EMPTY, 5, 1, transportCreatePartitionsAction, (request, listener) -> {
    }, IndexNameResolver.forTable(BULK_IMPORT_IDENT), RAW_SOURCE_REFERENCE, Collections.singletonList(ID_IDENT), Collections.<Symbol>singletonList(new InputColumn(1)), null, null, sourceInput, collectExpressions, 20, null, null, false, false, UUID.randomUUID(), UpsertResultContext.forRowCount(), false);
    RowN rowN = new RowN(new Object[] { new BytesRef("{\"y\": \"x\"}"), null });
    BatchIterator<Row> batchIterator = InMemoryBatchIterator.of(Collections.singletonList(rowN), SENTINEL, true);
    batchIterator = indexWriter.apply(batchIterator);
    TestingRowConsumer testingBatchConsumer = new TestingRowConsumer();
    testingBatchConsumer.accept(batchIterator, null);
    List<Object[]> result = testingBatchConsumer.getResult();
    // Zero affected rows as a NULL as a PK value will result in an exception.
    // It must never bubble up as other rows might already have been written.
    assertThat(result.get(0)[0], is(0L));
}
Also used : TransportCreatePartitionsAction(org.elasticsearch.action.admin.indices.create.TransportCreatePartitionsAction) ClusterSettings(org.elasticsearch.common.settings.ClusterSettings) CollectExpression(io.crate.execution.engine.collect.CollectExpression) InputCollectExpression(io.crate.execution.engine.collect.InputCollectExpression) RowN(io.crate.data.RowN) InputCollectExpression(io.crate.execution.engine.collect.InputCollectExpression) InputColumn(io.crate.expression.symbol.InputColumn) NodeLimits(io.crate.execution.jobs.NodeLimits) Row(io.crate.data.Row) NoopCircuitBreaker(org.elasticsearch.common.breaker.NoopCircuitBreaker) BytesRef(org.apache.lucene.util.BytesRef) TestingRowConsumer(io.crate.testing.TestingRowConsumer) CrateDummyClusterServiceUnitTest(io.crate.test.integration.CrateDummyClusterServiceUnitTest) Test(org.junit.Test)

Example 19 with CollectExpression

use of io.crate.execution.engine.collect.CollectExpression in project crate by crate.

the class DocRefResolver method getImplementation.

@Override
public CollectExpression<Doc, ?> getImplementation(Reference ref) {
    ColumnIdent columnIdent = ref.column();
    String fqn = columnIdent.fqn();
    switch(fqn) {
        case DocSysColumns.Names.VERSION:
            return forFunction(Doc::getVersion);
        case DocSysColumns.Names.SEQ_NO:
            return forFunction(Doc::getSeqNo);
        case DocSysColumns.Names.PRIMARY_TERM:
            return forFunction(Doc::getPrimaryTerm);
        case DocSysColumns.Names.ID:
            return NestableCollectExpression.forFunction(Doc::getId);
        case DocSysColumns.Names.DOCID:
            return forFunction(Doc::docId);
        case DocSysColumns.Names.RAW:
            return forFunction(Doc::getRaw);
        case DocSysColumns.Names.DOC:
            return forFunction(Doc::getSource);
        default:
            for (int i = 0; i < partitionedByColumns.size(); i++) {
                var pColumn = partitionedByColumns.get(i);
                if (pColumn.equals(columnIdent)) {
                    final int idx = i;
                    return forFunction(getResp -> ref.valueType().implicitCast(PartitionName.fromIndexOrTemplate(getResp.getIndex()).values().get(idx)));
                } else if (pColumn.isChildOf(columnIdent)) {
                    final int idx = i;
                    return forFunction(response -> {
                        if (response == null) {
                            return null;
                        }
                        var partitionName = PartitionName.fromIndexOrTemplate(response.getIndex());
                        var partitionValue = partitionName.values().get(idx);
                        var source = response.getSource();
                        Maps.mergeInto(source, pColumn.name(), pColumn.path(), partitionValue);
                        return ref.valueType().implicitCast(ValueExtractors.fromMap(source, columnIdent));
                    });
                }
            }
            return forFunction(response -> {
                if (response == null) {
                    return null;
                }
                return ref.valueType().implicitCast(ValueExtractors.fromMap(response.getSource(), ref.column()));
            });
    }
}
Also used : DocSysColumns(io.crate.metadata.doc.DocSysColumns) NestableCollectExpression.forFunction(io.crate.execution.engine.collect.NestableCollectExpression.forFunction) List(java.util.List) Maps(io.crate.common.collections.Maps) CollectExpression(io.crate.execution.engine.collect.CollectExpression) ColumnIdent(io.crate.metadata.ColumnIdent) Reference(io.crate.metadata.Reference) NestableCollectExpression(io.crate.execution.engine.collect.NestableCollectExpression) ValueExtractors(io.crate.expression.ValueExtractors) PartitionName(io.crate.metadata.PartitionName) ColumnIdent(io.crate.metadata.ColumnIdent)

Example 20 with CollectExpression

use of io.crate.execution.engine.collect.CollectExpression in project crate by crate.

the class DocRefResolverTest method testSystemColumnsCollectExpressions.

@Test
public void testSystemColumnsCollectExpressions() throws Exception {
    List<Reference> references = List.of(refInfo("t1._id", DocSysColumns.COLUMN_IDENTS.get(DocSysColumns.ID), RowGranularity.DOC), refInfo("t1._version", DocSysColumns.COLUMN_IDENTS.get(DocSysColumns.VERSION), RowGranularity.DOC), refInfo("t1._doc", DocSysColumns.COLUMN_IDENTS.get(DocSysColumns.DOC), RowGranularity.DOC), refInfo("t1._raw", DocSysColumns.COLUMN_IDENTS.get(DocSysColumns.RAW), RowGranularity.DOC), refInfo("t1._docid", DocSysColumns.COLUMN_IDENTS.get(DocSysColumns.DOCID), RowGranularity.DOC), refInfo("t1._seq_no", DocSysColumns.COLUMN_IDENTS.get(DocSysColumns.SEQ_NO), RowGranularity.DOC), refInfo("t1._primary_term", DocSysColumns.COLUMN_IDENTS.get(DocSysColumns.PRIMARY_TERM), RowGranularity.DOC));
    List<CollectExpression<Doc, ?>> collectExpressions = new ArrayList<>(4);
    for (Reference reference : references) {
        CollectExpression<Doc, ?> collectExpression = REF_RESOLVER.getImplementation(reference);
        collectExpression.setNextRow(GET_RESULT);
        collectExpressions.add(collectExpression);
    }
    assertThat(collectExpressions.get(0).value(), is("abc"));
    assertThat(collectExpressions.get(1).value(), is(1L));
    assertThat(collectExpressions.get(2).value(), is(XContentHelper.convertToMap(SOURCE, false, XContentType.JSON).v2()));
    assertThat(collectExpressions.get(3).value(), is(SOURCE.utf8ToString()));
    assertThat(collectExpressions.get(4).value(), is(2));
    assertThat(collectExpressions.get(5).value(), is(1L));
    assertThat(collectExpressions.get(6).value(), is(1L));
}
Also used : Reference(io.crate.metadata.Reference) BytesReference(org.elasticsearch.common.bytes.BytesReference) ArrayList(java.util.ArrayList) Doc(io.crate.expression.reference.Doc) CollectExpression(io.crate.execution.engine.collect.CollectExpression) Test(org.junit.Test)

Aggregations

CollectExpression (io.crate.execution.engine.collect.CollectExpression)21 Input (io.crate.data.Input)15 InputFactory (io.crate.expression.InputFactory)12 Symbol (io.crate.expression.symbol.Symbol)12 ArrayList (java.util.ArrayList)11 Row (io.crate.data.Row)9 InputColumn (io.crate.expression.symbol.InputColumn)8 RowN (io.crate.data.RowN)7 Reference (io.crate.metadata.Reference)7 List (java.util.List)6 ClusterState (org.elasticsearch.cluster.ClusterState)6 InputCollectExpression (io.crate.execution.engine.collect.InputCollectExpression)5 NestableCollectExpression (io.crate.execution.engine.collect.NestableCollectExpression)5 CrateDummyClusterServiceUnitTest (io.crate.test.integration.CrateDummyClusterServiceUnitTest)5 OrderBy (io.crate.analyze.OrderBy)4 RamAccounting (io.crate.breaker.RamAccounting)4 InMemoryBatchIterator (io.crate.data.InMemoryBatchIterator)4 Function (io.crate.expression.symbol.Function)4 DataType (io.crate.types.DataType)4 Test (org.junit.Test)4