Search in sources :

Example 1 with BulkShardProcessor

use of org.elasticsearch.action.bulk.BulkShardProcessor in project crate by crate.

the class IndexWriterCountBatchIteratorTest method getBulkShardProcessor.

private BulkShardProcessor<ShardUpsertRequest> getBulkShardProcessor() {
    UUID jobId = UUID.randomUUID();
    ShardUpsertRequest.Builder builder = new ShardUpsertRequest.Builder(CrateSettings.BULK_REQUEST_TIMEOUT.extractTimeValue(Settings.EMPTY), false, true, null, new Reference[] { new Reference(new ReferenceIdent(bulkImportIdent, DocSysColumns.RAW), RowGranularity.DOC, DataTypes.STRING) }, jobId, false);
    IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY);
    return new BulkShardProcessor<>(internalCluster().getInstance(ClusterService.class), internalCluster().getInstance(TransportBulkCreateIndicesAction.class), indexNameExpressionResolver, Settings.EMPTY, internalCluster().getInstance(BulkRetryCoordinatorPool.class), false, 2, builder, internalCluster().getInstance(TransportShardUpsertAction.class)::execute, jobId);
}
Also used : BulkRetryCoordinatorPool(org.elasticsearch.action.bulk.BulkRetryCoordinatorPool) ClusterService(org.elasticsearch.cluster.ClusterService) ShardUpsertRequest(io.crate.executor.transport.ShardUpsertRequest) IndexNameExpressionResolver(org.elasticsearch.cluster.metadata.IndexNameExpressionResolver) TransportBulkCreateIndicesAction(org.elasticsearch.action.admin.indices.create.TransportBulkCreateIndicesAction) UUID(java.util.UUID) BulkShardProcessor(org.elasticsearch.action.bulk.BulkShardProcessor)

Example 2 with BulkShardProcessor

use of org.elasticsearch.action.bulk.BulkShardProcessor in project crate by crate.

the class IndexWriterCountBatchIteratorTest method testIndexWriterIterator.

@Test
public void testIndexWriterIterator() throws Exception {
    execute("create table bulk_import (id int primary key) with (number_of_replicas=0)");
    ensureGreen();
    Supplier<BatchIterator> sourceSupplier = () -> RowsBatchIterator.newInstance(RowGenerator.fromSingleColValues(() -> IntStream.range(0, 10).mapToObj(i -> new BytesRef("{\"id\": " + i + "}")).iterator()), 1);
    Supplier<String> indexNameResolver = IndexNameResolver.forTable(new TableIdent(null, "bulk_import"));
    Input<?> sourceInput = new InputCollectExpression(0);
    List<CollectExpression<Row, ?>> collectExpressions = Collections.singletonList((InputCollectExpression) sourceInput);
    RowShardResolver rowShardResolver = getRowShardResolver();
    BulkShardProcessor bulkShardProcessor = getBulkShardProcessor();
    Supplier<ShardUpsertRequest.Item> updateItemSupplier = () -> new ShardUpsertRequest.Item(rowShardResolver.id(), null, new Object[] { sourceInput.value() }, null);
    List<Object[]> expectedResult = Collections.singletonList(new Object[] { 10L });
    BatchIteratorTester tester = new BatchIteratorTester(() -> IndexWriterCountBatchIterator.newIndexInstance(sourceSupplier.get(), indexNameResolver, collectExpressions, rowShardResolver, bulkShardProcessor, updateItemSupplier));
    tester.verifyResultAndEdgeCaseBehaviour(expectedResult);
}
Also used : CrateSettings(io.crate.metadata.settings.CrateSettings) TransportBulkCreateIndicesAction(org.elasticsearch.action.admin.indices.create.TransportBulkCreateIndicesAction) IntStream(java.util.stream.IntStream) Input(io.crate.data.Input) Arrays(java.util.Arrays) BulkShardProcessor(org.elasticsearch.action.bulk.BulkShardProcessor) InputColumn(io.crate.analyze.symbol.InputColumn) BatchIterator(io.crate.data.BatchIterator) BatchIteratorTester(io.crate.testing.BatchIteratorTester) Supplier(java.util.function.Supplier) SQLTransportIntegrationTest(io.crate.integrationtests.SQLTransportIntegrationTest) CollectExpression(io.crate.operation.collect.CollectExpression) Settings(org.elasticsearch.common.settings.Settings) Symbol(io.crate.analyze.symbol.Symbol) ClusterService(org.elasticsearch.cluster.ClusterService) io.crate.metadata(io.crate.metadata) DocSysColumns(io.crate.metadata.doc.DocSysColumns) RowsBatchIterator(io.crate.data.RowsBatchIterator) RowShardResolver(io.crate.operation.collect.RowShardResolver) BytesRef(org.apache.lucene.util.BytesRef) Test(org.junit.Test) UUID(java.util.UUID) RowGenerator(io.crate.testing.RowGenerator) ShardUpsertRequest(io.crate.executor.transport.ShardUpsertRequest) TransportShardUpsertAction(io.crate.executor.transport.TransportShardUpsertAction) List(java.util.List) Row(io.crate.data.Row) DataTypes(io.crate.types.DataTypes) BulkRetryCoordinatorPool(org.elasticsearch.action.bulk.BulkRetryCoordinatorPool) IndexNameExpressionResolver(org.elasticsearch.cluster.metadata.IndexNameExpressionResolver) InputCollectExpression(io.crate.operation.collect.InputCollectExpression) Collections(java.util.Collections) BatchIterator(io.crate.data.BatchIterator) RowsBatchIterator(io.crate.data.RowsBatchIterator) CollectExpression(io.crate.operation.collect.CollectExpression) InputCollectExpression(io.crate.operation.collect.InputCollectExpression) BulkShardProcessor(org.elasticsearch.action.bulk.BulkShardProcessor) InputCollectExpression(io.crate.operation.collect.InputCollectExpression) BatchIteratorTester(io.crate.testing.BatchIteratorTester) BytesRef(org.apache.lucene.util.BytesRef) RowShardResolver(io.crate.operation.collect.RowShardResolver) SQLTransportIntegrationTest(io.crate.integrationtests.SQLTransportIntegrationTest) Test(org.junit.Test)

Aggregations

ShardUpsertRequest (io.crate.executor.transport.ShardUpsertRequest)2 UUID (java.util.UUID)2 TransportBulkCreateIndicesAction (org.elasticsearch.action.admin.indices.create.TransportBulkCreateIndicesAction)2 BulkRetryCoordinatorPool (org.elasticsearch.action.bulk.BulkRetryCoordinatorPool)2 BulkShardProcessor (org.elasticsearch.action.bulk.BulkShardProcessor)2 ClusterService (org.elasticsearch.cluster.ClusterService)2 IndexNameExpressionResolver (org.elasticsearch.cluster.metadata.IndexNameExpressionResolver)2 InputColumn (io.crate.analyze.symbol.InputColumn)1 Symbol (io.crate.analyze.symbol.Symbol)1 BatchIterator (io.crate.data.BatchIterator)1 Input (io.crate.data.Input)1 Row (io.crate.data.Row)1 RowsBatchIterator (io.crate.data.RowsBatchIterator)1 TransportShardUpsertAction (io.crate.executor.transport.TransportShardUpsertAction)1 SQLTransportIntegrationTest (io.crate.integrationtests.SQLTransportIntegrationTest)1 io.crate.metadata (io.crate.metadata)1 DocSysColumns (io.crate.metadata.doc.DocSysColumns)1 CrateSettings (io.crate.metadata.settings.CrateSettings)1 CollectExpression (io.crate.operation.collect.CollectExpression)1 InputCollectExpression (io.crate.operation.collect.InputCollectExpression)1