Examples with ShardId org.elasticsearch.index.shard.ShardId used on opensource projects

Example 6 with ShardId

use of org.elasticsearch.index.shard.ShardId in project crate by crate.

the class ShardUpsertRequestTest method testStreaming.

@Test
public void testStreaming() throws Exception {
    ShardId shardId = new ShardId("test", 1);
    String[] assignmentColumns = new String[] { "id", "name" };
    UUID jobId = UUID.randomUUID();
    Reference[] missingAssignmentColumns = new Reference[] { ID_REF, NAME_REF };
    ShardUpsertRequest request = new ShardUpsertRequest.Builder(false, false, assignmentColumns, missingAssignmentColumns, jobId, false).newRequest(shardId, "42");
    request.validateConstraints(false);
    request.add(123, new ShardUpsertRequest.Item("99", null, new Object[] { 99, new BytesRef("Marvin") }, null));
    request.add(5, new ShardUpsertRequest.Item("42", new Symbol[] { Literal.of(42), Literal.of("Deep Thought") }, null, 2L));
    BytesStreamOutput out = new BytesStreamOutput();
    request.writeTo(out);
    StreamInput in = StreamInput.wrap(out.bytes());
    ShardUpsertRequest request2 = new ShardUpsertRequest();
    request2.readFrom(in);
    assertThat(request, equalTo(request2));
}

Example 7 with ShardId

use of org.elasticsearch.index.shard.ShardId in project crate by crate.

the class BulkShardProcessor method createPendingIndices.

private void createPendingIndices() {
    final List<PendingRequest> pendings = new ArrayList<>();
    final Set<String> indices;
    synchronized (requestsForNewIndices) {
        indices = ImmutableSet.copyOf(Iterables.filter(Sets.difference(requestsForNewIndices.keySet(), indicesCreated), shouldAutocreateIndexPredicate));
        for (Map.Entry<String, List<PendingRequest>> entry : requestsForNewIndices.entrySet()) {
            pendings.addAll(entry.getValue());
        }
        requestsForNewIndices.clear();
        pendingNewIndexRequests.set(0);
    }
    if (pendings.size() > 0 || indices.size() > 0) {
        LOGGER.debug("create {} pending indices in bulk...", indices.size());
        BulkCreateIndicesRequest bulkCreateIndicesRequest = new BulkCreateIndicesRequest(indices, jobId);
        final FutureCallback<Void> indicesCreatedCallback = new FutureCallback<Void>() {

            @Override
            public void onSuccess(@Nullable Void result) {
                if (failure.get() != null) {
                    return;
                }
                trace("applying pending requests for created indices...");
                Iterator<PendingRequest> it = pendings.iterator();
                while (it.hasNext()) {
                    PendingRequest pendingRequest = it.next();
                    // add pending requests for created indices
                    ShardId shardId = shardId(pendingRequest.indexName, pendingRequest.item.id(), pendingRequest.routing);
                    if (shardId == null) {
                        // seems like index is deleted meanwhile, mark item as failed and remove pending
                        indicesDeleted.add(pendingRequest.indexName);
                        it.remove();
                        synchronized (responsesLock) {
                            responses.set(globalCounter.getAndIncrement(), false);
                        }
                        setResultIfDone(1);
                        continue;
                    }
                    partitionRequestByShard(shardId, pendingRequest.item, pendingRequest.routing);
                }
                trace("added %d pending requests, lets see if we can execute them", pendings.size());
                executeRequestsIfNeeded();
            }

            @Override
            public void onFailure(Throwable t) {
                setFailure(t);
            }
        };
        if (indices.isEmpty()) {
            indicesCreatedCallback.onSuccess(null);
        } else {
            transportBulkCreateIndicesAction.execute(bulkCreateIndicesRequest, new ActionListener<BulkCreateIndicesResponse>() {

                @Override
                public void onResponse(BulkCreateIndicesResponse response) {
                    indicesCreated.addAll(indices);
                    indicesCreatedCallback.onSuccess(null);
                }

                @Override
                public void onFailure(Throwable t) {
                    indicesCreatedCallback.onFailure(t);
                }
            });
        }
    }
}

Example 8 with ShardId

use of org.elasticsearch.index.shard.ShardId in project crate by crate.

the class OrderedLuceneBatchIteratorFactoryTest method createOrderedCollector.

private LuceneOrderedDocCollector createOrderedCollector(IndexSearcher searcher, int shardId) {
    CollectorContext collectorContext = new CollectorContext(mock(IndexFieldDataService.class), new CollectorFieldsVisitor(0));
    List<LuceneCollectorExpression<?>> expressions = Collections.singletonList(new OrderByCollectorExpression(reference, orderBy));
    return new LuceneOrderedDocCollector(new ShardId("dummy", shardId), searcher, new MatchAllDocsQuery(), null, false, // batchSize < 10 to have at least one searchMore call.
    5, fieldTypeLookup, collectorContext, orderBy, new Sort(new SortedNumericSortField(columnName, SortField.Type.LONG, reverseFlags[0])), expressions, expressions);
}

Example 9 with ShardId

use of org.elasticsearch.index.shard.ShardId in project crate by crate.

the class ShardCollectorProviderTest method assertNoShardEntriesLeftInShardCollectSource.

public void assertNoShardEntriesLeftInShardCollectSource() throws Exception {
    final Field shards = ShardCollectSource.class.getDeclaredField("shards");
    shards.setAccessible(true);
    final List<ShardCollectSource> shardCollectSources = Lists.newArrayList((internalCluster().getInstances(ShardCollectSource.class)));
    for (ShardCollectSource shardCollectSource : shardCollectSources) {
        try {
            //noinspection unchecked
            Map<ShardId, ShardCollectorProvider> shardMap = (Map<ShardId, ShardCollectorProvider>) shards.get(shardCollectSource);
            assertThat(shardMap.size(), is(0));
        } catch (IllegalAccessException e) {
            Throwables.propagate(e);
        }
    }
}

Example 10 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class SearchHitTests method createTestItem.

public static SearchHit createTestItem(boolean withOptionalInnerHits) {
    int internalId = randomInt();
    String uid = randomAsciiOfLength(10);
    Text type = new Text(randomAsciiOfLengthBetween(5, 10));
    NestedIdentity nestedIdentity = null;
    if (randomBoolean()) {
        nestedIdentity = NestedIdentityTests.createTestItem(randomIntBetween(0, 2));
    }
    Map<String, SearchHitField> fields = new HashMap<>();
    if (randomBoolean()) {
        int size = randomIntBetween(0, 10);
        for (int i = 0; i < size; i++) {
            Tuple<List<Object>, List<Object>> values = RandomObjects.randomStoredFieldValues(random(), XContentType.JSON);
            if (randomBoolean()) {
                String metaField = randomFrom(META_FIELDS);
                fields.put(metaField, new SearchHitField(metaField, values.v1()));
            } else {
                String fieldName = randomAsciiOfLengthBetween(5, 10);
                fields.put(fieldName, new SearchHitField(fieldName, values.v1()));
            }
        }
    }
    SearchHit hit = new SearchHit(internalId, uid, type, nestedIdentity, fields);
    if (frequently()) {
        if (rarely()) {
            hit.score(Float.NaN);
        } else {
            hit.score(randomFloat());
        }
    }
    if (frequently()) {
        hit.sourceRef(RandomObjects.randomSource(random()));
    }
    if (randomBoolean()) {
        hit.version(randomLong());
    }
    if (randomBoolean()) {
        hit.sortValues(SearchSortValuesTests.createTestItem());
    }
    if (randomBoolean()) {
        int size = randomIntBetween(0, 5);
        Map<String, HighlightField> highlightFields = new HashMap<>(size);
        for (int i = 0; i < size; i++) {
            highlightFields.put(randomAsciiOfLength(5), HighlightFieldTests.createTestItem());
        }
        hit.highlightFields(highlightFields);
    }
    if (randomBoolean()) {
        int size = randomIntBetween(0, 5);
        String[] matchedQueries = new String[size];
        for (int i = 0; i < size; i++) {
            matchedQueries[i] = randomAsciiOfLength(5);
        }
        hit.matchedQueries(matchedQueries);
    }
    if (randomBoolean()) {
        hit.explanation(createExplanation(randomIntBetween(0, 5)));
    }
    if (withOptionalInnerHits) {
        int innerHitsSize = randomIntBetween(0, 3);
        Map<String, SearchHits> innerHits = new HashMap<>(innerHitsSize);
        for (int i = 0; i < innerHitsSize; i++) {
            innerHits.put(randomAsciiOfLength(5), SearchHitsTests.createTestItem());
        }
        hit.setInnerHits(innerHits);
    }
    if (randomBoolean()) {
        hit.shard(new SearchShardTarget(randomAsciiOfLengthBetween(5, 10), new ShardId(new Index(randomAsciiOfLengthBetween(5, 10), randomAsciiOfLengthBetween(5, 10)), randomInt())));
    }
    return hit;
}