Examples with ShardId org.elasticsearch.index.shard.ShardId used on opensource projects

Example 11 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class NestedAggregatorTests method testResetRootDocId.

public void testResetRootDocId() throws Exception {
    Directory directory = newDirectory();
    IndexWriterConfig iwc = new IndexWriterConfig(null);
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, iwc);
    List<Document> documents = new ArrayList<>();
    // 1 segment with, 1 root document, with 3 nested sub docs
    Document document = new Document();
    document.add(new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
    document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE));
    documents.add(document);
    document = new Document();
    document.add(new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
    document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE));
    documents.add(document);
    document = new Document();
    document.add(new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
    document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE));
    documents.add(document);
    document = new Document();
    document.add(new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.FIELD_TYPE));
    document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE));
    documents.add(document);
    indexWriter.addDocuments(documents);
    indexWriter.commit();
    documents.clear();
    // 1 segment with:
    // 1 document, with 1 nested subdoc
    document = new Document();
    document.add(new Field(UidFieldMapper.NAME, "type#2", UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
    document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE));
    documents.add(document);
    document = new Document();
    document.add(new Field(UidFieldMapper.NAME, "type#2", UidFieldMapper.Defaults.FIELD_TYPE));
    document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE));
    documents.add(document);
    indexWriter.addDocuments(documents);
    documents.clear();
    // and 1 document, with 1 nested subdoc
    document = new Document();
    document.add(new Field(UidFieldMapper.NAME, "type#3", UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
    document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE));
    documents.add(document);
    document = new Document();
    document.add(new Field(UidFieldMapper.NAME, "type#3", UidFieldMapper.Defaults.FIELD_TYPE));
    document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE));
    documents.add(document);
    indexWriter.addDocuments(documents);
    indexWriter.commit();
    indexWriter.close();
    IndexService indexService = createIndex("test");
    DirectoryReader directoryReader = DirectoryReader.open(directory);
    directoryReader = ElasticsearchDirectoryReader.wrap(directoryReader, new ShardId(indexService.index(), 0));
    IndexSearcher searcher = new IndexSearcher(directoryReader);
    indexService.mapperService().merge("test", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("test", "nested_field", "type=nested").string()), MapperService.MergeReason.MAPPING_UPDATE, false);
    SearchContext context = createSearchContext(indexService);
    AggregatorFactories.Builder builder = AggregatorFactories.builder();
    NestedAggregationBuilder factory = new NestedAggregationBuilder("test", "nested_field");
    builder.addAggregator(factory);
    AggregatorFactories factories = builder.build(context, null);
    context.aggregations(new SearchContextAggregations(factories));
    Aggregator[] aggs = factories.createTopLevelAggregators();
    BucketCollector collector = BucketCollector.wrap(Arrays.asList(aggs));
    collector.preCollection();
    // A regular search always exclude nested docs, so we use NonNestedDocsFilter.INSTANCE here (otherwise MatchAllDocsQuery would be sufficient)
    // We exclude root doc with uid type#2, this will trigger the bug if we don't reset the root doc when we process a new segment, because
    // root doc type#3 and root doc type#1 have the same segment docid
    BooleanQuery.Builder bq = new BooleanQuery.Builder();
    bq.add(Queries.newNonNestedFilter(), Occur.MUST);
    bq.add(new TermQuery(new Term(UidFieldMapper.NAME, "type#2")), Occur.MUST_NOT);
    searcher.search(new ConstantScoreQuery(bq.build()), collector);
    collector.postCollection();
    Nested nested = (Nested) aggs[0].buildAggregation(0);
    // The bug manifests if 6 docs are returned, because currentRootDoc isn't reset the previous child docs from the first segment are emitted as hits.
    assertThat(nested.getDocCount(), equalTo(4L));
    directoryReader.close();
    directory.close();
}

Example 12 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class ParentToChildrenAggregatorTests method testParentChild.

public void testParentChild() throws IOException {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    final Map<String, Tuple<Integer, Integer>> expectedParentChildRelations = setupIndex(indexWriter);
    indexWriter.close();
    IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), new ShardId(new Index("foo", "_na_"), 1));
    // TODO set "maybeWrap" to true for IndexSearcher once #23338 is resolved
    IndexSearcher indexSearcher = newSearcher(indexReader, false, true);
    testCase(new MatchAllDocsQuery(), indexSearcher, child -> {
        int expectedTotalChildren = 0;
        int expectedMinValue = Integer.MAX_VALUE;
        for (Tuple<Integer, Integer> expectedValues : expectedParentChildRelations.values()) {
            expectedTotalChildren += expectedValues.v1();
            expectedMinValue = Math.min(expectedMinValue, expectedValues.v2());
        }
        assertEquals(expectedTotalChildren, child.getDocCount());
        assertEquals(expectedMinValue, ((InternalMin) child.getAggregations().get("in_child")).getValue(), Double.MIN_VALUE);
    });
    for (String parent : expectedParentChildRelations.keySet()) {
        testCase(new TermInSetQuery(UidFieldMapper.NAME, new BytesRef(Uid.createUid(PARENT_TYPE, parent))), indexSearcher, child -> {
            assertEquals((long) expectedParentChildRelations.get(parent).v1(), child.getDocCount());
            assertEquals(expectedParentChildRelations.get(parent).v2(), ((InternalMin) child.getAggregations().get("in_child")).getValue(), Double.MIN_VALUE);
        });
    }
    indexReader.close();
    directory.close();
}

Example 13 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class ShardSearchTransportRequestTests method createShardSearchTransportRequest.

private ShardSearchTransportRequest createShardSearchTransportRequest() throws IOException {
    SearchRequest searchRequest = createSearchRequest();
    ShardId shardId = new ShardId(randomAsciiOfLengthBetween(2, 10), randomAsciiOfLengthBetween(2, 10), randomInt());
    final AliasFilter filteringAliases;
    if (randomBoolean()) {
        String[] strings = generateRandomStringArray(10, 10, false, false);
        filteringAliases = new AliasFilter(RandomQueryBuilder.createQuery(random()), strings);
    } else {
        filteringAliases = new AliasFilter(null, Strings.EMPTY_ARRAY);
    }
    return new ShardSearchTransportRequest(searchRequest, shardId, randomIntBetween(1, 100), filteringAliases, randomBoolean() ? 1.0f : randomFloat(), Math.abs(randomLong()));
}

Example 14 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class SharedClusterSnapshotRestoreIT method testDeleteOrphanSnapshot.

public void testDeleteOrphanSnapshot() throws Exception {
    Client client = client();
    logger.info("-->  creating repository");
    final String repositoryName = "test-repo";
    assertAcked(client.admin().cluster().preparePutRepository(repositoryName).setType("mock").setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean()).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
    logger.info("--> create the index");
    final String idxName = "test-idx";
    createIndex(idxName);
    ensureGreen();
    ClusterService clusterService = internalCluster().getInstance(ClusterService.class, internalCluster().getMasterName());
    final CountDownLatch countDownLatch = new CountDownLatch(1);
    logger.info("--> snapshot");
    final String snapshotName = "test-snap";
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshotName).setWaitForCompletion(true).setIndices(idxName).get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    logger.info("--> emulate an orphan snapshot");
    RepositoriesService repositoriesService = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
    final RepositoryData repositoryData = repositoriesService.repository(repositoryName).getRepositoryData();
    final IndexId indexId = repositoryData.resolveIndexId(idxName);
    clusterService.submitStateUpdateTask("orphan snapshot test", new ClusterStateUpdateTask() {

        @Override
        public ClusterState execute(ClusterState currentState) {
            // Simulate orphan snapshot
            ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
            shards.put(new ShardId(idxName, "_na_", 0), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            shards.put(new ShardId(idxName, "_na_", 1), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            shards.put(new ShardId(idxName, "_na_", 2), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            List<Entry> entries = new ArrayList<>();
            entries.add(new Entry(new Snapshot(repositoryName, createSnapshotResponse.getSnapshotInfo().snapshotId()), true, false, State.ABORTED, Collections.singletonList(indexId), System.currentTimeMillis(), repositoryData.getGenId(), shards.build()));
            return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
        }

        @Override
        public void onFailure(String source, Exception e) {
            fail();
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, final ClusterState newState) {
            countDownLatch.countDown();
        }
    });
    countDownLatch.await();
    logger.info("--> try deleting the orphan snapshot");
    assertAcked(client.admin().cluster().prepareDeleteSnapshot(repositoryName, snapshotName).get("10s"));
}

Example 15 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class RandomObjects method randomShardInfoFailure.

/**
     * Returns a tuple that contains a randomized {@link Failure} value (left side) and its corresponding
     * value (right side) after it has been printed out as a {@link ToXContent} and parsed back using a parsing
     * method like {@link ShardInfo.Failure#fromXContent(XContentParser)}.
     *
     * @param random Random generator
     */
private static Tuple<Failure, Failure> randomShardInfoFailure(Random random) {
    String index = randomAsciiOfLength(random, 5);
    String indexUuid = randomAsciiOfLength(random, 5);
    int shardId = randomIntBetween(random, 1, 10);
    String nodeId = randomAsciiOfLength(random, 5);
    RestStatus status = randomFrom(random, RestStatus.INTERNAL_SERVER_ERROR, RestStatus.FORBIDDEN, RestStatus.NOT_FOUND);
    boolean primary = random.nextBoolean();
    ShardId shard = new ShardId(index, indexUuid, shardId);
    Exception actualException;
    ElasticsearchException expectedException;
    int type = randomIntBetween(random, 0, 3);
    switch(type) {
        case 0:
            actualException = new ClusterBlockException(singleton(DiscoverySettings.NO_MASTER_BLOCK_WRITES));
            expectedException = new ElasticsearchException("Elasticsearch exception [type=cluster_block_exception, " + "reason=blocked by: [SERVICE_UNAVAILABLE/2/no master];]");
            break;
        case 1:
            actualException = new ShardNotFoundException(shard);
            expectedException = new ElasticsearchException("Elasticsearch exception [type=shard_not_found_exception, " + "reason=no such shard]");
            expectedException.setShard(shard);
            break;
        case 2:
            actualException = new IllegalArgumentException("Closed resource", new RuntimeException("Resource"));
            expectedException = new ElasticsearchException("Elasticsearch exception [type=illegal_argument_exception, " + "reason=Closed resource]", new ElasticsearchException("Elasticsearch exception [type=runtime_exception, reason=Resource]"));
            break;
        case 3:
            actualException = new IndexShardRecoveringException(shard);
            expectedException = new ElasticsearchException("Elasticsearch exception [type=index_shard_recovering_exception, " + "reason=CurrentState[RECOVERING] Already recovering]");
            expectedException.setShard(shard);
            break;
        default:
            throw new UnsupportedOperationException("No randomized exceptions generated for type [" + type + "]");
    }
    Failure actual = new Failure(shard, nodeId, actualException, status, primary);
    Failure expected = new Failure(new ShardId(index, INDEX_UUID_NA_VALUE, shardId), nodeId, expectedException, status, primary);
    return Tuple.tuple(actual, expected);
}