Examples with Index org.elasticsearch.index.Index used on opensource projects

Example 91 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class ParentChildFieldDataTests method testThreads.

public void testThreads() throws Exception {
    final ParentChildIndexFieldData indexFieldData = getForField(childType);
    final DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId(new Index("test", ""), 0));
    final IndexParentChildFieldData global = indexFieldData.loadGlobal(reader);
    final AtomicReference<Exception> error = new AtomicReference<>();
    final int numThreads = scaledRandomIntBetween(3, 8);
    final Thread[] threads = new Thread[numThreads];
    final CountDownLatch latch = new CountDownLatch(1);
    final Map<Object, BytesRef[]> expected = new HashMap<>();
    for (LeafReaderContext context : reader.leaves()) {
        AtomicParentChildFieldData leafData = global.load(context);
        SortedDocValues parentIds = leafData.getOrdinalsValues(parentType);
        final BytesRef[] ids = new BytesRef[parentIds.getValueCount()];
        for (int j = 0; j < parentIds.getValueCount(); ++j) {
            final BytesRef id = parentIds.lookupOrd(j);
            if (id != null) {
                ids[j] = BytesRef.deepCopyOf(id);
            }
        }
        expected.put(context.reader().getCoreCacheKey(), ids);
    }
    for (int i = 0; i < numThreads; ++i) {
        threads[i] = new Thread() {

            @Override
            public void run() {
                try {
                    latch.await();
                    for (int i = 0; i < 100000; ++i) {
                        for (LeafReaderContext context : reader.leaves()) {
                            AtomicParentChildFieldData leafData = global.load(context);
                            SortedDocValues parentIds = leafData.getOrdinalsValues(parentType);
                            final BytesRef[] expectedIds = expected.get(context.reader().getCoreCacheKey());
                            for (int j = 0; j < parentIds.getValueCount(); ++j) {
                                final BytesRef id = parentIds.lookupOrd(j);
                                assertEquals(expectedIds[j], id);
                            }
                        }
                    }
                } catch (Exception e) {
                    error.compareAndSet(null, e);
                }
            }
        };
        threads[i].start();
    }
    latch.countDown();
    for (Thread thread : threads) {
        thread.join();
    }
    if (error.get() != null) {
        throw error.get();
    }
}

Example 92 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class InternalEngineTests method testRecoverFromForeignTranslog.

public void testRecoverFromForeignTranslog() throws IOException {
    final int numDocs = randomIntBetween(1, 10);
    for (int i = 0; i < numDocs; i++) {
        ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
        Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
        Engine.IndexResult index = engine.index(firstIndexRequest);
        assertThat(index.getVersion(), equalTo(1L));
    }
    engine.refresh("test");
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
    Translog.TranslogGeneration generation = engine.getTranslog().getGeneration();
    engine.close();
    Translog translog = new Translog(new TranslogConfig(shardId, createTempDir(), INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), null, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO);
    translog.add(new Translog.Index("test", "SomeBogusId", "{}".getBytes(Charset.forName("UTF-8"))));
    assertEquals(generation.translogFileGeneration, translog.currentFileGeneration());
    translog.close();
    EngineConfig config = engine.config();
    /* create a TranslogConfig that has been created with a different UUID */
    TranslogConfig translogConfig = new TranslogConfig(shardId, translog.location(), config.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE);
    EngineConfig brokenConfig = new EngineConfig(EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG, shardId, threadPool, config.getIndexSettings(), null, store, createSnapshotDeletionPolicy(), newMergePolicy(), config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener(), config.getTranslogRecoveryPerformer(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), config.getRefreshListeners(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
    try {
        InternalEngine internalEngine = new InternalEngine(brokenConfig);
        fail("translog belongs to a different engine");
    } catch (EngineCreationFailureException ex) {
    }
    // and recover again!
    engine = createEngine(store, primaryTranslogDir);
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
}

Example 93 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class InternalEngineTests method testVersioningReplicaConflict1.

public void testVersioningReplicaConflict1() throws IOException {
    final ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
    final Engine.Index v1Index = indexForDoc(doc);
    final Engine.IndexResult v1Result = engine.index(v1Index);
    assertThat(v1Result.getVersion(), equalTo(1L));
    final Engine.Index v2Index = indexForDoc(doc);
    final Engine.IndexResult v2Result = engine.index(v2Index);
    assertThat(v2Result.getVersion(), equalTo(2L));
    // apply the second index to the replica, should work fine
    final Engine.Index replicaV2Index = new Engine.Index(newUid(doc), doc, v2Result.getSeqNo(), v2Index.primaryTerm(), v2Result.getVersion(), VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
    final Engine.IndexResult replicaV2Result = replicaEngine.index(replicaV2Index);
    assertThat(replicaV2Result.getVersion(), equalTo(2L));
    // now, the old one should produce an indexing result
    final Engine.Index replicaV1Index = new Engine.Index(newUid(doc), doc, v1Result.getSeqNo(), v1Index.primaryTerm(), v1Result.getVersion(), VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
    final Engine.IndexResult replicaV1Result = replicaEngine.index(replicaV1Index);
    assertFalse(replicaV1Result.hasFailure());
    assertFalse(replicaV1Result.isCreated());
    assertThat(replicaV1Result.getVersion(), equalTo(2L));
    // second version on replica should fail as well
    final Engine.IndexResult replicaV2ReplayResult = replicaEngine.index(replicaV2Index);
    assertFalse(replicaV2Result.hasFailure());
    assertFalse(replicaV1Result.isCreated());
    assertThat(replicaV2ReplayResult.getVersion(), equalTo(2L));
}

Example 94 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class InternalEngineTests method testAppendConcurrently.

public void testAppendConcurrently() throws InterruptedException, IOException {
    Thread[] thread = new Thread[randomIntBetween(3, 5)];
    int numDocs = randomIntBetween(1000, 10000);
    assertEquals(0, engine.getNumVersionLookups());
    assertEquals(0, engine.getNumIndexVersionsLookups());
    List<Engine.Index> docs = new ArrayList<>();
    for (int i = 0; i < numDocs; i++) {
        final ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
        Engine.Index index = randomAppendOnly(doc, false, i);
        docs.add(index);
    }
    Collections.shuffle(docs, random());
    CountDownLatch startGun = new CountDownLatch(thread.length);
    AtomicInteger offset = new AtomicInteger(-1);
    for (int i = 0; i < thread.length; i++) {
        thread[i] = new Thread() {

            @Override
            public void run() {
                startGun.countDown();
                try {
                    startGun.await();
                } catch (InterruptedException e) {
                    throw new AssertionError(e);
                }
                int docOffset;
                while ((docOffset = offset.incrementAndGet()) < docs.size()) {
                    try {
                        engine.index(docs.get(docOffset));
                    } catch (IOException e) {
                        throw new AssertionError(e);
                    }
                }
            }
        };
        thread[i].start();
    }
    for (int i = 0; i < thread.length; i++) {
        thread[i].join();
    }
    engine.refresh("test");
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10);
        assertEquals(docs.size(), topDocs.totalHits);
    }
    assertEquals(0, engine.getNumVersionLookups());
    assertEquals(0, engine.getNumIndexVersionsLookups());
    assertFalse(engine.indexWriterHasDeletions());
}

Example 95 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class InternalEngineTests method testVersioningReplicaConflict2.

public void testVersioningReplicaConflict2() throws IOException {
    final ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
    final Engine.Index v1Index = indexForDoc(doc);
    final Engine.IndexResult v1Result = engine.index(v1Index);
    assertThat(v1Result.getVersion(), equalTo(1L));
    // apply the first index to the replica, should work fine
    final Engine.Index replicaV1Index = new Engine.Index(newUid(doc), doc, v1Result.getSeqNo(), v1Index.primaryTerm(), v1Result.getVersion(), VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
    Engine.IndexResult replicaV1Result = replicaEngine.index(replicaV1Index);
    assertThat(replicaV1Result.getVersion(), equalTo(1L));
    // index it again
    final Engine.Index v2Index = indexForDoc(doc);
    final Engine.IndexResult v2Result = engine.index(v2Index);
    assertThat(v2Result.getVersion(), equalTo(2L));
    // now delete it
    final Engine.Delete delete = new Engine.Delete("test", "1", newUid(doc));
    final Engine.DeleteResult deleteResult = engine.delete(delete);
    assertThat(deleteResult.getVersion(), equalTo(3L));
    // apply the delete on the replica (skipping the second index)
    final Engine.Delete replicaDelete = new Engine.Delete("test", "1", newUid(doc), deleteResult.getSeqNo(), delete.primaryTerm(), deleteResult.getVersion(), VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0);
    final Engine.DeleteResult replicaDeleteResult = replicaEngine.delete(replicaDelete);
    assertThat(replicaDeleteResult.getVersion(), equalTo(3L));
    // second time delete with same version should just produce the same version
    final Engine.DeleteResult deleteReplayResult = replicaEngine.delete(replicaDelete);
    assertFalse(deleteReplayResult.hasFailure());
    assertTrue(deleteReplayResult.isFound());
    assertThat(deleteReplayResult.getVersion(), equalTo(3L));
    // now do the second index on the replica, it should result in the current version
    final Engine.Index replicaV2Index = new Engine.Index(newUid(doc), doc, v2Result.getSeqNo(), v2Index.primaryTerm(), v2Result.getVersion(), VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
    final Engine.IndexResult replicaV2Result = replicaEngine.index(replicaV2Index);
    assertFalse(replicaV2Result.hasFailure());
    assertFalse(replicaV2Result.isCreated());
    assertThat(replicaV2Result.getVersion(), equalTo(3L));
}