Examples with ParsedDocument org.elasticsearch.index.mapper.ParsedDocument used on opensource projects

Example 91 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testVersioningNewIndex.

@Test
public void testVersioningNewIndex() throws IOException {
    ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null);
    Engine.Index index = indexForDoc(doc);
    Engine.IndexResult indexResult = engine.index(index);
    assertThat(indexResult.getVersion(), equalTo(1L));
    index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), null, REPLICA, 0, -1, false, UNASSIGNED_SEQ_NO, 0);
    indexResult = replicaEngine.index(index);
    assertThat(indexResult.getVersion(), equalTo(1L));
}

Example 92 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testRandomOperations.

/**
 * A simple test to check that random combination of operations can coexist in segments and be lookup.
 * This is needed as some fields in Lucene may not exist if a segment misses operation types and this code is to check for that.
 * For example, a segment containing only no-ops does not have neither _uid or _version.
 */
@Test
public void testRandomOperations() throws Exception {
    int numOps = between(10, 100);
    for (int i = 0; i < numOps; i++) {
        String id = Integer.toString(randomIntBetween(1, 10));
        ParsedDocument doc = createParsedDoc(id, null);
        Engine.Operation.TYPE type = randomFrom(Engine.Operation.TYPE.values());
        switch(type) {
            case INDEX:
                Engine.IndexResult index = engine.index(replicaIndexForDoc(doc, between(1, 100), i, randomBoolean()));
                assertThat(index.getFailure(), nullValue());
                break;
            case DELETE:
                Engine.DeleteResult delete = engine.delete(replicaDeleteForDoc(doc.id(), between(1, 100), i, randomNonNegativeLong()));
                assertThat(delete.getFailure(), nullValue());
                break;
            case NO_OP:
                Engine.NoOpResult noOp = engine.noOp(new Engine.NoOp(i, primaryTerm.get(), randomFrom(Engine.Operation.Origin.values()), randomNonNegativeLong(), ""));
                assertThat(noOp.getFailure(), nullValue());
                break;
            default:
                throw new IllegalStateException("Invalid op [" + type + "]");
        }
        if (randomBoolean()) {
            engine.refresh("test");
        }
        if (randomBoolean()) {
            engine.flush();
        }
        if (randomBoolean()) {
            engine.forceMerge(randomBoolean(), between(1, 10), randomBoolean(), false, false, UUIDs.randomBase64UUID());
        }
    }
    if (engine.engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
        List<Translog.Operation> operations = readAllOperationsInLucene(engine, createMapperService("test"));
        assertThat(operations, hasSize(numOps));
    }
}

Example 93 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class LuceneChangesSnapshotTests method testBasics.

public void testBasics() throws Exception {
    long fromSeqNo = randomNonNegativeLong();
    long toSeqNo = randomLongBetween(fromSeqNo, Long.MAX_VALUE);
    // Empty engine
    try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", mapperService, fromSeqNo, toSeqNo, true)) {
        IllegalStateException error = expectThrows(IllegalStateException.class, () -> drainAll(snapshot));
        assertThat(error.getMessage(), containsString("Not all operations between from_seqno [" + fromSeqNo + "] and to_seqno [" + toSeqNo + "] found"));
    }
    try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", mapperService, fromSeqNo, toSeqNo, false)) {
        assertThat(snapshot, SnapshotMatchers.size(0));
    }
    int numOps = between(1, 100);
    int refreshedSeqNo = -1;
    for (int i = 0; i < numOps; i++) {
        String id = Integer.toString(randomIntBetween(i, i + 5));
        ParsedDocument doc = createParsedDoc(id, null, randomBoolean());
        if (randomBoolean()) {
            engine.index(indexForDoc(doc));
        } else {
            engine.delete(new Engine.Delete(doc.id(), newUid(doc.id()), primaryTerm.get()));
        }
        if (rarely()) {
            if (randomBoolean()) {
                engine.flush();
            } else {
                engine.refresh("test");
            }
            refreshedSeqNo = i;
        }
    }
    if (refreshedSeqNo == -1) {
        fromSeqNo = between(0, numOps);
        toSeqNo = randomLongBetween(fromSeqNo, numOps * 2);
        Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL);
        try (Translog.Snapshot snapshot = new LuceneChangesSnapshot(searcher, mapperService, between(1, LuceneChangesSnapshot.DEFAULT_BATCH_SIZE), fromSeqNo, toSeqNo, false)) {
            searcher = null;
            assertThat(snapshot, SnapshotMatchers.size(0));
        } finally {
            IOUtils.close(searcher);
        }
        searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL);
        try (Translog.Snapshot snapshot = new LuceneChangesSnapshot(searcher, mapperService, between(1, LuceneChangesSnapshot.DEFAULT_BATCH_SIZE), fromSeqNo, toSeqNo, true)) {
            searcher = null;
            IllegalStateException error = expectThrows(IllegalStateException.class, () -> drainAll(snapshot));
            assertThat(error.getMessage(), containsString("Not all operations between from_seqno [" + fromSeqNo + "] and to_seqno [" + toSeqNo + "] found"));
        } finally {
            IOUtils.close(searcher);
        }
    } else {
        fromSeqNo = randomLongBetween(0, refreshedSeqNo);
        toSeqNo = randomLongBetween(refreshedSeqNo + 1, numOps * 2);
        Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL);
        try (Translog.Snapshot snapshot = new LuceneChangesSnapshot(searcher, mapperService, between(1, LuceneChangesSnapshot.DEFAULT_BATCH_SIZE), fromSeqNo, toSeqNo, false)) {
            searcher = null;
            assertThat(snapshot, SnapshotMatchers.containsSeqNoRange(fromSeqNo, refreshedSeqNo));
        } finally {
            IOUtils.close(searcher);
        }
        searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL);
        try (Translog.Snapshot snapshot = new LuceneChangesSnapshot(searcher, mapperService, between(1, LuceneChangesSnapshot.DEFAULT_BATCH_SIZE), fromSeqNo, toSeqNo, true)) {
            searcher = null;
            IllegalStateException error = expectThrows(IllegalStateException.class, () -> drainAll(snapshot));
            assertThat(error.getMessage(), containsString("Not all operations between from_seqno [" + fromSeqNo + "] and to_seqno [" + toSeqNo + "] found"));
        } finally {
            IOUtils.close(searcher);
        }
        toSeqNo = randomLongBetween(fromSeqNo, refreshedSeqNo);
        searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL);
        try (Translog.Snapshot snapshot = new LuceneChangesSnapshot(searcher, mapperService, between(1, LuceneChangesSnapshot.DEFAULT_BATCH_SIZE), fromSeqNo, toSeqNo, true)) {
            searcher = null;
            assertThat(snapshot, SnapshotMatchers.containsSeqNoRange(fromSeqNo, toSeqNo));
        } finally {
            IOUtils.close(searcher);
        }
    }
    // Get snapshot via engine will auto refresh
    fromSeqNo = randomLongBetween(0, numOps - 1);
    toSeqNo = randomLongBetween(fromSeqNo, numOps - 1);
    try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", mapperService, fromSeqNo, toSeqNo, randomBoolean())) {
        assertThat(snapshot, SnapshotMatchers.containsSeqNoRange(fromSeqNo, toSeqNo));
    }
}

Example 94 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class ReadOnlyEngineTests method test_recover_from_translog_applies_no_operations.

@Test
public void test_recover_from_translog_applies_no_operations() throws IOException {
    IOUtils.close(engine, store);
    AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    try (Store store = createStore()) {
        EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
        int numDocs = scaledRandomIntBetween(10, 1000);
        try (InternalEngine engine = createEngine(config)) {
            for (int i = 0; i < numDocs; i++) {
                ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), new BytesArray("{}"), null);
                engine.index(new Engine.Index(newUid(doc), doc, i, primaryTerm.get(), 1, null, Engine.Operation.Origin.REPLICA, System.nanoTime(), -1, false, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
                if (rarely()) {
                    engine.flush();
                }
                globalCheckpoint.set(i);
            }
            engine.syncTranslog();
            engine.flushAndClose();
        }
        try (ReadOnlyEngine readOnlyEngine = new ReadOnlyEngine(config, null, null, true, x -> x)) {
            TranslogHandler translogHandler = new TranslogHandler(xContentRegistry(), config.getIndexSettings());
            readOnlyEngine.recoverFromTranslog(translogHandler, randomNonNegativeLong());
            int opsRecovered = translogHandler.run(readOnlyEngine, new Translog.Snapshot() {

                @Override
                public int totalOperations() {
                    return 0;
                }

                @Override
                public Translog.Operation next() {
                    return null;
                }

                @Override
                public void close() {
                }
            });
            assertThat(opsRecovered, is(0));
        }
    }
}

Example 95 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class ReadOnlyEngineTests method test_flushes.

@Test
public void test_flushes() throws IOException {
    IOUtils.close(engine, store);
    Engine readOnlyEngine = null;
    AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    try (Store store = createStore()) {
        EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
        int numDocs = scaledRandomIntBetween(10, 1000);
        try (InternalEngine engine = createEngine(config)) {
            for (int i = 0; i < numDocs; i++) {
                ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), new BytesArray("{}"), null);
                engine.index(new Engine.Index(newUid(doc), doc, i, primaryTerm.get(), 1, null, Engine.Operation.Origin.REPLICA, System.nanoTime(), -1, false, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
                if (rarely()) {
                    engine.flush();
                }
                // advance persisted local checkpoint
                engine.syncTranslog();
                globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
            }
            globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
            engine.syncTranslog();
            engine.flushAndClose();
            readOnlyEngine = new ReadOnlyEngine(engine.engineConfig, null, null, true, Function.identity());
            Engine.CommitId flush = readOnlyEngine.flush(randomBoolean(), true);
            assertThat(readOnlyEngine.flush(randomBoolean(), true), is(flush));
        } finally {
            IOUtils.close(readOnlyEngine);
        }
    }
}