Examples with ParsedDocument org.elasticsearch.index.mapper.ParsedDocument used on opensource projects

Example 61 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testSequenceNumberAdvancesToMaxSeqOnEngineOpenOnPrimary.

@Test
public void testSequenceNumberAdvancesToMaxSeqOnEngineOpenOnPrimary() throws BrokenBarrierException, InterruptedException, IOException {
    engine.close();
    final int docs = randomIntBetween(1, 32);
    InternalEngine initialEngine = null;
    try {
        final AtomicReference<CountDownLatch> latchReference = new AtomicReference<>(new CountDownLatch(1));
        final CyclicBarrier barrier = new CyclicBarrier(2);
        final AtomicBoolean stall = new AtomicBoolean();
        final AtomicLong expectedLocalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
        final List<Thread> threads = new ArrayList<>();
        initialEngine = createEngine(defaultSettings, store, primaryTranslogDir, newMergePolicy(), null, LocalCheckpointTracker::new, null, getStallingSeqNoGenerator(latchReference, barrier, stall, expectedLocalCheckpoint));
        final InternalEngine finalInitialEngine = initialEngine;
        for (int i = 0; i < docs; i++) {
            final String id = Integer.toString(i);
            final ParsedDocument doc = testParsedDocument(id, null, testDocumentWithTextField(), SOURCE, null);
            stall.set(randomBoolean());
            final Thread thread = new Thread(() -> {
                try {
                    finalInitialEngine.index(indexForDoc(doc));
                } catch (IOException e) {
                    throw new AssertionError(e);
                }
            });
            thread.start();
            if (stall.get()) {
                threads.add(thread);
                barrier.await();
            } else {
                thread.join();
            }
        }
        assertThat(initialEngine.getProcessedLocalCheckpoint(), equalTo(expectedLocalCheckpoint.get()));
        assertThat(initialEngine.getSeqNoStats(-1).getMaxSeqNo(), equalTo((long) (docs - 1)));
        initialEngine.flush(true, true);
        assertEquals(initialEngine.getProcessedLocalCheckpoint(), initialEngine.getPersistedLocalCheckpoint());
        latchReference.get().countDown();
        for (final Thread thread : threads) {
            thread.join();
        }
    } finally {
        IOUtils.close(initialEngine);
    }
    try (var recoveringEngine = new InternalEngine(initialEngine.config())) {
        recoveringEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
        recoveringEngine.fillSeqNoGaps(2);
        assertEquals(recoveringEngine.getProcessedLocalCheckpoint(), recoveringEngine.getPersistedLocalCheckpoint());
        assertThat(recoveringEngine.getProcessedLocalCheckpoint(), greaterThanOrEqualTo((long) (docs - 1)));
    }
}

Example 62 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testHandleDocumentFailure.

@Test
public void testHandleDocumentFailure() throws Exception {
    try (Store store = createStore()) {
        final ParsedDocument doc1 = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null);
        final ParsedDocument doc2 = testParsedDocument("2", null, testDocumentWithTextField(), B_1, null);
        final ParsedDocument doc3 = testParsedDocument("3", null, testDocumentWithTextField(), B_1, null);
        AtomicReference<ThrowingIndexWriter> throwingIndexWriter = new AtomicReference<>();
        try (InternalEngine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE, (directory, iwc) -> {
            throwingIndexWriter.set(new ThrowingIndexWriter(directory, iwc));
            return throwingIndexWriter.get();
        })) {
            // test document failure while indexing
            if (randomBoolean()) {
                throwingIndexWriter.get().setThrowFailure(() -> new IOException("simulated"));
            } else {
                throwingIndexWriter.get().setThrowFailure(() -> new IllegalArgumentException("simulated max token length"));
            }
            // test index with document failure
            Engine.IndexResult indexResult = engine.index(indexForDoc(doc1));
            assertNotNull(indexResult.getFailure());
            assertThat(indexResult.getSeqNo(), equalTo(0L));
            assertThat(indexResult.getVersion(), equalTo(Versions.MATCH_ANY));
            assertNotNull(indexResult.getTranslogLocation());
            throwingIndexWriter.get().clearFailure();
            indexResult = engine.index(indexForDoc(doc1));
            assertThat(indexResult.getSeqNo(), equalTo(1L));
            assertThat(indexResult.getVersion(), equalTo(1L));
            assertNull(indexResult.getFailure());
            assertNotNull(indexResult.getTranslogLocation());
            engine.index(indexForDoc(doc2));
            // test non document level failure is thrown
            if (randomBoolean()) {
                // simulate close by corruption
                throwingIndexWriter.get().setThrowFailure(null);
                UncheckedIOException uncheckedIOException = expectThrows(UncheckedIOException.class, () -> {
                    Engine.Index index = indexForDoc(doc3);
                    index.parsedDoc().rootDoc().add(new StoredField("foo", "bar") {

                        // this is a hack to add a failure during store document which triggers a tragic event
                        // and in turn fails the engine
                        @Override
                        public BytesRef binaryValue() {
                            throw new UncheckedIOException(new MockDirectoryWrapper.FakeIOException());
                        }
                    });
                    engine.index(index);
                });
                assertTrue(uncheckedIOException.getCause() instanceof MockDirectoryWrapper.FakeIOException);
            } else {
                // normal close
                engine.close();
            }
            // now the engine is closed check we respond correctly
            expectThrows(AlreadyClosedException.class, () -> engine.index(indexForDoc(doc1)));
            expectThrows(AlreadyClosedException.class, () -> engine.delete(new Engine.Delete("1", newUid(doc1), UNASSIGNED_SEQ_NO, primaryTerm.get(), Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0)));
            expectThrows(AlreadyClosedException.class, () -> engine.noOp(new Engine.NoOp(engine.getLocalCheckpointTracker().generateSeqNo(), engine.config().getPrimaryTermSupplier().getAsLong(), randomFrom(Engine.Operation.Origin.values()), randomNonNegativeLong(), "test")));
        }
    }
}

Example 63 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testTranslogRecoveryDoesNotReplayIntoTranslog.

@Test
public void testTranslogRecoveryDoesNotReplayIntoTranslog() throws IOException {
    final int docs = randomIntBetween(1, 32);
    Engine initialEngine = null;
    try {
        initialEngine = engine;
        for (int i = 0; i < docs; i++) {
            final String id = Integer.toString(i);
            final ParsedDocument doc = testParsedDocument(id, null, testDocumentWithTextField(), SOURCE, null);
            initialEngine.index(indexForDoc(doc));
        }
    } finally {
        IOUtils.close(initialEngine);
    }
    Engine recoveringEngine = null;
    try {
        final AtomicBoolean committed = new AtomicBoolean();
        recoveringEngine = new InternalEngine(initialEngine.config()) {

            @Override
            protected void commitIndexWriter(IndexWriter writer, Translog translog, String syncId) throws IOException {
                committed.set(true);
                super.commitIndexWriter(writer, translog, syncId);
            }
        };
        recoveringEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
        assertTrue(committed.get());
    } finally {
        IOUtils.close(recoveringEngine);
    }
}

Example 64 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testEnableGcDeletes.

@Test
public void testEnableGcDeletes() throws Exception {
    try (Store store = createStore();
        Engine engine = createEngine(config(defaultSettings, store, createTempDir(), newMergePolicy(), null))) {
        engine.config().setEnableGcDeletes(false);
        final BiFunction<String, Engine.SearcherScope, Searcher> searcherFactory = engine::acquireSearcher;
        // Add document
        Document document = testDocument();
        document.add(new TextField("value", "test1", Field.Store.YES));
        ParsedDocument doc = testParsedDocument("1", null, document, B_2, null);
        engine.index(new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, 0, 1, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0));
        // Delete document we just added:
        engine.delete(new Engine.Delete("1", newUid(doc), UNASSIGNED_SEQ_NO, 0, 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0));
        // Get should not find the document
        Engine.GetResult getResult = engine.get(newGet(doc), searcherFactory);
        assertThat(getResult.docIdAndVersion(), is(nullValue()));
        // Give the gc pruning logic a chance to kick in
        Thread.sleep(1000);
        if (randomBoolean()) {
            engine.refresh("test");
        }
        // Delete non-existent document
        engine.delete(new Engine.Delete("2", newUid("2"), UNASSIGNED_SEQ_NO, 0, 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0));
        // Get should not find the document (we never indexed uid=2):
        getResult = engine.get(new Engine.Get("2", newUid("2")), searcherFactory);
        assertThat(getResult.docIdAndVersion(), is(nullValue()));
        // Try to index uid=1 with a too-old version, should fail:
        Engine.Index index = new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);
        Engine.IndexResult indexResult = engine.index(index);
        assertThat(indexResult.getResultType(), equalTo(Engine.Result.Type.FAILURE));
        assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
        // Get should still not find the document
        getResult = engine.get(newGet(doc), searcherFactory);
        assertThat(getResult.docIdAndVersion(), is(nullValue()));
        // Try to index uid=2 with a too-old version, should fail:
        Engine.Index index1 = new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);
        indexResult = engine.index(index1);
        assertThat(indexResult.getResultType(), equalTo(Engine.Result.Type.FAILURE));
        assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
        // Get should not find the document
        getResult = engine.get(newGet(doc), searcherFactory);
        assertThat(getResult.docIdAndVersion(), is(nullValue()));
    }
}

Example 65 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testSegmentsWithSoftDeletes.

@Test
public void testSegmentsWithSoftDeletes() throws Exception {
    Settings.Builder settings = Settings.builder().put(defaultSettings.getSettings()).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
    final IndexMetadata indexMetadata = IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build();
    final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetadata);
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    try (Store store = createStore();
        InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get))) {
        assertThat(engine.segments(false), empty());
        int numDocsFirstSegment = randomIntBetween(5, 50);
        Set<String> liveDocsFirstSegment = new HashSet<>();
        for (int i = 0; i < numDocsFirstSegment; i++) {
            String id = Integer.toString(i);
            ParsedDocument doc = testParsedDocument(id, null, testDocument(), B_1, null);
            engine.index(indexForDoc(doc));
            liveDocsFirstSegment.add(id);
        }
        engine.refresh("test");
        List<Segment> segments = engine.segments(randomBoolean());
        assertThat(segments, hasSize(1));
        assertThat(segments.get(0).getNumDocs(), equalTo(liveDocsFirstSegment.size()));
        assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
        assertFalse(segments.get(0).committed);
        int deletes = 0;
        int updates = 0;
        int appends = 0;
        int iterations = scaledRandomIntBetween(1, 50);
        for (int i = 0; i < iterations && liveDocsFirstSegment.isEmpty() == false; i++) {
            String idToUpdate = randomFrom(liveDocsFirstSegment);
            liveDocsFirstSegment.remove(idToUpdate);
            ParsedDocument doc = testParsedDocument(idToUpdate, null, testDocument(), B_1, null);
            if (randomBoolean()) {
                engine.delete(new Engine.Delete(doc.id(), newUid(doc), primaryTerm.get()));
                deletes++;
            } else {
                engine.index(indexForDoc(doc));
                updates++;
            }
            if (randomBoolean()) {
                engine.index(indexForDoc(testParsedDocument(UUIDs.randomBase64UUID(), null, testDocument(), B_1, null)));
                appends++;
            }
        }
        boolean committed = randomBoolean();
        if (committed) {
            engine.flush();
        }
        engine.refresh("test");
        segments = engine.segments(randomBoolean());
        assertThat(segments, hasSize(2));
        assertThat(segments.get(0).getNumDocs(), equalTo(liveDocsFirstSegment.size()));
        assertThat(segments.get(0).getDeletedDocs(), equalTo(updates + deletes));
        assertThat(segments.get(0).committed, equalTo(committed));
        assertThat(segments.get(1).getNumDocs(), equalTo(updates + appends));
        // delete tombstones
        assertThat(segments.get(1).getDeletedDocs(), equalTo(deletes));
        assertThat(segments.get(1).committed, equalTo(committed));
    }
}