Examples with ParsedDocument org.elasticsearch.index.mapper.ParsedDocument used on opensource projects

Example 81 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testReplicatedVersioningWithFlush.

@Test
public void testReplicatedVersioningWithFlush() throws IOException {
    ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null);
    Engine.Index create = new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, primaryTerm.get(), Versions.MATCH_DELETED, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);
    Engine.IndexResult indexResult = engine.index(create);
    assertThat(indexResult.getVersion(), equalTo(1L));
    assertTrue(indexResult.isCreated());
    create = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), create.primaryTerm(), indexResult.getVersion(), null, REPLICA, 0, -1, false, UNASSIGNED_SEQ_NO, 0);
    indexResult = replicaEngine.index(create);
    assertThat(indexResult.getVersion(), equalTo(1L));
    assertTrue(indexResult.isCreated());
    if (randomBoolean()) {
        engine.flush();
    }
    if (randomBoolean()) {
        replicaEngine.flush();
    }
    Engine.Index update = new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, primaryTerm.get(), 1, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);
    Engine.IndexResult updateResult = engine.index(update);
    assertThat(updateResult.getVersion(), equalTo(2L));
    assertFalse(updateResult.isCreated());
    update = new Engine.Index(newUid(doc), doc, updateResult.getSeqNo(), update.primaryTerm(), updateResult.getVersion(), null, REPLICA, 0, -1, false, UNASSIGNED_SEQ_NO, 0);
    updateResult = replicaEngine.index(update);
    assertThat(updateResult.getVersion(), equalTo(2L));
    assertFalse(updateResult.isCreated());
    replicaEngine.refresh("test");
    try (Searcher searcher = replicaEngine.acquireSearcher("test")) {
        assertEquals(1, searcher.getDirectoryReader().numDocs());
    }
    engine.refresh("test");
    try (Searcher searcher = engine.acquireSearcher("test")) {
        assertEquals(1, searcher.getDirectoryReader().numDocs());
    }
}

Example 82 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testSeqNoGenerator.

@Test
public void testSeqNoGenerator() throws IOException {
    engine.close();
    final long seqNo = randomIntBetween(Math.toIntExact(SequenceNumbers.NO_OPS_PERFORMED), Integer.MAX_VALUE);
    final BiFunction<Long, Long, LocalCheckpointTracker> localCheckpointTrackerSupplier = (ms, lcp) -> new LocalCheckpointTracker(SequenceNumbers.NO_OPS_PERFORMED, SequenceNumbers.NO_OPS_PERFORMED);
    final AtomicLong seqNoGenerator = new AtomicLong(seqNo);
    try (Engine e = createEngine(defaultSettings, store, primaryTranslogDir, newMergePolicy(), null, localCheckpointTrackerSupplier, null, (engine, operation) -> seqNoGenerator.getAndIncrement())) {
        final String id = "id";
        final Field uidField = new Field("_id", id, IdFieldMapper.Defaults.FIELD_TYPE);
        final Field versionField = new NumericDocValuesField("_version", 0);
        final SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
        final ParseContext.Document document = new ParseContext.Document();
        document.add(uidField);
        document.add(versionField);
        document.add(seqID.seqNo);
        document.add(seqID.seqNoDocValue);
        document.add(seqID.primaryTerm);
        final BytesReference source = new BytesArray(new byte[] { 1 });
        final ParsedDocument parsedDocument = new ParsedDocument(versionField, seqID, id, "routing", Collections.singletonList(document), source, null);
        final Engine.Index index = new Engine.Index(new Term("_id", parsedDocument.id()), parsedDocument, UNASSIGNED_SEQ_NO, randomIntBetween(1, 8), Versions.NOT_FOUND, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, randomBoolean(), UNASSIGNED_SEQ_NO, 0);
        final Engine.IndexResult indexResult = e.index(index);
        assertThat(indexResult.getSeqNo(), equalTo(seqNo));
        assertThat(seqNoGenerator.get(), equalTo(seqNo + 1));
        final Engine.Delete delete = new Engine.Delete(id, new Term("_id", parsedDocument.id()), UNASSIGNED_SEQ_NO, randomIntBetween(1, 8), Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0);
        final Engine.DeleteResult deleteResult = e.delete(delete);
        assertThat(deleteResult.getSeqNo(), equalTo(seqNo + 1));
        assertThat(seqNoGenerator.get(), equalTo(seqNo + 2));
    }
}

Example 83 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testSyncedFlushVanishesOnReplay.

@Test
public void testSyncedFlushVanishesOnReplay() throws IOException {
    final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20);
    ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), new BytesArray("{}"), null);
    engine.index(indexForDoc(doc));
    final Engine.CommitId commitID = engine.flush();
    assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID), Engine.SyncedFlushResult.SUCCESS);
    assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId);
    assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId);
    doc = testParsedDocument("2", null, testDocumentWithTextField(), new BytesArray("{}"), null);
    engine.index(indexForDoc(doc));
    EngineConfig config = engine.config();
    engine.close();
    engine = new InternalEngine(config);
    engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
    assertNull("Sync ID must be gone since we have a document to replay", engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID));
}

Example 84 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testRetryWithAutogeneratedIdsAndWrongOrderWorksAndNoDuplicateDocs.

@Test
public void testRetryWithAutogeneratedIdsAndWrongOrderWorksAndNoDuplicateDocs() throws IOException {
    final ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
    boolean isRetry = true;
    long autoGeneratedIdTimestamp = 0;
    Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, 0, randomBoolean() ? Versions.MATCH_DELETED : Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry, UNASSIGNED_SEQ_NO, 0);
    Engine.IndexResult result = engine.index(firstIndexRequest);
    assertThat(result.getVersion(), equalTo(1L));
    Engine.Index firstIndexRequestReplica = new Engine.Index(newUid(doc), doc, result.getSeqNo(), firstIndexRequest.primaryTerm(), result.getVersion(), null, REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry, UNASSIGNED_SEQ_NO, 0);
    Engine.IndexResult indexReplicaResult = replicaEngine.index(firstIndexRequestReplica);
    assertThat(indexReplicaResult.getVersion(), equalTo(1L));
    isRetry = false;
    Engine.Index secondIndexRequest = new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry, UNASSIGNED_SEQ_NO, 0);
    Engine.IndexResult indexResult = engine.index(secondIndexRequest);
    assertFalse(indexResult.isCreated());
    engine.refresh("test");
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
        assertEquals(1, topDocs.totalHits.value);
    }
    Engine.Index secondIndexRequestReplica = new Engine.Index(newUid(doc), doc, result.getSeqNo(), secondIndexRequest.primaryTerm(), result.getVersion(), null, REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry, UNASSIGNED_SEQ_NO, 0);
    replicaEngine.index(secondIndexRequestReplica);
    replicaEngine.refresh("test");
    try (Engine.Searcher searcher = replicaEngine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
        assertEquals(1, topDocs.totalHits.value);
    }
}

Example 85 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testStoreHonorsLuceneVersion.

@Test
public void testStoreHonorsLuceneVersion() throws IOException {
    for (Version createdVersion : Arrays.asList(Version.CURRENT, VersionUtils.getPreviousMinorVersion(), VersionUtils.getFirstVersion())) {
        Settings settings = Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion).build();
        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings);
        try (Store store = createStore();
            InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null))) {
            ParsedDocument doc = testParsedDocument("1", null, new Document(), new BytesArray("{}".getBytes("UTF-8")), null);
            engine.index(appendOnlyPrimary(doc, false, 1));
            engine.refresh("test");
            try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
                LeafReader leafReader = getOnlyLeafReader(searcher.getIndexReader());
                assertEquals(createdVersion.luceneVersion.major, leafReader.getMetaData().getCreatedVersionMajor());
            }
        }
    }
}