Examples with ParsedDocument org.elasticsearch.index.mapper.ParsedDocument used on opensource projects

Example 56 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class ArrayMapperTest method testParseDynamicEmptyArray.

@Test
public void testParseDynamicEmptyArray() throws Exception {
    String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject(TYPE).startObject("properties").endObject().endObject().endObject());
    DocumentMapper mapper = mapper(INDEX, mapping);
    // parse source with empty array
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().array("new_array_field").endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument doc = mapper.parse(sourceToParse);
    assertThat(doc.docs().get(0).getField("new_array_field"), is(nullValue()));
    assertThat(mapper.mappers().getMapper("new_array_field"), is(nullValue()));
}

Example 57 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testMaxSeqNoInCommitUserData.

@Test
public void testMaxSeqNoInCommitUserData() throws Exception {
    AtomicBoolean running = new AtomicBoolean(true);
    Thread rollTranslog = new Thread(() -> {
        while (running.get() && engine.getTranslog().currentFileGeneration() < 500) {
            // make adding operations to translog slower
            engine.rollTranslogGeneration();
        }
    });
    rollTranslog.start();
    Thread indexing = new Thread(() -> {
        long seqNo = 0;
        while (running.get() && seqNo <= 1000) {
            try {
                String id = Long.toString(between(1, 50));
                if (randomBoolean()) {
                    ParsedDocument doc = testParsedDocument(id, null, testDocumentWithTextField(), SOURCE, null);
                    engine.index(replicaIndexForDoc(doc, 1L, seqNo, false));
                } else {
                    engine.delete(replicaDeleteForDoc(id, 1L, seqNo, 0L));
                }
                seqNo++;
            } catch (IOException e) {
                throw new AssertionError(e);
            }
        }
    });
    indexing.start();
    int numCommits = between(5, 20);
    for (int i = 0; i < numCommits; i++) {
        engine.flush(false, true);
    }
    running.set(false);
    indexing.join();
    rollTranslog.join();
    assertMaxSeqNoInCommitUserData(engine);
}

Example 58 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testLookupSeqNoByIdInLucene.

@Test
public void testLookupSeqNoByIdInLucene() throws Exception {
    int numOps = between(10, 100);
    long seqNo = 0;
    List<Engine.Operation> operations = new ArrayList<>(numOps);
    for (int i = 0; i < numOps; i++) {
        String id = Integer.toString(between(1, 50));
        boolean isIndexing = randomBoolean();
        int copies = frequently() ? 1 : between(2, 4);
        for (int c = 0; c < copies; c++) {
            final ParsedDocument doc = EngineTestCase.createParsedDoc(id, null);
            if (isIndexing) {
                operations.add(new Engine.Index(EngineTestCase.newUid(doc), doc, seqNo, primaryTerm.get(), i, null, Engine.Operation.Origin.REPLICA, threadPool.relativeTimeInMillis(), -1, true, UNASSIGNED_SEQ_NO, 0L));
            } else {
                operations.add(new Engine.Delete(doc.id(), EngineTestCase.newUid(doc), seqNo, primaryTerm.get(), i, null, Engine.Operation.Origin.REPLICA, threadPool.relativeTimeInMillis(), UNASSIGNED_SEQ_NO, 0L));
            }
        }
        seqNo++;
        if (rarely()) {
            seqNo++;
        }
    }
    Randomness.shuffle(operations);
    Settings.Builder settings = Settings.builder().put(defaultSettings.getSettings()).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
    final IndexMetadata indexMetadata = IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build();
    final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetadata);
    // id -> latest seq_no
    Map<String, Engine.Operation> latestOps = new HashMap<>();
    try (Store store = createStore();
        InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), newMergePolicy(), null))) {
        CheckedRunnable<IOException> lookupAndCheck = () -> {
            try (Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
                Map<String, Long> liveOps = latestOps.entrySet().stream().filter(e -> e.getValue().operationType() == Engine.Operation.TYPE.INDEX).collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().seqNo()));
                assertThat(getDocIds(engine, true).stream().collect(Collectors.toMap(e -> e.getId(), e -> e.getSeqNo())), equalTo(liveOps));
                for (String id : latestOps.keySet()) {
                    String msg = "latestOps=" + latestOps + " op=" + id;
                    DocIdAndSeqNo docIdAndSeqNo = VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.getIndexReader(), newUid(id));
                    if (liveOps.containsKey(id) == false) {
                        assertNull(msg, docIdAndSeqNo);
                    } else {
                        assertNotNull(msg, docIdAndSeqNo);
                        assertThat(msg, docIdAndSeqNo.seqNo, equalTo(latestOps.get(id).seqNo()));
                    }
                }
                String notFoundId = randomValueOtherThanMany(liveOps::containsKey, () -> Long.toString(randomNonNegativeLong()));
                assertNull(VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.getIndexReader(), newUid(notFoundId)));
            }
        };
        for (Engine.Operation op : operations) {
            if (op instanceof Engine.Index) {
                engine.index((Engine.Index) op);
                if (latestOps.containsKey(op.id()) == false || latestOps.get(op.id()).seqNo() < op.seqNo()) {
                    latestOps.put(op.id(), op);
                }
            } else if (op instanceof Engine.Delete) {
                engine.delete((Engine.Delete) op);
                if (latestOps.containsKey(op.id()) == false || latestOps.get(op.id()).seqNo() < op.seqNo()) {
                    latestOps.put(op.id(), op);
                }
            }
            if (randomInt(100) < 10) {
                engine.refresh("test");
                lookupAndCheck.run();
            }
            if (rarely()) {
                engine.flush(false, true);
                lookupAndCheck.run();
            }
        }
        engine.refresh("test");
        lookupAndCheck.run();
    }
}

Example 59 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testRetryWithAutogeneratedIdWorksAndNoDuplicateDocs.

@Test
public void testRetryWithAutogeneratedIdWorksAndNoDuplicateDocs() throws IOException {
    final ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
    boolean isRetry = false;
    long autoGeneratedIdTimestamp = 0;
    Engine.Index index = new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, 0, randomBoolean() ? Versions.MATCH_DELETED : Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry, UNASSIGNED_SEQ_NO, 0);
    Engine.IndexResult indexResult = engine.index(index);
    assertThat(indexResult.getVersion(), equalTo(1L));
    index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), null, REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry, UNASSIGNED_SEQ_NO, 0);
    indexResult = replicaEngine.index(index);
    assertThat(indexResult.getVersion(), equalTo(1L));
    isRetry = true;
    index = new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry, UNASSIGNED_SEQ_NO, 0);
    indexResult = engine.index(index);
    assertThat(indexResult.getVersion(), equalTo(1L));
    assertNotEquals(indexResult.getSeqNo(), UNASSIGNED_SEQ_NO);
    engine.refresh("test");
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
        assertEquals(1, topDocs.totalHits.value);
    }
    index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), null, REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry, UNASSIGNED_SEQ_NO, 0);
    indexResult = replicaEngine.index(index);
    assertThat(indexResult.getResultType(), equalTo(Engine.Result.Type.SUCCESS));
    replicaEngine.refresh("test");
    try (Engine.Searcher searcher = replicaEngine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
        assertEquals(1, topDocs.totalHits.value);
    }
}

Example 60 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testForceMergeWithSoftDeletesRetention.

@Test
public void testForceMergeWithSoftDeletesRetention() throws Exception {
    final long retainedExtraOps = randomLongBetween(0, 10);
    Settings.Builder settings = Settings.builder().put(defaultSettings.getSettings()).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), retainedExtraOps);
    final IndexMetadata indexMetadata = IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build();
    final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetadata);
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    final MapperService mapperService = createMapperService("test");
    final Set<String> liveDocs = new HashSet<>();
    try (Store store = createStore();
        InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get))) {
        int numDocs = scaledRandomIntBetween(10, 100);
        for (int i = 0; i < numDocs; i++) {
            ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), B_1, null);
            engine.index(indexForDoc(doc));
            liveDocs.add(doc.id());
        }
        for (int i = 0; i < numDocs; i++) {
            ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), B_1, null);
            if (randomBoolean()) {
                engine.delete(new Engine.Delete(doc.id(), newUid(doc.id()), primaryTerm.get()));
                liveDocs.remove(doc.id());
            }
            if (randomBoolean()) {
                engine.index(indexForDoc(doc));
                liveDocs.add(doc.id());
            }
            if (randomBoolean()) {
                engine.flush(randomBoolean(), true);
            }
        }
        engine.flush();
        long localCheckpoint = engine.getProcessedLocalCheckpoint();
        globalCheckpoint.set(randomLongBetween(0, localCheckpoint));
        engine.syncTranslog();
        final long safeCommitCheckpoint;
        try (Engine.IndexCommitRef safeCommit = engine.acquireSafeIndexCommit()) {
            safeCommitCheckpoint = Long.parseLong(safeCommit.getIndexCommit().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
        }
        engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
        assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, mapperService);
        Map<Long, Translog.Operation> ops = readAllOperationsInLucene(engine, mapperService).stream().collect(Collectors.toMap(Translog.Operation::seqNo, Function.identity()));
        for (long seqno = 0; seqno <= localCheckpoint; seqno++) {
            long minSeqNoToRetain = Math.min(globalCheckpoint.get() + 1 - retainedExtraOps, safeCommitCheckpoint + 1);
            String msg = "seq# [" + seqno + "], global checkpoint [" + globalCheckpoint + "], retained-ops [" + retainedExtraOps + "]";
            if (seqno < minSeqNoToRetain) {
                Translog.Operation op = ops.get(seqno);
                if (op != null) {
                    assertThat(op, instanceOf(Translog.Index.class));
                    assertThat(msg, ((Translog.Index) op).id(), isIn(liveDocs));
                    assertEquals(msg, ((Translog.Index) op).source(), B_1);
                }
            } else {
                assertThat(msg, ops.get(seqno), notNullValue());
            }
        }
        settings.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), 0);
        indexSettings.updateIndexMetadata(IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build());
        engine.onSettingsChanged(indexSettings.getTranslogRetentionAge(), indexSettings.getTranslogRetentionSize(), indexSettings.getSoftDeleteRetentionOperations());
        globalCheckpoint.set(localCheckpoint);
        engine.syncTranslog();
        engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
        assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, mapperService);
        assertThat(readAllOperationsInLucene(engine, mapperService), hasSize(liveDocs.size()));
    }
}