Examples with Document org.opensearch.index.mapper.ParseContext.Document used on opensource projects

Example 6 with Document

use of org.opensearch.index.mapper.ParseContext.Document in project OpenSearch by opensearch-project.

the class InternalEngineTests method testEnableGcDeletes.

public void testEnableGcDeletes() throws Exception {
    try (Store store = createStore();
        Engine engine = createEngine(config(defaultSettings, store, createTempDir(), newMergePolicy(), null))) {
        engine.config().setEnableGcDeletes(false);
        final BiFunction<String, Engine.SearcherScope, Engine.Searcher> searcherFactory = engine::acquireSearcher;
        // Add document
        Document document = testDocument();
        document.add(new TextField("value", "test1", Field.Store.YES));
        ParsedDocument doc = testParsedDocument("1", null, document, B_2, null);
        engine.index(new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, 0, 1, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0));
        // Delete document we just added:
        engine.delete(new Engine.Delete("test", "1", newUid(doc), UNASSIGNED_SEQ_NO, 0, 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0));
        // Get should not find the document
        Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory);
        assertThat(getResult.exists(), equalTo(false));
        // Give the gc pruning logic a chance to kick in
        Thread.sleep(1000);
        if (randomBoolean()) {
            engine.refresh("test");
        }
        // Delete non-existent document
        engine.delete(new Engine.Delete("test", "2", newUid("2"), UNASSIGNED_SEQ_NO, 0, 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0));
        // Get should not find the document (we never indexed uid=2):
        getResult = engine.get(new Engine.Get(true, false, "2", newUid("2")), searcherFactory);
        assertThat(getResult.exists(), equalTo(false));
        // Try to index uid=1 with a too-old version, should fail:
        Engine.Index index = new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);
        Engine.IndexResult indexResult = engine.index(index);
        assertThat(indexResult.getResultType(), equalTo(Engine.Result.Type.FAILURE));
        assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
        // Get should still not find the document
        getResult = engine.get(newGet(true, doc), searcherFactory);
        assertThat(getResult.exists(), equalTo(false));
        // Try to index uid=2 with a too-old version, should fail:
        Engine.Index index1 = new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);
        indexResult = engine.index(index1);
        assertThat(indexResult.getResultType(), equalTo(Engine.Result.Type.FAILURE));
        assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
        // Get should not find the document
        getResult = engine.get(newGet(true, doc), searcherFactory);
        assertThat(getResult.exists(), equalTo(false));
    }
}

Example 7 with Document

use of org.opensearch.index.mapper.ParseContext.Document in project OpenSearch by opensearch-project.

the class InternalEngineTests method testConcurrentOutOfOrderDocsOnReplica.

public void testConcurrentOutOfOrderDocsOnReplica() throws IOException, InterruptedException {
    final List<Engine.Operation> opsDoc1 = generateSingleDocHistory(true, randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL), 2, 100, 300, "1");
    final Engine.Operation lastOpDoc1 = opsDoc1.get(opsDoc1.size() - 1);
    final String lastFieldValueDoc1;
    if (lastOpDoc1 instanceof Engine.Index) {
        Engine.Index index = (Engine.Index) lastOpDoc1;
        lastFieldValueDoc1 = index.docs().get(0).get("value");
    } else {
        // delete
        lastFieldValueDoc1 = null;
    }
    final List<Engine.Operation> opsDoc2 = generateSingleDocHistory(true, randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL), 2, 100, 300, "2");
    final Engine.Operation lastOpDoc2 = opsDoc2.get(opsDoc2.size() - 1);
    final String lastFieldValueDoc2;
    if (lastOpDoc2 instanceof Engine.Index) {
        Engine.Index index = (Engine.Index) lastOpDoc2;
        lastFieldValueDoc2 = index.docs().get(0).get("value");
    } else {
        // delete
        lastFieldValueDoc2 = null;
    }
    // randomly interleave
    final AtomicLong seqNoGenerator = new AtomicLong();
    BiFunction<Engine.Operation, Long, Engine.Operation> seqNoUpdater = (operation, newSeqNo) -> {
        if (operation instanceof Engine.Index) {
            Engine.Index index = (Engine.Index) operation;
            Document doc = testDocumentWithTextField(index.docs().get(0).get("value"));
            ParsedDocument parsedDocument = testParsedDocument(index.id(), index.routing(), doc, index.source(), null);
            return new Engine.Index(index.uid(), parsedDocument, newSeqNo, index.primaryTerm(), index.version(), index.versionType(), index.origin(), index.startTime(), index.getAutoGeneratedIdTimestamp(), index.isRetry(), UNASSIGNED_SEQ_NO, 0);
        } else {
            Engine.Delete delete = (Engine.Delete) operation;
            return new Engine.Delete(delete.type(), delete.id(), delete.uid(), newSeqNo, delete.primaryTerm(), delete.version(), delete.versionType(), delete.origin(), delete.startTime(), UNASSIGNED_SEQ_NO, 0);
        }
    };
    final List<Engine.Operation> allOps = new ArrayList<>();
    Iterator<Engine.Operation> iter1 = opsDoc1.iterator();
    Iterator<Engine.Operation> iter2 = opsDoc2.iterator();
    while (iter1.hasNext() && iter2.hasNext()) {
        final Engine.Operation next = randomBoolean() ? iter1.next() : iter2.next();
        allOps.add(seqNoUpdater.apply(next, seqNoGenerator.getAndIncrement()));
    }
    iter1.forEachRemaining(o -> allOps.add(seqNoUpdater.apply(o, seqNoGenerator.getAndIncrement())));
    iter2.forEachRemaining(o -> allOps.add(seqNoUpdater.apply(o, seqNoGenerator.getAndIncrement())));
    // insert some duplicates
    randomSubsetOf(allOps).forEach(op -> allOps.add(seqNoUpdater.apply(op, op.seqNo())));
    shuffle(allOps, random());
    concurrentlyApplyOps(allOps, engine);
    engine.refresh("test");
    if (lastFieldValueDoc1 != null) {
        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
            final TotalHitCountCollector collector = new TotalHitCountCollector();
            searcher.search(new TermQuery(new Term("value", lastFieldValueDoc1)), collector);
            assertThat(collector.getTotalHits(), equalTo(1));
        }
    }
    if (lastFieldValueDoc2 != null) {
        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
            final TotalHitCountCollector collector = new TotalHitCountCollector();
            searcher.search(new TermQuery(new Term("value", lastFieldValueDoc2)), collector);
            assertThat(collector.getTotalHits(), equalTo(1));
        }
    }
    int totalExpectedOps = 0;
    if (lastFieldValueDoc1 != null) {
        totalExpectedOps++;
    }
    if (lastFieldValueDoc2 != null) {
        totalExpectedOps++;
    }
    assertVisibleCount(engine, totalExpectedOps);
}

Example 8 with Document

use of org.opensearch.index.mapper.ParseContext.Document in project OpenSearch by opensearch-project.

the class DateFieldTypeTests method testDateNanoDocValues.

public void testDateNanoDocValues() throws IOException {
    // Create an index with some docValues
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
    Document doc = new Document();
    NumericDocValuesField docValuesField = new NumericDocValuesField("my_date", 1444608000000L);
    doc.add(docValuesField);
    w.addDocument(doc);
    docValuesField.setLongValue(1459641600000L);
    w.addDocument(doc);
    // Create the doc values reader
    SortedNumericIndexFieldData fieldData = new SortedNumericIndexFieldData("my_date", IndexNumericFieldData.NumericType.DATE_NANOSECONDS);
    // Read index and check the doc values
    DirectoryReader reader = DirectoryReader.open(w);
    assertTrue(reader.leaves().size() > 0);
    LeafNumericFieldData a = fieldData.load(reader.leaves().get(0).reader().getContext());
    SortedNumericDocValues docValues = a.getLongValues();
    assertEquals(0, docValues.nextDoc());
    assertEquals(1, docValues.nextDoc());
    assertEquals(DocIdSetIterator.NO_MORE_DOCS, docValues.nextDoc());
    reader.close();
    w.close();
    dir.close();
}

Example 9 with Document

use of org.opensearch.index.mapper.ParseContext.Document in project OpenSearch by opensearch-project.

the class DocumentParserTests method testSimpleParser.

public void testSimpleParser() throws Exception {
    String mapping = copyToStringFromClasspath("/org/opensearch/index/mapper/simple/test-mapping.json");
    DocumentMapper docMapper = createDocumentMapper(MapperService.SINGLE_MAPPING_NAME, mapping);
    assertThat((String) docMapper.meta().get("param1"), equalTo("value1"));
    BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/opensearch/index/mapper/simple/test1.json"));
    Document doc = docMapper.parse(new SourceToParse("test", MapperService.SINGLE_MAPPING_NAME, "1", json, XContentType.JSON)).rootDoc();
    assertThat(doc.getBinaryValue(docMapper.idFieldMapper().name()), equalTo(Uid.encodeId("1")));
    assertThat(doc.get(docMapper.mappers().getMapper("name.first").name()), equalTo("fred"));
}

Example 10 with Document

use of org.opensearch.index.mapper.ParseContext.Document in project OpenSearch by opensearch-project.

the class DocumentParserTests method testNoDocumentSent.

public void testNoDocumentSent() throws Exception {
    DocumentMapper docMapper = createDocumentMapper(mapping(b -> {
    }));
    BytesReference json = new BytesArray("".getBytes(StandardCharsets.UTF_8));
    MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(new SourceToParse("test", MapperService.SINGLE_MAPPING_NAME, "1", json, XContentType.JSON)));
    assertThat(e.getMessage(), equalTo("failed to parse, document is empty"));
}