Examples with ParsedDocument org.elasticsearch.index.mapper.ParsedDocument used on opensource projects

Example 46 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class IndexShard method prepareIndex.

public static Engine.Index prepareIndex(DocumentMapper docMapper, SourceToParse source, long seqNo, long primaryTerm, long version, VersionType versionType, Engine.Operation.Origin origin, long autoGeneratedIdTimestamp, boolean isRetry, long ifSeqNo, long ifPrimaryTerm) {
    long startTime = System.nanoTime();
    ParsedDocument doc = docMapper.parse(source);
    Term uid = new Term(IdFieldMapper.NAME, Uid.encodeId(doc.id()));
    return new Engine.Index(uid, doc, seqNo, primaryTerm, version, versionType, origin, startTime, autoGeneratedIdTimestamp, isRetry, ifSeqNo, ifPrimaryTerm);
}

Example 47 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class EngineTestCase method generateHistoryOnReplica.

public List<Engine.Operation> generateHistoryOnReplica(int numOps, boolean allowGapInSeqNo, boolean allowDuplicate) throws Exception {
    long seqNo = 0;
    final int maxIdValue = randomInt(numOps * 2);
    final List<Engine.Operation> operations = new ArrayList<>(numOps);
    for (int i = 0; i < numOps; i++) {
        final String id = Integer.toString(randomInt(maxIdValue));
        final Engine.Operation.TYPE opType = randomFrom(Engine.Operation.TYPE.values());
        final long startTime = threadPool.relativeTimeInMillis();
        final int copies = allowDuplicate && rarely() ? between(2, 4) : 1;
        for (int copy = 0; copy < copies; copy++) {
            final ParsedDocument doc = createParsedDoc(id, null);
            switch(opType) {
                case INDEX:
                    operations.add(new Engine.Index(EngineTestCase.newUid(doc), doc, seqNo, primaryTerm.get(), i, null, randomFrom(REPLICA, PEER_RECOVERY), startTime, -1, true, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
                    break;
                case DELETE:
                    operations.add(new Engine.Delete(doc.id(), EngineTestCase.newUid(doc), seqNo, primaryTerm.get(), i, null, randomFrom(REPLICA, PEER_RECOVERY), startTime, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
                    break;
                case NO_OP:
                    operations.add(new Engine.NoOp(seqNo, primaryTerm.get(), randomFrom(REPLICA, PEER_RECOVERY), startTime, "test-" + i));
                    break;
                default:
                    throw new IllegalStateException("Unknown operation type [" + opType + "]");
            }
        }
        seqNo++;
        if (allowGapInSeqNo && rarely()) {
            seqNo++;
        }
    }
    Randomness.shuffle(operations);
    return operations;
}

Example 48 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class EngineTestCase method testParsedDocument.

protected static ParsedDocument testParsedDocument(String id, String routing, ParseContext.Document document, BytesReference source, Mapping mappingUpdate, boolean recoverySource) {
    Field uidField = new Field("_id", Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE);
    Field versionField = new NumericDocValuesField("_version", 0);
    SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
    document.add(uidField);
    document.add(versionField);
    document.add(seqID.seqNo);
    document.add(seqID.seqNoDocValue);
    document.add(seqID.primaryTerm);
    BytesRef ref = source.toBytesRef();
    if (recoverySource) {
        document.add(new StoredField(SourceFieldMapper.RECOVERY_SOURCE_NAME, ref.bytes, ref.offset, ref.length));
        document.add(new NumericDocValuesField(SourceFieldMapper.RECOVERY_SOURCE_NAME, 1));
    } else {
        document.add(new StoredField(SourceFieldMapper.NAME, ref.bytes, ref.offset, ref.length));
    }
    return new ParsedDocument(versionField, seqID, id, routing, Arrays.asList(document), source, mappingUpdate);
}

Example 49 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngine method deleteInLucene.

private DeleteResult deleteInLucene(Delete delete, DeletionStrategy plan) throws IOException {
    assert assertMaxSeqNoOfUpdatesIsAdvanced(delete.uid(), delete.seqNo(), false, false);
    try {
        if (softDeleteEnabled) {
            final ParsedDocument tombstone = engineConfig.getTombstoneDocSupplier().newDeleteTombstoneDoc(delete.id());
            assert tombstone.docs().size() == 1 : "Tombstone doc should have single doc [" + tombstone + "]";
            tombstone.updateSeqID(delete.seqNo(), delete.primaryTerm());
            tombstone.version().setLongValue(plan.versionOfDeletion);
            final ParseContext.Document doc = tombstone.docs().get(0);
            assert doc.getField(SeqNoFieldMapper.TOMBSTONE_NAME) != null : "Delete tombstone document but _tombstone field is not set [" + doc + " ]";
            doc.add(softDeletesField);
            if (plan.addStaleOpToLucene || plan.currentlyDeleted) {
                indexWriter.addDocument(doc);
            } else {
                indexWriter.softUpdateDocument(delete.uid(), doc, softDeletesField);
            }
        } else if (plan.currentlyDeleted == false) {
            // any exception that comes from this is a either an ACE or a fatal exception there
            // can't be any document failures  coming from this
            indexWriter.deleteDocuments(delete.uid());
        }
        if (plan.deleteFromLucene) {
            numDocDeletes.inc();
            versionMap.putDeleteUnderLock(delete.uid().bytes(), new DeleteVersionValue(plan.versionOfDeletion, delete.seqNo(), delete.primaryTerm(), engineConfig.getThreadPool().relativeTimeInMillis()));
        }
        return new DeleteResult(plan.versionOfDeletion, delete.primaryTerm(), delete.seqNo(), plan.currentlyDeleted == false);
    } catch (final Exception ex) {
        /*
             * Document level failures when deleting are unexpected, we likely
             * hit something fatal such as the Lucene index being corrupt, or
             * the Lucene document limit. We have already issued a sequence number
             * here so this is fatal, fail the engine.
             */
        if (ex instanceof AlreadyClosedException == false && indexWriter.getTragicException() == null) {
            final String reason = String.format(Locale.ROOT, "delete id[%s] origin [%s] seq#[%d] failed at the document level", delete.id(), delete.origin(), delete.seqNo());
            failEngine(reason, ex);
        }
        throw ex;
    }
}

Example 50 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngine method innerNoOp.

private NoOpResult innerNoOp(final NoOp noOp) throws IOException {
    assert readLock.isHeldByCurrentThread() || writeLock.isHeldByCurrentThread();
    assert noOp.seqNo() > SequenceNumbers.NO_OPS_PERFORMED;
    final long seqNo = noOp.seqNo();
    try (Releasable ignored = noOpKeyedLock.acquire(seqNo)) {
        NoOpResult noOpResult;
        final Optional<Exception> preFlightError = preFlightCheckForNoOp(noOp);
        if (preFlightError.isPresent()) {
            noOpResult = new NoOpResult(SequenceNumbers.UNASSIGNED_PRIMARY_TERM, SequenceNumbers.UNASSIGNED_SEQ_NO, preFlightError.get());
        } else {
            markSeqNoAsSeen(noOp.seqNo());
            if (softDeleteEnabled && hasBeenProcessedBefore(noOp) == false) {
                try {
                    final ParsedDocument tombstone = engineConfig.getTombstoneDocSupplier().newNoopTombstoneDoc(noOp.reason());
                    tombstone.updateSeqID(noOp.seqNo(), noOp.primaryTerm());
                    // A noop tombstone does not require a _version but it's added to have a fully dense docvalues for the version field.
                    // 1L is selected to optimize the compression because it might probably be the most common value in version field.
                    tombstone.version().setLongValue(1L);
                    assert tombstone.docs().size() == 1 : "Tombstone should have a single doc [" + tombstone + "]";
                    final ParseContext.Document doc = tombstone.docs().get(0);
                    assert doc.getField(SeqNoFieldMapper.TOMBSTONE_NAME) != null : "Noop tombstone document but _tombstone field is not set [" + doc + " ]";
                    doc.add(softDeletesField);
                    indexWriter.addDocument(doc);
                } catch (final Exception ex) {
                    /*
                         * Document level failures when adding a no-op are unexpected, we likely hit something fatal such as the Lucene
                         * index being corrupt, or the Lucene document limit. We have already issued a sequence number here so this is
                         * fatal, fail the engine.
                         */
                    if (ex instanceof AlreadyClosedException == false && indexWriter.getTragicException() == null) {
                        failEngine("no-op origin[" + noOp.origin() + "] seq#[" + noOp.seqNo() + "] failed at document level", ex);
                    }
                    throw ex;
                }
            }
            noOpResult = new NoOpResult(noOp.primaryTerm(), noOp.seqNo());
            if (noOp.origin().isFromTranslog() == false && noOpResult.getResultType() == Result.Type.SUCCESS) {
                final Translog.Location location = translog.add(new Translog.NoOp(noOp.seqNo(), noOp.primaryTerm(), noOp.reason()));
                noOpResult.setTranslogLocation(location);
            }
        }
        localCheckpointTracker.markSeqNoAsProcessed(noOpResult.getSeqNo());
        if (noOpResult.getTranslogLocation() == null) {
            // the op is coming from the translog (and is hence persisted already) or it does not have a sequence number
            assert noOp.origin().isFromTranslog() || noOpResult.getSeqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO;
            localCheckpointTracker.markSeqNoAsPersisted(noOpResult.getSeqNo());
        }
        noOpResult.setTook(System.nanoTime() - noOp.startTime());
        noOpResult.freeze();
        return noOpResult;
    }
}