Search in sources :

Example 1 with TriFunction

use of org.elasticsearch.common.TriFunction in project elasticsearch by elastic.

the class SimilarityService method addSimilarities.

private Map<String, SimilarityProvider> addSimilarities(Map<String, Settings> similaritySettings, Settings indexSettings, Map<String, TriFunction<String, Settings, Settings, SimilarityProvider>> similarities) {
    Map<String, SimilarityProvider> providers = new HashMap<>(similarities.size());
    for (Map.Entry<String, TriFunction<String, Settings, Settings, SimilarityProvider>> entry : similarities.entrySet()) {
        String name = entry.getKey();
        TriFunction<String, Settings, Settings, SimilarityProvider> factory = entry.getValue();
        Settings providerSettings = similaritySettings.get(name);
        if (providerSettings == null) {
            providerSettings = Settings.Builder.EMPTY_SETTINGS;
        }
        providers.put(name, factory.apply(name, providerSettings, indexSettings));
    }
    return providers;
}
Also used : HashMap(java.util.HashMap) TriFunction(org.elasticsearch.common.TriFunction) HashMap(java.util.HashMap) Map(java.util.Map) Settings(org.elasticsearch.common.settings.Settings) IndexSettings(org.elasticsearch.index.IndexSettings)

Example 2 with TriFunction

use of org.elasticsearch.common.TriFunction in project crate by crate.

the class InternalEngineTests method assertOpsOnPrimary.

private int assertOpsOnPrimary(List<Engine.Operation> ops, long currentOpVersion, boolean docDeleted, InternalEngine engine) throws IOException {
    String lastFieldValue = null;
    int opsPerformed = 0;
    long lastOpVersion = currentOpVersion;
    long lastOpSeqNo = UNASSIGNED_SEQ_NO;
    long lastOpTerm = UNASSIGNED_PRIMARY_TERM;
    PrimaryTermSupplier currentTerm = (PrimaryTermSupplier) engine.engineConfig.getPrimaryTermSupplier();
    BiFunction<Long, Engine.Index, Engine.Index> indexWithVersion = (version, index) -> new Engine.Index(index.uid(), index.parsedDoc(), UNASSIGNED_SEQ_NO, currentTerm.get(), version, index.versionType(), index.origin(), index.startTime(), index.getAutoGeneratedIdTimestamp(), index.isRetry(), UNASSIGNED_SEQ_NO, 0);
    BiFunction<Long, Engine.Delete, Engine.Delete> delWithVersion = (version, delete) -> new Engine.Delete(delete.id(), delete.uid(), UNASSIGNED_SEQ_NO, currentTerm.get(), version, delete.versionType(), delete.origin(), delete.startTime(), UNASSIGNED_SEQ_NO, 0);
    TriFunction<Long, Long, Engine.Index, Engine.Index> indexWithSeq = (seqNo, term, index) -> new Engine.Index(index.uid(), index.parsedDoc(), UNASSIGNED_SEQ_NO, currentTerm.get(), index.version(), index.versionType(), index.origin(), index.startTime(), index.getAutoGeneratedIdTimestamp(), index.isRetry(), seqNo, term);
    TriFunction<Long, Long, Engine.Delete, Engine.Delete> delWithSeq = (seqNo, term, delete) -> new Engine.Delete(delete.id(), delete.uid(), UNASSIGNED_SEQ_NO, currentTerm.get(), delete.version(), delete.versionType(), delete.origin(), delete.startTime(), seqNo, term);
    Function<Engine.Index, Engine.Index> indexWithCurrentTerm = index -> new Engine.Index(index.uid(), index.parsedDoc(), UNASSIGNED_SEQ_NO, currentTerm.get(), index.version(), index.versionType(), index.origin(), index.startTime(), index.getAutoGeneratedIdTimestamp(), index.isRetry(), index.getIfSeqNo(), index.getIfPrimaryTerm());
    Function<Engine.Delete, Engine.Delete> deleteWithCurrentTerm = delete -> new Engine.Delete(delete.id(), delete.uid(), UNASSIGNED_SEQ_NO, currentTerm.get(), delete.version(), delete.versionType(), delete.origin(), delete.startTime(), delete.getIfSeqNo(), delete.getIfPrimaryTerm());
    for (Engine.Operation op : ops) {
        final boolean versionConflict = rarely();
        final boolean versionedOp = versionConflict || randomBoolean();
        final long conflictingVersion = docDeleted || randomBoolean() ? lastOpVersion + (randomBoolean() ? 1 : -1) : Versions.MATCH_DELETED;
        final long conflictingSeqNo = lastOpSeqNo == UNASSIGNED_SEQ_NO || randomBoolean() ? // use 5 to go above 0 for magic numbers
        lastOpSeqNo + 5 : lastOpSeqNo;
        final long conflictingTerm = conflictingSeqNo == lastOpSeqNo || randomBoolean() ? lastOpTerm + 1 : lastOpTerm;
        if (rarely()) {
            currentTerm.set(currentTerm.get() + 1L);
            engine.rollTranslogGeneration();
        }
        final long correctVersion = docDeleted ? Versions.MATCH_DELETED : lastOpVersion;
        logger.info("performing [{}]{}{}", op.operationType().name().charAt(0), versionConflict ? " (conflict " + conflictingVersion + ")" : "", versionedOp ? " (versioned " + correctVersion + ", seqNo " + lastOpSeqNo + ", term " + lastOpTerm + " )" : "");
        if (op instanceof Engine.Index) {
            final Engine.Index index = (Engine.Index) op;
            if (versionConflict) {
                // generate a conflict
                final Engine.IndexResult result;
                if (randomBoolean()) {
                    result = engine.index(indexWithSeq.apply(conflictingSeqNo, conflictingTerm, index));
                } else {
                    result = engine.index(indexWithVersion.apply(conflictingVersion, index));
                }
                assertThat(result.isCreated(), equalTo(false));
                assertThat(result.getVersion(), equalTo(lastOpVersion));
                assertThat(result.getResultType(), equalTo(Engine.Result.Type.FAILURE));
                assertThat(result.getFailure(), instanceOf(VersionConflictEngineException.class));
            } else {
                final Engine.IndexResult result;
                if (versionedOp) {
                    // TODO: add support for non-existing docs
                    if (randomBoolean() && lastOpSeqNo != SequenceNumbers.UNASSIGNED_SEQ_NO && docDeleted == false) {
                        result = engine.index(indexWithSeq.apply(lastOpSeqNo, lastOpTerm, index));
                    } else {
                        result = engine.index(indexWithVersion.apply(correctVersion, index));
                    }
                } else {
                    result = engine.index(indexWithCurrentTerm.apply(index));
                }
                assertThat(result.isCreated(), equalTo(docDeleted));
                assertThat(result.getVersion(), equalTo(Math.max(lastOpVersion + 1, 1)));
                assertThat(result.getResultType(), equalTo(Engine.Result.Type.SUCCESS));
                assertThat(result.getFailure(), nullValue());
                lastFieldValue = index.docs().get(0).get("value");
                docDeleted = false;
                lastOpVersion = result.getVersion();
                lastOpSeqNo = result.getSeqNo();
                lastOpTerm = result.getTerm();
                opsPerformed++;
            }
        } else {
            final Engine.Delete delete = (Engine.Delete) op;
            if (versionConflict) {
                // generate a conflict
                Engine.DeleteResult result;
                if (randomBoolean()) {
                    result = engine.delete(delWithSeq.apply(conflictingSeqNo, conflictingTerm, delete));
                } else {
                    result = engine.delete(delWithVersion.apply(conflictingVersion, delete));
                }
                assertThat(result.isFound(), equalTo(docDeleted == false));
                assertThat(result.getVersion(), equalTo(lastOpVersion));
                assertThat(result.getResultType(), equalTo(Engine.Result.Type.FAILURE));
                assertThat(result.getFailure(), instanceOf(VersionConflictEngineException.class));
            } else {
                final Engine.DeleteResult result;
                long correctSeqNo = docDeleted ? UNASSIGNED_SEQ_NO : lastOpSeqNo;
                if (versionedOp && lastOpSeqNo != UNASSIGNED_SEQ_NO && randomBoolean()) {
                    result = engine.delete(delWithSeq.apply(correctSeqNo, lastOpTerm, delete));
                } else if (versionedOp) {
                    result = engine.delete(delWithVersion.apply(correctVersion, delete));
                } else {
                    result = engine.delete(deleteWithCurrentTerm.apply(delete));
                }
                assertThat(result.isFound(), equalTo(docDeleted == false));
                assertThat(result.getVersion(), equalTo(Math.max(lastOpVersion + 1, 1)));
                assertThat(result.getResultType(), equalTo(Engine.Result.Type.SUCCESS));
                assertThat(result.getFailure(), nullValue());
                docDeleted = true;
                lastOpVersion = result.getVersion();
                lastOpSeqNo = result.getSeqNo();
                lastOpTerm = result.getTerm();
                opsPerformed++;
            }
        }
        if (randomBoolean()) {
            // refresh and take the chance to check everything is ok so far
            assertVisibleCount(engine, docDeleted ? 0 : 1);
            // first op and it failed.
            if (docDeleted == false && lastFieldValue != null) {
                try (Searcher searcher = engine.acquireSearcher("test")) {
                    final TotalHitCountCollector collector = new TotalHitCountCollector();
                    searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector);
                    assertThat(collector.getTotalHits(), equalTo(1));
                }
            }
        }
        if (randomBoolean()) {
            engine.flush();
            engine.refresh("test");
        }
        if (rarely()) {
            // simulate GC deletes
            engine.refresh("gc_simulation", Engine.SearcherScope.INTERNAL, true);
            engine.clearDeletedTombstones();
            if (docDeleted) {
                lastOpVersion = Versions.NOT_FOUND;
                lastOpSeqNo = UNASSIGNED_SEQ_NO;
                lastOpTerm = UNASSIGNED_PRIMARY_TERM;
            }
        }
    }
    assertVisibleCount(engine, docDeleted ? 0 : 1);
    if (docDeleted == false) {
        try (Searcher searcher = engine.acquireSearcher("test")) {
            final TotalHitCountCollector collector = new TotalHitCountCollector();
            searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector);
            assertThat(collector.getTotalHits(), equalTo(1));
        }
    }
    return opsPerformed;
}
Also used : ShardId(org.elasticsearch.index.shard.ShardId) NoMergePolicy(org.apache.lucene.index.NoMergePolicy) Arrays(java.util.Arrays) Versions(org.elasticsearch.common.lucene.uid.Versions) LongSupplier(java.util.function.LongSupplier) PEER_RECOVERY(org.elasticsearch.index.engine.Engine.Operation.Origin.PEER_RECOVERY) BigArrays(org.elasticsearch.common.util.BigArrays) IndexSettingsModule(org.elasticsearch.test.IndexSettingsModule) IndexMetadata(org.elasticsearch.cluster.metadata.IndexMetadata) Matchers.not(org.hamcrest.Matchers.not) Term(org.apache.lucene.index.Term) Level(org.apache.logging.log4j.Level) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) ElasticsearchDirectoryReader(org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader) LogEvent(org.apache.logging.log4j.core.LogEvent) ReferenceManager(org.apache.lucene.search.ReferenceManager) ParseContext(org.elasticsearch.index.mapper.ParseContext) CoreMatchers.instanceOf(org.hamcrest.CoreMatchers.instanceOf) SeqNoStats(org.elasticsearch.index.seqno.SeqNoStats) LOCAL_TRANSLOG_RECOVERY(org.elasticsearch.index.engine.Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY) RandomNumbers(com.carrotsearch.randomizedtesting.generators.RandomNumbers) MergePolicy(org.apache.lucene.index.MergePolicy) TermsEnum(org.apache.lucene.index.TermsEnum) Matchers.nullValue(org.hamcrest.Matchers.nullValue) Map(java.util.Map) TestTranslog(org.elasticsearch.index.translog.TestTranslog) CheckedRunnable(org.elasticsearch.common.CheckedRunnable) FieldsVisitor(org.elasticsearch.index.fieldvisitor.FieldsVisitor) Path(java.nio.file.Path) REPLICA(org.elasticsearch.index.engine.Engine.Operation.Origin.REPLICA) NumericDocValuesField(org.apache.lucene.document.NumericDocValuesField) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) SoftDeletesRetentionMergePolicy(org.apache.lucene.index.SoftDeletesRetentionMergePolicy) UUIDs(org.elasticsearch.common.UUIDs) Set(java.util.Set) MatchAllDocsQuery(org.apache.lucene.search.MatchAllDocsQuery) SnapshotMatchers(org.elasticsearch.index.translog.SnapshotMatchers) UncheckedIOException(java.io.UncheckedIOException) PointValues(org.apache.lucene.index.PointValues) CountDownLatch(java.util.concurrent.CountDownLatch) SeqNoFieldMapper(org.elasticsearch.index.mapper.SeqNoFieldMapper) AbstractRunnable(org.elasticsearch.common.util.concurrent.AbstractRunnable) Logger(org.apache.logging.log4j.Logger) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) ReplicationTracker(org.elasticsearch.index.seqno.ReplicationTracker) Matchers.containsString(org.hamcrest.Matchers.containsString) TestShardRouting(org.elasticsearch.cluster.routing.TestShardRouting) IndexCommit(org.apache.lucene.index.IndexCommit) Tuple(io.crate.common.collections.Tuple) LiveIndexWriterConfig(org.apache.lucene.index.LiveIndexWriterConfig) LogDocMergePolicy(org.apache.lucene.index.LogDocMergePolicy) FixedBitSet(org.apache.lucene.util.FixedBitSet) RegexFilter(org.apache.logging.log4j.core.filter.RegexFilter) CodecService(org.elasticsearch.index.codec.CodecService) Mockito.spy(org.mockito.Mockito.spy) ShardRoutingState(org.elasticsearch.cluster.routing.ShardRoutingState) Supplier(java.util.function.Supplier) CheckedBiConsumer(org.elasticsearch.common.CheckedBiConsumer) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) ToLongBiFunction(java.util.function.ToLongBiFunction) BytesArray(org.elasticsearch.common.bytes.BytesArray) RetentionLease(org.elasticsearch.index.seqno.RetentionLease) RetentionLeases(org.elasticsearch.index.seqno.RetentionLeases) Lock(org.apache.lucene.store.Lock) Store(org.elasticsearch.index.store.Store) Matchers.hasSize(org.hamcrest.Matchers.hasSize) Matchers.isIn(org.hamcrest.Matchers.isIn) Bits(org.apache.lucene.util.Bits) TranslogConfig(org.elasticsearch.index.translog.TranslogConfig) TieredMergePolicy(org.apache.lucene.index.TieredMergePolicy) Loggers(org.elasticsearch.common.logging.Loggers) TopDocs(org.apache.lucene.search.TopDocs) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) LongStream(java.util.stream.LongStream) SequenceNumbers(org.elasticsearch.index.seqno.SequenceNumbers) Files(java.nio.file.Files) IdFieldMapper(org.elasticsearch.index.mapper.IdFieldMapper) AbstractAppender(org.apache.logging.log4j.core.appender.AbstractAppender) DocIdAndSeqNo(org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo) IOUtils(io.crate.common.io.IOUtils) SequentialStoredFieldsLeafReader(org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader) IOException(java.io.IOException) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) Test(org.junit.Test) ParsedDocument(org.elasticsearch.index.mapper.ParsedDocument) AtomicLong(java.util.concurrent.atomic.AtomicLong) SourceFieldMapper(org.elasticsearch.index.mapper.SourceFieldMapper) VersionFieldMapper(org.elasticsearch.index.mapper.VersionFieldMapper) Matchers.hasItem(org.hamcrest.Matchers.hasItem) Matchers.sameInstance(org.hamcrest.Matchers.sameInstance) Phaser(java.util.concurrent.Phaser) Matcher(org.hamcrest.Matcher) TextField(org.apache.lucene.document.TextField) TranslogDeletionPolicies.createTranslogDeletionPolicy(org.elasticsearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy) Lucene87StoredFieldsFormat(org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat) ActionListener(org.elasticsearch.action.ActionListener) Randomness(org.elasticsearch.common.Randomness) Collections.shuffle(java.util.Collections.shuffle) CoreMatchers.is(org.hamcrest.CoreMatchers.is) ElasticsearchException(org.elasticsearch.ElasticsearchException) BiFunction(java.util.function.BiFunction) IndexableField(org.apache.lucene.index.IndexableField) ConcurrentCollections(org.elasticsearch.common.util.concurrent.ConcurrentCollections) StoredField(org.apache.lucene.document.StoredField) Matchers.hasKey(org.hamcrest.Matchers.hasKey) VersionType(org.elasticsearch.index.VersionType) Settings(org.elasticsearch.common.settings.Settings) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Directory(org.apache.lucene.store.Directory) ThreadPool(org.elasticsearch.threadpool.ThreadPool) LeafReaderContext(org.apache.lucene.index.LeafReaderContext) ShardUtils(org.elasticsearch.index.shard.ShardUtils) TotalHitCountCollector(org.apache.lucene.search.TotalHitCountCollector) ByteSizeValue(org.elasticsearch.common.unit.ByteSizeValue) CyclicBarrier(java.util.concurrent.CyclicBarrier) Terms(org.apache.lucene.index.Terms) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) BytesRef(org.apache.lucene.util.BytesRef) DirectoryReader(org.apache.lucene.index.DirectoryReader) UNASSIGNED_PRIMARY_TERM(org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM) UNASSIGNED_SEQ_NO(org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO) IndexShardRoutingTable(org.elasticsearch.cluster.routing.IndexShardRoutingTable) BytesReference(org.elasticsearch.common.bytes.BytesReference) Collectors(java.util.stream.Collectors) SegmentInfos(org.apache.lucene.index.SegmentInfos) Searcher(org.elasticsearch.index.engine.Engine.Searcher) MapperService(org.elasticsearch.index.mapper.MapperService) Base64(java.util.Base64) List(java.util.List) IndexWriter(org.apache.lucene.index.IndexWriter) Version(org.elasticsearch.Version) MatcherAssert(org.hamcrest.MatcherAssert) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) TriFunction(org.elasticsearch.common.TriFunction) Matchers.equalTo(org.hamcrest.Matchers.equalTo) LeafReader(org.apache.lucene.index.LeafReader) TimeValue(io.crate.common.unit.TimeValue) Queue(java.util.Queue) IndexWriterConfig(org.apache.lucene.index.IndexWriterConfig) LogByteSizeMergePolicy(org.apache.lucene.index.LogByteSizeMergePolicy) MockDirectoryWrapper(org.apache.lucene.store.MockDirectoryWrapper) IndexReader(org.apache.lucene.index.IndexReader) IndexSearcher(org.apache.lucene.search.IndexSearcher) ShardRouting(org.elasticsearch.cluster.routing.ShardRouting) LongPoint(org.apache.lucene.document.LongPoint) NumericDocValues(org.apache.lucene.index.NumericDocValues) PRIMARY(org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Document(org.elasticsearch.index.mapper.ParseContext.Document) HashMap(java.util.HashMap) VersionsAndSeqNoResolver(org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver) Lucene(org.elasticsearch.common.lucene.Lucene) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) TransportActions(org.elasticsearch.action.support.TransportActions) Strings(org.elasticsearch.common.Strings) HashSet(java.util.HashSet) LOCAL_RESET(org.elasticsearch.index.engine.Engine.Operation.Origin.LOCAL_RESET) Charset(java.nio.charset.Charset) NoneCircuitBreakerService(org.elasticsearch.indices.breaker.NoneCircuitBreakerService) IndexSettings(org.elasticsearch.index.IndexSettings) LocalCheckpointTracker(org.elasticsearch.index.seqno.LocalCheckpointTracker) IntSupplier(java.util.function.IntSupplier) Matchers.empty(org.hamcrest.Matchers.empty) Iterator(java.util.Iterator) Uid(org.elasticsearch.index.mapper.Uid) Matchers(org.hamcrest.Matchers) Mockito.when(org.mockito.Mockito.when) VersionUtils(org.elasticsearch.test.VersionUtils) TimeUnit(java.util.concurrent.TimeUnit) TermQuery(org.apache.lucene.search.TermQuery) NO_OPS_PERFORMED(org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED) Field(org.apache.lucene.document.Field) Closeable(java.io.Closeable) Translog(org.elasticsearch.index.translog.Translog) Comparator(java.util.Comparator) Collections(java.util.Collections) LogManager(org.apache.logging.log4j.LogManager) Matchers.containsString(org.hamcrest.Matchers.containsString) TotalHitCountCollector(org.apache.lucene.search.TotalHitCountCollector) TermQuery(org.apache.lucene.search.TermQuery) Searcher(org.elasticsearch.index.engine.Engine.Searcher) IndexSearcher(org.apache.lucene.search.IndexSearcher) Term(org.apache.lucene.index.Term) LongPoint(org.apache.lucene.document.LongPoint) AtomicLong(java.util.concurrent.atomic.AtomicLong)

Aggregations

HashMap (java.util.HashMap)2 Map (java.util.Map)2 TriFunction (org.elasticsearch.common.TriFunction)2 Settings (org.elasticsearch.common.settings.Settings)2 IndexSettings (org.elasticsearch.index.IndexSettings)2 RandomNumbers (com.carrotsearch.randomizedtesting.generators.RandomNumbers)1 Tuple (io.crate.common.collections.Tuple)1 IOUtils (io.crate.common.io.IOUtils)1 TimeValue (io.crate.common.unit.TimeValue)1 Closeable (java.io.Closeable)1 IOException (java.io.IOException)1 UncheckedIOException (java.io.UncheckedIOException)1 Charset (java.nio.charset.Charset)1 Files (java.nio.file.Files)1 Path (java.nio.file.Path)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Base64 (java.util.Base64)1 Collections (java.util.Collections)1 Collections.shuffle (java.util.Collections.shuffle)1