Search in sources :

Example 1 with CheckedRunnable

use of org.opensearch.common.CheckedRunnable in project OpenSearch by opensearch-project.

the class TestEventHandlerTests method testLogOnElapsedTime.

public void testLogOnElapsedTime() throws Exception {
    try (MockLogAppender appender = MockLogAppender.createForLoggers(LogManager.getLogger(MockNioTransport.class))) {
        long start = System.nanoTime();
        long end = start + TimeUnit.MILLISECONDS.toNanos(400);
        AtomicBoolean isStart = new AtomicBoolean(true);
        LongSupplier timeSupplier = () -> {
            if (isStart.compareAndSet(true, false)) {
                return start;
            } else if (isStart.compareAndSet(false, true)) {
                return end;
            }
            throw new IllegalStateException("Cannot update isStart");
        };
        final ThreadPool threadPool = mock(ThreadPool.class);
        doAnswer(i -> timeSupplier.getAsLong()).when(threadPool).relativeTimeInNanos();
        TestEventHandler eventHandler = new TestEventHandler(e -> {
        }, () -> null, new MockNioTransport.TransportThreadWatchdog(threadPool, Settings.EMPTY));
        ServerChannelContext serverChannelContext = mock(ServerChannelContext.class);
        SocketChannelContext socketChannelContext = mock(SocketChannelContext.class);
        RuntimeException exception = new RuntimeException("boom");
        Map<String, CheckedRunnable<Exception>> tests = new HashMap<>();
        tests.put("acceptChannel", () -> eventHandler.acceptChannel(serverChannelContext));
        tests.put("acceptException", () -> eventHandler.acceptException(serverChannelContext, exception));
        tests.put("registrationException", () -> eventHandler.registrationException(socketChannelContext, exception));
        tests.put("handleConnect", () -> eventHandler.handleConnect(socketChannelContext));
        tests.put("connectException", () -> eventHandler.connectException(socketChannelContext, exception));
        tests.put("handleRead", () -> eventHandler.handleRead(socketChannelContext));
        tests.put("readException", () -> eventHandler.readException(socketChannelContext, exception));
        tests.put("handleWrite", () -> eventHandler.handleWrite(socketChannelContext));
        tests.put("writeException", () -> eventHandler.writeException(socketChannelContext, exception));
        tests.put("handleTask", () -> eventHandler.handleTask(mock(Runnable.class)));
        tests.put("taskException", () -> eventHandler.taskException(exception));
        tests.put("handleClose", () -> eventHandler.handleClose(socketChannelContext));
        tests.put("closeException", () -> eventHandler.closeException(socketChannelContext, exception));
        tests.put("genericChannelException", () -> eventHandler.genericChannelException(socketChannelContext, exception));
        for (Map.Entry<String, CheckedRunnable<Exception>> entry : tests.entrySet()) {
            String message = "*Slow execution on network thread*";
            MockLogAppender.LoggingExpectation slowExpectation = new MockLogAppender.SeenEventExpectation(entry.getKey(), MockNioTransport.class.getCanonicalName(), Level.WARN, message);
            appender.addExpectation(slowExpectation);
            entry.getValue().run();
            appender.assertAllExpectationsMatched();
        }
    }
}
Also used : ServerChannelContext(org.opensearch.nio.ServerChannelContext) MockLogAppender(org.opensearch.test.MockLogAppender) CheckedRunnable(org.opensearch.common.CheckedRunnable) HashMap(java.util.HashMap) ThreadPool(org.opensearch.threadpool.ThreadPool) SocketChannelContext(org.opensearch.nio.SocketChannelContext) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) LongSupplier(java.util.function.LongSupplier) HashMap(java.util.HashMap) Map(java.util.Map)

Example 2 with CheckedRunnable

use of org.opensearch.common.CheckedRunnable in project OpenSearch by opensearch-project.

the class InternalEngineTests method testLookupSeqNoByIdInLucene.

public void testLookupSeqNoByIdInLucene() throws Exception {
    int numOps = between(10, 100);
    long seqNo = 0;
    List<Engine.Operation> operations = new ArrayList<>(numOps);
    for (int i = 0; i < numOps; i++) {
        String id = Integer.toString(between(1, 50));
        boolean isIndexing = randomBoolean();
        int copies = frequently() ? 1 : between(2, 4);
        for (int c = 0; c < copies; c++) {
            final ParsedDocument doc = EngineTestCase.createParsedDoc(id, null);
            if (isIndexing) {
                operations.add(new Engine.Index(EngineTestCase.newUid(doc), doc, seqNo, primaryTerm.get(), i, null, Engine.Operation.Origin.REPLICA, threadPool.relativeTimeInMillis(), -1, true, UNASSIGNED_SEQ_NO, 0L));
            } else {
                operations.add(new Engine.Delete(doc.type(), doc.id(), EngineTestCase.newUid(doc), seqNo, primaryTerm.get(), i, null, Engine.Operation.Origin.REPLICA, threadPool.relativeTimeInMillis(), UNASSIGNED_SEQ_NO, 0L));
            }
        }
        seqNo++;
        if (rarely()) {
            seqNo++;
        }
    }
    Randomness.shuffle(operations);
    // id -> latest seq_no
    Map<String, Engine.Operation> latestOps = new HashMap<>();
    try (Store store = createStore();
        InternalEngine engine = createEngine(config(defaultSettings, store, createTempDir(), newMergePolicy(), null))) {
        CheckedRunnable<IOException> lookupAndCheck = () -> {
            try (Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
                Map<String, Long> liveOps = latestOps.entrySet().stream().filter(e -> e.getValue().operationType() == Engine.Operation.TYPE.INDEX).collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().seqNo()));
                assertThat(getDocIds(engine, true).stream().collect(Collectors.toMap(e -> e.getId(), e -> e.getSeqNo())), equalTo(liveOps));
                for (String id : latestOps.keySet()) {
                    String msg = "latestOps=" + latestOps + " op=" + id;
                    DocIdAndSeqNo docIdAndSeqNo = VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.getIndexReader(), newUid(id));
                    if (liveOps.containsKey(id) == false) {
                        assertNull(msg, docIdAndSeqNo);
                    } else {
                        assertNotNull(msg, docIdAndSeqNo);
                        assertThat(msg, docIdAndSeqNo.seqNo, equalTo(latestOps.get(id).seqNo()));
                    }
                }
                String notFoundId = randomValueOtherThanMany(liveOps::containsKey, () -> Long.toString(randomNonNegativeLong()));
                assertNull(VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.getIndexReader(), newUid(notFoundId)));
            }
        };
        for (Engine.Operation op : operations) {
            if (op instanceof Engine.Index) {
                engine.index((Engine.Index) op);
                if (latestOps.containsKey(op.id()) == false || latestOps.get(op.id()).seqNo() < op.seqNo()) {
                    latestOps.put(op.id(), op);
                }
            } else if (op instanceof Engine.Delete) {
                engine.delete((Engine.Delete) op);
                if (latestOps.containsKey(op.id()) == false || latestOps.get(op.id()).seqNo() < op.seqNo()) {
                    latestOps.put(op.id(), op);
                }
            }
            if (randomInt(100) < 10) {
                engine.refresh("test");
                lookupAndCheck.run();
            }
            if (rarely()) {
                engine.flush(false, true);
                lookupAndCheck.run();
            }
        }
        engine.refresh("test");
        lookupAndCheck.run();
    }
}
Also used : Term(org.apache.lucene.index.Term) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) Version(org.opensearch.Version) PEER_RECOVERY(org.opensearch.index.engine.Engine.Operation.Origin.PEER_RECOVERY) MergePolicy(org.apache.lucene.index.MergePolicy) Map(java.util.Map) Mockito.doAnswer(org.mockito.Mockito.doAnswer) DefaultTranslogDeletionPolicy(org.opensearch.index.translog.DefaultTranslogDeletionPolicy) Path(java.nio.file.Path) TimeValue(org.opensearch.common.unit.TimeValue) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) SoftDeletesRetentionMergePolicy(org.apache.lucene.index.SoftDeletesRetentionMergePolicy) MatchAllDocsQuery(org.apache.lucene.search.MatchAllDocsQuery) PointValues(org.apache.lucene.index.PointValues) CountDownLatch(java.util.concurrent.CountDownLatch) Logger(org.apache.logging.log4j.Logger) Randomness(org.opensearch.common.Randomness) BytesArray(org.opensearch.common.bytes.BytesArray) DocIdAndSeqNo(org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo) XContentType(org.opensearch.common.xcontent.XContentType) CodecService(org.opensearch.index.codec.CodecService) ThreadPool(org.opensearch.threadpool.ThreadPool) LogDocMergePolicy(org.apache.lucene.index.LogDocMergePolicy) FixedBitSet(org.apache.lucene.util.FixedBitSet) RegexFilter(org.apache.logging.log4j.core.filter.RegexFilter) Mockito.spy(org.mockito.Mockito.spy) Supplier(java.util.function.Supplier) LinkedHashMap(java.util.LinkedHashMap) ToLongBiFunction(java.util.function.ToLongBiFunction) IndexWriterMaxDocsChanger(org.apache.lucene.index.IndexWriterMaxDocsChanger) Lock(org.apache.lucene.store.Lock) Mapping(org.opensearch.index.mapper.Mapping) VersionFieldMapper(org.opensearch.index.mapper.VersionFieldMapper) Matchers.hasSize(org.hamcrest.Matchers.hasSize) Bits(org.apache.lucene.util.Bits) TieredMergePolicy(org.apache.lucene.index.TieredMergePolicy) Versions(org.opensearch.common.lucene.uid.Versions) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) TestTranslog(org.opensearch.index.translog.TestTranslog) IOException(java.io.IOException) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) UNASSIGNED_PRIMARY_TERM(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM) SourceFieldMapper(org.opensearch.index.mapper.SourceFieldMapper) AtomicLong(java.util.concurrent.atomic.AtomicLong) Phaser(java.util.concurrent.Phaser) TextField(org.apache.lucene.document.TextField) SeqNoFieldMapper(org.opensearch.index.mapper.SeqNoFieldMapper) Matchers.emptyArray(org.hamcrest.Matchers.emptyArray) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) BiFunction(java.util.function.BiFunction) StoredField(org.apache.lucene.document.StoredField) Matchers.hasKey(org.hamcrest.Matchers.hasKey) Matchers.everyItem(org.hamcrest.Matchers.everyItem) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Terms(org.apache.lucene.index.Terms) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Store(org.opensearch.index.store.Store) Collectors(java.util.stream.Collectors) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Matchers.equalTo(org.hamcrest.Matchers.equalTo) LeafReader(org.apache.lucene.index.LeafReader) IndexSettings(org.opensearch.index.IndexSettings) MetadataFieldMapper(org.opensearch.index.mapper.MetadataFieldMapper) ShardUtils(org.opensearch.index.shard.ShardUtils) Queue(java.util.Queue) BigArrays(org.opensearch.common.util.BigArrays) IndexReader(org.apache.lucene.index.IndexReader) IndexSearcher(org.apache.lucene.search.IndexSearcher) IndexSettingsModule(org.opensearch.test.IndexSettingsModule) BytesReference(org.opensearch.common.bytes.BytesReference) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) CheckedBiConsumer(org.opensearch.common.CheckedBiConsumer) LOCAL_RESET(org.opensearch.index.engine.Engine.Operation.Origin.LOCAL_RESET) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Function(java.util.function.Function) HashSet(java.util.HashSet) Charset(java.nio.charset.Charset) Translog(org.opensearch.index.translog.Translog) PRIMARY(org.opensearch.index.engine.Engine.Operation.Origin.PRIMARY) IntSupplier(java.util.function.IntSupplier) RetentionLease(org.opensearch.index.seqno.RetentionLease) CoreMatchers.sameInstance(org.hamcrest.CoreMatchers.sameInstance) OpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader) Collections.emptyMap(java.util.Collections.emptyMap) Matchers.empty(org.hamcrest.Matchers.empty) Semaphore(java.util.concurrent.Semaphore) Mockito.when(org.mockito.Mockito.when) ShardRouting(org.opensearch.cluster.routing.ShardRouting) IOUtils(org.opensearch.core.internal.io.IOUtils) ShardId(org.opensearch.index.shard.ShardId) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting) VersionsAndSeqNoResolver(org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver) Field(org.apache.lucene.document.Field) TransportActions(org.opensearch.action.support.TransportActions) Comparator(java.util.Comparator) LogManager(org.apache.logging.log4j.LogManager) NoMergePolicy(org.apache.lucene.index.NoMergePolicy) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) Arrays(java.util.Arrays) LongSupplier(java.util.function.LongSupplier) Matchers.not(org.hamcrest.Matchers.not) Level(org.apache.logging.log4j.Level) ContentPath(org.opensearch.index.mapper.ContentPath) LogEvent(org.apache.logging.log4j.core.LogEvent) ReferenceManager(org.apache.lucene.search.ReferenceManager) Document(org.opensearch.index.mapper.ParseContext.Document) Strings(org.opensearch.common.Strings) CoreMatchers.instanceOf(org.hamcrest.CoreMatchers.instanceOf) RandomNumbers(com.carrotsearch.randomizedtesting.generators.RandomNumbers) REPLICA(org.opensearch.index.engine.Engine.Operation.Origin.REPLICA) TermsEnum(org.apache.lucene.index.TermsEnum) Matchers.nullValue(org.hamcrest.Matchers.nullValue) Lucene(org.opensearch.common.lucene.Lucene) ActionListener(org.opensearch.action.ActionListener) SequentialStoredFieldsLeafReader(org.opensearch.common.lucene.index.SequentialStoredFieldsLeafReader) FieldsVisitor(org.opensearch.index.fieldvisitor.FieldsVisitor) NumericDocValuesField(org.apache.lucene.document.NumericDocValuesField) NO_OPS_PERFORMED(org.opensearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED) Mockito.atLeastOnce(org.mockito.Mockito.atLeastOnce) Set(java.util.Set) SortedSetSortField(org.apache.lucene.search.SortedSetSortField) Settings(org.opensearch.common.settings.Settings) ReplicationTracker(org.opensearch.index.seqno.ReplicationTracker) BuilderContext(org.opensearch.index.mapper.Mapper.BuilderContext) UncheckedIOException(java.io.UncheckedIOException) VersionType(org.opensearch.index.VersionType) Matchers.contains(org.hamcrest.Matchers.contains) CheckedRunnable(org.opensearch.common.CheckedRunnable) LocalCheckpointTracker(org.opensearch.index.seqno.LocalCheckpointTracker) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) Matchers.containsString(org.hamcrest.Matchers.containsString) Matchers.in(org.hamcrest.Matchers.in) TriFunction(org.opensearch.common.TriFunction) IndexCommit(org.apache.lucene.index.IndexCommit) LiveIndexWriterConfig(org.apache.lucene.index.LiveIndexWriterConfig) TranslogDeletionPolicies.createTranslogDeletionPolicy(org.opensearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy) ArrayList(java.util.ArrayList) UNASSIGNED_SEQ_NO(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO) VersionUtils(org.opensearch.test.VersionUtils) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) ParsedDocument(org.opensearch.index.mapper.ParsedDocument) TopDocs(org.apache.lucene.search.TopDocs) ParseContext(org.opensearch.index.mapper.ParseContext) LongStream(java.util.stream.LongStream) SetOnce(org.apache.lucene.util.SetOnce) Files(java.nio.file.Files) AbstractAppender(org.apache.logging.log4j.core.appender.AbstractAppender) Matchers.hasItem(org.hamcrest.Matchers.hasItem) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) SnapshotMatchers(org.opensearch.index.translog.SnapshotMatchers) LOCAL_TRANSLOG_RECOVERY(org.opensearch.index.engine.Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY) Collections.shuffle(java.util.Collections.shuffle) IdFieldMapper(org.opensearch.index.mapper.IdFieldMapper) IndexableField(org.apache.lucene.index.IndexableField) NoneCircuitBreakerService(org.opensearch.indices.breaker.NoneCircuitBreakerService) OpenSearchException(org.opensearch.OpenSearchException) ConcurrentCollections(org.opensearch.common.util.concurrent.ConcurrentCollections) ObjectObjectCursor(com.carrotsearch.hppc.cursors.ObjectObjectCursor) MapperService(org.opensearch.index.mapper.MapperService) Directory(org.apache.lucene.store.Directory) LeafReaderContext(org.apache.lucene.index.LeafReaderContext) TotalHitCountCollector(org.apache.lucene.search.TotalHitCountCollector) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) CyclicBarrier(java.util.concurrent.CyclicBarrier) Sort(org.apache.lucene.search.Sort) BytesRef(org.apache.lucene.util.BytesRef) DirectoryReader(org.apache.lucene.index.DirectoryReader) TranslogConfig(org.opensearch.index.translog.TranslogConfig) TranslogDeletionPolicyFactory(org.opensearch.index.translog.TranslogDeletionPolicyFactory) SegmentInfos(org.apache.lucene.index.SegmentInfos) Tuple(org.opensearch.common.collect.Tuple) IndexWriter(org.apache.lucene.index.IndexWriter) List(java.util.List) MatcherAssert(org.hamcrest.MatcherAssert) IndexWriterConfig(org.apache.lucene.index.IndexWriterConfig) MockDirectoryWrapper(org.apache.lucene.store.MockDirectoryWrapper) Uid(org.opensearch.index.mapper.Uid) LongPoint(org.apache.lucene.document.LongPoint) NumericDocValues(org.apache.lucene.index.NumericDocValues) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) HashMap(java.util.HashMap) ReleasableLock(org.opensearch.common.util.concurrent.ReleasableLock) AtomicReference(java.util.concurrent.atomic.AtomicReference) Loggers(org.opensearch.common.logging.Loggers) UUIDs(org.opensearch.common.UUIDs) Iterator(java.util.Iterator) Matchers(org.hamcrest.Matchers) RootObjectMapper(org.opensearch.index.mapper.RootObjectMapper) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) TermQuery(org.apache.lucene.search.TermQuery) Closeable(java.io.Closeable) IndexRequest(org.opensearch.action.index.IndexRequest) Collections(java.util.Collections) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) IndexSearcher(org.apache.lucene.search.IndexSearcher) ArrayList(java.util.ArrayList) Store(org.opensearch.index.store.Store) Matchers.containsString(org.hamcrest.Matchers.containsString) IOException(java.io.IOException) UncheckedIOException(java.io.UncheckedIOException) DocIdAndSeqNo(org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo) LongPoint(org.apache.lucene.document.LongPoint) ParsedDocument(org.opensearch.index.mapper.ParsedDocument) Map(java.util.Map) LinkedHashMap(java.util.LinkedHashMap) Collections.emptyMap(java.util.Collections.emptyMap) HashMap(java.util.HashMap)

Aggregations

HashMap (java.util.HashMap)2 Map (java.util.Map)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 LongSupplier (java.util.function.LongSupplier)2 ObjectObjectCursor (com.carrotsearch.hppc.cursors.ObjectObjectCursor)1 RandomNumbers (com.carrotsearch.randomizedtesting.generators.RandomNumbers)1 Closeable (java.io.Closeable)1 IOException (java.io.IOException)1 UncheckedIOException (java.io.UncheckedIOException)1 Charset (java.nio.charset.Charset)1 Files (java.nio.file.Files)1 Path (java.nio.file.Path)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collections (java.util.Collections)1 Collections.emptyMap (java.util.Collections.emptyMap)1 Collections.shuffle (java.util.Collections.shuffle)1 Comparator (java.util.Comparator)1 HashSet (java.util.HashSet)1 Iterator (java.util.Iterator)1