Examples with TranslogStats org.opensearch.index.translog.TranslogStats used on opensource projects

Example 1 with TranslogStats

use of org.opensearch.index.translog.TranslogStats in project OpenSearch by opensearch-project.

the class IndexShardIT method testMaybeFlush.

public void testMaybeFlush() throws Exception {
    createIndex("test", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST).build());
    ensureGreen();
    IndicesService indicesService = getInstanceFromNode(IndicesService.class);
    IndexService test = indicesService.indexService(resolveIndex("test"));
    IndexShard shard = test.getShardOrNull(0);
    assertFalse(shard.shouldPeriodicallyFlush());
    client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(135, /* size of the operation + one generation header&footer*/
    ByteSizeUnit.BYTES)).build()).get();
    client().prepareIndex("test").setId("0").setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
    assertFalse(shard.shouldPeriodicallyFlush());
    shard.applyIndexOperationOnPrimary(Versions.MATCH_ANY, VersionType.INTERNAL, new SourceToParse("test", "_doc", "1", new BytesArray("{}"), XContentType.JSON), SequenceNumbers.UNASSIGNED_SEQ_NO, 0, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false);
    assertTrue(shard.shouldPeriodicallyFlush());
    final Translog translog = getTranslog(shard);
    assertEquals(2, translog.stats().getUncommittedOperations());
    assertThat(shard.flushStats().getTotal(), equalTo(0L));
    client().prepareIndex("test").setId("2").setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
    assertThat(shard.getLastKnownGlobalCheckpoint(), equalTo(2L));
    assertBusy(() -> {
        // this is async
        assertFalse(shard.shouldPeriodicallyFlush());
        assertThat(shard.flushStats().getPeriodic(), equalTo(1L));
        assertThat(shard.flushStats().getTotal(), equalTo(1L));
    });
    shard.sync();
    assertThat(shard.getLastSyncedGlobalCheckpoint(), equalTo(2L));
    assertThat("last commit [" + shard.commitStats().getUserData() + "]", translog.stats().getUncommittedOperations(), equalTo(0));
    long size = Math.max(translog.stats().getUncommittedSizeInBytes(), Translog.DEFAULT_HEADER_SIZE_IN_BYTES + 1);
    logger.info("--> current translog size: [{}] num_ops [{}] generation [{}]", translog.stats().getUncommittedSizeInBytes(), translog.stats().getUncommittedOperations(), translog.getGeneration());
    client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(size, ByteSizeUnit.BYTES)).build()).get();
    client().prepareDelete("test", "2").get();
    logger.info("--> translog size after delete: [{}] num_ops [{}] generation [{}]", translog.stats().getUncommittedSizeInBytes(), translog.stats().getUncommittedOperations(), translog.getGeneration());
    assertBusy(() -> {
        // this is async
        final TranslogStats translogStats = translog.stats();
        final CommitStats commitStats = shard.commitStats();
        final FlushStats flushStats = shard.flushStats();
        logger.info("--> translog stats [{}] gen [{}] commit_stats [{}] flush_stats [{}/{}]", Strings.toString(translogStats), translog.getGeneration().translogFileGeneration, commitStats.getUserData(), flushStats.getPeriodic(), flushStats.getTotal());
        assertFalse(shard.shouldPeriodicallyFlush());
    });
    shard.sync();
    assertEquals(0, translog.stats().getUncommittedOperations());
}

Example 2 with TranslogStats

use of org.opensearch.index.translog.TranslogStats in project OpenSearch by opensearch-project.

the class CommonStats method add.

public void add(CommonStats stats) {
    if (docs == null) {
        if (stats.getDocs() != null) {
            docs = new DocsStats();
            docs.add(stats.getDocs());
        }
    } else {
        docs.add(stats.getDocs());
    }
    if (store == null) {
        if (stats.getStore() != null) {
            store = new StoreStats();
            store.add(stats.getStore());
        }
    } else {
        store.add(stats.getStore());
    }
    if (indexing == null) {
        if (stats.getIndexing() != null) {
            indexing = new IndexingStats();
            indexing.add(stats.getIndexing());
        }
    } else {
        indexing.add(stats.getIndexing());
    }
    if (get == null) {
        if (stats.getGet() != null) {
            get = new GetStats();
            get.add(stats.getGet());
        }
    } else {
        get.add(stats.getGet());
    }
    if (search == null) {
        if (stats.getSearch() != null) {
            search = new SearchStats();
            search.add(stats.getSearch());
        }
    } else {
        search.add(stats.getSearch());
    }
    if (merge == null) {
        if (stats.getMerge() != null) {
            merge = new MergeStats();
            merge.add(stats.getMerge());
        }
    } else {
        merge.add(stats.getMerge());
    }
    if (refresh == null) {
        if (stats.getRefresh() != null) {
            refresh = new RefreshStats();
            refresh.add(stats.getRefresh());
        }
    } else {
        refresh.add(stats.getRefresh());
    }
    if (flush == null) {
        if (stats.getFlush() != null) {
            flush = new FlushStats();
            flush.add(stats.getFlush());
        }
    } else {
        flush.add(stats.getFlush());
    }
    if (warmer == null) {
        if (stats.getWarmer() != null) {
            warmer = new WarmerStats();
            warmer.add(stats.getWarmer());
        }
    } else {
        warmer.add(stats.getWarmer());
    }
    if (queryCache == null) {
        if (stats.getQueryCache() != null) {
            queryCache = new QueryCacheStats();
            queryCache.add(stats.getQueryCache());
        }
    } else {
        queryCache.add(stats.getQueryCache());
    }
    if (fieldData == null) {
        if (stats.getFieldData() != null) {
            fieldData = new FieldDataStats();
            fieldData.add(stats.getFieldData());
        }
    } else {
        fieldData.add(stats.getFieldData());
    }
    if (completion == null) {
        if (stats.getCompletion() != null) {
            completion = new CompletionStats();
            completion.add(stats.getCompletion());
        }
    } else {
        completion.add(stats.getCompletion());
    }
    if (segments == null) {
        if (stats.getSegments() != null) {
            segments = new SegmentsStats();
            segments.add(stats.getSegments());
        }
    } else {
        segments.add(stats.getSegments());
    }
    if (translog == null) {
        if (stats.getTranslog() != null) {
            translog = new TranslogStats();
            translog.add(stats.getTranslog());
        }
    } else {
        translog.add(stats.getTranslog());
    }
    if (requestCache == null) {
        if (stats.getRequestCache() != null) {
            requestCache = new RequestCacheStats();
            requestCache.add(stats.getRequestCache());
        }
    } else {
        requestCache.add(stats.getRequestCache());
    }
    if (recoveryStats == null) {
        if (stats.getRecoveryStats() != null) {
            recoveryStats = new RecoveryStats();
            recoveryStats.add(stats.getRecoveryStats());
        }
    } else {
        recoveryStats.add(stats.getRecoveryStats());
    }
}

Example 3 with TranslogStats

use of org.opensearch.index.translog.TranslogStats in project OpenSearch by opensearch-project.

the class ReadOnlyEngineTests method testVerifyShardBeforeIndexClosingIsNoOp.

/**
 * Test that {@link ReadOnlyEngine#verifyEngineBeforeIndexClosing()} never fails
 * whatever the value of the global checkpoint to check is.
 */
public void testVerifyShardBeforeIndexClosingIsNoOp() throws IOException {
    IOUtils.close(engine, store);
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    try (Store store = createStore()) {
        EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
        store.createEmpty(Version.CURRENT.luceneVersion);
        try (ReadOnlyEngine readOnlyEngine = new ReadOnlyEngine(config, null, new TranslogStats(), true, Function.identity(), true)) {
            globalCheckpoint.set(randomNonNegativeLong());
            try {
                readOnlyEngine.verifyEngineBeforeIndexClosing();
            } catch (final IllegalStateException e) {
                fail("Read-only engine pre-closing verifications failed");
            }
        }
    }
}

Example 4 with TranslogStats

use of org.opensearch.index.translog.TranslogStats in project OpenSearch by opensearch-project.

the class ReadOnlyEngineTests method testReadOnly.

public void testReadOnly() throws IOException {
    IOUtils.close(engine, store);
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    try (Store store = createStore()) {
        EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
        store.createEmpty(Version.CURRENT.luceneVersion);
        try (ReadOnlyEngine readOnlyEngine = new ReadOnlyEngine(config, null, new TranslogStats(), true, Function.identity(), true)) {
            Class<? extends Throwable> expectedException = LuceneTestCase.TEST_ASSERTS_ENABLED ? AssertionError.class : UnsupportedOperationException.class;
            expectThrows(expectedException, () -> readOnlyEngine.index(null));
            expectThrows(expectedException, () -> readOnlyEngine.delete(null));
            expectThrows(expectedException, () -> readOnlyEngine.noOp(null));
        }
    }
}

Example 5 with TranslogStats

use of org.opensearch.index.translog.TranslogStats in project OpenSearch by opensearch-project.

the class IndexShardTests method testResetEngine.

public void testResetEngine() throws Exception {
    IndexShard shard = newStartedShard(false);
    indexOnReplicaWithGaps(shard, between(0, 1000), Math.toIntExact(shard.getLocalCheckpoint()));
    long maxSeqNoBeforeRollback = shard.seqNoStats().getMaxSeqNo();
    final long globalCheckpoint = randomLongBetween(shard.getLastKnownGlobalCheckpoint(), shard.getLocalCheckpoint());
    shard.updateGlobalCheckpointOnReplica(globalCheckpoint, "test");
    Set<String> docBelowGlobalCheckpoint = getShardDocUIDs(shard).stream().filter(id -> Long.parseLong(id) <= globalCheckpoint).collect(Collectors.toSet());
    TranslogStats translogStats = shard.translogStats();
    AtomicBoolean done = new AtomicBoolean();
    CountDownLatch latch = new CountDownLatch(1);
    Thread thread = new Thread(() -> {
        latch.countDown();
        int hitClosedExceptions = 0;
        while (done.get() == false) {
            try {
                List<String> exposedDocIds = EngineTestCase.getDocIds(getEngine(shard), rarely()).stream().map(DocIdSeqNoAndSource::getId).collect(Collectors.toList());
                assertThat("every operations before the global checkpoint must be reserved", docBelowGlobalCheckpoint, everyItem(is(in(exposedDocIds))));
            } catch (AlreadyClosedException ignored) {
                hitClosedExceptions++;
            } catch (IOException e) {
                throw new AssertionError(e);
            }
        }
        // engine reference was switched twice: current read/write engine -> ready-only engine -> new read/write engine
        assertThat(hitClosedExceptions, lessThanOrEqualTo(2));
    });
    thread.start();
    latch.await();
    final CountDownLatch engineResetLatch = new CountDownLatch(1);
    shard.acquireAllReplicaOperationsPermits(shard.getOperationPrimaryTerm(), globalCheckpoint, 0L, ActionListener.wrap(r -> {
        try {
            shard.resetEngineToGlobalCheckpoint();
        } finally {
            r.close();
            engineResetLatch.countDown();
        }
    }, Assert::assertNotNull), TimeValue.timeValueMinutes(1L));
    engineResetLatch.await();
    assertThat(getShardDocUIDs(shard), equalTo(docBelowGlobalCheckpoint));
    assertThat(shard.seqNoStats().getMaxSeqNo(), equalTo(globalCheckpoint));
    if (shard.indexSettings.isSoftDeleteEnabled()) {
        // we might have trimmed some operations if the translog retention policy is ignored (when soft-deletes enabled).
        assertThat(shard.translogStats().estimatedNumberOfOperations(), lessThanOrEqualTo(translogStats.estimatedNumberOfOperations()));
    } else {
        assertThat(shard.translogStats().estimatedNumberOfOperations(), equalTo(translogStats.estimatedNumberOfOperations()));
    }
    assertThat(shard.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(maxSeqNoBeforeRollback));
    done.set(true);
    thread.join();
    closeShard(shard, false);
}