Examples with TranslogConfig org.opensearch.index.translog.TranslogConfig used on opensource projects

Example 6 with TranslogConfig

use of org.opensearch.index.translog.TranslogConfig in project OpenSearch by opensearch-project.

the class EngineTestCase method config.

protected EngineConfig config(EngineConfig config, Store store, Path translogPath, EngineConfig.TombstoneDocSupplier tombstoneDocSupplier) {
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", Settings.builder().put(config.getIndexSettings().getSettings()).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).build());
    TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
    return new EngineConfig(config.getShardId(), config.getThreadPool(), indexSettings, config.getWarmer(), store, config.getMergePolicy(), config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener(), config.getQueryCache(), config.getQueryCachingPolicy(), translogConfig, config.getFlushMergesAfter(), config.getExternalRefreshListener(), config.getInternalRefreshListener(), config.getIndexSort(), config.getCircuitBreakerService(), config.getGlobalCheckpointSupplier(), config.retentionLeasesSupplier(), config.getPrimaryTermSupplier(), tombstoneDocSupplier);
}

Example 7 with TranslogConfig

use of org.opensearch.index.translog.TranslogConfig in project OpenSearch by opensearch-project.

the class EngineTestCase method config.

public EngineConfig config(final IndexSettings indexSettings, final Store store, final Path translogPath, final MergePolicy mergePolicy, final ReferenceManager.RefreshListener externalRefreshListener, final ReferenceManager.RefreshListener internalRefreshListener, final Sort indexSort, @Nullable final LongSupplier maybeGlobalCheckpointSupplier, @Nullable final Supplier<RetentionLeases> maybeRetentionLeasesSupplier, final CircuitBreakerService breakerService) {
    final IndexWriterConfig iwc = newIndexWriterConfig();
    final TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
    final Engine.EventListener eventListener = new Engine.EventListener() {
    };
    // we don't need to notify anybody in this test
    final List<ReferenceManager.RefreshListener> extRefreshListenerList = externalRefreshListener == null ? emptyList() : Collections.singletonList(externalRefreshListener);
    final List<ReferenceManager.RefreshListener> intRefreshListenerList = internalRefreshListener == null ? emptyList() : Collections.singletonList(internalRefreshListener);
    final LongSupplier globalCheckpointSupplier;
    final Supplier<RetentionLeases> retentionLeasesSupplier;
    if (maybeGlobalCheckpointSupplier == null) {
        assert maybeRetentionLeasesSupplier == null;
        final ReplicationTracker replicationTracker = new ReplicationTracker(shardId, allocationId.getId(), indexSettings, randomNonNegativeLong(), SequenceNumbers.NO_OPS_PERFORMED, update -> {
        }, () -> 0L, (leases, listener) -> listener.onResponse(new ReplicationResponse()), () -> SafeCommitInfo.EMPTY);
        globalCheckpointSupplier = replicationTracker;
        retentionLeasesSupplier = replicationTracker::getRetentionLeases;
    } else {
        assert maybeRetentionLeasesSupplier != null;
        globalCheckpointSupplier = maybeGlobalCheckpointSupplier;
        retentionLeasesSupplier = maybeRetentionLeasesSupplier;
    }
    return new EngineConfig(shardId, threadPool, indexSettings, null, store, mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), eventListener, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), extRefreshListenerList, intRefreshListenerList, indexSort, breakerService, globalCheckpointSupplier, retentionLeasesSupplier, primaryTerm, tombstoneDocSupplier());
}

Example 8 with TranslogConfig

use of org.opensearch.index.translog.TranslogConfig in project OpenSearch by opensearch-project.

the class InternalEngine method openTranslog.

private Translog openTranslog(EngineConfig engineConfig, TranslogDeletionPolicy translogDeletionPolicy, LongSupplier globalCheckpointSupplier, LongConsumer persistedSequenceNumberConsumer) throws IOException {
    final TranslogConfig translogConfig = engineConfig.getTranslogConfig();
    final Map<String, String> userData = store.readLastCommittedSegmentsInfo().getUserData();
    final String translogUUID = Objects.requireNonNull(userData.get(Translog.TRANSLOG_UUID_KEY));
    // We expect that this shard already exists, so it must already have an existing translog else something is badly wrong!
    return new Translog(translogConfig, translogUUID, translogDeletionPolicy, globalCheckpointSupplier, engineConfig.getPrimaryTermSupplier(), persistedSequenceNumberConsumer);
}

Example 9 with TranslogConfig

use of org.opensearch.index.translog.TranslogConfig in project OpenSearch by opensearch-project.

the class NRTReplicationEngine method openTranslog.

private Translog openTranslog(EngineConfig engineConfig, TranslogDeletionPolicy translogDeletionPolicy, LongSupplier globalCheckpointSupplier, LongConsumer persistedSequenceNumberConsumer) throws IOException {
    final TranslogConfig translogConfig = engineConfig.getTranslogConfig();
    final Map<String, String> userData = lastCommittedSegmentInfos.getUserData();
    final String translogUUID = Objects.requireNonNull(userData.get(Translog.TRANSLOG_UUID_KEY));
    // We expect that this shard already exists, so it must already have an existing translog else something is badly wrong!
    return new Translog(translogConfig, translogUUID, translogDeletionPolicy, globalCheckpointSupplier, engineConfig.getPrimaryTermSupplier(), persistedSequenceNumberConsumer);
}

Example 10 with TranslogConfig

use of org.opensearch.index.translog.TranslogConfig in project OpenSearch by opensearch-project.

the class NoOpEngine method trimUnreferencedTranslogFiles.

/**
 * This implementation will trim existing translog files using a {@link TranslogDeletionPolicy}
 * that retains nothing but the last translog generation from safe commit.
 */
@Override
public void trimUnreferencedTranslogFiles() {
    final Store store = this.engineConfig.getStore();
    store.incRef();
    try (ReleasableLock lock = readLock.acquire()) {
        ensureOpen();
        final List<IndexCommit> commits = DirectoryReader.listCommits(store.directory());
        if (commits.size() == 1 && translogStats.getTranslogSizeInBytes() > translogStats.getUncommittedSizeInBytes()) {
            final Map<String, String> commitUserData = getLastCommittedSegmentInfos().getUserData();
            final String translogUuid = commitUserData.get(Translog.TRANSLOG_UUID_KEY);
            if (translogUuid == null) {
                throw new IllegalStateException("commit doesn't contain translog unique id");
            }
            final TranslogConfig translogConfig = engineConfig.getTranslogConfig();
            final long localCheckpoint = Long.parseLong(commitUserData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
            final TranslogDeletionPolicy translogDeletionPolicy = new DefaultTranslogDeletionPolicy(-1, -1, 0);
            translogDeletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpoint);
            try (Translog translog = new Translog(translogConfig, translogUuid, translogDeletionPolicy, engineConfig.getGlobalCheckpointSupplier(), engineConfig.getPrimaryTermSupplier(), seqNo -> {
            })) {
                translog.trimUnreferencedReaders();
                // refresh the translog stats
                this.translogStats = translog.stats();
                assert translog.currentFileGeneration() == translog.getMinFileGeneration() : "translog was not trimmed " + " current gen " + translog.currentFileGeneration() + " != min gen " + translog.getMinFileGeneration();
            }
        }
    } catch (final Exception e) {
        try {
            failEngine("translog trimming failed", e);
        } catch (Exception inner) {
            e.addSuppressed(inner);
        }
        throw new EngineException(shardId, "failed to trim translog", e);
    } finally {
        store.decRef();
    }
}