Examples with IndexCommit org.apache.lucene.index.IndexCommit used on opensource projects

Example 56 with IndexCommit

use of org.apache.lucene.index.IndexCommit in project crate by crate.

the class InternalEngineTests method testTrimUnsafeCommits.

@Test
public void testTrimUnsafeCommits() throws Exception {
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    final int maxSeqNo = 40;
    final List<Long> seqNos = LongStream.rangeClosed(0, maxSeqNo).boxed().collect(Collectors.toList());
    Collections.shuffle(seqNos, random());
    try (Store store = createStore()) {
        EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
        final List<Long> commitMaxSeqNo = new ArrayList<>();
        final long minTranslogGen;
        try (InternalEngine engine = createEngine(config)) {
            for (int i = 0; i < seqNos.size(); i++) {
                ParsedDocument doc = testParsedDocument(Long.toString(seqNos.get(i)), null, testDocument(), new BytesArray("{}"), null);
                Engine.Index index = new Engine.Index(newUid(doc), doc, seqNos.get(i), 0, 1, null, REPLICA, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);
                engine.index(index);
                if (randomBoolean()) {
                    engine.flush();
                    final Long maxSeqNoInCommit = seqNos.subList(0, i + 1).stream().max(Long::compareTo).orElse(-1L);
                    commitMaxSeqNo.add(maxSeqNoInCommit);
                }
            }
            globalCheckpoint.set(randomInt(maxSeqNo));
            engine.syncTranslog();
            minTranslogGen = engine.getTranslog().getMinFileGeneration();
        }
        store.trimUnsafeCommits(globalCheckpoint.get(), minTranslogGen, config.getIndexSettings().getIndexVersionCreated());
        long safeMaxSeqNo = commitMaxSeqNo.stream().filter(s -> s <= globalCheckpoint.get()).reduce(// get the last one.
        (s1, s2) -> s2).orElse(SequenceNumbers.NO_OPS_PERFORMED);
        final List<IndexCommit> commits = DirectoryReader.listCommits(store.directory());
        assertThat(commits, hasSize(1));
        assertThat(commits.get(0).getUserData().get(SequenceNumbers.MAX_SEQ_NO), equalTo(Long.toString(safeMaxSeqNo)));
        try (IndexReader reader = DirectoryReader.open(commits.get(0))) {
            for (LeafReaderContext context : reader.leaves()) {
                final NumericDocValues values = context.reader().getNumericDocValues(SeqNoFieldMapper.NAME);
                if (values != null) {
                    for (int docID = 0; docID < context.reader().maxDoc(); docID++) {
                        if (values.advanceExact(docID) == false) {
                            throw new AssertionError("Document does not have a seq number: " + docID);
                        }
                        assertThat(values.longValue(), lessThanOrEqualTo(globalCheckpoint.get()));
                    }
                }
            }
        }
    }
}

Example 57 with IndexCommit

use of org.apache.lucene.index.IndexCommit in project crate by crate.

the class InternalEngineTests method testKeepTranslogAfterGlobalCheckpoint.

@Test
public void testKeepTranslogAfterGlobalCheckpoint() throws Exception {
    IOUtils.close(engine, store);
    final IndexSettings indexSettings = new IndexSettings(defaultSettings.getIndexMetadata(), defaultSettings.getNodeSettings(), defaultSettings.getScopedSettings());
    IndexMetadata.Builder builder = IndexMetadata.builder(indexSettings.getIndexMetadata()).settings(Settings.builder().put(indexSettings.getSettings()).put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomFrom("-1", "100micros", "30m")).put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), randomFrom("-1", "512b", "1gb")));
    indexSettings.updateIndexMetadata(builder.build());
    final Path translogPath = createTempDir();
    store = createStore();
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    store.createEmpty(Version.CURRENT.luceneVersion);
    final String translogUUID = Translog.createEmptyTranslog(translogPath, globalCheckpoint.get(), shardId, primaryTerm.get());
    store.associateIndexWithNewTranslog(translogUUID);
    final EngineConfig engineConfig = config(indexSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, () -> globalCheckpoint.get());
    final AtomicLong lastSyncedGlobalCheckpointBeforeCommit = new AtomicLong(Translog.readGlobalCheckpoint(translogPath, translogUUID));
    try (InternalEngine engine = new InternalEngine(engineConfig) {

        @Override
        protected void commitIndexWriter(IndexWriter writer, Translog translog, String syncId) throws IOException {
            lastSyncedGlobalCheckpointBeforeCommit.set(Translog.readGlobalCheckpoint(translogPath, translogUUID));
            // (this value is visible to the deletion policy) and an in memory global checkpoint in the SequenceNumbersService.
            if (rarely()) {
                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), getPersistedLocalCheckpoint()));
            }
            super.commitIndexWriter(writer, translog, syncId);
        }
    }) {
        engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
        int numDocs = scaledRandomIntBetween(10, 100);
        for (int docId = 0; docId < numDocs; docId++) {
            ParseContext.Document document = testDocumentWithTextField();
            document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
            engine.index(indexForDoc(testParsedDocument(Integer.toString(docId), null, document, B_1, null)));
            if (frequently()) {
                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
                engine.syncTranslog();
            }
            if (frequently()) {
                engine.flush(randomBoolean(), true);
                final List<IndexCommit> commits = DirectoryReader.listCommits(store.directory());
                // Keep only one safe commit as the oldest commit.
                final IndexCommit safeCommit = commits.get(0);
                if (lastSyncedGlobalCheckpointBeforeCommit.get() == UNASSIGNED_SEQ_NO) {
                    // If the global checkpoint is still unassigned, we keep an empty(eg. initial) commit as a safe commit.
                    assertThat(Long.parseLong(safeCommit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
                } else {
                    assertThat(Long.parseLong(safeCommit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)), lessThanOrEqualTo(lastSyncedGlobalCheckpointBeforeCommit.get()));
                }
                for (int i = 1; i < commits.size(); i++) {
                    assertThat(Long.parseLong(commits.get(i).getUserData().get(SequenceNumbers.MAX_SEQ_NO)), greaterThan(lastSyncedGlobalCheckpointBeforeCommit.get()));
                }
                // Make sure we keep all translog operations after the local checkpoint of the safe commit.
                long localCheckpointFromSafeCommit = Long.parseLong(safeCommit.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
                try (Translog.Snapshot snapshot = getTranslog(engine).newSnapshot()) {
                    assertThat(snapshot, SnapshotMatchers.containsSeqNoRange(localCheckpointFromSafeCommit + 1, docId));
                }
            }
        }
    }
}

Example 58 with IndexCommit

use of org.apache.lucene.index.IndexCommit in project crate by crate.

the class InternalEngineTests method testCleanupCommitsWhenReleaseSnapshot.

@Test
public void testCleanupCommitsWhenReleaseSnapshot() throws Exception {
    IOUtils.close(engine, store);
    store = createStore();
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    try (InternalEngine engine = createEngine(store, createTempDir(), globalCheckpoint::get)) {
        final int numDocs = scaledRandomIntBetween(10, 100);
        for (int docId = 0; docId < numDocs; docId++) {
            index(engine, docId);
            if (frequently()) {
                engine.flush(randomBoolean(), true);
            }
        }
        engine.flush(false, randomBoolean());
        int numSnapshots = between(1, 10);
        final List<Engine.IndexCommitRef> snapshots = new ArrayList<>();
        for (int i = 0; i < numSnapshots; i++) {
            // taking snapshots from the safe commit.
            snapshots.add(engine.acquireSafeIndexCommit());
        }
        globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
        engine.syncTranslog();
        final List<IndexCommit> commits = DirectoryReader.listCommits(store.directory());
        for (int i = 0; i < numSnapshots - 1; i++) {
            snapshots.get(i).close();
            // pending snapshots - should not release any commit.
            assertThat(DirectoryReader.listCommits(store.directory()), equalTo(commits));
        }
        // release the last snapshot - delete all except the last commit
        snapshots.get(numSnapshots - 1).close();
        assertThat(DirectoryReader.listCommits(store.directory()), hasSize(1));
    }
}

Example 59 with IndexCommit

use of org.apache.lucene.index.IndexCommit in project crate by crate.

the class CombinedDeletionPolicyTests method mockIndexCommit.

IndexCommit mockIndexCommit(long localCheckpoint, long maxSeqNo, UUID translogUUID) throws IOException {
    final Map<String, String> userData = new HashMap<>();
    userData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(localCheckpoint));
    userData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(maxSeqNo));
    userData.put(Translog.TRANSLOG_UUID_KEY, translogUUID.toString());
    final IndexCommit commit = mock(IndexCommit.class);
    final Directory directory = mock(Directory.class);
    when(commit.getUserData()).thenReturn(userData);
    when(commit.getDirectory()).thenReturn(directory);
    resetDeletion(commit);
    return commit;
}

Example 60 with IndexCommit

use of org.apache.lucene.index.IndexCommit in project crate by crate.

the class CombinedDeletionPolicyTests method testDeleteInvalidCommits.

public void testDeleteInvalidCommits() throws Exception {
    final AtomicLong globalCheckpoint = new AtomicLong(randomNonNegativeLong());
    final SoftDeletesPolicy softDeletesPolicy = new SoftDeletesPolicy(globalCheckpoint::get, -1, 0, () -> RetentionLeases.EMPTY);
    TranslogDeletionPolicy translogPolicy = createTranslogDeletionPolicy();
    CombinedDeletionPolicy indexPolicy = newCombinedDeletionPolicy(translogPolicy, softDeletesPolicy, globalCheckpoint);
    final int invalidCommits = between(1, 10);
    final List<IndexCommit> commitList = new ArrayList<>();
    for (int i = 0; i < invalidCommits; i++) {
        long maxSeqNo = randomNonNegativeLong();
        commitList.add(mockIndexCommit(randomLongBetween(-1, maxSeqNo), maxSeqNo, UUID.randomUUID()));
    }
    final UUID expectedTranslogUUID = UUID.randomUUID();
    final int validCommits = between(1, 10);
    long lastMaxSeqNo = between(1, 1000);
    long lastCheckpoint = randomLongBetween(-1, lastMaxSeqNo);
    for (int i = 0; i < validCommits; i++) {
        lastMaxSeqNo += between(1, 1000);
        lastCheckpoint = randomLongBetween(lastCheckpoint, lastMaxSeqNo);
        commitList.add(mockIndexCommit(lastCheckpoint, lastMaxSeqNo, expectedTranslogUUID));
    }
    // We should never keep invalid commits regardless of the value of the global checkpoint.
    indexPolicy.onCommit(commitList);
    for (int i = 0; i < invalidCommits - 1; i++) {
        verify(commitList.get(i), times(1)).delete();
    }
    assertThat(softDeletesPolicy.getMinRetainedSeqNo(), equalTo(getLocalCheckpoint(CombinedDeletionPolicy.findSafeCommitPoint(commitList, globalCheckpoint.get())) + 1));
}