Examples with StoreFindToken com.github.ambry.store.StoreFindToken used on opensource projects

Example 31 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method rebuildTokenWhenLastSegmentAutoClosedTestCase2.

/**
 * Test generating an index based token for second last index segment and close last log segment, should return index based
 * token point to the beginning of last Index Segment.
 * @throws StoreException
 */
@Test
public void rebuildTokenWhenLastSegmentAutoClosedTestCase2() throws StoreException {
    assumeTrue(isLogSegmented);
    IndexSegment lastIndexSegment = stateForTokenTest.index.getIndexSegments().lastEntry().getValue();
    IndexSegment secondLastIndexSegment = stateForTokenTest.index.getIndexSegments().lowerEntry(lastIndexSegment.getStartOffset()).getValue();
    StoreKey secondLastStoreKeyInSecondLastIndexSegment = secondLastIndexSegment.listIterator(secondLastIndexSegment.size()).previous().getKey();
    StoreFindToken startToken = new StoreFindToken(secondLastStoreKeyInSecondLastIndexSegment, secondLastIndexSegment.getStartOffset(), stateForTokenTest.sessionId, stateForTokenTest.incarnationId, secondLastIndexSegment.getResetKey(), secondLastIndexSegment.getResetKeyType(), secondLastIndexSegment.getResetKeyLifeVersion());
    autoCloseLastLogSegmentAndCleanUpJournal();
    FindInfo findInfo = stateForTokenTest.index.findEntriesSince(startToken, 1);
    StoreFindToken token = (StoreFindToken) findInfo.getFindToken();
    StoreFindToken expectedToken = new StoreFindToken(lastIndexSegment.listIterator(1).previous().getKey(), lastIndexSegment.getStartOffset(), stateForTokenTest.sessionId, stateForTokenTest.incarnationId, lastIndexSegment.getResetKey(), lastIndexSegment.getResetKeyType(), lastIndexSegment.getResetKeyLifeVersion());
    compareTokens(expectedToken, token);
}

Example 32 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method findDeletedEntriesSinceOneByOneTest.

/**
 * Uses {@link PersistentIndex#findDeletedEntriesSince(FindToken, long, long)} to get entries one by one.
 * @throws StoreException
 */
private void findDeletedEntriesSinceOneByOneTest() throws StoreException {
    Offset journalStartOffset = state.index.journal.getFirstOffset();
    StoreFindToken startToken = new StoreFindToken();
    Offset stoppedAt = null;
    for (Map.Entry<Offset, TreeMap<MockId, TreeSet<IndexValue>>> indexEntry : state.referenceIndex.entrySet()) {
        Offset indexSegmentStartOffset = indexEntry.getKey();
        // 2. The size of entries being obtained is <= the size of records in the current index segment
        if (indexSegmentStartOffset.compareTo(journalStartOffset) >= 0) {
            stoppedAt = indexSegmentStartOffset;
            break;
        }
        IndexSegment segmentOfToken = state.index.getIndexSegments().get(indexSegmentStartOffset);
        for (Map.Entry<MockId, TreeSet<IndexValue>> indexSegmentEntry : indexEntry.getValue().entrySet()) {
            MockId id = indexSegmentEntry.getKey();
            boolean isDeleted = indexSegmentEntry.getValue().last().isDelete() && state.getExpectedValue(indexSegmentEntry.getKey(), EnumSet.of(PersistentIndex.IndexEntryType.UNDELETE), null) == null;
            StoreFindToken expectedEndToken = new StoreFindToken(id, indexSegmentStartOffset, state.sessionId, state.incarnationId, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
            long size = getSizeOfAllValues(indexSegmentEntry.getValue());
            doFindDeletedEntriesSinceTest(startToken, size, isDeleted ? Collections.singleton(id) : Collections.emptySet(), expectedEndToken);
            startToken = expectedEndToken;
        }
    }
    Map.Entry<Offset, Pair<MockId, CuratedLogIndexState.LogEntry>> logEntry = state.logOrder.floorEntry(stoppedAt);
    while (logEntry != null) {
        Offset startOffset = logEntry.getKey();
        MockId id = logEntry.getValue().getFirst();
        IndexValue value = state.getExpectedValue(id, false);
        boolean isDeleted = value.isDelete() && state.getExpectedValue(id, EnumSet.of(PersistentIndex.IndexEntryType.UNDELETE), null) == null;
        // size returned is the size of the delete if the key has been deleted.
        long size = value.getSize();
        IndexSegment segmentOfToken = state.index.getIndexSegments().floorEntry(startOffset).getValue();
        StoreFindToken expectedEndToken = new StoreFindToken(startOffset, state.sessionId, state.incarnationId, false, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
        doFindDeletedEntriesSinceTest(startToken, size, isDeleted ? Collections.singleton(id) : Collections.emptySet(), expectedEndToken);
        startToken = expectedEndToken;
        logEntry = state.logOrder.higherEntry(logEntry.getKey());
    }
}

Example 33 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method findEntriesSinceAfterAutoCloseLastLogSegmentTest.

/**
 * Tests {@link PersistentIndex#findEntriesSince(FindToken, long)} when last log segment gets auto closed and token
 * points to the last index segment.
 * @throws StoreException
 */
@Test
public void findEntriesSinceAfterAutoCloseLastLogSegmentTest() throws StoreException {
    state.addPutEntries(7, CuratedLogIndexState.PUT_RECORD_SIZE, Utils.Infinite_Time);
    state.addDeleteEntry(state.getIdToDeleteFromIndexSegment(state.referenceIndex.lastKey(), false));
    state.addPutEntries(3, CuratedLogIndexState.PUT_RECORD_SIZE, Utils.Infinite_Time);
    if (isLogSegmented) {
        CompactionPolicySwitchInfo compactionPolicySwitchInfo = new CompactionPolicySwitchInfo(System.currentTimeMillis(), true);
        backUpCompactionPolicyInfo(tempDir.toString(), compactionPolicySwitchInfo);
        if (state.log.autoCloseLastLogSegmentIfQualified()) {
            // refresh journal.
            state.index.journal.cleanUpJournal();
        }
        // 1. index based token points to last entry of second last index segment + 1 -> index based token points to first
        // entry of last Index segment
        Offset lastIndexSegmentStartOffset = state.referenceIndex.lastKey();
        Offset secondLastSegmentStartOffset = state.referenceIndex.lowerKey(lastIndexSegmentStartOffset);
        TreeMap<MockId, TreeSet<IndexValue>> lastIndexSegment = state.referenceIndex.get(lastIndexSegmentStartOffset);
        Map.Entry<MockId, TreeSet<IndexValue>> lastIndexSegmentFirstEntry = lastIndexSegment.firstEntry();
        long maxTotalSizeOfEntries = getSizeOfAllValues(state.referenceIndex.get(lastIndexSegmentStartOffset).firstEntry().getValue());
        MockId lastId = state.referenceIndex.get(secondLastSegmentStartOffset).lastKey();
        StoreFindToken startToken = new StoreFindToken(lastId, secondLastSegmentStartOffset, state.sessionId, state.incarnationId, null, null, UNINITIALIZED_RESET_KEY_VERSION);
        IndexSegment segmentOfToken = state.index.getIndexSegments().get(lastIndexSegmentStartOffset);
        StoreFindToken expectedEndToken = new StoreFindToken(lastIndexSegmentFirstEntry.getKey(), lastIndexSegmentStartOffset, state.sessionId, state.incarnationId, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
        try {
            expectedEndToken.setBytesRead(state.index.getAbsoluteReadBytesFromIndexBasedToken(expectedEndToken));
        } catch (StoreException e) {
            expectedEndToken.setBytesRead(state.index.getAbsolutePositionInLogForOffset(lastIndexSegmentStartOffset));
        }
        Set<MockId> expectedKeys = new HashSet<>();
        expectedKeys.add(lastIndexSegmentFirstEntry.getKey());
        doFindEntriesSinceTest(startToken, maxTotalSizeOfEntries, expectedKeys, expectedEndToken);
        // 2. index based token points to last entry of last index segment -> index based token points to last entry of last
        // Index segment
        lastId = state.referenceIndex.get(lastIndexSegmentStartOffset).lastKey();
        startToken = new StoreFindToken(lastId, lastIndexSegmentStartOffset, state.sessionId, state.incarnationId, null, null, UNINITIALIZED_RESET_KEY_VERSION);
        Map.Entry<MockId, TreeSet<IndexValue>> lastIndexSegmentLastEntry = lastIndexSegment.lastEntry();
        expectedEndToken = new StoreFindToken(lastIndexSegmentLastEntry.getKey(), lastIndexSegmentStartOffset, state.sessionId, state.incarnationId, null, null, UNINITIALIZED_RESET_KEY_VERSION);
        try {
            expectedEndToken.setBytesRead(state.index.getAbsoluteReadBytesFromIndexBasedToken(expectedEndToken));
        } catch (StoreException e) {
            expectedEndToken.setBytesRead(state.index.getAbsolutePositionInLogForOffset(lastIndexSegmentStartOffset));
        }
        expectedKeys = new HashSet<>();
        maxTotalSizeOfEntries = getSizeOfAllValues(state.referenceIndex.get(lastIndexSegmentStartOffset).lastEntry().getValue());
        doFindEntriesSinceTest(startToken, maxTotalSizeOfEntries, expectedKeys, expectedEndToken);
    }
}

Example 34 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method rebuildTokenWhenLastSegmentAutoClosedTestCase1.

/**
 * Test generating a journal based token and close last log segment, should return the index based token point to first
 * index entry of last index segment.
 * Then input the output index based token and check if we can move the token forward until it reached to the end of
 * last index segment.
 * @throws StoreException
 */
@Test
public void rebuildTokenWhenLastSegmentAutoClosedTestCase1() throws StoreException {
    assumeTrue(isLogSegmented);
    IndexSegment lastIndexSegment = stateForTokenTest.index.getIndexSegments().lastEntry().getValue();
    Offset validOffset = lastIndexSegment.getStartOffset();
    StoreFindToken startToken = new StoreFindToken(validOffset, stateForTokenTest.sessionId, stateForTokenTest.incarnationId, false, lastIndexSegment.getResetKey(), lastIndexSegment.getResetKeyType(), lastIndexSegment.getResetKeyLifeVersion());
    autoCloseLastLogSegmentAndCleanUpJournal();
    StoreFindToken token = startToken;
    StoreFindToken expectedToken;
    int totalIndexEntryInLastIndexSegment = lastIndexSegment.size();
    for (int i = 0; i <= totalIndexEntryInLastIndexSegment; i++) {
        FindInfo findInfo = stateForTokenTest.index.findEntriesSince(token, 1);
        token = (StoreFindToken) findInfo.getFindToken();
        if (i < totalIndexEntryInLastIndexSegment) {
            // the index based token is moving forward after call findEntriesSince until it reached to the end of last index segment.
            expectedToken = new StoreFindToken(lastIndexSegment.listIterator(i + 1).previous().getKey(), lastIndexSegment.getStartOffset(), stateForTokenTest.sessionId, stateForTokenTest.incarnationId, lastIndexSegment.getResetKey(), lastIndexSegment.getResetKeyType(), lastIndexSegment.getResetKeyLifeVersion());
        } else {
            // after index based token point to the last entry of index segment, should return directly return the original index based token.
            expectedToken = new StoreFindToken(lastIndexSegment.listIterator(totalIndexEntryInLastIndexSegment).previous().getKey(), lastIndexSegment.getStartOffset(), stateForTokenTest.sessionId, stateForTokenTest.incarnationId, lastIndexSegment.getResetKey(), lastIndexSegment.getResetKeyType(), lastIndexSegment.getResetKeyLifeVersion());
        }
        compareTokens(expectedToken, token);
    }
}

Example 35 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method findEntriesSinceToIndexBasedTest.

// findEntriesSinceTest() helpers
/**
 * Tests all cases of {@link PersistentIndex#findEntriesSince(FindToken, long)} that result in an index based
 * {@link StoreFindToken} being returned.
 * 1. Uninited -> Index
 * 2. Index -> Index
 * 3. Journal -> Index
 * @throws StoreException
 */
private void findEntriesSinceToIndexBasedTest() throws StoreException {
    // ------------------
    // 1. Index -> Index
    Offset firstIndexSegmentStartOffset = state.referenceIndex.firstKey();
    Offset secondIndexSegmentStartOffset = state.referenceIndex.higherKey(firstIndexSegmentStartOffset);
    MockId firstId = state.referenceIndex.get(firstIndexSegmentStartOffset).firstKey();
    // All elements from first index segment and two from the second to be returned (because of size restrictions)
    Set<MockId> expectedKeys = new HashSet<>();
    long maxTotalSizeOfEntries = 0;
    for (Map.Entry<MockId, TreeSet<IndexValue>> segmentEntry : state.referenceIndex.get(firstIndexSegmentStartOffset).entrySet()) {
        if (!segmentEntry.getKey().equals(firstId)) {
            expectedKeys.add(segmentEntry.getKey());
            maxTotalSizeOfEntries += getSizeOfAllValues(segmentEntry.getValue());
        }
    }
    TreeMap<MockId, TreeSet<IndexValue>> secondIndexSegment = state.referenceIndex.get(secondIndexSegmentStartOffset);
    Map.Entry<MockId, TreeSet<IndexValue>> secondIndexSegmentEntry = secondIndexSegment.firstEntry();
    expectedKeys.add(secondIndexSegmentEntry.getKey());
    maxTotalSizeOfEntries += getSizeOfAllValues(secondIndexSegmentEntry.getValue());
    secondIndexSegmentEntry = secondIndexSegment.higherEntry(secondIndexSegmentEntry.getKey());
    expectedKeys.add(secondIndexSegmentEntry.getKey());
    maxTotalSizeOfEntries += getSizeOfAllValues(secondIndexSegmentEntry.getValue());
    StoreFindToken startToken = new StoreFindToken(firstId, firstIndexSegmentStartOffset, state.sessionId, state.incarnationId, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    IndexSegment segmentOfToken = state.index.getIndexSegments().get(secondIndexSegmentStartOffset);
    StoreFindToken expectedEndToken = new StoreFindToken(secondIndexSegmentEntry.getKey(), secondIndexSegmentStartOffset, state.sessionId, state.incarnationId, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
    expectedEndToken.setBytesRead(state.index.getAbsolutePositionInLogForOffset(secondIndexSegmentStartOffset));
    doFindEntriesSinceTest(startToken, maxTotalSizeOfEntries, expectedKeys, expectedEndToken);
    // ------------------
    // 2. Uninitialized -> Index
    // add firstStoreKey and its size
    expectedKeys.add(firstId);
    maxTotalSizeOfEntries += getSizeOfAllValues(state.referenceIndex.get(firstIndexSegmentStartOffset).firstEntry().getValue());
    doFindEntriesSinceTest(new StoreFindToken(), maxTotalSizeOfEntries, expectedKeys, expectedEndToken);
    // ------------------
    // 3. Journal -> Index
    // create a journal based token for an offset that isn't in the journal
    startToken = new StoreFindToken(state.logOrder.firstKey(), state.sessionId, state.incarnationId, false, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    doFindEntriesSinceTest(startToken, maxTotalSizeOfEntries, expectedKeys, expectedEndToken);
}