Examples with StoreFindToken com.github.ambry.store.StoreFindToken used on opensource projects

Example 26 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method doFindDeletedEntriesSinceTest.

/**
 * Does the test for {@link PersistentIndex#findDeletedEntriesSince(FindToken, long, long)}  and compares the token
 * received to the expected token. It also verifies that the index entries obtained are as expected.
 * @param startToken the {@link StoreFindToken} to provide to the function.
 * @param maxTotalSizeOfEntries the total size of entries to fetch
 * @param expectedKeys the keys expected in the returned entries.
 * @param expectedEndToken the {@link StoreFindToken} expected to be returned.
 * @throws StoreException
 */
private void doFindDeletedEntriesSinceTest(StoreFindToken startToken, long maxTotalSizeOfEntries, Set<MockId> expectedKeys, StoreFindToken expectedEndToken) throws StoreException {
    FindInfo findInfo = state.index.findDeletedEntriesSince(startToken, maxTotalSizeOfEntries, Long.MAX_VALUE);
    StoreFindToken token = (StoreFindToken) findInfo.getFindToken();
    compareTokens(expectedEndToken, token);
    List<MessageInfo> infos = findInfo.getMessageEntries();
    Set<StoreKey> keysExamined = new HashSet<>();
    for (MessageInfo info : infos) {
        IndexValue value = state.getExpectedValue((MockId) info.getStoreKey(), false);
        assertEquals("Inconsistent size", value.getSize(), info.getSize());
        assertTrue("Not deleted", info.isDeleted());
        assertEquals("Inconsistent expiresAtMs", value.getExpiresAtMs(), info.getExpirationTimeInMs());
        keysExamined.add(info.getStoreKey());
    }
    assertEquals("All keys should be present " + expectedKeys + "\n" + keysExamined, expectedKeys, keysExamined);
}

Example 27 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method rebuildTokenWhenInputIndexBasedTokenTest.

/**
 * Test generating a index based token mapping to last indexEntry not in journal. it will return the first journal
 * based token in journal.
 * @throws StoreException
 */
@Test
public void rebuildTokenWhenInputIndexBasedTokenTest() throws StoreException {
    assumeTrue(isLogSegmented);
    IndexSegment lastIndexSegment = stateForTokenTest.index.getIndexSegments().lastEntry().getValue();
    IndexSegment secondLastIndexSegment = stateForTokenTest.index.getIndexSegments().lowerEntry(lastIndexSegment.getStartOffset()).getValue();
    StoreKey lastStoreKeyInLastIndexSegment = secondLastIndexSegment.listIterator(DEFAULT_MAX_IN_MEM_ELEMENTS - 1).next().getKey();
    Offset firstOffsetInJournal = stateForTokenTest.index.journal.getFirstOffset();
    StoreFindToken startToken = new StoreFindToken(lastStoreKeyInLastIndexSegment, secondLastIndexSegment.getStartOffset(), stateForTokenTest.sessionId, stateForTokenTest.incarnationId, secondLastIndexSegment.getResetKey(), secondLastIndexSegment.getResetKeyType(), secondLastIndexSegment.getResetKeyLifeVersion());
    FindInfo findInfo = stateForTokenTest.index.findEntriesSince(startToken, 1);
    StoreFindToken token = (StoreFindToken) findInfo.getFindToken();
    StoreFindToken expectedToken;
    expectedToken = new StoreFindToken(firstOffsetInJournal, stateForTokenTest.sessionId, stateForTokenTest.incarnationId, false, lastIndexSegment.getResetKey(), lastIndexSegment.getResetKeyType(), lastIndexSegment.getResetKeyLifeVersion());
    compareTokens(expectedToken, token);
}

Example 28 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method findDeletedEntriesSinceToIndexBasedTest.

// findDeletedEntriesSinceTest() helpers
/**
 * Tests all cases of {@link PersistentIndex#findDeletedEntriesSince(FindToken, long, long)} that result in an index
 * based {@link StoreFindToken} being returned.
 * 1. Uninitialized -> Index
 * 2. Index -> Index
 * 3. Journal -> Index
 * @throws StoreException
 */
private void findDeletedEntriesSinceToIndexBasedTest() throws StoreException {
    // ------------------
    // 1. Index -> Index
    Offset secondIndexSegmentStartOffset = state.referenceIndex.higherKey(state.referenceIndex.firstKey());
    Map.Entry<MockId, TreeSet<IndexValue>> firstSegmentEntry = state.referenceIndex.get(secondIndexSegmentStartOffset).firstEntry();
    // Most elements from the second to be returned (because of size restrictions)
    Set<MockId> expectedKeys = new HashSet<>();
    long maxTotalSizeOfEntries = 0;
    MockId lastKey = null;
    for (Map.Entry<MockId, TreeSet<IndexValue>> segmentEntry : state.referenceIndex.get(secondIndexSegmentStartOffset).entrySet()) {
        if (!segmentEntry.equals(firstSegmentEntry)) {
            if (segmentEntry.getValue().last().isDelete() && state.getExpectedValue(segmentEntry.getKey(), EnumSet.of(PersistentIndex.IndexEntryType.UNDELETE), null) == null) {
                expectedKeys.add(segmentEntry.getKey());
            }
            maxTotalSizeOfEntries += getSizeOfAllValues(segmentEntry.getValue());
        }
        lastKey = segmentEntry.getKey();
    }
    StoreFindToken startToken = new StoreFindToken(firstSegmentEntry.getKey(), secondIndexSegmentStartOffset, state.sessionId, state.incarnationId, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    IndexSegment segmentOfToken = state.index.getIndexSegments().floorEntry(secondIndexSegmentStartOffset).getValue();
    StoreFindToken expectedEndToken = new StoreFindToken(lastKey, secondIndexSegmentStartOffset, state.sessionId, state.incarnationId, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
    doFindDeletedEntriesSinceTest(startToken, maxTotalSizeOfEntries, expectedKeys, expectedEndToken);
    // add size of values and any keys that are supposed to be returned from the first index segment
    for (Map.Entry<MockId, TreeSet<IndexValue>> segmentEntry : state.referenceIndex.firstEntry().getValue().entrySet()) {
        if (segmentEntry.getValue().last().isDelete() && state.getExpectedValue(segmentEntry.getKey(), EnumSet.of(PersistentIndex.IndexEntryType.UNDELETE), null) == null) {
            expectedKeys.add(segmentEntry.getKey());
        }
        maxTotalSizeOfEntries += getSizeOfAllValues(segmentEntry.getValue());
    }
    // add size of value of firstIdInSegment
    maxTotalSizeOfEntries += getSizeOfAllValues(firstSegmentEntry.getValue());
    if (firstSegmentEntry.getValue().last().isDelete() && state.getExpectedValue(firstSegmentEntry.getKey(), EnumSet.of(PersistentIndex.IndexEntryType.UNDELETE), null) == null) {
        expectedKeys.add(firstSegmentEntry.getKey());
    }
    doFindDeletedEntriesSinceTest(new StoreFindToken(), maxTotalSizeOfEntries, expectedKeys, expectedEndToken);
    // ------------------
    // 3. Journal -> Index
    // create a journal based token for an offset that isn't in the journal
    startToken = new StoreFindToken(state.logOrder.firstKey(), state.sessionId, state.incarnationId, false, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    doFindDeletedEntriesSinceTest(startToken, maxTotalSizeOfEntries, expectedKeys, expectedEndToken);
}

Example 29 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method getExpectedJournalEndToken.

/**
 * Create a journal-based find token that can be compared against a token returned by
 * {@link PersistentIndex#findEntriesSince}.
 * @param tokenOffset the offset to include in the token
 * @param lastKey the {@link StoreKey} of the last message that will be returned in the query.
 * @return a {@link StoreFindToken} with bytes read set.
 */
private StoreFindToken getExpectedJournalEndToken(Offset tokenOffset, StoreKey lastKey) throws StoreException {
    IndexSegment segmentOfToken = state.index.getIndexSegments().floorEntry(tokenOffset).getValue();
    StoreFindToken expectedEndToken = new StoreFindToken(tokenOffset, state.sessionId, state.incarnationId, false, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
    // the last message's size will be the size of the most recent log entry for its key, not necessarily the entry at
    // tokenOffset. This size is used to calculate bytes read. even if it is not strictly the entry at the log offset
    // of the last journal entry processed.
    long lastMessageSize = state.getExpectedValue((MockId) lastKey, EnumSet.allOf(PersistentIndex.IndexEntryType.class), null).getSize();
    Offset endOffset = state.log.getFileSpanForMessage(tokenOffset, lastMessageSize).getEndOffset();
    expectedEndToken.setBytesRead(state.index.getAbsolutePositionInLogForOffset(endOffset));
    return expectedEndToken;
}

Example 30 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method rebuildTokenWhenInputJournalBasedTokenTestCase2.

/**
 * Test generating the last journal based token, it will return the original token.
 * @throws StoreException
 */
@Test
public void rebuildTokenWhenInputJournalBasedTokenTestCase2() throws StoreException {
    assumeTrue(isLogSegmented);
    IndexSegment lastIndexSegment = stateForTokenTest.index.getIndexSegments().lastEntry().getValue();
    Offset validOffset = stateForTokenTest.index.journal.getLastOffset();
    StoreFindToken startToken = new StoreFindToken(validOffset, stateForTokenTest.sessionId, stateForTokenTest.incarnationId, false, lastIndexSegment.getResetKey(), lastIndexSegment.getResetKeyType(), lastIndexSegment.getResetKeyLifeVersion());
    FindInfo findInfo = stateForTokenTest.index.findEntriesSince(startToken, 1);
    StoreFindToken token = (StoreFindToken) findInfo.getFindToken();
    StoreFindToken expectedToken = startToken;
    compareTokens(expectedToken, token);
}