Examples with StoreFindToken com.github.ambry.store.StoreFindToken used on opensource projects

Example 21 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method findEntriesSinceToJournalBasedTest.

/**
 * Tests all cases of {@link PersistentIndex#findEntriesSince(FindToken, long)} that result in an journal based
 * {@link StoreFindToken} being returned.
 * 1. Uninited -> Journal
 * 2. Index -> Journal
 * 3. Journal -> Journal
 * 4. No movement.
 * @throws StoreException
 */
private void findEntriesSinceToJournalBasedTest() throws StoreException {
    IndexSegment segmentOfToken = state.index.getIndexSegments().lastEntry().getValue();
    StoreFindToken absoluteEndToken = new StoreFindToken(state.logOrder.lastKey(), state.sessionId, state.incarnationId, false, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
    absoluteEndToken.setBytesRead(state.index.getLogUsedCapacity());
    // ------------------
    // 1. Uninitialized -> Journal
    doFindEntriesSinceTest(new StoreFindToken(), Long.MAX_VALUE, state.allKeys.keySet(), absoluteEndToken);
    // ------------------
    // 2. Index -> Journal
    Offset firstIndexSegmentStartOffset = state.referenceIndex.firstKey();
    StoreKey firstStoreKey = state.referenceIndex.get(firstIndexSegmentStartOffset).firstKey();
    StoreFindToken startToken = new StoreFindToken(firstStoreKey, firstIndexSegmentStartOffset, state.sessionId, state.incarnationId, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    Set<MockId> expectedKeys = new HashSet<>(state.allKeys.keySet());
    if (!state.deletedKeys.contains(firstStoreKey)) {
        // if firstStoreKey has not been deleted, it will not show up in findEntries since its PUT record is ignored
        expectedKeys.remove(firstStoreKey);
    }
    doFindEntriesSinceTest(startToken, Long.MAX_VALUE, expectedKeys, absoluteEndToken);
    // ------------------
    // 3. Journal -> Journal
    // a. Token no longer in journal
    startToken = new StoreFindToken(state.logOrder.firstKey(), state.sessionId, state.incarnationId, false, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    doFindEntriesSinceTest(startToken, Long.MAX_VALUE, state.allKeys.keySet(), absoluteEndToken);
    // b. Token still in journal
    startToken = new StoreFindToken(state.index.journal.getFirstOffset(), state.sessionId, state.incarnationId, false, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    expectedKeys = new HashSet<>();
    for (Map.Entry<Offset, Pair<MockId, CuratedLogIndexState.LogEntry>> entry : state.logOrder.tailMap(startToken.getOffset(), false).entrySet()) {
        expectedKeys.add(entry.getValue().getFirst());
    }
    doFindEntriesSinceTest(startToken, Long.MAX_VALUE, expectedKeys, absoluteEndToken);
    // c. Token still in journal with inclusiveness set to true
    startToken = new StoreFindToken(state.index.journal.getFirstOffset(), state.sessionId, state.incarnationId, true, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    expectedKeys.add(state.logOrder.tailMap(startToken.getOffset(), true).firstEntry().getValue().getFirst());
    doFindEntriesSinceTest(startToken, Long.MAX_VALUE, expectedKeys, absoluteEndToken);
    // ------------------
    // 4. Journal no change
    doFindEntriesSinceTest(absoluteEndToken, Long.MAX_VALUE, Collections.emptySet(), absoluteEndToken);
}

Example 22 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method rebuildTokenWhenInputJournalBasedTokenTestCase1.

/**
 * Tests generating a journal based token mapping to first indexEntry in last index segment. it will return the next
 * journal based token in journal.
 * @throws StoreException
 */
@Test
public void rebuildTokenWhenInputJournalBasedTokenTestCase1() throws StoreException {
    assumeTrue(isLogSegmented);
    IndexSegment lastIndexSegment = stateForTokenTest.index.getIndexSegments().lastEntry().getValue();
    Offset validOffset = new Offset(lastIndexSegment.getLogSegmentName(), lastIndexSegment.getStartOffset().getOffset());
    Offset nextOffset = stateForTokenTest.index.journal.getEntriesSince(validOffset, false).get(0).getOffset();
    StoreFindToken startToken = new StoreFindToken(validOffset, stateForTokenTest.sessionId, stateForTokenTest.incarnationId, false, lastIndexSegment.getResetKey(), lastIndexSegment.getResetKeyType(), lastIndexSegment.getResetKeyLifeVersion());
    FindInfo findInfo = stateForTokenTest.index.findEntriesSince(startToken, 1);
    StoreFindToken token = (StoreFindToken) findInfo.getFindToken();
    StoreFindToken expectedToken = new StoreFindToken(nextOffset, stateForTokenTest.sessionId, stateForTokenTest.incarnationId, false, lastIndexSegment.getResetKey(), lastIndexSegment.getResetKeyType(), lastIndexSegment.getResetKeyLifeVersion());
    compareTokens(expectedToken, token);
}

Example 23 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method findEntriesSinceOneByOneTest.

/**
 * Uses {@link PersistentIndex#findEntriesSince(FindToken, long)} to get entries one by one.
 * @throws StoreException
 */
private void findEntriesSinceOneByOneTest() throws StoreException {
    Offset journalStartOffset = state.index.journal.getFirstOffset();
    StoreFindToken startToken = new StoreFindToken();
    Offset stoppedAt = null;
    for (Map.Entry<Offset, TreeMap<MockId, TreeSet<IndexValue>>> indexEntry : state.referenceIndex.entrySet()) {
        Offset indexSegmentStartOffset = indexEntry.getKey();
        // 2. The size of entries being obtained is <= the size of records in the current index segment
        if (indexSegmentStartOffset.compareTo(journalStartOffset) >= 0) {
            stoppedAt = indexSegmentStartOffset;
            break;
        }
        IndexSegment segmentOfToken = state.index.getIndexSegments().get(indexSegmentStartOffset);
        for (Map.Entry<MockId, TreeSet<IndexValue>> indexSegmentEntry : indexEntry.getValue().entrySet()) {
            MockId id = indexSegmentEntry.getKey();
            StoreFindToken expectedEndToken = new StoreFindToken(id, indexSegmentStartOffset, state.sessionId, state.incarnationId, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
            expectedEndToken.setBytesRead(state.index.getAbsolutePositionInLogForOffset(indexSegmentStartOffset));
            doFindEntriesSinceTest(startToken, getSizeOfAllValues(indexSegmentEntry.getValue()), Collections.singleton(id), expectedEndToken);
            startToken = expectedEndToken;
        }
    }
    Map.Entry<Offset, Pair<MockId, CuratedLogIndexState.LogEntry>> logEntry = state.logOrder.floorEntry(stoppedAt);
    while (logEntry != null) {
        Offset startOffset = logEntry.getKey();
        MockId id = logEntry.getValue().getFirst();
        // size returned is the size of the most recent record
        long size = state.getExpectedValue(id, EnumSet.allOf(PersistentIndex.IndexEntryType.class), null).getSize();
        IndexSegment segmentOfToken = state.index.getIndexSegments().floorEntry(startOffset).getValue();
        StoreFindToken expectedEndToken = new StoreFindToken(startOffset, state.sessionId, state.incarnationId, false, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
        Offset endOffset = state.log.getFileSpanForMessage(startOffset, size).getEndOffset();
        expectedEndToken.setBytesRead(state.index.getAbsolutePositionInLogForOffset(endOffset));
        doFindEntriesSinceTest(startToken, size, Collections.singleton(id), expectedEndToken);
        startToken = expectedEndToken;
        logEntry = state.logOrder.higherEntry(logEntry.getKey());
    }
}

Example 24 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method findDeletedEntriesSinceIndexBasedTokenForOffsetInJournalTest.

/**
 * Tests {@link PersistentIndex#findDeletedEntriesSince(FindToken, long, long)} when an index based
 * {@link StoreFindToken} has been given out for an offset in the {@link Journal}.
 * @throws StoreException
 */
private void findDeletedEntriesSinceIndexBasedTokenForOffsetInJournalTest() throws StoreException {
    Map.Entry<Offset, TreeMap<MockId, TreeSet<IndexValue>>> indexEntry = state.referenceIndex.floorEntry(state.index.journal.getFirstOffset());
    Offset nextIndexSegmentStartOffset = state.referenceIndex.higherKey(indexEntry.getKey());
    MockId firstIdInSegment = indexEntry.getValue().firstKey();
    StoreFindToken startToken = new StoreFindToken(firstIdInSegment, indexEntry.getKey(), state.sessionId, state.incarnationId, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    long maxSize = 0;
    Set<MockId> expectedKeys = new HashSet<>();
    for (Map.Entry<MockId, TreeSet<IndexValue>> indexSegmentEntry : indexEntry.getValue().entrySet()) {
        if (!firstIdInSegment.equals(indexSegmentEntry.getKey())) {
            if (indexSegmentEntry.getValue().last().isDelete()) {
                expectedKeys.add(indexSegmentEntry.getKey());
            }
            maxSize += getSizeOfAllValues(indexSegmentEntry.getValue());
        }
    }
    MockId logId = state.logOrder.get(nextIndexSegmentStartOffset).getFirst();
    long size = state.logOrder.get(nextIndexSegmentStartOffset).getSecond().indexValue.getSize();
    if (state.deletedKeys.contains(logId)) {
        expectedKeys.add(logId);
        size = CuratedLogIndexState.DELETE_RECORD_SIZE;
    }
    maxSize += size;
    IndexSegment segmentOfToken = state.index.getIndexSegments().get(nextIndexSegmentStartOffset);
    StoreFindToken expectedEndToken = new StoreFindToken(nextIndexSegmentStartOffset, state.sessionId, state.incarnationId, false, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
    doFindDeletedEntriesSinceTest(startToken, maxSize, expectedKeys, expectedEndToken);
}

Example 25 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method rebuildTokenWhenLastSegmentAutoClosedTestCase4.

/**
 * Test generate a journal based token point to last entry of index segment, and close the last log segment and clean up journal.
 * It should first return the index based token point to the last IndexSegment startOffset and first key.
 * Test after two more entries added in current last log segment, when put the index based token point to last entry of last index segment,
 * should return the journal based token point to first entry of journal.
 * @throws StoreException
 */
@Test
public void rebuildTokenWhenLastSegmentAutoClosedTestCase4() throws StoreException {
    assumeTrue(isLogSegmented);
    IndexSegment lastIndexSegment = stateForTokenTest.index.getIndexSegments().lastEntry().getValue();
    Offset validOffset = stateForTokenTest.index.journal.getLastOffset();
    StoreFindToken startToken = new StoreFindToken(validOffset, stateForTokenTest.sessionId, stateForTokenTest.incarnationId, false, lastIndexSegment.getResetKey(), lastIndexSegment.getResetKeyType(), lastIndexSegment.getResetKeyLifeVersion());
    autoCloseLastLogSegmentAndCleanUpJournal();
    FindInfo findInfo = stateForTokenTest.index.findEntriesSince(startToken, 1);
    StoreFindToken token = (StoreFindToken) findInfo.getFindToken();
    StoreFindToken expectedToken = new StoreFindToken(lastIndexSegment.iterator().next().getKey(), lastIndexSegment.getStartOffset(), stateForTokenTest.sessionId, stateForTokenTest.incarnationId, lastIndexSegment.getResetKey(), lastIndexSegment.getResetKeyType(), lastIndexSegment.getResetKeyLifeVersion());
    compareTokens(expectedToken, token);
    stateForTokenTest.addPutEntries(2, CuratedLogIndexState.PUT_RECORD_SIZE, Utils.Infinite_Time);
    // put the index based token point to last entry of last index segment, should return the journal based token point to first entry of journal.
    lastIndexSegment = stateForTokenTest.index.getIndexSegments().lastEntry().getValue();
    IndexSegment secondLastIndexSegment = stateForTokenTest.index.getIndexSegments().lowerEntry(lastIndexSegment.getStartOffset()).getValue();
    StoreKey lastStoreKeyInSecondLastIndexSegment = secondLastIndexSegment.listIterator(secondLastIndexSegment.size()).previous().getKey();
    startToken = new StoreFindToken(lastStoreKeyInSecondLastIndexSegment, secondLastIndexSegment.getStartOffset(), stateForTokenTest.sessionId, stateForTokenTest.incarnationId, secondLastIndexSegment.getResetKey(), secondLastIndexSegment.getResetKeyType(), secondLastIndexSegment.getResetKeyLifeVersion());
    findInfo = stateForTokenTest.index.findEntriesSince(startToken, 1);
    token = (StoreFindToken) findInfo.getFindToken();
    expectedToken = new StoreFindToken(stateForTokenTest.index.journal.getFirstOffset(), stateForTokenTest.sessionId, stateForTokenTest.incarnationId, false, lastIndexSegment.getResetKey(), lastIndexSegment.getResetKeyType(), lastIndexSegment.getResetKeyLifeVersion());
    compareTokens(expectedToken, token);
}