Examples with StoreFindToken com.github.ambry.store.StoreFindToken used on opensource projects

Example 11 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method doFindEntriesSinceTest.

/**
 * Does the test for {@link PersistentIndex#findEntriesSince(FindToken, long)} and compares the token received to
 * the expected token. It also verifies that the index entries obtained are as expected.
 * @param startToken the {@link StoreFindToken} to provide to the function.
 * @param maxTotalSizeOfEntries the total size of entries to fetch
 * @param expectedKeys the keys expected in the returned entries.
 * @param expectedEndToken the {@link StoreFindToken} expected to be returned.
 * @throws StoreException
 */
private void doFindEntriesSinceTest(StoreFindToken startToken, long maxTotalSizeOfEntries, Set<MockId> expectedKeys, StoreFindToken expectedEndToken) throws StoreException {
    FindInfo findInfo = state.index.findEntriesSince(startToken, maxTotalSizeOfEntries);
    StoreFindToken token = (StoreFindToken) findInfo.getFindToken();
    compareTokens(expectedEndToken, token);
    assertEquals("Returned token should have the right number of bytes read", expectedEndToken.getBytesRead(), token.getBytesRead());
    List<MessageInfo> infos = findInfo.getMessageEntries();
    Set<StoreKey> keysExamined = new HashSet<>();
    for (MessageInfo info : infos) {
        MockId id = (MockId) info.getStoreKey();
        IndexValue putValue = state.getExpectedValue(id, EnumSet.of(PersistentIndex.IndexEntryType.PUT), null);
        IndexValue ttlUpdateValue = state.getExpectedValue(id, EnumSet.of(PersistentIndex.IndexEntryType.TTL_UPDATE), null);
        IndexValue deleteValue = state.getExpectedValue(id, EnumSet.of(PersistentIndex.IndexEntryType.DELETE), null);
        IndexValue undeleteValue = state.getExpectedValue(id, EnumSet.of(PersistentIndex.IndexEntryType.UNDELETE), null);
        // size returned is hard to predict if the key has been updated - it depends on the locations of the PUT and
        // update entries and whether all or some of them are present in the return list. It is not useful to recompute
        // the situations here and check
        long expiresAtMs = undeleteValue != null ? undeleteValue.getExpiresAtMs() : deleteValue != null ? deleteValue.getExpiresAtMs() : ttlUpdateValue != null ? ttlUpdateValue.getExpiresAtMs() : putValue.getExpiresAtMs();
        short accountId = undeleteValue != null ? undeleteValue.getAccountId() : deleteValue != null ? deleteValue.getAccountId() : ttlUpdateValue != null ? ttlUpdateValue.getAccountId() : putValue.getAccountId();
        short containerId = undeleteValue != null ? undeleteValue.getContainerId() : deleteValue != null ? deleteValue.getContainerId() : ttlUpdateValue != null ? ttlUpdateValue.getContainerId() : putValue.getContainerId();
        long operationTimeMs = undeleteValue != null ? undeleteValue.getOperationTimeInMs() : deleteValue != null ? deleteValue.getOperationTimeInMs() : ttlUpdateValue != null ? ttlUpdateValue.getOperationTimeInMs() : putValue.getOperationTimeInMs();
        boolean isTtlUpdated = undeleteValue != null ? undeleteValue.isTtlUpdate() : deleteValue != null ? deleteValue.isTtlUpdate() : ttlUpdateValue != null;
        // the one that is returned.
        if (undeleteValue != null || deleteValue != null) {
            // there is either delete or undelete
            IndexValue lastValue = state.getExpectedValue(id, EnumSet.of(PersistentIndex.IndexEntryType.DELETE, PersistentIndex.IndexEntryType.UNDELETE), null);
            assertEquals("Inconsistent delete state for key " + id, lastValue.isDelete(), info.isDeleted());
            assertEquals("Inconsistent undelete state for key " + id, lastValue.isUndelete(), info.isUndeleted());
        }
        assertEquals("Inconsistent TTL update state", isTtlUpdated, info.isTtlUpdated());
        assertEquals("Inconsistent expiresAtMs", expiresAtMs, info.getExpirationTimeInMs());
        assertEquals("Inconsistent accountId", accountId, info.getAccountId());
        assertEquals("Inconsistent containerId", containerId, info.getContainerId());
        assertEquals("Inconsistent operationTimeMs", operationTimeMs, info.getOperationTimeMs());
        keysExamined.add(info.getStoreKey());
    }
    assertEquals("All keys should be present", expectedKeys, keysExamined);
}

Example 12 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class CompactionLogTest method iterationWithReloadTest.

/**
 * Tests the use of {@link CompactionLog} when it is closed and reloaded b/w operations and compaction cycles.
 * @throws IOException
 */
@Test
public void iterationWithReloadTest() throws IOException {
    String storeName = "store";
    List<CompactionDetails> detailsList = getCompactionDetailsList(5);
    CompactionDetails combined = combineListOfDetails(detailsList);
    assertFalse("Compaction should not be in progress", CompactionLog.isCompactionInProgress(tempDirStr, storeName));
    CompactionLog cLog = new CompactionLog(tempDirStr, storeName, time, combined, config);
    assertTrue("Compaction should should be in progress", CompactionLog.isCompactionInProgress(tempDirStr, storeName));
    int currentIdx = 0;
    Iterator<CompactionDetails> detailsIterator = detailsList.iterator();
    CompactionDetails currDetails = detailsIterator.next();
    while (currDetails != null) {
        detailsIterator.remove();
        assertEquals("CurrentIdx not as expected", currentIdx, cLog.getCurrentIdx());
        cLog.close();
        cLog = new CompactionLog(tempDirStr, storeName, STORE_KEY_FACTORY, time, config);
        verifyEquality(combined, cLog.getCompactionDetails());
        assertEquals("Should be in the PREPARE phase", CompactionLog.Phase.PREPARE, cLog.getCompactionPhase());
        cLog.markCopyStart();
        cLog.close();
        cLog = new CompactionLog(tempDirStr, storeName, STORE_KEY_FACTORY, time, config);
        assertEquals("Should be in the COPY phase", CompactionLog.Phase.COPY, cLog.getCompactionPhase());
        Offset offset = new Offset(LogSegmentName.generateFirstSegmentName(true), Utils.getRandomLong(TestUtils.RANDOM, Long.MAX_VALUE));
        cLog.setStartOffsetOfLastIndexSegmentForDeleteCheck(offset);
        cLog.close();
        cLog = new CompactionLog(tempDirStr, storeName, STORE_KEY_FACTORY, time, config);
        assertEquals("Offset that was set was not the one returned", offset, cLog.getStartOffsetOfLastIndexSegmentForDeleteCheck());
        StoreFindToken safeToken = new StoreFindToken(new MockId("dummy"), new Offset(LogSegmentName.generateFirstSegmentName(true), 0), new UUID(1, 1), new UUID(1, 1), null, null, UNINITIALIZED_RESET_KEY_VERSION);
        cLog.setSafeToken(safeToken);
        cLog.close();
        cLog = new CompactionLog(tempDirStr, storeName, STORE_KEY_FACTORY, time, config);
        assertEquals("Returned token not the same as the one that was set", safeToken, cLog.getSafeToken());
        CompactionDetails nextDetails = detailsIterator.hasNext() ? detailsIterator.next() : null;
        if (nextDetails != null) {
            cLog.splitCurrentCycle(nextDetails.getLogSegmentsUnderCompaction().get(0));
            verifyEquality(currDetails, cLog.getCompactionDetails());
            cLog.close();
            cLog = new CompactionLog(tempDirStr, storeName, STORE_KEY_FACTORY, time, config);
            verifyEquality(currDetails, cLog.getCompactionDetails());
        }
        cLog.markCommitStart();
        cLog.close();
        cLog = new CompactionLog(tempDirStr, storeName, STORE_KEY_FACTORY, time, config);
        assertEquals("Should be in the SWITCH phase", CompactionLog.Phase.COMMIT, cLog.getCompactionPhase());
        cLog.markCleanupStart();
        cLog.close();
        cLog = new CompactionLog(tempDirStr, storeName, STORE_KEY_FACTORY, time, config);
        assertEquals("Should be in the CLEANUP phase", CompactionLog.Phase.CLEANUP, cLog.getCompactionPhase());
        cLog.markCycleComplete();
        currentIdx++;
        currDetails = nextDetails;
        if (nextDetails != null) {
            combined = combineListOfDetails(detailsList);
        }
    }
    assertEquals("CurrentIdx not as expected", -1, cLog.getCurrentIdx());
    assertEquals("Should be in the DONE phase", CompactionLog.Phase.DONE, cLog.getCompactionPhase());
    cLog.close();
    assertFalse("Compaction should not be in progress", CompactionLog.isCompactionInProgress(tempDirStr, storeName));
}

Example 13 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class CompactionLogTest method checkTransitionFailure.

// phaseTransitionEnforcementTest() helpers
/**
 * Checks that transitions to phases in {@code transitionToFailures} fails.
 * @param cLog the {@link CompactionLog} being used.
 * @param transitionToFailures the list of phases, transition to which should fail.
 */
private void checkTransitionFailure(CompactionLog cLog, CompactionLog.Phase... transitionToFailures) {
    for (CompactionLog.Phase transitionToFailure : transitionToFailures) {
        try {
            switch(transitionToFailure) {
                case COPY:
                    cLog.markCopyStart();
                    break;
                case COMMIT:
                    cLog.markCommitStart();
                    break;
                case CLEANUP:
                    cLog.markCleanupStart();
                    break;
                case DONE:
                    cLog.markCycleComplete();
                    break;
                default:
                    throw new IllegalArgumentException("Unknown Phase: " + transitionToFailure);
            }
            fail("Transition should have failed");
        } catch (IllegalStateException e) {
        // expected. Nothing to do.
        }
    }
    if (!cLog.getCompactionPhase().equals(CompactionLog.Phase.COPY)) {
        StoreFindToken safeToken = new StoreFindToken(new MockId("dummy"), new Offset(LogSegmentName.generateFirstSegmentName(true), 0), new UUID(1, 1), new UUID(1, 1), null, null, UNINITIALIZED_RESET_KEY_VERSION);
        try {
            cLog.setSafeToken(safeToken);
            fail("Setting safe token should have failed");
        } catch (IllegalStateException e) {
        // expected. Nothing to do.
        }
    }
}

Example 14 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method rebuildTokenWhenLastSegmentAutoClosedTestCase3.

/**
 * Test closing last log segment and generating an index based token pointing to last entry of last index segment,
 * it should return the same index token.
 * @throws StoreException
 */
@Test
public void rebuildTokenWhenLastSegmentAutoClosedTestCase3() throws StoreException {
    assumeTrue(isLogSegmented);
    autoCloseLastLogSegmentAndCleanUpJournal();
    IndexSegment lastIndexSegment = stateForTokenTest.index.getIndexSegments().lastEntry().getValue();
    Offset validOffset = new Offset(lastIndexSegment.getLogSegmentName(), lastIndexSegment.getStartOffset().getOffset());
    StoreFindToken startToken = new StoreFindToken(lastIndexSegment.listIterator(lastIndexSegment.size()).previous().getKey(), validOffset, stateForTokenTest.sessionId, stateForTokenTest.incarnationId, lastIndexSegment.getResetKey(), lastIndexSegment.getResetKeyType(), lastIndexSegment.getResetKeyLifeVersion());
    FindInfo findInfo = stateForTokenTest.index.findEntriesSince(startToken, 1);
    StoreFindToken token = (StoreFindToken) findInfo.getFindToken();
    StoreFindToken expectedToken = startToken;
    compareTokens(expectedToken, token);
}

Example 15 with StoreFindToken

use of com.github.ambry.store.StoreFindToken in project ambry by linkedin.

the class IndexTest method findEntriesSinceIndexBasedTokenForOffsetInJournalTest.

/**
 * Tests {@link PersistentIndex#findEntriesSince(FindToken, long)} when an index based {@link StoreFindToken} has been
 * given out for an offset in the {@link Journal}.
 * @throws StoreException
 */
private void findEntriesSinceIndexBasedTokenForOffsetInJournalTest() throws StoreException {
    Map.Entry<Offset, TreeMap<MockId, TreeSet<IndexValue>>> indexEntry = state.referenceIndex.floorEntry(state.index.journal.getFirstOffset());
    Offset nextIndexSegmentStartOffset = state.referenceIndex.higherKey(indexEntry.getKey());
    MockId firstIdInSegment = indexEntry.getValue().firstKey();
    StoreFindToken startToken = new StoreFindToken(firstIdInSegment, indexEntry.getKey(), state.sessionId, state.incarnationId, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    long maxSize = 0;
    Set<MockId> expectedKeys = new HashSet<>();
    for (Map.Entry<MockId, TreeSet<IndexValue>> indexSegmentEntry : indexEntry.getValue().entrySet()) {
        if (!firstIdInSegment.equals(indexSegmentEntry.getKey())) {
            expectedKeys.add(indexSegmentEntry.getKey());
            maxSize += getSizeOfAllValues(indexSegmentEntry.getValue());
        }
    }
    MockId logId = state.logOrder.get(nextIndexSegmentStartOffset).getFirst();
    expectedKeys.add(logId);
    long size = state.deletedKeys.contains(logId) ? CuratedLogIndexState.DELETE_RECORD_SIZE : state.logOrder.get(nextIndexSegmentStartOffset).getSecond().indexValue.getSize();
    maxSize += size;
    Offset endOffset = state.log.getFileSpanForMessage(nextIndexSegmentStartOffset, size).getEndOffset();
    IndexSegment segmentOfToken = state.index.getIndexSegments().get(nextIndexSegmentStartOffset);
    StoreFindToken expectedEndToken = new StoreFindToken(nextIndexSegmentStartOffset, state.sessionId, state.incarnationId, false, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
    expectedEndToken.setBytesRead(state.index.getAbsolutePositionInLogForOffset(endOffset));
    doFindEntriesSinceTest(startToken, maxSize, expectedKeys, expectedEndToken);
}