Examples with CuratedLogIndexState com.github.ambry.store.CuratedLogIndexState used on opensource projects

Example 1 with CuratedLogIndexState

use of com.github.ambry.store.CuratedLogIndexState in project ambry by linkedin.

the class IndexTest method getAbsoluteEndPositionOfLastPutTest.

/**
 * Test getting absolute end position of last PUT under different circumstances (i.e. last PUT is out of Journal)
 * @throws Exception
 */
@Test
public void getAbsoluteEndPositionOfLastPutTest() throws Exception {
    File testDir = StoreTestUtils.createTempDirectory("indexDirTest-" + TestUtils.getRandomString(10));
    CuratedLogIndexState indexState = new CuratedLogIndexState(isLogSegmented, testDir, false, false, true, true, false, false);
    assertEquals("There should be no PUT record since index is empty", -1, indexState.index.getAbsoluteEndPositionOfLastPut());
    long expiresAtMs = indexState.time.milliseconds() + TimeUnit.HOURS.toMillis(1);
    // DEFAULT_MAX_IN_MEM_ELEMENTS = 5, here we put 5 entries into the log
    List<IndexEntry> indexEntries = indexState.addPutEntries(5, PUT_RECORD_SIZE, expiresAtMs);
    assertEquals("Number of index segments should be 1", 1, indexState.index.getIndexSegments().size());
    // update ttl for above 5 entries and the ttl entries fall in second index segment.
    for (IndexEntry entry : indexEntries) {
        indexState.makePermanent((MockId) entry.getKey(), false);
    }
    assertEquals("Number of index segments (after ttl update) should be 2", 2, indexState.index.getIndexSegments().size());
    // default journal size should be 2 * DEFAULT_MAX_IN_MEM_ELEMENTS = 10
    assertEquals("Journal size not expected", 2 * DEFAULT_MAX_IN_MEM_ELEMENTS, indexState.index.journal.getAllEntries().size());
    // delete above 5 entries, these delete entries should fall in third index segment
    for (IndexEntry entry : indexEntries) {
        indexState.addDeleteEntry((MockId) entry.getKey());
    }
    assertEquals("Number of index segments (after deletion) should be 3", 3, indexState.index.getIndexSegments().size());
    // although there are now 15 entries in total, journal size is still 10 (old entries are removed from journal)
    assertEquals("Journal size (after deletion) shouldn't change", 2 * DEFAULT_MAX_IN_MEM_ELEMENTS, indexState.index.journal.getAllEntries().size());
    // after deletion, the last PUT record falls in first segment (out of journal)
    // for segmented log, there is a header size = 18
    assertEquals("Absolute end position of last PUT record not expected", 5 * PUT_RECORD_SIZE + (isLogSegmented ? 18 : 0), indexState.index.getAbsoluteEndPositionOfLastPut());
    // close the index to seal all index segments
    indexState.index.close(false);
    // get end position of last PUT again, it should return same result (this is to test getting last PUT in sealed index segment)
    assertEquals("Absolute end position of last PUT record not expected", 5 * PUT_RECORD_SIZE + (isLogSegmented ? 18 : 0), indexState.index.getAbsoluteEndPositionOfLastPut());
    // calculate current end position in log segment (note that there are 5 PUT, 5 TTL update and 5 DELETE entries)
    long currentEndPosition = 5 * PUT_RECORD_SIZE + 5 * TTL_UPDATE_RECORD_SIZE + 5 * DELETE_RECORD_SIZE + (isLogSegmented ? 18 : 0);
    // add one more PUT entry and delete it afterwards
    indexEntries = indexState.addPutEntries(1, PUT_RECORD_SIZE, Utils.Infinite_Time);
    indexState.addDeleteEntry((MockId) indexEntries.get(0).getKey());
    assertEquals("Number of index segments after new PUT and delete should be 4", 4, indexState.index.getIndexSegments().size());
    // now, the latest PUT entry should be in the journal
    assertEquals("Absolute end position of last PUT record not expected", currentEndPosition + PUT_RECORD_SIZE, indexState.index.getAbsoluteEndPositionOfLastPut());
    indexState.destroy();
}

Example 2 with CuratedLogIndexState

use of com.github.ambry.store.CuratedLogIndexState in project ambry by linkedin.

the class BlobStoreStatsTest method testBucketingWithEmptyIndexToBegin.

@Test
public void testBucketingWithEmptyIndexToBegin() throws InterruptedException, StoreException, IOException, TimeoutException {
    assumeTrue(bucketingEnabled);
    state.destroy();
    assertTrue(tempDir.getAbsolutePath() + " could not be deleted", StoreTestUtils.cleanDirectory(tempDir, true));
    tempDir = StoreTestUtils.createTempDirectory("blobStoreStatsDir-" + TestUtils.getRandomString(10));
    state = new CuratedLogIndexState(true, tempDir, false, false, true, true, false, false);
    MockThrottler mockThrottler = new MockThrottler(new CountDownLatch(0), new CountDownLatch(0));
    throttlers.put(BlobStoreStats.IO_SCHEDULER_JOB_TYPE, mockThrottler);
    int bucketCount = 50;
    BlobStoreStats blobStoreStats = setupBlobStoreStats(bucketCount, 0);
    // Make sure the index scanner is finished and we can enqueue
    if (!TestUtils.checkAndSleep(() -> blobStoreStats.isRecentEntryQueueEnabled(), 10000)) {
        throw new TimeoutException("Time out to wait for IndexScanner to finish");
    }
    verifyContainerStorageStatsAndGetTotalValidSize(blobStoreStats, state.time.milliseconds());
    verifyAndGetLogSegmentValidSize(blobStoreStats, new TimeRange(state.time.milliseconds(), 0));
    long expiresAtInMs = state.time.milliseconds() + ((long) bucketCount - 2) * BUCKET_SPAN_IN_MS;
    // 3 new puts with expiry
    List<IndexEntry> newPutEntries = state.addPutEntries(3, PUT_RECORD_SIZE, expiresAtInMs);
    // 3 new put with no expiry
    newPutEntries.addAll(state.addPutEntries(3, PUT_RECORD_SIZE, Utils.Infinite_Time));
    for (IndexEntry entry : newPutEntries) {
        blobStoreStats.handleNewPutEntry(entry.getKey(), entry.getValue());
    }
    // delete one of the put with expiry
    MockId putWithExpiry = getIdToDelete(newPutEntries.get(0).getKey());
    newDelete(blobStoreStats, putWithExpiry);
    // delete one of the put without expiry
    MockId putWithoutExpiry = getIdToDelete(newPutEntries.get(newPutEntries.size() - 1).getKey());
    newDelete(blobStoreStats, putWithoutExpiry);
    // a probe put with a latch to inform us about the state of the queue
    CountDownLatch queueProcessedLatch = new CountDownLatch(1);
    blobStoreStats.handleNewPutEntry(null, new MockIndexValue(queueProcessedLatch, state.index.getCurrentEndOffset()));
    assertTrue("QueueProcessor took too long to process the new entries", queueProcessedLatch.await(3, TimeUnit.SECONDS));
    verifyContainerStorageStatsAndGetTotalValidSize(blobStoreStats, state.time.milliseconds());
    verifyAndGetLogSegmentValidSize(blobStoreStats, new TimeRange(state.time.milliseconds(), 0));
    advanceTimeToNextSecond();
    verifyContainerStorageStatsAndGetTotalValidSize(blobStoreStats, state.time.milliseconds());
    verifyAndGetLogSegmentValidSize(blobStoreStats, new TimeRange(state.time.milliseconds(), 0));
    long timeToLiveInMs = expiresAtInMs - state.time.milliseconds() < 0 ? 0 : expiresAtInMs - state.time.milliseconds();
    state.advanceTime(timeToLiveInMs + Time.MsPerSec);
    verifyContainerStorageStatsAndGetTotalValidSize(blobStoreStats, state.time.milliseconds());
    verifyAndGetLogSegmentValidSize(blobStoreStats, new TimeRange(state.time.milliseconds(), 0));
    blobStoreStats.close();
}

Example 3 with CuratedLogIndexState

use of com.github.ambry.store.CuratedLogIndexState in project ambry by linkedin.

the class BlobStoreStatsTest method testWithLowIndexEntries.

@Test
public void testWithLowIndexEntries() throws StoreException, IOException {
    state.destroy();
    assertTrue(tempDir.getAbsolutePath() + " could not be deleted", StoreTestUtils.cleanDirectory(tempDir, true));
    tempDir = StoreTestUtils.createTempDirectory("blobStoreStatsDir-" + TestUtils.getRandomString(10));
    state = new CuratedLogIndexState(true, tempDir, false, false, true, true, false, false);
    int bucketCount = bucketingEnabled ? 1 : 0;
    BlobStoreStats blobStoreStats = setupBlobStoreStats(bucketCount, 0);
    verifyContainerStorageStatsAndGetTotalValidSize(blobStoreStats, state.time.milliseconds());
    verifyAndGetLogSegmentValidSize(blobStoreStats, new TimeRange(state.time.milliseconds(), 0));
    blobStoreStats.close();
    state.addPutEntries(3, PUT_RECORD_SIZE, Utils.Infinite_Time);
    blobStoreStats = setupBlobStoreStats(bucketCount, 0);
    verifyContainerStorageStatsAndGetTotalValidSize(blobStoreStats, state.time.milliseconds());
    verifyAndGetLogSegmentValidSize(blobStoreStats, new TimeRange(state.time.milliseconds(), 0));
    blobStoreStats.close();
}