Examples with StoreConfig com.github.ambry.config.StoreConfig used on opensource projects

Example 51 with StoreConfig

use of com.github.ambry.config.StoreConfig in project ambry by linkedin.

the class CompactionManagerTest method testCompactionWithMisbehavingStores.

/**
 * Tests that compaction proceeds on all non misbehaving stores even in the presence of some misbehaving stores
 * (not started/throwing exceptions).
 * @throws Exception
 */
@Test
public void testCompactionWithMisbehavingStores() throws Exception {
    int numStores = 5;
    List<BlobStore> stores = new ArrayList<>();
    // one store that isn't started isn't going to get compact calls.
    // another store that throws an exception on resumeCompaction() isn't going to get a compact call.
    // the new added store in storesDisabledCompaction isn't going to get compact calls.
    CountDownLatch compactCallsCountdown = new CountDownLatch(numStores - 3);
    properties.setProperty("store.compaction.triggers", ALL_COMPACTION_TRIGGERS);
    properties.setProperty("store.compaction.check.frequency.in.hours", Integer.toString(100));
    config = new StoreConfig(new VerifiableProperties(properties));
    MetricRegistry metricRegistry = new MetricRegistry();
    StoreMetrics metrics = new StoreMetrics(metricRegistry);
    for (int i = 0; i < numStores; i++) {
        MockBlobStore store = new MockBlobStore(config, metrics, time, compactCallsCountdown, null);
        store.details = generateRandomCompactionDetails(2);
        store.storeId = String.valueOf(i);
        if (i == 0) {
            // one store should not be started
            store.started = false;
        } else if (i == 1) {
            // one store should throw on resumeCompaction()
            store.exceptionToThrowOnResume = new RuntimeException("Misbehaving store");
        } else if (i == 3) {
            // one store should throw on compact()
            store.exceptionToThrowOnCompact = new RuntimeException("Misbehaving store");
        }
        stores.add(store);
    }
    // This CountDownLatch is to ensure that current test thread won't advance until the exception thrown on compact()
    // is caught and processed. (Mainly to address race condition that store 3 is the last store being checked for
    // compaction. The compact() method is called but exception hasn't been captured. At this point of time, cpu switches
    // back to this test thread and checks scheduleNextForCompaction() on each store. Since exception of store 3 hasn't
    // been captured, store 3 is not in storesToSkip set and assert fails)
    MockStorageManagerMetrics mockMetrics = new MockStorageManagerMetrics(metricRegistry);
    // The reason to set latch count = numStore + 2 is, stores[1 - 4] plus new added will be checked if resume compaction
    // (store 0 won't be checked because it is not started). So markCompactionStop() is called on 4 + 1 stores when checking
    // whether to resume compaction. After that only store 3 and store 4 are eligible to perform compact() and call
    // markCompactionStop() twice.
    CountDownLatch compactStopCountdown = new CountDownLatch(4 + 1 + 2);
    mockMetrics.compactStopCountdown = compactStopCountdown;
    // using real time here so that compaction is not scheduled more than once for a store during the test unless
    // asked for.
    compactionManager = new CompactionManager(MOUNT_PATH, config, stores, mockMetrics, SystemTime.getInstance());
    // disable the third blobstore for compaction before starting the CompactionExecutor thread
    MockBlobStore controlCompactionTestStore = (MockBlobStore) stores.get(2);
    compactionManager.controlCompactionForBlobStore(controlCompactionTestStore, false);
    // add a new blobstore into compaction manager but not started yet.
    MockBlobStore newAddedStore = new MockBlobStore(config, metrics, time, compactCallsCountdown, generateRandomCompactionDetails(2));
    newAddedStore.storeId = "newStore";
    compactionManager.addBlobStore(newAddedStore);
    compactionManager.enable();
    assertNotNull("Compaction thread should be created", TestUtils.getThreadByThisName(CompactionManager.THREAD_NAME_PREFIX));
    assertTrue("Compaction calls did not come within the expected time", compactCallsCountdown.await(1, TimeUnit.SECONDS));
    assertTrue("Compaction stop latch didn't count down to 0 within expected time", compactStopCountdown.await(1, TimeUnit.SECONDS));
    for (int i = 0; i < numStores; i++) {
        MockBlobStore store = (MockBlobStore) stores.get(i);
        if (store.callOrderException != null) {
            throw store.callOrderException;
        }
        if (i > 2) {
            assertTrue("Compact was not called", store.compactCalled);
        } else {
            // should not call for i == 0 because store has not been started.
            // should not call for i == 1 because resumeCompaction() would have marked this as a misbehaving store.
            // should not call for i == 2 because store has been disabled for compaction.
            assertFalse("Compact should not have been called", store.compactCalled);
        }
    }
    // ensure compact is not called on new added store
    assertFalse("Compact should not have been called", newAddedStore.compactCalled);
    // set all compact called to false
    for (BlobStore store : stores) {
        ((MockBlobStore) store).compactCalled = false;
    }
    // stores that are not started, disabled or failed on resumeCompaction() and compact() cannot be scheduled for compaction
    for (int i = 0; i < numStores; i++) {
        MockBlobStore store = (MockBlobStore) stores.get(i);
        if (i < 4) {
            assertFalse("Should not schedule compaction for " + store, compactionManager.scheduleNextForCompaction(store));
            assertFalse("compact() should not have been called on " + store, store.compactCalled);
        } else {
            store.compactCallsCountdown = new CountDownLatch(1);
            assertFalse("compactCalled should be reset", store.compactCalled);
            assertTrue("Should schedule compaction for " + store, compactionManager.scheduleNextForCompaction(store));
            assertTrue("Compaction call did not come within the expected time", store.compactCallsCountdown.await(1, TimeUnit.SECONDS));
            if (store.callOrderException != null) {
                throw store.callOrderException;
            }
            assertTrue("compact() should have been called on " + store, store.compactCalled);
        }
    }
    // disable Compaction Manager for restart
    compactionManager.disable();
    compactionManager.awaitTermination();
    assertFalse("Compaction thread should not be running", compactionManager.isCompactionExecutorRunning());
    compactCallsCountdown = new CountDownLatch(3);
    // set all compact called to false
    for (BlobStore store : stores) {
        ((MockBlobStore) store).compactCalled = false;
        ((MockBlobStore) store).resumeCompactionCalled = false;
        ((MockBlobStore) store).compactCallsCountdown = compactCallsCountdown;
    }
    newAddedStore.compactCallsCountdown = compactCallsCountdown;
    compactionManager.controlCompactionForBlobStore(controlCompactionTestStore, true);
    compactionManager.controlCompactionForBlobStore(newAddedStore, true);
    // restart Compaction Manager to trigger Periodic Compaction
    compactionManager.enable();
    assertNotNull("Compaction thread should be created", TestUtils.getThreadByThisName(CompactionManager.THREAD_NAME_PREFIX));
    assertTrue("Compaction calls did not come within the expected time", compactCallsCountdown.await(1, TimeUnit.SECONDS));
    for (int i = 0; i < numStores; i++) {
        MockBlobStore store = (MockBlobStore) stores.get(i);
        if (store.callOrderException != null) {
            throw store.callOrderException;
        }
        if (i == 2 || i == 4) {
            assertTrue("Compact was not called", store.compactCalled);
        } else {
            // should not call for i == 0 because store has not been started.
            // should not call for i == 1 because resumeCompaction() would have marked this as a misbehaving store.
            // should not call for i == 3 because it encountered exception during last compaction.
            assertFalse("Compact should not have been called", store.compactCalled);
        }
    }
    // ensure compact is called for new added store
    assertTrue("Compact was not called", newAddedStore.compactCalled);
    compactionManager.disable();
    compactionManager.awaitTermination();
    assertFalse("Compaction thread should not be running", compactionManager.isCompactionExecutorRunning());
}

Example 52 with StoreConfig

use of com.github.ambry.config.StoreConfig in project ambry by linkedin.

the class MockBlobStoreStats method testDifferentMessageRetentionDays.

/**
 * Tests {@link CompactionManager#getCompactionDetails(BlobStore)} for different values for
 * {@link StoreConfig#storeDeletedMessageRetentionHours}
 */
@Test
public void testDifferentMessageRetentionDays() throws StoreException, InterruptedException {
    List<LogSegmentName> bestCandidates = null;
    int[] messageRetentionHoursValues = new int[] { 1, 2, 3, 6, 9 };
    for (int messageRetentionHours : messageRetentionHoursValues) {
        time = new MockTime();
        Pair<MockBlobStore, StoreConfig> initState = initializeBlobStore(properties, time, -1, messageRetentionHours, DEFAULT_MAX_BLOB_SIZE);
        if (compactionPolicy instanceof StatsBasedCompactionPolicy) {
            bestCandidates = setUpStateForStatsBasedCompactionPolicy(blobStore, mockBlobStoreStats);
            compactionPolicy = new StatsBasedCompactionPolicy(initState.getSecond(), time);
        } else if (compactionPolicy instanceof CompactAllPolicy) {
            blobStore.logSegmentsNotInJournal = generateRandomLogSegmentName(3);
            bestCandidates = blobStore.logSegmentsNotInJournal;
            compactionPolicy = new CompactAllPolicy(initState.getSecond(), time);
        }
        verifyCompactionDetails(new CompactionDetails(time.milliseconds() - TimeUnit.HOURS.toMillis(messageRetentionHours), bestCandidates, null), blobStore, compactionPolicy);
    }
}

Example 53 with StoreConfig

use of com.github.ambry.config.StoreConfig in project ambry by linkedin.

the class MockBlobStoreStats method testHybridCompactionPolicy.

/**
 * Tests HybridCompactionPolicy which runs {@link StatsBasedCompactionPolicy} more frequently and {@link CompactAllPolicy} regularly.
 * @throws StoreException
 */
@Test
public void testHybridCompactionPolicy() throws StoreException, IOException {
    cleanupBackupFiles();
    // with compaction enabled.
    properties.setProperty("store.compaction.triggers", "Periodic");
    properties.setProperty("store.compaction.policy.switch.timestamp.days", "6");
    properties.setProperty("store.compaction.policy.factory", "com.github.ambry.store.HybridCompactionPolicyFactory");
    config = new StoreConfig(new VerifiableProperties(properties));
    MetricRegistry metricRegistry = new MetricRegistry();
    CompactionManager compactionManager = new CompactionManager(MOUNT_PATH, config, Collections.singleton(blobStore), new StorageManagerMetrics(metricRegistry), time);
    StoreMetrics metrics = new StoreMetrics(metricRegistry);
    MockBlobStoreStats mockBlobStoreStats = new MockBlobStoreStats(DEFAULT_MAX_BLOB_SIZE, 1);
    MockBlobStore blobStore = new MockBlobStore(config, metrics, time, CAPACITY_IN_BYTES, SEGMENT_CAPACITY_IN_BYTES, SEGMENT_HEADER_SIZE, DEFAULT_USED_CAPACITY_IN_BYTES, mockBlobStoreStats);
    String blobId = mockBlobStoreStats.getStoreId();
    File tmpDir = new File(MOUNT_PATH + blobId);
    if (!tmpDir.exists()) {
        tmpDir.mkdir();
    }
    compactionManager.getCompactionDetails(blobStore);
    File compactionPolicyInfoFile = new File(tmpDir.toString(), COMPACT_POLICY_INFO_FILE_NAME_V2);
    CompactionPolicySwitchInfo compactionPolicySwitchInfo = objectMapper.readValue(compactionPolicyInfoFile, CompactionPolicySwitchInfo.class);
    assertFalse("Next round of compaction is not compactAll", compactionPolicySwitchInfo.isNextRoundCompactAllPolicy());
    // set last compactAll kick off time to current time to trigger compactAllPolicy
    compactionPolicySwitchInfo.setLastCompactAllTime(compactionPolicySwitchInfo.getLastCompactAllTime() - TimeUnit.DAYS.toMillis(config.storeCompactionPolicySwitchTimestampDays));
    ((HybridCompactionPolicy) compactionManager.getCompactionPolicy()).getBlobToCompactionPolicySwitchInfoMap().put(mockBlobStoreStats.getStoreId(), compactionPolicySwitchInfo);
    objectMapper.writerWithDefaultPrettyPrinter().writeValue(compactionPolicyInfoFile, compactionPolicySwitchInfo);
    compactionManager.getCompactionDetails(blobStore);
    compactionPolicySwitchInfo = objectMapper.readValue(compactionPolicyInfoFile, CompactionPolicySwitchInfo.class);
    assertTrue("Next round of compaction is not compactAll", compactionPolicySwitchInfo.isNextRoundCompactAllPolicy());
    // add one more round to check if compaction will be recovered from backup file
    ((HybridCompactionPolicy) compactionManager.getCompactionPolicy()).getBlobToCompactionPolicySwitchInfoMap().clear();
    compactionManager.getCompactionDetails(blobStore);
    compactionPolicySwitchInfo = objectMapper.readValue(compactionPolicyInfoFile, CompactionPolicySwitchInfo.class);
    assertFalse("Next round of compaction is not compactAll", compactionPolicySwitchInfo.isNextRoundCompactAllPolicy());
}

Example 54 with StoreConfig

use of com.github.ambry.config.StoreConfig in project ambry by linkedin.

the class BlobStoreTest method multiReplicaStatusDelegatesTest.

/**
 * Test store is able to correctly seal/unseal replica with multiple participants.
 * @throws Exception
 */
@Test
public void multiReplicaStatusDelegatesTest() throws Exception {
    Set<ReplicaId> sealedReplicas1 = new HashSet<>();
    ReplicaStatusDelegate mockDelegate1 = Mockito.mock(ReplicaStatusDelegate.class);
    doAnswer(invocation -> {
        sealedReplicas1.add(invocation.getArgument(0));
        return true;
    }).when(mockDelegate1).seal(any());
    Set<ReplicaId> sealedReplicas2 = new HashSet<>();
    ReplicaStatusDelegate mockDelegate2 = Mockito.mock(ReplicaStatusDelegate.class);
    doAnswer(invocation -> {
        sealedReplicas2.add(invocation.getArgument(0));
        return true;
    }).when(mockDelegate2).seal(any());
    doAnswer(invocation -> {
        sealedReplicas1.remove((ReplicaId) invocation.getArgument(0));
        return true;
    }).when(mockDelegate1).unseal(any());
    doAnswer(invocation -> {
        sealedReplicas2.remove((ReplicaId) invocation.getArgument(0));
        return true;
    }).when(mockDelegate2).unseal(any());
    doAnswer(invocation -> sealedReplicas1.stream().map(r -> r.getPartitionId().toPathString()).collect(Collectors.toList())).when(mockDelegate1).getSealedReplicas();
    doAnswer(invocation -> sealedReplicas2.stream().map(r -> r.getPartitionId().toPathString()).collect(Collectors.toList())).when(mockDelegate2).getSealedReplicas();
    StoreConfig defaultConfig = changeThreshold(65, 5, true);
    StoreTestUtils.MockReplicaId replicaId = getMockReplicaId(tempDirStr);
    reloadStore(defaultConfig, replicaId, Arrays.asList(mockDelegate1, mockDelegate2));
    // make the replica sealed
    put(4, (long) (SEGMENT_CAPACITY * 0.8), Utils.Infinite_Time);
    assertEquals("Sealed replica lists are different", sealedReplicas1, sealedReplicas2);
    assertEquals("Sealed replica is not correct", replicaId, sealedReplicas1.iterator().next());
    // try to bump the readonly threshold so as to unseal the replica
    replicaId.setSealedState(true);
    reloadStore(changeThreshold(99, 1, true), replicaId, Arrays.asList(mockDelegate1, mockDelegate2));
    assertTrue("Replica should be unsealed", sealedReplicas1.isEmpty() && sealedReplicas2.isEmpty());
    assertEquals("After startup, store should be in STANDBY state", STANDBY, store.getCurrentState());
    // verify store still updates sealed lists even though replica state is already sealed. ("replicaId.setSealedState(true)")
    // lower the threshold to make replica sealed again
    reloadStore(changeThreshold(50, 5, true), replicaId, Arrays.asList(mockDelegate1, mockDelegate2));
    assertEquals("Sealed replica lists are different", sealedReplicas1, sealedReplicas2);
    assertEquals("Sealed replica is not correct", replicaId, sealedReplicas1.iterator().next());
    // verify reconciliation case: we make read-write delta a wide range and clear sealedReplicas2 to make them reconcile
    sealedReplicas2.clear();
    reloadStore(changeThreshold(99, 90, true), replicaId, Arrays.asList(mockDelegate1, mockDelegate2));
    assertEquals("Sealed replica lists are different", sealedReplicas1, sealedReplicas2);
    assertEquals("Sealed replica is not correct", replicaId, sealedReplicas2.iterator().next());
    store.shutdown();
}

Example 55 with StoreConfig

use of com.github.ambry.config.StoreConfig in project ambry by linkedin.

the class LogSegmentTest method setFilePermissionsTest.

/**
 * Test that the file permissions are correctly set based on permissions specified in {@link StoreConfig}.
 * @throws Exception
 */
@Test
public void setFilePermissionsTest() throws Exception {
    Properties props = new Properties();
    props.setProperty("store.set.file.permission.enabled", Boolean.toString(true));
    props.setProperty("store.data.file.permission", "rw-rw-r--");
    StoreConfig initialConfig = new StoreConfig(new VerifiableProperties(props));
    File segmentFile = new File(tempDir, "test_segment");
    LogSegmentName segmentName = LogSegmentName.generateFirstSegmentName(true);
    assertTrue("Fail to create segment file", segmentFile.createNewFile());
    segmentFile.deleteOnExit();
    // create log segment instance by writing into brand new file (using initialConfig, file permission = "rw-rw-r--")
    new LogSegment(segmentName, segmentFile, STANDARD_SEGMENT_SIZE, initialConfig, metrics, true);
    Set<PosixFilePermission> filePerm = Files.getPosixFilePermissions(segmentFile.toPath());
    assertEquals("File permissions are not expected", "rw-rw-r--", PosixFilePermissions.toString(filePerm));
    // create log segment instance by reading from existing file (using default store config, file permission = "rw-rw----")
    new LogSegment(segmentName, segmentFile, config, metrics);
    filePerm = Files.getPosixFilePermissions(segmentFile.toPath());
    assertEquals("File permissions are not expected", "rw-rw----", PosixFilePermissions.toString(filePerm));
}