Examples with MockDataNodeId com.github.ambry.clustermap.MockDataNodeId used on opensource projects

Example 61 with MockDataNodeId

use of com.github.ambry.clustermap.MockDataNodeId in project ambry by linkedin.

the class StorageManagerTest method replicaFromOfflineToBootstrapTest.

/**
 * test that both success and failure in storage manager when replica becomes BOOTSTRAP from OFFLINE (update
 * InstanceConfig in Helix is turned off in this test)
 * @throws Exception
 */
@Test
public void replicaFromOfflineToBootstrapTest() throws Exception {
    generateConfigs(true, false);
    MockDataNodeId localNode = clusterMap.getDataNodes().get(0);
    List<PartitionId> partitionIds = clusterMap.getAllPartitionIds(null);
    List<ReplicaId> localReplicas = clusterMap.getReplicaIds(localNode);
    MockClusterParticipant mockHelixParticipant = new MockClusterParticipant();
    StorageManager storageManager = createStorageManager(localNode, metricRegistry, Collections.singletonList(mockHelixParticipant));
    storageManager.start();
    // 1. get listeners from Helix participant and verify there is a storageManager listener.
    Map<StateModelListenerType, PartitionStateChangeListener> listeners = mockHelixParticipant.getPartitionStateChangeListeners();
    assertTrue("Should contain storage manager listener", listeners.containsKey(StateModelListenerType.StorageManagerListener));
    // 2. if new bootstrap replica is not found, there should be an exception
    try {
        mockHelixParticipant.onPartitionBecomeBootstrapFromOffline(String.valueOf(partitionIds.size() + 1));
        fail("should fail due to bootstrap replica not found");
    } catch (StateTransitionException e) {
        assertEquals("Error code doesn't match", ReplicaNotFound, e.getErrorCode());
    }
    // 3. test regular store didn't start up (which triggers StoreNotStarted exception)
    ReplicaId replicaId = localReplicas.get(0);
    Store localStore = storageManager.getStore(replicaId.getPartitionId(), true);
    localStore.shutdown();
    try {
        mockHelixParticipant.onPartitionBecomeBootstrapFromOffline(replicaId.getPartitionId().toPathString());
        fail("should fail due to store not started");
    } catch (StateTransitionException e) {
        assertEquals("Error code doesn't match", StoreNotStarted, e.getErrorCode());
    }
    localStore.start();
    // 4. test both failure and success cases regarding new replica addition
    PartitionId newPartition = clusterMap.createNewPartition(Collections.singletonList(localNode));
    assertNull("There should not be any store associated with new partition", storageManager.getStore(newPartition, true));
    // find an existing replica that shares disk with new replica
    ReplicaId newReplica = newPartition.getReplicaIds().get(0);
    ReplicaId replicaOnSameDisk = localReplicas.stream().filter(r -> r.getDiskId().equals(newReplica.getDiskId())).findFirst().get();
    // test add new store failure by shutting down target diskManager
    storageManager.getDiskManager(replicaOnSameDisk.getPartitionId()).shutdown();
    try {
        mockHelixParticipant.onPartitionBecomeBootstrapFromOffline(newPartition.toPathString());
    } catch (StateTransitionException e) {
        assertEquals("Error code doesn't match", ReplicaOperationFailure, e.getErrorCode());
    }
    // restart disk manager to test case where new replica(store) is successfully added into StorageManager
    storageManager.getDiskManager(replicaOnSameDisk.getPartitionId()).start();
    mockHelixParticipant.onPartitionBecomeBootstrapFromOffline(newPartition.toPathString());
    BlobStore newAddedStore = (BlobStore) storageManager.getStore(newPartition);
    assertNotNull("There should be a started store associated with new partition", newAddedStore);
    // 5. verify that new added store has bootstrap file
    assertTrue("There should be a bootstrap file indicating store is in BOOTSTRAP state", newAddedStore.isBootstrapInProgress());
    assertEquals("The store's current state should be BOOTSTRAP", ReplicaState.BOOTSTRAP, newAddedStore.getCurrentState());
    // 6. test that state transition should succeed for existing non-empty replicas (we write some data into store beforehand)
    MockId id = new MockId(TestUtils.getRandomString(MOCK_ID_STRING_LENGTH), Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM));
    MessageInfo info = new MessageInfo(id, PUT_RECORD_SIZE, id.getAccountId(), id.getContainerId(), Utils.Infinite_Time);
    MessageWriteSet writeSet = new MockMessageWriteSet(Collections.singletonList(info), Collections.singletonList(ByteBuffer.allocate(PUT_RECORD_SIZE)));
    Store storeToWrite = storageManager.getStore(localReplicas.get(1).getPartitionId());
    storeToWrite.put(writeSet);
    mockHelixParticipant.onPartitionBecomeBootstrapFromOffline(localReplicas.get(1).getPartitionId().toPathString());
    assertFalse("There should not be any bootstrap file for existing non-empty store", storeToWrite.isBootstrapInProgress());
    assertEquals("The store's current state should be BOOTSTRAP", ReplicaState.BOOTSTRAP, storeToWrite.getCurrentState());
    // 7. test that for new created (empty) store, state transition puts it into BOOTSTRAP state
    mockHelixParticipant.onPartitionBecomeBootstrapFromOffline(localReplicas.get(0).getPartitionId().toPathString());
    assertTrue("There should be a bootstrap file because store is empty and probably recreated", localStore.isBootstrapInProgress());
    assertEquals("The store's current state should be BOOTSTRAP", ReplicaState.BOOTSTRAP, localStore.getCurrentState());
    shutdownAndAssertStoresInaccessible(storageManager, localReplicas);
}

Example 62 with MockDataNodeId

use of com.github.ambry.clustermap.MockDataNodeId in project ambry by linkedin.

the class StorageManagerTest method storeStartFailureOnOneDiskTest.

/**
 * Tests that {@link StorageManager} can start when all of the stores on one disk fail to start. Checks that these
 * stores are not accessible. We can make the replica path non-readable to induce a store starting failure.
 * @throws Exception
 */
@Test
public void storeStartFailureOnOneDiskTest() throws Exception {
    MockDataNodeId dataNode = clusterMap.getDataNodes().get(0);
    List<ReplicaId> replicas = clusterMap.getReplicaIds(dataNode);
    List<String> mountPaths = dataNode.getMountPaths();
    String badDiskMountPath = mountPaths.get(RANDOM.nextInt(mountPaths.size()));
    int downReplicaCount = 0;
    for (ReplicaId replica : replicas) {
        if (replica.getMountPath().equals(badDiskMountPath)) {
            new File(replica.getReplicaPath()).setReadable(false);
            downReplicaCount++;
        }
    }
    StorageManager storageManager = createStorageManager(dataNode, metricRegistry, null);
    storageManager.start();
    assertEquals("There should be no unexpected partitions reported", 0, getNumUnrecognizedPartitionsReported());
    Map<String, Counter> counters = metricRegistry.getCounters();
    assertEquals(0, getCounterValue(counters, DiskSpaceAllocator.class.getName(), "DiskSpaceAllocatorInitFailureCount"));
    assertEquals(downReplicaCount, getCounterValue(counters, DiskManager.class.getName(), "TotalStoreStartFailures"));
    assertEquals(0, getCounterValue(counters, DiskManager.class.getName(), "DiskMountPathFailures"));
    checkStoreAccessibility(replicas, badDiskMountPath, storageManager);
    assertEquals("Compaction thread count is incorrect", mountPaths.size(), TestUtils.numThreadsByThisName(CompactionManager.THREAD_NAME_PREFIX));
    verifyCompactionThreadCount(storageManager, mountPaths.size());
    shutdownAndAssertStoresInaccessible(storageManager, replicas);
    assertEquals("Compaction thread count is incorrect", 0, storageManager.getCompactionThreadCount());
}

Example 63 with MockDataNodeId

use of com.github.ambry.clustermap.MockDataNodeId in project ambry by linkedin.

the class StorageManagerTest method scheduleAndControlCompactionTest.

/**
 * Tests that schedule compaction and control compaction in StorageManager
 * @throws Exception
 */
@Test
public void scheduleAndControlCompactionTest() throws Exception {
    MockDataNodeId dataNode = clusterMap.getDataNodes().get(0);
    List<ReplicaId> replicas = clusterMap.getReplicaIds(dataNode);
    List<MockDataNodeId> dataNodes = new ArrayList<>();
    dataNodes.add(dataNode);
    MockPartitionId invalidPartition = new MockPartitionId(Long.MAX_VALUE, MockClusterMap.DEFAULT_PARTITION_CLASS, dataNodes, 0);
    List<? extends ReplicaId> invalidPartitionReplicas = invalidPartition.getReplicaIds();
    StorageManager storageManager = createStorageManager(dataNode, metricRegistry, null);
    storageManager.start();
    assertEquals("There should be 1 unexpected partition reported", 1, getNumUnrecognizedPartitionsReported());
    // add invalid replica id
    replicas.add(invalidPartitionReplicas.get(0));
    for (int i = 0; i < replicas.size(); i++) {
        ReplicaId replica = replicas.get(i);
        PartitionId id = replica.getPartitionId();
        if (i == replicas.size() - 1) {
            assertFalse("Schedule compaction should fail", storageManager.scheduleNextForCompaction(id));
            assertFalse("Disable compaction should fail", storageManager.controlCompactionForBlobStore(id, false));
            assertFalse("Enable compaction should fail", storageManager.controlCompactionForBlobStore(id, true));
        } else {
            assertTrue("Enable compaction should succeed", storageManager.controlCompactionForBlobStore(id, true));
            assertTrue("Schedule compaction should succeed", storageManager.scheduleNextForCompaction(id));
        }
    }
    ReplicaId replica = replicas.get(0);
    PartitionId id = replica.getPartitionId();
    assertTrue("Disable compaction should succeed", storageManager.controlCompactionForBlobStore(id, false));
    assertFalse("Schedule compaction should fail", storageManager.scheduleNextForCompaction(id));
    assertTrue("Enable compaction should succeed", storageManager.controlCompactionForBlobStore(id, true));
    assertTrue("Schedule compaction should succeed", storageManager.scheduleNextForCompaction(id));
    replica = replicas.get(1);
    id = replica.getPartitionId();
    assertTrue("Schedule compaction should succeed", storageManager.scheduleNextForCompaction(id));
    replicas.remove(replicas.size() - 1);
    shutdownAndAssertStoresInaccessible(storageManager, replicas);
}