Examples with DiskId com.github.ambry.clustermap.DiskId used on opensource projects

Example 1 with DiskId

use of com.github.ambry.clustermap.DiskId in project ambry by linkedin.

the class StorageManagerTest method isDiskAvailableTest.

/**
 * Tests that{@link StorageManager} can correctly determine if disk is unavailable based on states of all stores.
 */
@Test
public void isDiskAvailableTest() throws Exception {
    MockDataNodeId dataNode = clusterMap.getDataNodes().get(0);
    List<ReplicaId> replicas = clusterMap.getReplicaIds(dataNode);
    Map<DiskId, List<ReplicaId>> diskToReplicas = new HashMap<>();
    StorageManager storageManager = createStorageManager(dataNode, metricRegistry, null);
    storageManager.start();
    assertEquals("There should be no unexpected partitions reported", 0, getNumUnrecognizedPartitionsReported());
    for (ReplicaId replica : replicas) {
        diskToReplicas.computeIfAbsent(replica.getDiskId(), disk -> new ArrayList<>()).add(replica);
    }
    // for each disk, shutdown all the stores except for the last one
    for (List<ReplicaId> replicasOnDisk : diskToReplicas.values()) {
        for (int i = 0; i < replicasOnDisk.size() - 1; ++i) {
            storageManager.getStore(replicasOnDisk.get(i).getPartitionId(), false).shutdown();
        }
    }
    // verify all disks are still available because at least one store on them is up
    for (List<ReplicaId> replicasOnDisk : diskToReplicas.values()) {
        assertTrue("Disk should be available", storageManager.isDiskAvailable(replicasOnDisk.get(0).getDiskId()));
        assertEquals("Disk state be available", HardwareState.AVAILABLE, replicasOnDisk.get(0).getDiskId().getState());
    }
    // now, shutdown the last store on each disk
    for (List<ReplicaId> replicasOnDisk : diskToReplicas.values()) {
        storageManager.getStore(replicasOnDisk.get(replicasOnDisk.size() - 1).getPartitionId(), false).shutdown();
    }
    // verify all disks are unavailable because all stores are down
    for (List<ReplicaId> replicasOnDisk : diskToReplicas.values()) {
        assertFalse("Disk should be unavailable", storageManager.isDiskAvailable(replicasOnDisk.get(0).getDiskId()));
    }
    // then, start the one store on each disk to test if disk is up again
    for (List<ReplicaId> replicasOnDisk : diskToReplicas.values()) {
        storageManager.startBlobStore(replicasOnDisk.get(0).getPartitionId());
    }
    // verify all disks are available again because one store is started
    for (List<ReplicaId> replicasOnDisk : diskToReplicas.values()) {
        assertTrue("Disk should be available", storageManager.isDiskAvailable(replicasOnDisk.get(0).getDiskId()));
        assertEquals("Disk state be available", HardwareState.AVAILABLE, replicasOnDisk.get(0).getDiskId().getState());
    }
    shutdownAndAssertStoresInaccessible(storageManager, replicas);
}

Example 2 with DiskId

use of com.github.ambry.clustermap.DiskId in project ambry by linkedin.

the class StorageManagerTest method setBlobStoreStoppedStateSuccessTest.

/**
 * Test successfully set stopped state of blobstore with given list of {@link PartitionId}.
 */
@Test
public void setBlobStoreStoppedStateSuccessTest() throws Exception {
    MockDataNodeId dataNode = clusterMap.getDataNodes().get(0);
    List<ReplicaId> replicas = clusterMap.getReplicaIds(dataNode);
    List<PartitionId> partitionIds = new ArrayList<>();
    Map<DiskId, List<ReplicaId>> diskToReplicas = new HashMap<>();
    // test setting the state of store via instantiated MockClusterParticipant
    ClusterParticipant participant = new MockClusterParticipant();
    ClusterParticipant participantSpy = Mockito.spy(participant);
    StorageManager storageManager = createStorageManager(dataNode, metricRegistry, Collections.singletonList(participantSpy));
    storageManager.start();
    assertEquals("There should be no unexpected partitions reported", 0, getNumUnrecognizedPartitionsReported());
    for (ReplicaId replica : replicas) {
        partitionIds.add(replica.getPartitionId());
        diskToReplicas.computeIfAbsent(replica.getDiskId(), disk -> new ArrayList<>()).add(replica);
    }
    List<PartitionId> failToUpdateList;
    // add a list of stores to STOPPED list. Note that the stores are residing on 3 disks.
    failToUpdateList = storageManager.setBlobStoreStoppedState(partitionIds, true);
    // make sure the update operation succeeds
    assertTrue("Add stores to stopped list should succeed, failToUpdateList should be empty", failToUpdateList.isEmpty());
    // make sure the stopped list contains all the added stores
    Set<String> stoppedReplicasCopy = new HashSet<>(participantSpy.getStoppedReplicas());
    for (ReplicaId replica : replicas) {
        assertTrue("The stopped list should contain the replica: " + replica.getPartitionId().toPathString(), stoppedReplicasCopy.contains(replica.getPartitionId().toPathString()));
    }
    // make sure replicaStatusDelegate is invoked 3 times and each time the input replica list conforms with stores on particular disk
    for (List<ReplicaId> replicasPerDisk : diskToReplicas.values()) {
        verify(participantSpy, times(1)).setReplicaStoppedState(replicasPerDisk, true);
    }
    // remove a list of stores from STOPPED list. Note that the stores are residing on 3 disks.
    storageManager.setBlobStoreStoppedState(partitionIds, false);
    // make sure the update operation succeeds
    assertTrue("Remove stores from stopped list should succeed, failToUpdateList should be empty", failToUpdateList.isEmpty());
    // make sure the stopped list is empty because all the stores are successfully removed.
    assertTrue("The stopped list should be empty after removing all stores", participantSpy.getStoppedReplicas().isEmpty());
    // make sure replicaStatusDelegate is invoked 3 times and each time the input replica list conforms with stores on particular disk
    for (List<ReplicaId> replicasPerDisk : diskToReplicas.values()) {
        verify(participantSpy, times(1)).setReplicaStoppedState(replicasPerDisk, false);
    }
    shutdownAndAssertStoresInaccessible(storageManager, replicas);
}

Example 3 with DiskId

use of com.github.ambry.clustermap.DiskId in project ambry by linkedin.

the class CryptoJobMetricsTracker method initializeResourceToHistogramMap.

/**
 * Initialize resource-to-latency-histogram maps based on given resource type. Here resource can be {@link PartitionId},
 * {@link DataNodeId}, etc. The resource type is defined by {@link RouterConfig#routerOperationTrackerMetricScope}.
 * @param clusterMap the {@link ClusterMap} that contains info of all resources.
 * @param routerConfig the {@link RouterConfig} that specifies histogram parameters.
 */
private void initializeResourceToHistogramMap(ClusterMap clusterMap, RouterConfig routerConfig) {
    String localDatacenterName = clusterMap.getDatacenterName(clusterMap.getLocalDatacenterId());
    switch(routerConfig.routerOperationTrackerMetricScope) {
        case Partition:
            for (PartitionId partitionId : clusterMap.getAllPartitionIds(null)) {
                getBlobLocalDcResourceToLatency.put(partitionId, createHistogram(routerConfig, false));
                getBlobInfoLocalDcResourceToLatency.put(partitionId, createHistogram(routerConfig, false));
                getBlobCrossDcResourceToLatency.put(partitionId, createHistogram(routerConfig, false));
                getBlobInfoCrossDcResourceToLatency.put(partitionId, createHistogram(routerConfig, false));
                putBlobResourceToLatency.put(partitionId, createHistogram(routerConfig, false));
            }
            break;
        case DataNode:
            List<? extends DataNodeId> dataNodeIds = clusterMap.getDataNodeIds();
            for (DataNodeId dataNodeId : dataNodeIds) {
                if (dataNodeId.getDatacenterName().equals(localDatacenterName)) {
                    getBlobLocalDcResourceToLatency.put(dataNodeId, createHistogram(routerConfig, false));
                    getBlobInfoLocalDcResourceToLatency.put(dataNodeId, createHistogram(routerConfig, false));
                    // Put blob only cares abou local db data nodes.
                    putBlobResourceToLatency.put(dataNodeId, createHistogram(routerConfig, false));
                } else {
                    getBlobCrossDcResourceToLatency.put(dataNodeId, createHistogram(routerConfig, false));
                    getBlobInfoCrossDcResourceToLatency.put(dataNodeId, createHistogram(routerConfig, false));
                }
            }
            break;
        case Disk:
            for (PartitionId partitionId : clusterMap.getAllPartitionIds(null)) {
                for (ReplicaId replicaId : partitionId.getReplicaIds()) {
                    DiskId diskId = replicaId.getDiskId();
                    if (getBlobLocalDcResourceToLatency.containsKey(diskId) || getBlobCrossDcResourceToLatency.containsKey(diskId)) {
                        continue;
                    }
                    if (replicaId.getDataNodeId().getDatacenterName().equals(localDatacenterName)) {
                        getBlobLocalDcResourceToLatency.put(diskId, createHistogram(routerConfig, false));
                        getBlobInfoLocalDcResourceToLatency.put(diskId, createHistogram(routerConfig, false));
                        putBlobResourceToLatency.put(diskId, createHistogram(routerConfig, false));
                    } else {
                        getBlobCrossDcResourceToLatency.put(diskId, createHistogram(routerConfig, false));
                        getBlobInfoCrossDcResourceToLatency.put(diskId, createHistogram(routerConfig, false));
                    }
                }
            }
        default:
    }
}

Example 4 with DiskId

use of com.github.ambry.clustermap.DiskId in project ambry by linkedin.

the class AdaptiveOperationTracker method getLatencyHistogram.

/**
 * Gets the {@link Histogram} that tracks request latencies to the class of replicas (intra or inter DC) that
 * {@code replicaId} belongs to.
 * @param replicaId the {@link ReplicaId} whose request latency is going to be tracked.
 * @return the {@link CachedHistogram} associated with this replica.
 */
CachedHistogram getLatencyHistogram(ReplicaId replicaId) {
    boolean isLocalReplica = replicaId.getDataNodeId().getDatacenterName().equals(datacenterName);
    CachedHistogram histogramToReturn;
    switch(routerConfig.routerOperationTrackerMetricScope) {
        case Datacenter:
            histogramToReturn = isLocalReplica ? localDcHistogram : crossDcHistogram;
            break;
        case Partition:
            PartitionId partitionId = replicaId.getPartitionId();
            histogramToReturn = isLocalReplica ? localDcResourceToHistogram.computeIfAbsent(partitionId, k -> createHistogram(routerConfig, false)) : crossDcResourceToHistogram.computeIfAbsent(partitionId, k -> createHistogram(routerConfig, false));
            break;
        case DataNode:
            DataNodeId dataNodeId = replicaId.getDataNodeId();
            histogramToReturn = isLocalReplica ? localDcResourceToHistogram.computeIfAbsent(dataNodeId, k -> createHistogram(routerConfig, false)) : crossDcResourceToHistogram.computeIfAbsent(dataNodeId, k -> createHistogram(routerConfig, false));
            break;
        case Disk:
            DiskId diskId = replicaId.getDiskId();
            histogramToReturn = isLocalReplica ? localDcResourceToHistogram.computeIfAbsent(diskId, k -> createHistogram(routerConfig, false)) : crossDcResourceToHistogram.computeIfAbsent(diskId, k -> createHistogram(routerConfig, false));
            break;
        default:
            throw new IllegalArgumentException("Unsupported operation tracker metric scope.");
    }
    return histogramToReturn;
}