Examples with SCMNodeMetric org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric used on opensource projects

Example 1 with SCMNodeMetric

use of org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric in project ozone by apache.

the class TestDatanodeMetrics method testSCMNodeMetric.

@Test
public void testSCMNodeMetric() {
    SCMNodeStat stat = new SCMNodeStat(100L, 10L, 90L);
    assertEquals((long) stat.getCapacity().get(), 100L);
    assertEquals(10L, (long) stat.getScmUsed().get());
    assertEquals(90L, (long) stat.getRemaining().get());
    SCMNodeMetric metric = new SCMNodeMetric(stat);
    SCMNodeStat newStat = new SCMNodeStat(100L, 10L, 90L);
    assertEquals(100L, (long) stat.getCapacity().get());
    assertEquals(10L, (long) stat.getScmUsed().get());
    assertEquals(90L, (long) stat.getRemaining().get());
    SCMNodeMetric newMetric = new SCMNodeMetric(newStat);
    assertTrue(metric.isEqual(newMetric.get()));
    newMetric.add(stat);
    assertTrue(newMetric.isGreater(metric.get()));
    SCMNodeMetric zeroMetric = new SCMNodeMetric(new SCMNodeStat());
    // Assert we can handle zero capacity.
    assertTrue(metric.isGreater(zeroMetric.get()));
}

Example 2 with SCMNodeMetric

use of org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric in project ozone by apache.

the class TestStatisticsUpdate method testStatisticsUpdate.

@Test
public void testStatisticsUpdate() throws Exception {
    // GIVEN
    DatanodeDetails datanode1 = MockDatanodeDetails.randomDatanodeDetails();
    DatanodeDetails datanode2 = MockDatanodeDetails.randomDatanodeDetails();
    String storagePath1 = GenericTestUtils.getRandomizedTempPath().concat("/" + datanode1.getUuidString());
    String storagePath2 = GenericTestUtils.getRandomizedTempPath().concat("/" + datanode2.getUuidString());
    StorageReportProto storageOne = HddsTestUtils.createStorageReport(datanode1.getUuid(), storagePath1, 100, 10, 90, null);
    StorageReportProto storageTwo = HddsTestUtils.createStorageReport(datanode2.getUuid(), storagePath2, 200, 20, 180, null);
    nodeManager.register(datanode1, HddsTestUtils.createNodeReport(Arrays.asList(storageOne), Collections.emptyList()), null);
    nodeManager.register(datanode2, HddsTestUtils.createNodeReport(Arrays.asList(storageTwo), Collections.emptyList()), null);
    NodeReportProto nodeReportProto1 = HddsTestUtils.createNodeReport(Arrays.asList(storageOne), Collections.emptyList());
    NodeReportProto nodeReportProto2 = HddsTestUtils.createNodeReport(Arrays.asList(storageTwo), Collections.emptyList());
    nodeReportHandler.onMessage(new NodeReportFromDatanode(datanode1, nodeReportProto1), Mockito.mock(EventPublisher.class));
    nodeReportHandler.onMessage(new NodeReportFromDatanode(datanode2, nodeReportProto2), Mockito.mock(EventPublisher.class));
    SCMNodeStat stat = nodeManager.getStats();
    Assert.assertEquals(300L, stat.getCapacity().get().longValue());
    Assert.assertEquals(270L, stat.getRemaining().get().longValue());
    Assert.assertEquals(30L, stat.getScmUsed().get().longValue());
    SCMNodeMetric nodeStat = nodeManager.getNodeStat(datanode1);
    Assert.assertEquals(100L, nodeStat.get().getCapacity().get().longValue());
    Assert.assertEquals(90L, nodeStat.get().getRemaining().get().longValue());
    Assert.assertEquals(10L, nodeStat.get().getScmUsed().get().longValue());
    // TODO: Support logic to mark a node as dead in NodeManager.
    LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
    StorageContainerDatanodeProtocolProtos.LayoutVersionProto layoutInfo = StorageContainerDatanodeProtocolProtos.LayoutVersionProto.newBuilder().setSoftwareLayoutVersion(versionManager.getSoftwareLayoutVersion()).setMetadataLayoutVersion(versionManager.getMetadataLayoutVersion()).build();
    nodeManager.processHeartbeat(datanode2, layoutInfo);
    Thread.sleep(1000);
    nodeManager.processHeartbeat(datanode2, layoutInfo);
    Thread.sleep(1000);
    nodeManager.processHeartbeat(datanode2, layoutInfo);
    Thread.sleep(1000);
    nodeManager.processHeartbeat(datanode2, layoutInfo);
    // THEN statistics in SCM should changed.
    stat = nodeManager.getStats();
    Assert.assertEquals(200L, stat.getCapacity().get().longValue());
    Assert.assertEquals(180L, stat.getRemaining().get().longValue());
    Assert.assertEquals(20L, stat.getScmUsed().get().longValue());
}

Example 3 with SCMNodeMetric

use of org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric in project ozone by apache.

the class TestNodeReportHandler method testNodeReport.

@Test
public void testNodeReport() throws IOException {
    DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
    StorageReportProto storageOne = HddsTestUtils.createStorageReport(dn.getUuid(), storagePath, 100, 10, 90, null);
    MetadataStorageReportProto metaStorageOne = HddsTestUtils.createMetadataStorageReport(metaStoragePath, 100, 10, 90, null);
    SCMNodeMetric nodeMetric = nodeManager.getNodeStat(dn);
    Assert.assertNull(nodeMetric);
    nodeManager.register(dn, getNodeReport(dn, Arrays.asList(storageOne), Arrays.asList(metaStorageOne)).getReport(), null);
    nodeMetric = nodeManager.getNodeStat(dn);
    Assert.assertTrue(nodeMetric.get().getCapacity().get() == 100);
    Assert.assertTrue(nodeMetric.get().getRemaining().get() == 90);
    Assert.assertTrue(nodeMetric.get().getScmUsed().get() == 10);
    StorageReportProto storageTwo = HddsTestUtils.createStorageReport(dn.getUuid(), storagePath, 100, 10, 90, null);
    nodeReportHandler.onMessage(getNodeReport(dn, Arrays.asList(storageOne, storageTwo), Arrays.asList(metaStorageOne)), this);
    nodeMetric = nodeManager.getNodeStat(dn);
    Assert.assertTrue(nodeMetric.get().getCapacity().get() == 200);
    Assert.assertTrue(nodeMetric.get().getRemaining().get() == 180);
    Assert.assertTrue(nodeMetric.get().getScmUsed().get() == 20);
}

Example 4 with SCMNodeMetric

use of org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric in project ozone by apache.

the class TestSCMContainerPlacementCapacity method chooseDatanodes.

@Test
public void chooseDatanodes() throws SCMException {
    // given
    OzoneConfiguration conf = new OzoneConfiguration();
    // We are using small units here
    conf.setStorageSize(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN, 1, StorageUnit.BYTES);
    List<DatanodeInfo> datanodes = new ArrayList<>();
    for (int i = 0; i < 7; i++) {
        DatanodeInfo datanodeInfo = new DatanodeInfo(MockDatanodeDetails.randomDatanodeDetails(), NodeStatus.inServiceHealthy(), UpgradeUtils.defaultLayoutVersionProto());
        StorageReportProto storage1 = HddsTestUtils.createStorageReport(datanodeInfo.getUuid(), "/data1-" + datanodeInfo.getUuidString(), 100L, 0, 100L, null);
        MetadataStorageReportProto metaStorage1 = HddsTestUtils.createMetadataStorageReport("/metadata1-" + datanodeInfo.getUuidString(), 100L, 0, 100L, null);
        datanodeInfo.updateStorageReports(new ArrayList<>(Arrays.asList(storage1)));
        datanodeInfo.updateMetaDataStorageReports(new ArrayList<>(Arrays.asList(metaStorage1)));
        datanodes.add(datanodeInfo);
    }
    StorageReportProto storage2 = HddsTestUtils.createStorageReport(datanodes.get(2).getUuid(), "/data1-" + datanodes.get(2).getUuidString(), 100L, 90L, 10L, null);
    datanodes.get(2).updateStorageReports(new ArrayList<>(Arrays.asList(storage2)));
    StorageReportProto storage3 = HddsTestUtils.createStorageReport(datanodes.get(3).getUuid(), "/data1-" + datanodes.get(3).getUuidString(), 100L, 80L, 20L, null);
    datanodes.get(3).updateStorageReports(new ArrayList<>(Arrays.asList(storage3)));
    StorageReportProto storage4 = HddsTestUtils.createStorageReport(datanodes.get(4).getUuid(), "/data1-" + datanodes.get(4).getUuidString(), 100L, 70L, 30L, null);
    datanodes.get(4).updateStorageReports(new ArrayList<>(Arrays.asList(storage4)));
    NodeManager mockNodeManager = Mockito.mock(NodeManager.class);
    when(mockNodeManager.getNodes(NodeStatus.inServiceHealthy())).thenReturn(new ArrayList<>(datanodes));
    when(mockNodeManager.getNodeStat(anyObject())).thenReturn(new SCMNodeMetric(100L, 0L, 100L));
    when(mockNodeManager.getNodeStat(datanodes.get(2))).thenReturn(new SCMNodeMetric(100L, 90L, 10L));
    when(mockNodeManager.getNodeStat(datanodes.get(3))).thenReturn(new SCMNodeMetric(100L, 80L, 20L));
    when(mockNodeManager.getNodeStat(datanodes.get(4))).thenReturn(new SCMNodeMetric(100L, 70L, 30L));
    SCMContainerPlacementCapacity scmContainerPlacementRandom = new SCMContainerPlacementCapacity(mockNodeManager, conf, null, true, null);
    List<DatanodeDetails> existingNodes = new ArrayList<>();
    existingNodes.add(datanodes.get(0));
    existingNodes.add(datanodes.get(1));
    Map<DatanodeDetails, Integer> selectedCount = new HashMap<>();
    for (DatanodeDetails datanode : datanodes) {
        selectedCount.put(datanode, 0);
    }
    for (int i = 0; i < 1000; i++) {
        // when
        List<DatanodeDetails> datanodeDetails = scmContainerPlacementRandom.chooseDatanodes(existingNodes, null, 1, 15, 15);
        // then
        Assert.assertEquals(1, datanodeDetails.size());
        DatanodeDetails datanode0Details = datanodeDetails.get(0);
        Assert.assertNotEquals("Datanode 0 should not been selected: excluded by parameter", datanodes.get(0), datanode0Details);
        Assert.assertNotEquals("Datanode 1 should not been selected: excluded by parameter", datanodes.get(1), datanode0Details);
        Assert.assertNotEquals("Datanode 2 should not been selected: not enough space there", datanodes.get(2), datanode0Details);
        selectedCount.put(datanode0Details, selectedCount.get(datanode0Details) + 1);
    }
    // datanode 6 has more space than datanode 3 and datanode 4.
    Assert.assertTrue(selectedCount.get(datanodes.get(3)) < selectedCount.get(datanodes.get(6)));
    Assert.assertTrue(selectedCount.get(datanodes.get(4)) < selectedCount.get(datanodes.get(6)));
}

Example 5 with SCMNodeMetric

use of org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric in project ozone by apache.

the class SCMContainerPlacementCapacity method chooseNode.

/**
 * Find a node from the healthy list and return it after removing it from the
 * list that we are operating on.
 *
 * @param healthyNodes - List of healthy nodes that meet the size
 * requirement.
 * @return DatanodeDetails that is chosen.
 */
@Override
public DatanodeDetails chooseNode(List<DatanodeDetails> healthyNodes) {
    int firstNodeNdx = getRand().nextInt(healthyNodes.size());
    int secondNodeNdx = getRand().nextInt(healthyNodes.size());
    DatanodeDetails datanodeDetails;
    // if that is so, we just return the node.
    if (firstNodeNdx == secondNodeNdx) {
        datanodeDetails = healthyNodes.get(firstNodeNdx);
    } else {
        DatanodeDetails firstNodeDetails = healthyNodes.get(firstNodeNdx);
        DatanodeDetails secondNodeDetails = healthyNodes.get(secondNodeNdx);
        SCMNodeMetric firstNodeMetric = getNodeManager().getNodeStat(firstNodeDetails);
        SCMNodeMetric secondNodeMetric = getNodeManager().getNodeStat(secondNodeDetails);
        datanodeDetails = !firstNodeMetric.isGreater(secondNodeMetric.get()) ? firstNodeDetails : secondNodeDetails;
    }
    healthyNodes.remove(datanodeDetails);
    return datanodeDetails;
}