Examples with StorageReportProto org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto used on opensource projects

Example 6 with StorageReportProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto in project ozone by apache.

the class TestOpenContainerCount method setUp.

@Before
public void setUp() throws Exception {
    // The following setup runs only once
    if (!isSetupDone) {
        initializeInjector();
        isSetupDone = true;
    }
    datanodeId = datanodeDetails.getUuid().toString();
    // initialize container report
    builder = ContainerReportsProto.newBuilder();
    for (long i = 1L; i <= 10L; i++) {
        builder.addReports(ContainerReplicaProto.newBuilder().setContainerID(i).setState(ContainerReplicaProto.State.OPEN).setOriginNodeId(datanodeId).build());
    }
    containerReportsProto = builder.build();
    UUID pipelineUuid = UUID.fromString(pipelineId);
    HddsProtos.UUID uuid128 = HddsProtos.UUID.newBuilder().setMostSigBits(pipelineUuid.getMostSignificantBits()).setLeastSigBits(pipelineUuid.getLeastSignificantBits()).build();
    UUID pipelineUuid2 = UUID.fromString(pipelineId2);
    HddsProtos.UUID uuid1282 = HddsProtos.UUID.newBuilder().setMostSigBits(pipelineUuid2.getMostSignificantBits()).setLeastSigBits(pipelineUuid2.getLeastSignificantBits()).build();
    PipelineReport pipelineReport = PipelineReport.newBuilder().setPipelineID(PipelineID.newBuilder().setId(pipelineId).setUuid128(uuid128).build()).setIsLeader(true).build();
    PipelineReport pipelineReport2 = PipelineReport.newBuilder().setPipelineID(PipelineID.newBuilder().setId(pipelineId2).setUuid128(uuid1282).build()).setIsLeader(false).build();
    pipelineReportsProto = PipelineReportsProto.newBuilder().addPipelineReport(pipelineReport).addPipelineReport(pipelineReport2).build();
    DatanodeDetailsProto datanodeDetailsProto = DatanodeDetailsProto.newBuilder().setHostName(HOST1).setUuid(datanodeId).setIpAddress(IP1).build();
    extendedDatanodeDetailsProto = HddsProtos.ExtendedDatanodeDetailsProto.newBuilder().setDatanodeDetails(datanodeDetailsProto).setVersion("0.6.0").setSetupTime(1596347628802L).setBuildDate("2020-08-01T08:50Z").setRevision("3346f493fa1690358add7bb9f3e5b52545993f36").build();
    StorageReportProto storageReportProto1 = StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK).setStorageLocation("/disk1").setScmUsed(10 * OzoneConsts.GB).setRemaining(90 * OzoneConsts.GB).setCapacity(100 * OzoneConsts.GB).setStorageUuid(UUID.randomUUID().toString()).setFailed(false).build();
    StorageReportProto storageReportProto2 = StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK).setStorageLocation("/disk2").setScmUsed(10 * OzoneConsts.GB).setRemaining(90 * OzoneConsts.GB).setCapacity(100 * OzoneConsts.GB).setStorageUuid(UUID.randomUUID().toString()).setFailed(false).build();
    nodeReportProto = NodeReportProto.newBuilder().addStorageReport(storageReportProto1).addStorageReport(storageReportProto2).build();
    try {
        reconScm.getDatanodeProtocolServer().register(extendedDatanodeDetailsProto, nodeReportProto, containerReportsProto, pipelineReportsProto, defaultLayoutVersionProto());
        // Process all events in the event queue
        reconScm.getEventQueue().processAll(1000);
    } catch (Exception ex) {
        Assert.fail(ex.getMessage());
    }
}

Example 7 with StorageReportProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto in project ozone by apache.

the class ScmTestMock method updateNodeReport.

/**
 * Update nodeReport.
 * @param datanodeDetailsProto
 * @param nodeReport
 */
public void updateNodeReport(DatanodeDetailsProto datanodeDetailsProto, NodeReportProto nodeReport) {
    DatanodeDetails datanode = DatanodeDetails.getFromProtoBuf(datanodeDetailsProto);
    NodeReportProto.Builder nodeReportProto = NodeReportProto.newBuilder();
    List<StorageReportProto> storageReports = nodeReport.getStorageReportList();
    for (StorageReportProto report : storageReports) {
        nodeReportProto.addStorageReport(report);
    }
    nodeReports.put(datanode, nodeReportProto.build());
}

Example 8 with StorageReportProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto in project ozone by apache.

the class TestSCMNodeManager method testGetNodeInfo.

@Test
public void testGetNodeInfo() throws IOException, InterruptedException, NodeNotFoundException, AuthenticationException {
    OzoneConfiguration conf = getConf();
    final int nodeCount = 6;
    SCMNodeManager nodeManager = createNodeManager(conf);
    for (int i = 0; i < nodeCount; i++) {
        DatanodeDetails datanodeDetails = MockDatanodeDetails.randomDatanodeDetails();
        final long capacity = 2000;
        final long used = 100;
        final long remaining = 1900;
        UUID dnId = datanodeDetails.getUuid();
        String storagePath = testDir.getAbsolutePath() + "/" + dnId;
        StorageReportProto report = HddsTestUtils.createStorageReport(dnId, storagePath, capacity, used, remaining, null);
        nodeManager.register(datanodeDetails, HddsTestUtils.createNodeReport(Arrays.asList(report), Collections.emptyList()), HddsTestUtils.getRandomPipelineReports());
        LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
        LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
        nodeManager.register(datanodeDetails, HddsTestUtils.createNodeReport(Arrays.asList(report), Collections.emptyList()), HddsTestUtils.getRandomPipelineReports(), layoutInfo);
        nodeManager.processHeartbeat(datanodeDetails, layoutInfo);
        if (i == 5) {
            nodeManager.setNodeOperationalState(datanodeDetails, HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE);
        }
        if (i == 3 || i == 4) {
            nodeManager.setNodeOperationalState(datanodeDetails, HddsProtos.NodeOperationalState.DECOMMISSIONED);
        }
    }
    Thread.sleep(100);
    Map<String, Long> stats = nodeManager.getNodeInfo();
    // 3 IN_SERVICE nodes:
    assertEquals(6000, stats.get("DiskCapacity").longValue());
    assertEquals(300, stats.get("DiskUsed").longValue());
    assertEquals(5700, stats.get("DiskRemaining").longValue());
    // 2 Decommissioned nodes
    assertEquals(4000, stats.get("DecommissionedDiskCapacity").longValue());
    assertEquals(200, stats.get("DecommissionedDiskUsed").longValue());
    assertEquals(3800, stats.get("DecommissionedDiskRemaining").longValue());
    // 1 Maintenance node
    assertEquals(2000, stats.get("MaintenanceDiskCapacity").longValue());
    assertEquals(100, stats.get("MaintenanceDiskUsed").longValue());
    assertEquals(1900, stats.get("MaintenanceDiskRemaining").longValue());
}

Example 9 with StorageReportProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto in project ozone by apache.

the class TestSCMNodeManager method testHandlingSCMCommandEvent.

@Test
public void testHandlingSCMCommandEvent() throws IOException, AuthenticationException {
    OzoneConfiguration conf = getConf();
    conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100, TimeUnit.MILLISECONDS);
    DatanodeDetails datanodeDetails = randomDatanodeDetails();
    UUID dnId = datanodeDetails.getUuid();
    String storagePath = testDir.getAbsolutePath() + "/" + dnId;
    StorageReportProto report = HddsTestUtils.createStorageReport(dnId, storagePath, 100, 10, 90, null);
    EventQueue eq = new EventQueue();
    try (SCMNodeManager nodemanager = createNodeManager(conf)) {
        eq.addHandler(DATANODE_COMMAND, nodemanager);
        nodemanager.register(datanodeDetails, HddsTestUtils.createNodeReport(Arrays.asList(report), Collections.emptyList()), HddsTestUtils.getRandomPipelineReports());
        eq.fireEvent(DATANODE_COMMAND, new CommandForDatanode<>(datanodeDetails.getUuid(), new CloseContainerCommand(1L, PipelineID.randomId())));
        LayoutVersionManager versionManager = nodemanager.getLayoutVersionManager();
        LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
        eq.processAll(1000L);
        List<SCMCommand> command = nodemanager.processHeartbeat(datanodeDetails, layoutInfo);
        // With dh registered, SCM will send create pipeline command to dn
        Assert.assertTrue(command.size() >= 1);
        Assert.assertTrue(command.get(0).getClass().equals(CloseContainerCommand.class) || command.get(1).getClass().equals(CloseContainerCommand.class));
    } catch (IOException e) {
        e.printStackTrace();
        throw e;
    }
}

Example 10 with StorageReportProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto in project ozone by apache.

the class TestSCMNodeManager method testScmNodeReportUpdate.

/**
 * Test single node stat update based on nodereport from different heartbeat
 * status (healthy, stale and dead).
 * @throws IOException
 * @throws InterruptedException
 * @throws TimeoutException
 */
@Test
public void testScmNodeReportUpdate() throws IOException, InterruptedException, TimeoutException, AuthenticationException {
    OzoneConfiguration conf = getConf();
    final int heartbeatCount = 5;
    final int nodeCount = 1;
    final int interval = 100;
    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval, MILLISECONDS);
    conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
    conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
    conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
    try (SCMNodeManager nodeManager = createNodeManager(conf)) {
        DatanodeDetails datanodeDetails = HddsTestUtils.createRandomDatanodeAndRegister(nodeManager);
        NodeReportHandler nodeReportHandler = new NodeReportHandler(nodeManager);
        EventPublisher publisher = mock(EventPublisher.class);
        final long capacity = 2000;
        final long usedPerHeartbeat = 100;
        UUID dnId = datanodeDetails.getUuid();
        for (int x = 0; x < heartbeatCount; x++) {
            long scmUsed = x * usedPerHeartbeat;
            long remaining = capacity - scmUsed;
            String storagePath = testDir.getAbsolutePath() + "/" + dnId;
            StorageReportProto report = HddsTestUtils.createStorageReport(dnId, storagePath, capacity, scmUsed, remaining, null);
            NodeReportProto nodeReportProto = HddsTestUtils.createNodeReport(Arrays.asList(report), Collections.emptyList());
            nodeReportHandler.onMessage(new NodeReportFromDatanode(datanodeDetails, nodeReportProto), publisher);
            LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
            LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
            nodeManager.processHeartbeat(datanodeDetails, layoutInfo);
            Thread.sleep(100);
        }
        final long expectedScmUsed = usedPerHeartbeat * (heartbeatCount - 1);
        final long expectedRemaining = capacity - expectedScmUsed;
        GenericTestUtils.waitFor(() -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed, 100, 4 * 1000);
        long foundCapacity = nodeManager.getStats().getCapacity().get();
        assertEquals(capacity, foundCapacity);
        long foundScmUsed = nodeManager.getStats().getScmUsed().get();
        assertEquals(expectedScmUsed, foundScmUsed);
        long foundRemaining = nodeManager.getStats().getRemaining().get();
        assertEquals(expectedRemaining, foundRemaining);
        // Test NodeManager#getNodeStats
        assertEquals(nodeCount, nodeManager.getNodeStats().size());
        long nodeCapacity = nodeManager.getNodeStat(datanodeDetails).get().getCapacity().get();
        assertEquals(capacity, nodeCapacity);
        foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get().getScmUsed().get();
        assertEquals(expectedScmUsed, foundScmUsed);
        foundRemaining = nodeManager.getNodeStat(datanodeDetails).get().getRemaining().get();
        assertEquals(expectedRemaining, foundRemaining);
        // Compare the result from
        // NodeManager#getNodeStats and NodeManager#getNodeStat
        SCMNodeStat stat1 = nodeManager.getNodeStats().get(datanodeDetails);
        SCMNodeStat stat2 = nodeManager.getNodeStat(datanodeDetails).get();
        assertEquals(stat1, stat2);
        // Wait up to 4s so that the node becomes stale
        // Verify the usage info should be unchanged.
        GenericTestUtils.waitFor(() -> nodeManager.getNodeCount(NodeStatus.inServiceStale()) == 1, 100, 4 * 1000);
        assertEquals(nodeCount, nodeManager.getNodeStats().size());
        foundCapacity = nodeManager.getNodeStat(datanodeDetails).get().getCapacity().get();
        assertEquals(capacity, foundCapacity);
        foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get().getScmUsed().get();
        assertEquals(expectedScmUsed, foundScmUsed);
        foundRemaining = nodeManager.getNodeStat(datanodeDetails).get().getRemaining().get();
        assertEquals(expectedRemaining, foundRemaining);
        // Wait up to 4 more seconds so the node becomes dead
        // Verify usage info should be updated.
        GenericTestUtils.waitFor(() -> nodeManager.getNodeCount(NodeStatus.inServiceDead()) == 1, 100, 4 * 1000);
        assertEquals(0, nodeManager.getNodeStats().size());
        foundCapacity = nodeManager.getStats().getCapacity().get();
        assertEquals(0, foundCapacity);
        foundScmUsed = nodeManager.getStats().getScmUsed().get();
        assertEquals(0, foundScmUsed);
        foundRemaining = nodeManager.getStats().getRemaining().get();
        assertEquals(0, foundRemaining);
        LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
        LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
        nodeManager.processHeartbeat(datanodeDetails, layoutInfo);
        // Wait up to 5 seconds so that the dead node becomes healthy
        // Verify usage info should be updated.
        GenericTestUtils.waitFor(() -> nodeManager.getNodeCount(NodeStatus.inServiceHealthy()) == 1, 100, 5 * 1000);
        GenericTestUtils.waitFor(() -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed, 100, 4 * 1000);
        assertEquals(nodeCount, nodeManager.getNodeStats().size());
        foundCapacity = nodeManager.getNodeStat(datanodeDetails).get().getCapacity().get();
        assertEquals(capacity, foundCapacity);
        foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get().getScmUsed().get();
        assertEquals(expectedScmUsed, foundScmUsed);
        foundRemaining = nodeManager.getNodeStat(datanodeDetails).get().getRemaining().get();
        assertEquals(expectedRemaining, foundRemaining);
    }
}