Examples with NodeReportProto org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto used on opensource projects

Example 6 with NodeReportProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto in project ozone by apache.

the class StorageContainerDatanodeProtocolServerSideTranslatorPB method register.

public SCMRegisteredResponseProto register(SCMRegisterRequestProto request) throws IOException {
    ContainerReportsProto containerRequestProto = request.getContainerReport();
    NodeReportProto dnNodeReport = request.getNodeReport();
    PipelineReportsProto pipelineReport = request.getPipelineReports();
    LayoutVersionProto layoutInfo = null;
    if (request.hasDataNodeLayoutVersion()) {
        layoutInfo = request.getDataNodeLayoutVersion();
    } else {
        // Backward compatibility to make sure old Datanodes can still talk to
        // SCM.
        layoutInfo = toLayoutVersionProto(INITIAL_VERSION.layoutVersion(), INITIAL_VERSION.layoutVersion());
    }
    return impl.register(request.getExtendedDatanodeDetails(), dnNodeReport, containerRequestProto, pipelineReport, layoutInfo);
}

Example 7 with NodeReportProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto in project ozone by apache.

the class TestSCMNodeManager method testScmNodeReportUpdate.

/**
 * Test single node stat update based on nodereport from different heartbeat
 * status (healthy, stale and dead).
 * @throws IOException
 * @throws InterruptedException
 * @throws TimeoutException
 */
@Test
public void testScmNodeReportUpdate() throws IOException, InterruptedException, TimeoutException, AuthenticationException {
    OzoneConfiguration conf = getConf();
    final int heartbeatCount = 5;
    final int nodeCount = 1;
    final int interval = 100;
    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval, MILLISECONDS);
    conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
    conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
    conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
    try (SCMNodeManager nodeManager = createNodeManager(conf)) {
        DatanodeDetails datanodeDetails = HddsTestUtils.createRandomDatanodeAndRegister(nodeManager);
        NodeReportHandler nodeReportHandler = new NodeReportHandler(nodeManager);
        EventPublisher publisher = mock(EventPublisher.class);
        final long capacity = 2000;
        final long usedPerHeartbeat = 100;
        UUID dnId = datanodeDetails.getUuid();
        for (int x = 0; x < heartbeatCount; x++) {
            long scmUsed = x * usedPerHeartbeat;
            long remaining = capacity - scmUsed;
            String storagePath = testDir.getAbsolutePath() + "/" + dnId;
            StorageReportProto report = HddsTestUtils.createStorageReport(dnId, storagePath, capacity, scmUsed, remaining, null);
            NodeReportProto nodeReportProto = HddsTestUtils.createNodeReport(Arrays.asList(report), Collections.emptyList());
            nodeReportHandler.onMessage(new NodeReportFromDatanode(datanodeDetails, nodeReportProto), publisher);
            LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
            LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
            nodeManager.processHeartbeat(datanodeDetails, layoutInfo);
            Thread.sleep(100);
        }
        final long expectedScmUsed = usedPerHeartbeat * (heartbeatCount - 1);
        final long expectedRemaining = capacity - expectedScmUsed;
        GenericTestUtils.waitFor(() -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed, 100, 4 * 1000);
        long foundCapacity = nodeManager.getStats().getCapacity().get();
        assertEquals(capacity, foundCapacity);
        long foundScmUsed = nodeManager.getStats().getScmUsed().get();
        assertEquals(expectedScmUsed, foundScmUsed);
        long foundRemaining = nodeManager.getStats().getRemaining().get();
        assertEquals(expectedRemaining, foundRemaining);
        // Test NodeManager#getNodeStats
        assertEquals(nodeCount, nodeManager.getNodeStats().size());
        long nodeCapacity = nodeManager.getNodeStat(datanodeDetails).get().getCapacity().get();
        assertEquals(capacity, nodeCapacity);
        foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get().getScmUsed().get();
        assertEquals(expectedScmUsed, foundScmUsed);
        foundRemaining = nodeManager.getNodeStat(datanodeDetails).get().getRemaining().get();
        assertEquals(expectedRemaining, foundRemaining);
        // Compare the result from
        // NodeManager#getNodeStats and NodeManager#getNodeStat
        SCMNodeStat stat1 = nodeManager.getNodeStats().get(datanodeDetails);
        SCMNodeStat stat2 = nodeManager.getNodeStat(datanodeDetails).get();
        assertEquals(stat1, stat2);
        // Wait up to 4s so that the node becomes stale
        // Verify the usage info should be unchanged.
        GenericTestUtils.waitFor(() -> nodeManager.getNodeCount(NodeStatus.inServiceStale()) == 1, 100, 4 * 1000);
        assertEquals(nodeCount, nodeManager.getNodeStats().size());
        foundCapacity = nodeManager.getNodeStat(datanodeDetails).get().getCapacity().get();
        assertEquals(capacity, foundCapacity);
        foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get().getScmUsed().get();
        assertEquals(expectedScmUsed, foundScmUsed);
        foundRemaining = nodeManager.getNodeStat(datanodeDetails).get().getRemaining().get();
        assertEquals(expectedRemaining, foundRemaining);
        // Wait up to 4 more seconds so the node becomes dead
        // Verify usage info should be updated.
        GenericTestUtils.waitFor(() -> nodeManager.getNodeCount(NodeStatus.inServiceDead()) == 1, 100, 4 * 1000);
        assertEquals(0, nodeManager.getNodeStats().size());
        foundCapacity = nodeManager.getStats().getCapacity().get();
        assertEquals(0, foundCapacity);
        foundScmUsed = nodeManager.getStats().getScmUsed().get();
        assertEquals(0, foundScmUsed);
        foundRemaining = nodeManager.getStats().getRemaining().get();
        assertEquals(0, foundRemaining);
        LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
        LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
        nodeManager.processHeartbeat(datanodeDetails, layoutInfo);
        // Wait up to 5 seconds so that the dead node becomes healthy
        // Verify usage info should be updated.
        GenericTestUtils.waitFor(() -> nodeManager.getNodeCount(NodeStatus.inServiceHealthy()) == 1, 100, 5 * 1000);
        GenericTestUtils.waitFor(() -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed, 100, 4 * 1000);
        assertEquals(nodeCount, nodeManager.getNodeStats().size());
        foundCapacity = nodeManager.getNodeStat(datanodeDetails).get().getCapacity().get();
        assertEquals(capacity, foundCapacity);
        foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get().getScmUsed().get();
        assertEquals(expectedScmUsed, foundScmUsed);
        foundRemaining = nodeManager.getNodeStat(datanodeDetails).get().getRemaining().get();
        assertEquals(expectedRemaining, foundRemaining);
    }
}

Example 8 with NodeReportProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto in project ozone by apache.

the class TestStatisticsUpdate method testStatisticsUpdate.

@Test
public void testStatisticsUpdate() throws Exception {
    // GIVEN
    DatanodeDetails datanode1 = MockDatanodeDetails.randomDatanodeDetails();
    DatanodeDetails datanode2 = MockDatanodeDetails.randomDatanodeDetails();
    String storagePath1 = GenericTestUtils.getRandomizedTempPath().concat("/" + datanode1.getUuidString());
    String storagePath2 = GenericTestUtils.getRandomizedTempPath().concat("/" + datanode2.getUuidString());
    StorageReportProto storageOne = HddsTestUtils.createStorageReport(datanode1.getUuid(), storagePath1, 100, 10, 90, null);
    StorageReportProto storageTwo = HddsTestUtils.createStorageReport(datanode2.getUuid(), storagePath2, 200, 20, 180, null);
    nodeManager.register(datanode1, HddsTestUtils.createNodeReport(Arrays.asList(storageOne), Collections.emptyList()), null);
    nodeManager.register(datanode2, HddsTestUtils.createNodeReport(Arrays.asList(storageTwo), Collections.emptyList()), null);
    NodeReportProto nodeReportProto1 = HddsTestUtils.createNodeReport(Arrays.asList(storageOne), Collections.emptyList());
    NodeReportProto nodeReportProto2 = HddsTestUtils.createNodeReport(Arrays.asList(storageTwo), Collections.emptyList());
    nodeReportHandler.onMessage(new NodeReportFromDatanode(datanode1, nodeReportProto1), Mockito.mock(EventPublisher.class));
    nodeReportHandler.onMessage(new NodeReportFromDatanode(datanode2, nodeReportProto2), Mockito.mock(EventPublisher.class));
    SCMNodeStat stat = nodeManager.getStats();
    Assert.assertEquals(300L, stat.getCapacity().get().longValue());
    Assert.assertEquals(270L, stat.getRemaining().get().longValue());
    Assert.assertEquals(30L, stat.getScmUsed().get().longValue());
    SCMNodeMetric nodeStat = nodeManager.getNodeStat(datanode1);
    Assert.assertEquals(100L, nodeStat.get().getCapacity().get().longValue());
    Assert.assertEquals(90L, nodeStat.get().getRemaining().get().longValue());
    Assert.assertEquals(10L, nodeStat.get().getScmUsed().get().longValue());
    // TODO: Support logic to mark a node as dead in NodeManager.
    LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
    StorageContainerDatanodeProtocolProtos.LayoutVersionProto layoutInfo = StorageContainerDatanodeProtocolProtos.LayoutVersionProto.newBuilder().setSoftwareLayoutVersion(versionManager.getSoftwareLayoutVersion()).setMetadataLayoutVersion(versionManager.getMetadataLayoutVersion()).build();
    nodeManager.processHeartbeat(datanode2, layoutInfo);
    Thread.sleep(1000);
    nodeManager.processHeartbeat(datanode2, layoutInfo);
    Thread.sleep(1000);
    nodeManager.processHeartbeat(datanode2, layoutInfo);
    Thread.sleep(1000);
    nodeManager.processHeartbeat(datanode2, layoutInfo);
    // THEN statistics in SCM should changed.
    stat = nodeManager.getStats();
    Assert.assertEquals(200L, stat.getCapacity().get().longValue());
    Assert.assertEquals(180L, stat.getRemaining().get().longValue());
    Assert.assertEquals(20L, stat.getScmUsed().get().longValue());
}

Example 9 with NodeReportProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto in project ozone by apache.

the class TestSCMDatanodeHeartbeatDispatcher method testNodeReportDispatcher.

@Test
public void testNodeReportDispatcher() throws IOException {
    AtomicInteger eventReceived = new AtomicInteger();
    NodeReportProto nodeReport = NodeReportProto.getDefaultInstance();
    NodeManager mockNodeManager = Mockito.mock(NodeManager.class);
    Mockito.when(mockNodeManager.isNodeRegistered(Mockito.any())).thenReturn(true);
    SCMDatanodeHeartbeatDispatcher dispatcher = new SCMDatanodeHeartbeatDispatcher(mockNodeManager, new EventPublisher() {

        @Override
        public <PAYLOAD, EVENT extends Event<PAYLOAD>> void fireEvent(EVENT event, PAYLOAD payload) {
            Assert.assertEquals(event, NODE_REPORT);
            eventReceived.incrementAndGet();
            Assert.assertEquals(nodeReport, ((NodeReportFromDatanode) payload).getReport());
        }
    });
    DatanodeDetails datanodeDetails = randomDatanodeDetails();
    SCMHeartbeatRequestProto heartbeat = SCMHeartbeatRequestProto.newBuilder().setDatanodeDetails(datanodeDetails.getProtoBufMessage()).setNodeReport(nodeReport).build();
    dispatcher.dispatch(heartbeat);
    Assert.assertEquals(1, eventReceived.get());
}

Example 10 with NodeReportProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto in project ozone by apache.

the class TestSCMNodeMetrics method testNodeCountAndInfoMetricsReported.

/**
 * Verify that datanode aggregated state and capacity metrics are
 * reported.
 */
@Test
public void testNodeCountAndInfoMetricsReported() throws Exception {
    StorageReportProto storageReport = HddsTestUtils.createStorageReport(registeredDatanode.getUuid(), "/tmp", 100, 10, 90, null);
    NodeReportProto nodeReport = NodeReportProto.newBuilder().addStorageReport(storageReport).build();
    nodeManager.processNodeReport(registeredDatanode, nodeReport);
    MetricsRecordBuilder metricsSource = getMetrics(SCMNodeMetrics.SOURCE_NAME);
    assertGauge("InServiceHealthyNodes", 1, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("InServiceHealthyReadonlyNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("InServiceStaleNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("InServiceDeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissioningHealthyNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissioningStaleNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissioningDeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedHealthyNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedStaleNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedDeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("EnteringMaintenanceHealthyNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("EnteringMaintenanceStaleNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("EnteringMaintenanceDeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("InMaintenanceHealthyNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("InMaintenanceStaleNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("InMaintenanceDeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("MaintenanceDiskCapacity", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("MaintenanceDiskUsed", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("MaintenanceDiskRemaining", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("MaintenanceSSDCapacity", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("MaintenanceSSDUsed", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("MaintenanceSSDRemaining", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedDiskCapacity", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedDiskUsed", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedDiskRemaining", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedSSDCapacity", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedSSDUsed", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedSSDRemaining", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
    LayoutVersionProto layoutInfo = LayoutVersionProto.newBuilder().setSoftwareLayoutVersion(versionManager.getSoftwareLayoutVersion()).setMetadataLayoutVersion(versionManager.getMetadataLayoutVersion()).build();
    nodeManager.processHeartbeat(registeredDatanode, layoutInfo);
    sleep(4000);
    metricsSource = getMetrics(SCMNodeMetrics.SOURCE_NAME);
    assertGauge("InServiceHealthyReadonlyNodes", 0, metricsSource);
    assertGauge("InServiceHealthyNodes", 1, metricsSource);
}