Examples with LayoutVersionProto org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto used on opensource projects

Example 16 with LayoutVersionProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto in project ozone by apache.

the class TestSCMNodeMetrics method testNodeCountAndInfoMetricsReported.

/**
 * Verify that datanode aggregated state and capacity metrics are
 * reported.
 */
@Test
public void testNodeCountAndInfoMetricsReported() throws Exception {
    StorageReportProto storageReport = HddsTestUtils.createStorageReport(registeredDatanode.getUuid(), "/tmp", 100, 10, 90, null);
    NodeReportProto nodeReport = NodeReportProto.newBuilder().addStorageReport(storageReport).build();
    nodeManager.processNodeReport(registeredDatanode, nodeReport);
    MetricsRecordBuilder metricsSource = getMetrics(SCMNodeMetrics.SOURCE_NAME);
    assertGauge("InServiceHealthyNodes", 1, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("InServiceHealthyReadonlyNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("InServiceStaleNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("InServiceDeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissioningHealthyNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissioningStaleNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissioningDeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedHealthyNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedStaleNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedDeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("EnteringMaintenanceHealthyNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("EnteringMaintenanceStaleNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("EnteringMaintenanceDeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("InMaintenanceHealthyNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("InMaintenanceStaleNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("InMaintenanceDeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("MaintenanceDiskCapacity", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("MaintenanceDiskUsed", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("MaintenanceDiskRemaining", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("MaintenanceSSDCapacity", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("MaintenanceSSDUsed", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("MaintenanceSSDRemaining", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedDiskCapacity", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedDiskUsed", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedDiskRemaining", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedSSDCapacity", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedSSDUsed", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    assertGauge("DecommissionedSSDRemaining", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName()));
    LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
    LayoutVersionProto layoutInfo = LayoutVersionProto.newBuilder().setSoftwareLayoutVersion(versionManager.getSoftwareLayoutVersion()).setMetadataLayoutVersion(versionManager.getMetadataLayoutVersion()).build();
    nodeManager.processHeartbeat(registeredDatanode, layoutInfo);
    sleep(4000);
    metricsSource = getMetrics(SCMNodeMetrics.SOURCE_NAME);
    assertGauge("InServiceHealthyReadonlyNodes", 0, metricsSource);
    assertGauge("InServiceHealthyNodes", 1, metricsSource);
}

Example 17 with LayoutVersionProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto in project ozone by apache.

the class TestSCMNodeManager method testSetNodeOpStateAndCommandFired.

/**
 * For leader SCM, ensure that a change to the operationalState of a node
 * fires a SCMCommand of type SetNodeOperationalStateCommand.
 *
 * For follower SCM, no SetNodeOperationalStateCommand should be fired, yet
 * operationalState of the node will be updated according to the heartbeat.
 */
@Test
public void testSetNodeOpStateAndCommandFired() throws IOException, NodeNotFoundException, AuthenticationException {
    final int interval = 100;
    OzoneConfiguration conf = getConf();
    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval, MILLISECONDS);
    // If factor 1 pipelines are auto created, registering the new node will
    // trigger a pipeline creation command which may interfere with command
    // checking in this test.
    conf.setBoolean(OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE, false);
    try (SCMNodeManager nodeManager = createNodeManager(conf)) {
        DatanodeDetails dn = HddsTestUtils.createRandomDatanodeAndRegister(nodeManager);
        LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
        final LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
        long expiry = System.currentTimeMillis() / 1000 + 1000;
        nodeManager.setNodeOperationalState(dn, HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, expiry);
        // If found mismatch, leader SCM fires a SetNodeOperationalStateCommand
        // to update the opState persisted in Datanode.
        scm.getScmContext().updateLeaderAndTerm(true, 1);
        List<SCMCommand> commands = nodeManager.processHeartbeat(dn, layoutInfo);
        Assert.assertTrue(commands.get(0).getClass().equals(SetNodeOperationalStateCommand.class));
        assertEquals(1, commands.size());
        // If found mismatch, follower SCM update its own opState according
        // to the heartbeat, and no SCMCommand will be fired.
        scm.getScmContext().updateLeaderAndTerm(false, 2);
        commands = nodeManager.processHeartbeat(dn, layoutInfo);
        assertEquals(0, commands.size());
        NodeStatus scmStatus = nodeManager.getNodeStatus(dn);
        assertTrue(scmStatus.getOperationalState() == dn.getPersistedOpState() && scmStatus.getOpStateExpiryEpochSeconds() == dn.getPersistedOpStateExpiryEpochSec());
    }
}

Example 18 with LayoutVersionProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto in project ozone by apache.

the class TestSCMNodeManager method testScmHeartbeat.

/**
 * Tests that Node manager handles heartbeats correctly, and comes out of
 * safe Mode.
 *
 * @throws IOException
 * @throws InterruptedException
 * @throws TimeoutException
 */
@Test
public void testScmHeartbeat() throws IOException, InterruptedException, AuthenticationException {
    try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
        LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
        LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
        int registeredNodes = 5;
        // Send some heartbeats from different nodes.
        for (int x = 0; x < registeredNodes; x++) {
            DatanodeDetails datanodeDetails = HddsTestUtils.createRandomDatanodeAndRegister(nodeManager);
            nodeManager.processHeartbeat(datanodeDetails, layoutInfo);
        }
        // TODO: wait for heartbeat to be processed
        Thread.sleep(4 * 1000);
        assertTrue("Heartbeat thread should have picked up the" + "scheduled heartbeats.", nodeManager.getAllNodes().size() == registeredNodes);
    }
}

Example 19 with LayoutVersionProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto in project ozone by apache.

the class TestSCMNodeManager method testScmHealthyNodeCount.

/**
 * Asserts that we detect as many healthy nodes as we have generated heartbeat
 * for.
 *
 * @throws IOException
 * @throws InterruptedException
 * @throws TimeoutException
 */
@Test
public void testScmHealthyNodeCount() throws IOException, InterruptedException, AuthenticationException {
    OzoneConfiguration conf = getConf();
    final int count = 10;
    try (SCMNodeManager nodeManager = createNodeManager(conf)) {
        LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
        LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
        for (int x = 0; x < count; x++) {
            DatanodeDetails datanodeDetails = HddsTestUtils.createRandomDatanodeAndRegister(nodeManager);
            nodeManager.processHeartbeat(datanodeDetails, layoutInfo);
        }
        // TODO: wait for heartbeat to be processed
        Thread.sleep(4 * 1000);
        assertEquals(count, nodeManager.getNodeCount(NodeStatus.inServiceHealthy()));
        Map<String, Map<String, Integer>> nodeCounts = nodeManager.getNodeCount();
        assertEquals(count, nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name()).get(HddsProtos.NodeState.HEALTHY.name()).intValue());
    }
}

Example 20 with LayoutVersionProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto in project ozone by apache.

the class TestSCMNodeManager method tesVolumeInfoFromNodeReport.

/**
 * Test multiple nodes sending initial heartbeat with their node report
 * with multiple volumes.
 *
 * @throws IOException
 * @throws InterruptedException
 * @throws TimeoutException
 */
@Test
public void tesVolumeInfoFromNodeReport() throws IOException, InterruptedException, AuthenticationException {
    OzoneConfiguration conf = getConf();
    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 1000, MILLISECONDS);
    final int volumeCount = 10;
    final long capacity = 2000;
    final long used = 100;
    List<DatanodeDetails> dnList = new ArrayList<>(1);
    try (SCMNodeManager nodeManager = createNodeManager(conf)) {
        EventQueue eventQueue = (EventQueue) scm.getEventQueue();
        DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
        dnList.add(dn);
        UUID dnId = dn.getUuid();
        long free = capacity - used;
        List<StorageReportProto> reports = new ArrayList<>(volumeCount);
        boolean failed = true;
        for (int x = 0; x < volumeCount; x++) {
            String storagePath = testDir.getAbsolutePath() + "/" + dnId;
            reports.add(HddsTestUtils.createStorageReport(dnId, storagePath, capacity, used, free, null, failed));
            failed = !failed;
        }
        nodeManager.register(dn, HddsTestUtils.createNodeReport(reports, Collections.emptyList()), null);
        LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
        LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
        nodeManager.processHeartbeat(dn, layoutInfo);
        // TODO: wait for EventQueue to be processed
        eventQueue.processAll(8000L);
        assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceHealthy()));
        assertEquals(volumeCount / 2, nodeManager.minHealthyVolumeNum(dnList));
        dnList.clear();
    }
}