use of org.apache.hadoop.ozone.upgrade.LayoutVersionManager in project ozone by apache.
the class TestSCMNodeMetrics method testHBProcessingFailure.
/**
* Verifies heartbeat processing failure count.
*/
@Test
public void testHBProcessingFailure() {
long hbProcessedFailed = getCounter("NumHBProcessingFailed");
LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
LayoutVersionProto layoutInfo = LayoutVersionProto.newBuilder().setSoftwareLayoutVersion(versionManager.getSoftwareLayoutVersion()).setMetadataLayoutVersion(versionManager.getMetadataLayoutVersion()).build();
nodeManager.processHeartbeat(MockDatanodeDetails.randomDatanodeDetails(), layoutInfo);
assertEquals("NumHBProcessingFailed", hbProcessedFailed + 1, getCounter("NumHBProcessingFailed"));
}
use of org.apache.hadoop.ozone.upgrade.LayoutVersionManager in project ozone by apache.
the class TestSCMNodeManager method testGetNodeInfo.
@Test
public void testGetNodeInfo() throws IOException, InterruptedException, NodeNotFoundException, AuthenticationException {
OzoneConfiguration conf = getConf();
final int nodeCount = 6;
SCMNodeManager nodeManager = createNodeManager(conf);
for (int i = 0; i < nodeCount; i++) {
DatanodeDetails datanodeDetails = MockDatanodeDetails.randomDatanodeDetails();
final long capacity = 2000;
final long used = 100;
final long remaining = 1900;
UUID dnId = datanodeDetails.getUuid();
String storagePath = testDir.getAbsolutePath() + "/" + dnId;
StorageReportProto report = HddsTestUtils.createStorageReport(dnId, storagePath, capacity, used, remaining, null);
nodeManager.register(datanodeDetails, HddsTestUtils.createNodeReport(Arrays.asList(report), Collections.emptyList()), HddsTestUtils.getRandomPipelineReports());
LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
nodeManager.register(datanodeDetails, HddsTestUtils.createNodeReport(Arrays.asList(report), Collections.emptyList()), HddsTestUtils.getRandomPipelineReports(), layoutInfo);
nodeManager.processHeartbeat(datanodeDetails, layoutInfo);
if (i == 5) {
nodeManager.setNodeOperationalState(datanodeDetails, HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE);
}
if (i == 3 || i == 4) {
nodeManager.setNodeOperationalState(datanodeDetails, HddsProtos.NodeOperationalState.DECOMMISSIONED);
}
}
Thread.sleep(100);
Map<String, Long> stats = nodeManager.getNodeInfo();
// 3 IN_SERVICE nodes:
assertEquals(6000, stats.get("DiskCapacity").longValue());
assertEquals(300, stats.get("DiskUsed").longValue());
assertEquals(5700, stats.get("DiskRemaining").longValue());
// 2 Decommissioned nodes
assertEquals(4000, stats.get("DecommissionedDiskCapacity").longValue());
assertEquals(200, stats.get("DecommissionedDiskUsed").longValue());
assertEquals(3800, stats.get("DecommissionedDiskRemaining").longValue());
// 1 Maintenance node
assertEquals(2000, stats.get("MaintenanceDiskCapacity").longValue());
assertEquals(100, stats.get("MaintenanceDiskUsed").longValue());
assertEquals(1900, stats.get("MaintenanceDiskRemaining").longValue());
}
use of org.apache.hadoop.ozone.upgrade.LayoutVersionManager in project ozone by apache.
the class TestSCMNodeManager method heartbeatNodeSet.
/**
* Heartbeat a given set of nodes at a specified frequency.
*
* @param manager - Node Manager
* @param list - List of datanodeIDs
* @param sleepDuration - Duration to sleep between heartbeats.
* @throws InterruptedException
*/
private void heartbeatNodeSet(SCMNodeManager manager, List<DatanodeDetails> list, int sleepDuration) throws InterruptedException {
LayoutVersionManager versionManager = manager.getLayoutVersionManager();
LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
while (!Thread.currentThread().isInterrupted()) {
for (DatanodeDetails dn : list) {
manager.processHeartbeat(dn, layoutInfo);
}
Thread.sleep(sleepDuration);
}
}
use of org.apache.hadoop.ozone.upgrade.LayoutVersionManager in project ozone by apache.
the class TestSCMNodeManager method testHandlingSCMCommandEvent.
@Test
public void testHandlingSCMCommandEvent() throws IOException, AuthenticationException {
OzoneConfiguration conf = getConf();
conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100, TimeUnit.MILLISECONDS);
DatanodeDetails datanodeDetails = randomDatanodeDetails();
UUID dnId = datanodeDetails.getUuid();
String storagePath = testDir.getAbsolutePath() + "/" + dnId;
StorageReportProto report = HddsTestUtils.createStorageReport(dnId, storagePath, 100, 10, 90, null);
EventQueue eq = new EventQueue();
try (SCMNodeManager nodemanager = createNodeManager(conf)) {
eq.addHandler(DATANODE_COMMAND, nodemanager);
nodemanager.register(datanodeDetails, HddsTestUtils.createNodeReport(Arrays.asList(report), Collections.emptyList()), HddsTestUtils.getRandomPipelineReports());
eq.fireEvent(DATANODE_COMMAND, new CommandForDatanode<>(datanodeDetails.getUuid(), new CloseContainerCommand(1L, PipelineID.randomId())));
LayoutVersionManager versionManager = nodemanager.getLayoutVersionManager();
LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
eq.processAll(1000L);
List<SCMCommand> command = nodemanager.processHeartbeat(datanodeDetails, layoutInfo);
// With dh registered, SCM will send create pipeline command to dn
Assert.assertTrue(command.size() >= 1);
Assert.assertTrue(command.get(0).getClass().equals(CloseContainerCommand.class) || command.get(1).getClass().equals(CloseContainerCommand.class));
} catch (IOException e) {
e.printStackTrace();
throw e;
}
}
use of org.apache.hadoop.ozone.upgrade.LayoutVersionManager in project ozone by apache.
the class TestSCMNodeManager method testScmNodeReportUpdate.
/**
* Test single node stat update based on nodereport from different heartbeat
* status (healthy, stale and dead).
* @throws IOException
* @throws InterruptedException
* @throws TimeoutException
*/
@Test
public void testScmNodeReportUpdate() throws IOException, InterruptedException, TimeoutException, AuthenticationException {
OzoneConfiguration conf = getConf();
final int heartbeatCount = 5;
final int nodeCount = 1;
final int interval = 100;
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval, MILLISECONDS);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
try (SCMNodeManager nodeManager = createNodeManager(conf)) {
DatanodeDetails datanodeDetails = HddsTestUtils.createRandomDatanodeAndRegister(nodeManager);
NodeReportHandler nodeReportHandler = new NodeReportHandler(nodeManager);
EventPublisher publisher = mock(EventPublisher.class);
final long capacity = 2000;
final long usedPerHeartbeat = 100;
UUID dnId = datanodeDetails.getUuid();
for (int x = 0; x < heartbeatCount; x++) {
long scmUsed = x * usedPerHeartbeat;
long remaining = capacity - scmUsed;
String storagePath = testDir.getAbsolutePath() + "/" + dnId;
StorageReportProto report = HddsTestUtils.createStorageReport(dnId, storagePath, capacity, scmUsed, remaining, null);
NodeReportProto nodeReportProto = HddsTestUtils.createNodeReport(Arrays.asList(report), Collections.emptyList());
nodeReportHandler.onMessage(new NodeReportFromDatanode(datanodeDetails, nodeReportProto), publisher);
LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
nodeManager.processHeartbeat(datanodeDetails, layoutInfo);
Thread.sleep(100);
}
final long expectedScmUsed = usedPerHeartbeat * (heartbeatCount - 1);
final long expectedRemaining = capacity - expectedScmUsed;
GenericTestUtils.waitFor(() -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed, 100, 4 * 1000);
long foundCapacity = nodeManager.getStats().getCapacity().get();
assertEquals(capacity, foundCapacity);
long foundScmUsed = nodeManager.getStats().getScmUsed().get();
assertEquals(expectedScmUsed, foundScmUsed);
long foundRemaining = nodeManager.getStats().getRemaining().get();
assertEquals(expectedRemaining, foundRemaining);
// Test NodeManager#getNodeStats
assertEquals(nodeCount, nodeManager.getNodeStats().size());
long nodeCapacity = nodeManager.getNodeStat(datanodeDetails).get().getCapacity().get();
assertEquals(capacity, nodeCapacity);
foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get().getScmUsed().get();
assertEquals(expectedScmUsed, foundScmUsed);
foundRemaining = nodeManager.getNodeStat(datanodeDetails).get().getRemaining().get();
assertEquals(expectedRemaining, foundRemaining);
// Compare the result from
// NodeManager#getNodeStats and NodeManager#getNodeStat
SCMNodeStat stat1 = nodeManager.getNodeStats().get(datanodeDetails);
SCMNodeStat stat2 = nodeManager.getNodeStat(datanodeDetails).get();
assertEquals(stat1, stat2);
// Wait up to 4s so that the node becomes stale
// Verify the usage info should be unchanged.
GenericTestUtils.waitFor(() -> nodeManager.getNodeCount(NodeStatus.inServiceStale()) == 1, 100, 4 * 1000);
assertEquals(nodeCount, nodeManager.getNodeStats().size());
foundCapacity = nodeManager.getNodeStat(datanodeDetails).get().getCapacity().get();
assertEquals(capacity, foundCapacity);
foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get().getScmUsed().get();
assertEquals(expectedScmUsed, foundScmUsed);
foundRemaining = nodeManager.getNodeStat(datanodeDetails).get().getRemaining().get();
assertEquals(expectedRemaining, foundRemaining);
// Wait up to 4 more seconds so the node becomes dead
// Verify usage info should be updated.
GenericTestUtils.waitFor(() -> nodeManager.getNodeCount(NodeStatus.inServiceDead()) == 1, 100, 4 * 1000);
assertEquals(0, nodeManager.getNodeStats().size());
foundCapacity = nodeManager.getStats().getCapacity().get();
assertEquals(0, foundCapacity);
foundScmUsed = nodeManager.getStats().getScmUsed().get();
assertEquals(0, foundScmUsed);
foundRemaining = nodeManager.getStats().getRemaining().get();
assertEquals(0, foundRemaining);
LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
nodeManager.processHeartbeat(datanodeDetails, layoutInfo);
// Wait up to 5 seconds so that the dead node becomes healthy
// Verify usage info should be updated.
GenericTestUtils.waitFor(() -> nodeManager.getNodeCount(NodeStatus.inServiceHealthy()) == 1, 100, 5 * 1000);
GenericTestUtils.waitFor(() -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed, 100, 4 * 1000);
assertEquals(nodeCount, nodeManager.getNodeStats().size());
foundCapacity = nodeManager.getNodeStat(datanodeDetails).get().getCapacity().get();
assertEquals(capacity, foundCapacity);
foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get().getScmUsed().get();
assertEquals(expectedScmUsed, foundScmUsed);
foundRemaining = nodeManager.getNodeStat(datanodeDetails).get().getRemaining().get();
assertEquals(expectedRemaining, foundRemaining);
}
}
Aggregations