Examples with LayoutVersionProto org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto used on opensource projects

Example 1 with LayoutVersionProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto in project ozone by apache.

the class RegisterEndpointTask method call.

/**
 * Computes a result, or throws an exception if unable to do so.
 *
 * @return computed result
 * @throws Exception if unable to compute a result
 */
@Override
public EndpointStateMachine.EndPointStates call() throws Exception {
    if (getDatanodeDetails() == null) {
        LOG.error("DatanodeDetails cannot be null in RegisterEndpoint task, " + "shutting down the endpoint.");
        return rpcEndPoint.setState(EndpointStateMachine.EndPointStates.SHUTDOWN);
    }
    rpcEndPoint.lock();
    try {
        if (rpcEndPoint.getState().equals(EndpointStateMachine.EndPointStates.REGISTER)) {
            LayoutVersionProto layoutInfo = LayoutVersionProto.newBuilder().setMetadataLayoutVersion(layoutVersionManager.getMetadataLayoutVersion()).setSoftwareLayoutVersion(layoutVersionManager.getSoftwareLayoutVersion()).build();
            ContainerReportsProto containerReport = datanodeContainerManager.getController().getContainerReport();
            NodeReportProto nodeReport = datanodeContainerManager.getNodeReport();
            PipelineReportsProto pipelineReportsProto = datanodeContainerManager.getPipelineReport();
            // TODO : Add responses to the command Queue.
            SCMRegisteredResponseProto response = rpcEndPoint.getEndPoint().register(datanodeDetails.getExtendedProtoBufMessage(), nodeReport, containerReport, pipelineReportsProto, layoutInfo);
            Preconditions.checkState(UUID.fromString(response.getDatanodeUUID()).equals(datanodeDetails.getUuid()), "Unexpected datanode ID in the response.");
            Preconditions.checkState(!StringUtils.isBlank(response.getClusterID()), "Invalid cluster ID in the response.");
            Preconditions.checkState(response.getErrorCode() == success, "DataNode has higher Software Layout Version than SCM.");
            if (response.hasHostname() && response.hasIpAddress()) {
                datanodeDetails.setHostName(response.getHostname());
                datanodeDetails.setIpAddress(response.getIpAddress());
            }
            if (response.hasNetworkName() && response.hasNetworkLocation()) {
                datanodeDetails.setNetworkName(response.getNetworkName());
                datanodeDetails.setNetworkLocation(response.getNetworkLocation());
            }
            EndpointStateMachine.EndPointStates nextState = rpcEndPoint.getState().getNextState();
            rpcEndPoint.setState(nextState);
            rpcEndPoint.zeroMissedCount();
            this.stateContext.configureHeartbeatFrequency();
        }
    } catch (IOException ex) {
        rpcEndPoint.logIfNeeded(ex);
    } finally {
        rpcEndPoint.unlock();
    }
    return rpcEndPoint.getState();
}

Example 2 with LayoutVersionProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto in project ozone by apache.

the class TestSCMNodeMetrics method testHBProcessingFailure.

/**
 * Verifies heartbeat processing failure count.
 */
@Test
public void testHBProcessingFailure() {
    long hbProcessedFailed = getCounter("NumHBProcessingFailed");
    LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
    LayoutVersionProto layoutInfo = LayoutVersionProto.newBuilder().setSoftwareLayoutVersion(versionManager.getSoftwareLayoutVersion()).setMetadataLayoutVersion(versionManager.getMetadataLayoutVersion()).build();
    nodeManager.processHeartbeat(MockDatanodeDetails.randomDatanodeDetails(), layoutInfo);
    assertEquals("NumHBProcessingFailed", hbProcessedFailed + 1, getCounter("NumHBProcessingFailed"));
}

Example 3 with LayoutVersionProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto in project ozone by apache.

the class TestEndpoints method setUp.

@Before
public void setUp() throws Exception {
    // The following setup runs only once
    if (!isSetupDone) {
        initializeInjector();
        isSetupDone = true;
    }
    String datanodeId = datanodeDetails.getUuid().toString();
    String datanodeId2 = datanodeDetails2.getUuid().toString();
    containerReportsProto = ContainerReportsProto.newBuilder().addReports(ContainerReplicaProto.newBuilder().setContainerID(containerId).setState(ContainerReplicaProto.State.OPEN).setOriginNodeId(datanodeId).build()).build();
    UUID pipelineUuid = UUID.fromString(pipelineId);
    HddsProtos.UUID uuid128 = HddsProtos.UUID.newBuilder().setMostSigBits(pipelineUuid.getMostSignificantBits()).setLeastSigBits(pipelineUuid.getLeastSignificantBits()).build();
    PipelineReport pipelineReport = PipelineReport.newBuilder().setPipelineID(PipelineID.newBuilder().setId(pipelineId).setUuid128(uuid128).build()).setIsLeader(true).build();
    PipelineReportsProto pipelineReportsProto = PipelineReportsProto.newBuilder().addPipelineReport(pipelineReport).build();
    DatanodeDetailsProto datanodeDetailsProto = DatanodeDetailsProto.newBuilder().setHostName(HOST1).setUuid(datanodeId).setIpAddress(IP1).build();
    extendedDatanodeDetailsProto = HddsProtos.ExtendedDatanodeDetailsProto.newBuilder().setDatanodeDetails(datanodeDetailsProto).setVersion("0.6.0").setSetupTime(1596347628802L).setBuildDate("2020-08-01T08:50Z").setRevision("3346f493fa1690358add7bb9f3e5b52545993f36").build();
    StorageReportProto storageReportProto1 = StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK).setStorageLocation("/disk1").setScmUsed(10000).setRemaining(5400).setCapacity(25000).setStorageUuid(UUID.randomUUID().toString()).setFailed(false).build();
    StorageReportProto storageReportProto2 = StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK).setStorageLocation("/disk2").setScmUsed(25000).setRemaining(10000).setCapacity(50000).setStorageUuid(UUID.randomUUID().toString()).setFailed(false).build();
    NodeReportProto nodeReportProto = NodeReportProto.newBuilder().addStorageReport(storageReportProto1).addStorageReport(storageReportProto2).build();
    DatanodeDetailsProto datanodeDetailsProto2 = DatanodeDetailsProto.newBuilder().setHostName(HOST2).setUuid(datanodeId2).setIpAddress(IP2).build();
    ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto2 = ExtendedDatanodeDetailsProto.newBuilder().setDatanodeDetails(datanodeDetailsProto2).setVersion("0.6.0").setSetupTime(1596347636802L).setBuildDate("2020-08-01T08:50Z").setRevision("3346f493fa1690358add7bb9f3e5b52545993f36").build();
    StorageReportProto storageReportProto3 = StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK).setStorageLocation("/disk1").setScmUsed(20000).setRemaining(7800).setCapacity(50000).setStorageUuid(UUID.randomUUID().toString()).setFailed(false).build();
    StorageReportProto storageReportProto4 = StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK).setStorageLocation("/disk2").setScmUsed(60000).setRemaining(10000).setCapacity(80000).setStorageUuid(UUID.randomUUID().toString()).setFailed(false).build();
    NodeReportProto nodeReportProto2 = NodeReportProto.newBuilder().addStorageReport(storageReportProto3).addStorageReport(storageReportProto4).build();
    LayoutVersionProto layoutInfo = defaultLayoutVersionProto();
    try {
        reconScm.getDatanodeProtocolServer().register(extendedDatanodeDetailsProto, nodeReportProto, containerReportsProto, pipelineReportsProto, layoutInfo);
        reconScm.getDatanodeProtocolServer().register(extendedDatanodeDetailsProto2, nodeReportProto2, ContainerReportsProto.newBuilder().build(), PipelineReportsProto.newBuilder().build(), defaultLayoutVersionProto());
        // Process all events in the event queue
        reconScm.getEventQueue().processAll(1000);
    } catch (Exception ex) {
        Assert.fail(ex.getMessage());
    }
    // Write Data to OM
    // A sample volume (sampleVol) and a bucket (bucketOne) is already created
    // in AbstractOMMetadataManagerTest.
    // Create a new volume and bucket and then write keys to the bucket.
    String volumeKey = reconOMMetadataManager.getVolumeKey("sampleVol2");
    OmVolumeArgs args = OmVolumeArgs.newBuilder().setVolume("sampleVol2").setAdminName("TestUser").setOwnerName("TestUser").build();
    reconOMMetadataManager.getVolumeTable().put(volumeKey, args);
    OmBucketInfo bucketInfo = OmBucketInfo.newBuilder().setVolumeName("sampleVol2").setBucketName("bucketOne").build();
    String bucketKey = reconOMMetadataManager.getBucketKey(bucketInfo.getVolumeName(), bucketInfo.getBucketName());
    reconOMMetadataManager.getBucketTable().put(bucketKey, bucketInfo);
    // key = key_one
    writeDataToOm(reconOMMetadataManager, "key_one");
    // key = key_two
    writeDataToOm(reconOMMetadataManager, "key_two");
    // key = key_three
    writeDataToOm(reconOMMetadataManager, "key_three");
    // Truncate global stats table before running each test
    dslContext.truncate(GLOBAL_STATS);
}

Example 4 with LayoutVersionProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto in project ozone by apache.

the class StorageContainerDatanodeProtocolServerSideTranslatorPB method register.

public SCMRegisteredResponseProto register(SCMRegisterRequestProto request) throws IOException {
    ContainerReportsProto containerRequestProto = request.getContainerReport();
    NodeReportProto dnNodeReport = request.getNodeReport();
    PipelineReportsProto pipelineReport = request.getPipelineReports();
    LayoutVersionProto layoutInfo = null;
    if (request.hasDataNodeLayoutVersion()) {
        layoutInfo = request.getDataNodeLayoutVersion();
    } else {
        // Backward compatibility to make sure old Datanodes can still talk to
        // SCM.
        layoutInfo = toLayoutVersionProto(INITIAL_VERSION.layoutVersion(), INITIAL_VERSION.layoutVersion());
    }
    return impl.register(request.getExtendedDatanodeDetails(), dnNodeReport, containerRequestProto, pipelineReport, layoutInfo);
}

Example 5 with LayoutVersionProto

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto in project ozone by apache.

the class TestSCMNodeManager method testGetNodeInfo.

@Test
public void testGetNodeInfo() throws IOException, InterruptedException, NodeNotFoundException, AuthenticationException {
    OzoneConfiguration conf = getConf();
    final int nodeCount = 6;
    SCMNodeManager nodeManager = createNodeManager(conf);
    for (int i = 0; i < nodeCount; i++) {
        DatanodeDetails datanodeDetails = MockDatanodeDetails.randomDatanodeDetails();
        final long capacity = 2000;
        final long used = 100;
        final long remaining = 1900;
        UUID dnId = datanodeDetails.getUuid();
        String storagePath = testDir.getAbsolutePath() + "/" + dnId;
        StorageReportProto report = HddsTestUtils.createStorageReport(dnId, storagePath, capacity, used, remaining, null);
        nodeManager.register(datanodeDetails, HddsTestUtils.createNodeReport(Arrays.asList(report), Collections.emptyList()), HddsTestUtils.getRandomPipelineReports());
        LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
        LayoutVersionProto layoutInfo = toLayoutVersionProto(versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion());
        nodeManager.register(datanodeDetails, HddsTestUtils.createNodeReport(Arrays.asList(report), Collections.emptyList()), HddsTestUtils.getRandomPipelineReports(), layoutInfo);
        nodeManager.processHeartbeat(datanodeDetails, layoutInfo);
        if (i == 5) {
            nodeManager.setNodeOperationalState(datanodeDetails, HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE);
        }
        if (i == 3 || i == 4) {
            nodeManager.setNodeOperationalState(datanodeDetails, HddsProtos.NodeOperationalState.DECOMMISSIONED);
        }
    }
    Thread.sleep(100);
    Map<String, Long> stats = nodeManager.getNodeInfo();
    // 3 IN_SERVICE nodes:
    assertEquals(6000, stats.get("DiskCapacity").longValue());
    assertEquals(300, stats.get("DiskUsed").longValue());
    assertEquals(5700, stats.get("DiskRemaining").longValue());
    // 2 Decommissioned nodes
    assertEquals(4000, stats.get("DecommissionedDiskCapacity").longValue());
    assertEquals(200, stats.get("DecommissionedDiskUsed").longValue());
    assertEquals(3800, stats.get("DecommissionedDiskRemaining").longValue());
    // 1 Maintenance node
    assertEquals(2000, stats.get("MaintenanceDiskCapacity").longValue());
    assertEquals(100, stats.get("MaintenanceDiskUsed").longValue());
    assertEquals(1900, stats.get("MaintenanceDiskRemaining").longValue());
}