Search in sources :

Example 1 with ReplicationManagerReport

use of org.apache.hadoop.hdds.scm.container.ReplicationManagerReport in project ozone by apache.

the class StorageContainerLocationProtocolClientSideTranslatorPB method getReplicationManagerReport.

@Override
public ReplicationManagerReport getReplicationManagerReport() throws IOException {
    ReplicationManagerReportRequestProto request = ReplicationManagerReportRequestProto.newBuilder().setTraceID(TracingUtil.exportCurrentSpan()).build();
    ReplicationManagerReportResponseProto response = submitRequest(Type.GetReplicationManagerReport, builder -> builder.setReplicationManagerReportRequest(request)).getGetReplicationManagerReportResponse();
    return ReplicationManagerReport.fromProtobuf(response.getReport());
}
Also used : DatanodeAdminErrorResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeAdminErrorResponseProto) GetContainerWithPipelineRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineRequestProto) HddsProtos(org.apache.hadoop.hdds.protocol.proto.HddsProtos) RecommissionNodesRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.RecommissionNodesRequestProto) StartReplicationManagerRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartReplicationManagerRequestProto) UpgradeFinalizer(org.apache.hadoop.ozone.upgrade.UpgradeFinalizer) SCMContainerLocationFailoverProxyProvider(org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider) ServiceException(com.google.protobuf.ServiceException) SCMListContainerRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SCMListContainerRequestProto) StopContainerBalancerRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StopContainerBalancerRequestProto) GetContainerWithPipelineBatchRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineBatchRequestProto) ScmContainerLocationResponse(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ScmContainerLocationResponse) Pair(org.apache.commons.lang3.tuple.Pair) ProtocolTranslator(org.apache.hadoop.ipc.ProtocolTranslator) ProtobufHelper(org.apache.hadoop.ipc.ProtobufHelper) StartMaintenanceNodesRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartMaintenanceNodesRequestProto) Map(java.util.Map) StatusAndMessages(org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages) ScmInfo(org.apache.hadoop.hdds.scm.ScmInfo) RecommissionNodesResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.RecommissionNodesResponseProto) SCMListContainerResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SCMListContainerResponseProto) NodeQueryRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.NodeQueryRequestProto) ForceExitSafeModeRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ForceExitSafeModeRequestProto) PipelineRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.PipelineRequestProto) Builder(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ScmContainerLocationRequest.Builder) StartContainerBalancerRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerRequestProto) ActivatePipelineRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ActivatePipelineRequestProto) RpcController(com.google.protobuf.RpcController) FinalizeScmUpgradeResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.FinalizeScmUpgradeResponseProto) SCMCloseContainerRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SCMCloseContainerRequestProto) SCMDeleteContainerRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SCMDeleteContainerRequestProto) Type(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.Type) ClosePipelineRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ClosePipelineRequestProto) ContainerResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerResponseProto) DeactivatePipelineRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto) List(java.util.List) GetContainerReplicasRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerReplicasRequestProto) ListPipelineRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ListPipelineRequestProto) QueryUpgradeFinalizationProgressRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.QueryUpgradeFinalizationProgressRequestProto) QueryUpgradeFinalizationProgressResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.QueryUpgradeFinalizationProgressResponseProto) ContainerRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerRequestProto) ReplicationManagerReportResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerReportResponseProto) GetContainerCountResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerCountResponseProto) StopReplicationManagerRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StopReplicationManagerRequestProto) RPC(org.apache.hadoop.ipc.RPC) Optional(java.util.Optional) ReplicationManagerReport(org.apache.hadoop.hdds.scm.container.ReplicationManagerReport) GetContainerCountRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerCountRequestProto) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo) GetScmInfoResponseProto(org.apache.hadoop.hdds.protocol.proto.HddsProtos.GetScmInfoResponseProto) CURRENT_VERSION(org.apache.hadoop.ozone.ClientVersions.CURRENT_VERSION) StorageContainerLocationProtocolProtos(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos) InSafeModeRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.InSafeModeRequestProto) HashMap(java.util.HashMap) GetSafeModeRuleStatusesRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetSafeModeRuleStatusesRequestProto) UpgradeFinalizationStatus(org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus) GetContainerTokenResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerTokenResponseProto) ArrayList(java.util.ArrayList) FinalizeScmUpgradeRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.FinalizeScmUpgradeRequestProto) DatanodeUsageInfoRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoRequestProto) GetContainerRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerRequestProto) ReplicationManagerStatusRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerStatusRequestProto) NodeQueryResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.NodeQueryResponseProto) ScmContainerLocationRequest(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ScmContainerLocationRequest) GetPipelineResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineResponseProto) DecommissionNodesRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto) ContainerBalancerStatusRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusRequestProto) ReplicationManagerStatusResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerStatusResponseProto) GetExistContainerWithPipelinesInBatchRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetExistContainerWithPipelinesInBatchRequestProto) GetPipelineRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineRequestProto) DatanodeUsageInfoResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoResponseProto) StartMaintenanceNodesResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartMaintenanceNodesResponseProto) DatanodeAdminError(org.apache.hadoop.hdds.scm.DatanodeAdminError) ForceExitSafeModeResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ForceExitSafeModeResponseProto) ContainerBalancerStatusResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusResponseProto) ContainerWithPipeline(org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline) ContainerID(org.apache.hadoop.hdds.scm.container.ContainerID) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) IOException(java.io.IOException) Token(org.apache.hadoop.security.token.Token) PipelineResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.PipelineResponseProto) Consumer(java.util.function.Consumer) GetSafeModeRuleStatusesResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetSafeModeRuleStatusesResponseProto) DecommissionNodesResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesResponseProto) ListPipelineResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ListPipelineResponseProto) InterfaceAudience(org.apache.hadoop.hdds.annotation.InterfaceAudience) SafeModeRuleStatusProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SafeModeRuleStatusProto) TracingUtil(org.apache.hadoop.hdds.tracing.TracingUtil) StorageContainerLocationProtocol(org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol) Closeable(java.io.Closeable) GetContainerTokenRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerTokenRequestProto) Preconditions(com.google.common.base.Preconditions) ReplicationManagerReportRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerReportRequestProto) RetryProxy(org.apache.hadoop.io.retry.RetryProxy) StartContainerBalancerResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerResponseProto) ReplicationManagerReportRequestProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerReportRequestProto) ReplicationManagerReportResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerReportResponseProto)

Example 2 with ReplicationManagerReport

use of org.apache.hadoop.hdds.scm.container.ReplicationManagerReport in project ozone by apache.

the class TestReportSubCommand method testCorrectValuesAppearInEmptyReport.

@Test
public void testCorrectValuesAppearInEmptyReport() throws IOException {
    ScmClient scmClient = mock(ScmClient.class);
    Mockito.when(scmClient.getReplicationManagerReport()).thenAnswer(invocation -> new ReplicationManagerReport());
    cmd.execute(scmClient);
    for (HddsProtos.LifeCycleState state : HddsProtos.LifeCycleState.values()) {
        Pattern p = Pattern.compile("^" + state.toString() + ": 0$", Pattern.MULTILINE);
        Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
        assertTrue(m.find());
    }
    for (ReplicationManagerReport.HealthState state : ReplicationManagerReport.HealthState.values()) {
        Pattern p = Pattern.compile("^" + state.toString() + ": 0$", Pattern.MULTILINE);
        Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
        assertTrue(m.find());
    }
}
Also used : Pattern(java.util.regex.Pattern) HddsProtos(org.apache.hadoop.hdds.protocol.proto.HddsProtos) Matcher(java.util.regex.Matcher) ScmClient(org.apache.hadoop.hdds.scm.client.ScmClient) ReplicationManagerReport(org.apache.hadoop.hdds.scm.container.ReplicationManagerReport) Test(org.junit.Test)

Example 3 with ReplicationManagerReport

use of org.apache.hadoop.hdds.scm.container.ReplicationManagerReport in project ozone by apache.

the class TestReportSubCommand method createReport.

private ReplicationManagerReport createReport() {
    ReplicationManagerReport report = new ReplicationManagerReport();
    int counter = SEED;
    for (HddsProtos.LifeCycleState state : HddsProtos.LifeCycleState.values()) {
        for (int i = 0; i < counter; i++) {
            report.increment(state);
        }
        counter++;
    }
    // Add samples
    counter = SEED;
    for (ReplicationManagerReport.HealthState state : ReplicationManagerReport.HealthState.values()) {
        for (int i = 0; i < counter; i++) {
            report.incrementAndSample(state, ContainerID.valueOf(i));
        }
        counter++;
    }
    return report;
}
Also used : HddsProtos(org.apache.hadoop.hdds.protocol.proto.HddsProtos) ReplicationManagerReport(org.apache.hadoop.hdds.scm.container.ReplicationManagerReport)

Example 4 with ReplicationManagerReport

use of org.apache.hadoop.hdds.scm.container.ReplicationManagerReport in project ozone by apache.

the class ReplicationManagerMetrics method getMetrics.

@Override
public void getMetrics(MetricsCollector collector, boolean all) {
    MetricsRecordBuilder builder = collector.addRecord(METRICS_SOURCE_NAME).addGauge(INFLIGHT_REPLICATION, getInflightReplication()).addGauge(INFLIGHT_DELETION, getInflightDeletion()).addGauge(INFLIGHT_MOVE, getInflightMove());
    ReplicationManagerReport report = replicationManager.getContainerReport();
    for (Map.Entry<HddsProtos.LifeCycleState, MetricsInfo> e : LIFECYCLE_STATE_METRICS.entrySet()) {
        builder.addGauge(e.getValue(), report.getStat(e.getKey()));
    }
    for (Map.Entry<ReplicationManagerReport.HealthState, MetricsInfo> e : CONTAINER_HEALTH_STATE_METRICS.entrySet()) {
        builder.addGauge(e.getValue(), report.getStat(e.getKey()));
    }
    numReplicationCmdsSent.snapshot(builder, all);
    numReplicationCmdsCompleted.snapshot(builder, all);
    numReplicationCmdsTimeout.snapshot(builder, all);
    numDeletionCmdsSent.snapshot(builder, all);
    numDeletionCmdsCompleted.snapshot(builder, all);
    numDeletionCmdsTimeout.snapshot(builder, all);
    numReplicationBytesTotal.snapshot(builder, all);
    numReplicationBytesCompleted.snapshot(builder, all);
    numDeletionBytesTotal.snapshot(builder, all);
    numDeletionBytesCompleted.snapshot(builder, all);
    replicationTime.snapshot(builder, all);
    deletionTime.snapshot(builder, all);
}
Also used : MetricsInfo(org.apache.hadoop.metrics2.MetricsInfo) LifeCycleState(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map) HealthState(org.apache.hadoop.hdds.scm.container.ReplicationManagerReport.HealthState) MetricsRecordBuilder(org.apache.hadoop.metrics2.MetricsRecordBuilder) ReplicationManagerReport(org.apache.hadoop.hdds.scm.container.ReplicationManagerReport)

Example 5 with ReplicationManagerReport

use of org.apache.hadoop.hdds.scm.container.ReplicationManagerReport in project ozone by apache.

the class TestReplicationManagerMetrics method setup.

@Before
public void setup() {
    ReplicationManagerReport report = new ReplicationManagerReport();
    // to the value by incrementing the counter that number of times.
    for (HddsProtos.LifeCycleState s : HddsProtos.LifeCycleState.values()) {
        for (int i = 0; i < s.getNumber(); i++) {
            report.increment(s);
        }
    }
    // The ordinal starts from 0, so each state will have a value of its ordinal
    for (ReplicationManagerReport.HealthState s : ReplicationManagerReport.HealthState.values()) {
        for (int i = 0; i < s.ordinal(); i++) {
            report.increment(s);
        }
    }
    replicationManager = Mockito.mock(ReplicationManager.class);
    Mockito.when(replicationManager.getContainerReport()).thenReturn(report);
    metrics = ReplicationManagerMetrics.create(replicationManager);
}
Also used : ReplicationManager(org.apache.hadoop.hdds.scm.container.ReplicationManager) HddsProtos(org.apache.hadoop.hdds.protocol.proto.HddsProtos) ReplicationManagerReport(org.apache.hadoop.hdds.scm.container.ReplicationManagerReport) Before(org.junit.Before)

Aggregations

ReplicationManagerReport (org.apache.hadoop.hdds.scm.container.ReplicationManagerReport)6 HddsProtos (org.apache.hadoop.hdds.protocol.proto.HddsProtos)4 Map (java.util.Map)2 Preconditions (com.google.common.base.Preconditions)1 RpcController (com.google.protobuf.RpcController)1 ServiceException (com.google.protobuf.ServiceException)1 Closeable (java.io.Closeable)1 IOException (java.io.IOException)1 ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 LinkedHashMap (java.util.LinkedHashMap)1 List (java.util.List)1 Optional (java.util.Optional)1 Consumer (java.util.function.Consumer)1 Matcher (java.util.regex.Matcher)1 Pattern (java.util.regex.Pattern)1 Pair (org.apache.commons.lang3.tuple.Pair)1 InterfaceAudience (org.apache.hadoop.hdds.annotation.InterfaceAudience)1 GetScmInfoResponseProto (org.apache.hadoop.hdds.protocol.proto.HddsProtos.GetScmInfoResponseProto)1 LifeCycleState (org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState)1