Search in sources :

Example 1 with XceiverClientGrpc

use of org.apache.hadoop.hdds.scm.XceiverClientGrpc in project ozone by apache.

the class TestReplicatedFileChecksumHelper method testOneBlock.

@Test
public void testOneBlock() throws IOException {
    // test the file checksum of a file with one block.
    OzoneConfiguration conf = new OzoneConfiguration();
    RpcClient mockRpcClient = Mockito.mock(RpcClient.class);
    List<DatanodeDetails> dns = Arrays.asList(DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).build());
    Pipeline pipeline;
    pipeline = Pipeline.newBuilder().setId(PipelineID.randomId()).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)).setState(Pipeline.PipelineState.CLOSED).setNodes(dns).build();
    XceiverClientGrpc xceiverClientGrpc = new XceiverClientGrpc(pipeline, conf) {

        @Override
        public XceiverClientReply sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request, DatanodeDetails dn) {
            return buildValidResponse();
        }
    };
    XceiverClientFactory factory = Mockito.mock(XceiverClientFactory.class);
    when(factory.acquireClientForReadData(ArgumentMatchers.any())).thenReturn(xceiverClientGrpc);
    when(mockRpcClient.getXceiverClientManager()).thenReturn(factory);
    OzoneManagerProtocol om = Mockito.mock(OzoneManagerProtocol.class);
    when(mockRpcClient.getOzoneManagerClient()).thenReturn(om);
    BlockID blockID = new BlockID(1, 1);
    OmKeyLocationInfo omKeyLocationInfo = new OmKeyLocationInfo.Builder().setPipeline(pipeline).setBlockID(blockID).build();
    List<OmKeyLocationInfo> omKeyLocationInfoList = Arrays.asList(omKeyLocationInfo);
    OmKeyInfo omKeyInfo = new OmKeyInfo.Builder().setVolumeName(null).setBucketName(null).setKeyName(null).setOmKeyLocationInfos(Collections.singletonList(new OmKeyLocationInfoGroup(0, omKeyLocationInfoList))).setCreationTime(Time.now()).setModificationTime(Time.now()).setDataSize(0).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setFileEncryptionInfo(null).setAcls(null).build();
    when(om.lookupKey(ArgumentMatchers.any())).thenReturn(omKeyInfo);
    OzoneVolume mockVolume = Mockito.mock(OzoneVolume.class);
    when(mockVolume.getName()).thenReturn("vol1");
    OzoneBucket bucket = Mockito.mock(OzoneBucket.class);
    when(bucket.getName()).thenReturn("bucket1");
    ReplicatedFileChecksumHelper helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", 10, mockRpcClient);
    helper.compute();
    FileChecksum fileChecksum = helper.getFileChecksum();
    assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
    assertEquals(1, helper.getKeyLocationInfoList().size());
}
Also used : OzoneManagerProtocol(org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) MD5MD5CRC32GzipFileChecksum(org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum) MockXceiverClientFactory(org.apache.hadoop.ozone.client.MockXceiverClientFactory) XceiverClientFactory(org.apache.hadoop.hdds.scm.XceiverClientFactory) OmKeyLocationInfo(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo) MD5MD5CRC32GzipFileChecksum(org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum) FileChecksum(org.apache.hadoop.fs.FileChecksum) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) OzoneVolume(org.apache.hadoop.ozone.client.OzoneVolume) OzoneBucket(org.apache.hadoop.ozone.client.OzoneBucket) OmKeyLocationInfoGroup(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) OmKeyInfo(org.apache.hadoop.ozone.om.helpers.OmKeyInfo) BlockID(org.apache.hadoop.hdds.client.BlockID) RpcClient(org.apache.hadoop.ozone.client.rpc.RpcClient) Test(org.junit.Test)

Example 2 with XceiverClientGrpc

use of org.apache.hadoop.hdds.scm.XceiverClientGrpc in project ozone by apache.

the class TestXceiverClientGrpc method testRandomFirstNodeIsCommandTarget.

@Test(timeout = 5000)
public void testRandomFirstNodeIsCommandTarget() throws IOException {
    final ArrayList<DatanodeDetails> allDNs = new ArrayList<>(dns);
    // is not happening.
    while (allDNs.size() > 0) {
        XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf) {

            @Override
            public XceiverClientReply sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request, DatanodeDetails dn) {
                allDNs.remove(dn);
                return buildValidResponse();
            }
        };
        invokeXceiverClientGetBlock(client);
    }
}
Also used : DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) MockDatanodeDetails(org.apache.hadoop.hdds.protocol.MockDatanodeDetails) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) ArrayList(java.util.ArrayList) Test(org.junit.Test)

Example 3 with XceiverClientGrpc

use of org.apache.hadoop.hdds.scm.XceiverClientGrpc in project ozone by apache.

the class TestXceiverClientGrpc method testFirstNodeIsCorrectWithTopologyForCommandTarget.

@Test
public void testFirstNodeIsCorrectWithTopologyForCommandTarget() throws IOException {
    final Set<DatanodeDetails> seenDNs = new HashSet<>();
    conf.setBoolean(OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY, true);
    // only a single DN is ever seen after 100 calls.
    for (int i = 0; i < 100; i++) {
        XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf) {

            @Override
            public XceiverClientReply sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request, DatanodeDetails dn) {
                seenDNs.add(dn);
                return buildValidResponse();
            }
        };
        invokeXceiverClientGetBlock(client);
    }
    Assert.assertEquals(1, seenDNs.size());
}
Also used : DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) MockDatanodeDetails(org.apache.hadoop.hdds.protocol.MockDatanodeDetails) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 4 with XceiverClientGrpc

use of org.apache.hadoop.hdds.scm.XceiverClientGrpc in project ozone by apache.

the class TestReconAsPassiveScm method testDatanodeRegistrationAndReports.

@Test
public void testDatanodeRegistrationAndReports() throws Exception {
    ReconStorageContainerManagerFacade reconScm = (ReconStorageContainerManagerFacade) cluster.getReconServer().getReconStorageContainerManager();
    StorageContainerManager scm = cluster.getStorageContainerManager();
    PipelineManager reconPipelineManager = reconScm.getPipelineManager();
    PipelineManager scmPipelineManager = scm.getPipelineManager();
    LambdaTestUtils.await(60000, 5000, () -> (reconPipelineManager.getPipelines().size() >= 4));
    // Verify if Recon has all the pipelines from SCM.
    scmPipelineManager.getPipelines().forEach(p -> {
        try {
            assertNotNull(reconPipelineManager.getPipeline(p.getId()));
        } catch (PipelineNotFoundException e) {
            Assert.fail();
        }
    });
    // Verify we can never create a pipeline in Recon.
    LambdaTestUtils.intercept(UnsupportedOperationException.class, "Trying to create pipeline in Recon, which is prohibited!", () -> reconPipelineManager.createPipeline(RatisReplicationConfig.getInstance(ONE)));
    ContainerManager scmContainerManager = scm.getContainerManager();
    assertTrue(scmContainerManager.getContainers().isEmpty());
    // Verify if all the 3 nodes are registered with Recon.
    NodeManager reconNodeManager = reconScm.getScmNodeManager();
    NodeManager scmNodeManager = scm.getScmNodeManager();
    assertEquals(scmNodeManager.getAllNodes().size(), reconNodeManager.getAllNodes().size());
    // Create container
    ContainerManager reconContainerManager = reconScm.getContainerManager();
    ContainerInfo containerInfo = scmContainerManager.allocateContainer(RatisReplicationConfig.getInstance(ONE), "test");
    long containerID = containerInfo.getContainerID();
    Pipeline pipeline = scmPipelineManager.getPipeline(containerInfo.getPipelineID());
    XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf);
    runTestOzoneContainerViaDataNode(containerID, client);
    // Verify Recon picked up the new container that was created.
    assertEquals(scmContainerManager.getContainerIDs(), reconContainerManager.getContainerIDs());
    GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer.captureLogs(ReconNodeManager.LOG);
    GenericTestUtils.setLogLevel(ReconNodeManager.LOG, Level.DEBUG);
    reconScm.getEventQueue().fireEvent(CLOSE_CONTAINER, containerInfo.containerID());
    GenericTestUtils.waitFor(() -> logCapturer.getOutput().contains("Ignoring unsupported command closeContainerCommand"), 1000, 20000);
}
Also used : ReconStorageContainerManagerFacade(org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade) ReconNodeManager(org.apache.hadoop.ozone.recon.scm.ReconNodeManager) NodeManager(org.apache.hadoop.hdds.scm.node.NodeManager) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) StorageContainerManager(org.apache.hadoop.hdds.scm.server.StorageContainerManager) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) ContainerManager(org.apache.hadoop.hdds.scm.container.ContainerManager) StorageContainerManager(org.apache.hadoop.hdds.scm.server.StorageContainerManager) PipelineManager(org.apache.hadoop.hdds.scm.pipeline.PipelineManager) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo) PipelineNotFoundException(org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException) GenericTestUtils(org.apache.ozone.test.GenericTestUtils) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) Test(org.junit.Test)

Example 5 with XceiverClientGrpc

use of org.apache.hadoop.hdds.scm.XceiverClientGrpc in project ozone by apache.

the class TestReconAsPassiveScm method testReconRestart.

@Test
public void testReconRestart() throws Exception {
    final OzoneStorageContainerManager reconScm = cluster.getReconServer().getReconStorageContainerManager();
    StorageContainerManager scm = cluster.getStorageContainerManager();
    // Stop Recon
    ContainerManager scmContainerManager = scm.getContainerManager();
    assertTrue(scmContainerManager.getContainers().isEmpty());
    ContainerManager reconContainerManager = reconScm.getContainerManager();
    assertTrue(reconContainerManager.getContainers().isEmpty());
    LambdaTestUtils.await(60000, 5000, () -> (reconScm.getScmNodeManager().getAllNodes().size() == 3));
    cluster.stopRecon();
    // Create container in SCM.
    ContainerInfo containerInfo = scmContainerManager.allocateContainer(RatisReplicationConfig.getInstance(ONE), "test");
    long containerID = containerInfo.getContainerID();
    PipelineManager scmPipelineManager = scm.getPipelineManager();
    Pipeline pipeline = scmPipelineManager.getPipeline(containerInfo.getPipelineID());
    XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf);
    runTestOzoneContainerViaDataNode(containerID, client);
    assertFalse(scmContainerManager.getContainers().isEmpty());
    // Close a pipeline
    Optional<Pipeline> pipelineToClose = scmPipelineManager.getPipelines(RatisReplicationConfig.getInstance(ONE)).stream().filter(p -> !p.getId().equals(containerInfo.getPipelineID())).findFirst();
    assertTrue(pipelineToClose.isPresent());
    scmPipelineManager.closePipeline(pipelineToClose.get(), false);
    // Start Recon
    cluster.startRecon();
    // Verify if Recon has all the nodes on restart (even if heartbeats are
    // not yet received).
    NodeManager reconNodeManager = reconScm.getScmNodeManager();
    NodeManager scmNodeManager = scm.getScmNodeManager();
    assertEquals(scmNodeManager.getAllNodes().size(), reconNodeManager.getAllNodes().size());
    // Verify Recon picks up new container, close pipeline SCM actions.
    OzoneStorageContainerManager newReconScm = cluster.getReconServer().getReconStorageContainerManager();
    PipelineManager reconPipelineManager = newReconScm.getPipelineManager();
    assertFalse(reconPipelineManager.containsPipeline(pipelineToClose.get().getId()));
    LambdaTestUtils.await(90000, 5000, () -> (newReconScm.getContainerManager().containerExist(ContainerID.valueOf(containerID))));
}
Also used : OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) ReconNodeManager(org.apache.hadoop.ozone.recon.scm.ReconNodeManager) NodeManager(org.apache.hadoop.hdds.scm.node.NodeManager) MiniOzoneCluster(org.apache.hadoop.ozone.MiniOzoneCluster) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) ONE(org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE) PipelineNotFoundException(org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException) After(org.junit.After) Level(org.slf4j.event.Level) Timeout(org.junit.rules.Timeout) ContainerManager(org.apache.hadoop.hdds.scm.container.ContainerManager) StorageContainerManager(org.apache.hadoop.hdds.scm.server.StorageContainerManager) LambdaTestUtils(org.apache.ozone.test.LambdaTestUtils) TestOzoneContainer.runTestOzoneContainerViaDataNode(org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer.runTestOzoneContainerViaDataNode) Before(org.junit.Before) HDDS_CONTAINER_REPORT_INTERVAL(org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL) RatisReplicationConfig(org.apache.hadoop.hdds.client.RatisReplicationConfig) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) ContainerID(org.apache.hadoop.hdds.scm.container.ContainerID) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) Test(org.junit.Test) PipelineManager(org.apache.hadoop.hdds.scm.pipeline.PipelineManager) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) CLOSE_CONTAINER(org.apache.hadoop.hdds.scm.events.SCMEvents.CLOSE_CONTAINER) Rule(org.junit.Rule) HDDS_PIPELINE_REPORT_INTERVAL(org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL) Assert.assertFalse(org.junit.Assert.assertFalse) Optional(java.util.Optional) ReconStorageContainerManagerFacade(org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade) Assert(org.junit.Assert) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo) GenericTestUtils(org.apache.ozone.test.GenericTestUtils) TemporaryFolder(org.junit.rules.TemporaryFolder) Assert.assertEquals(org.junit.Assert.assertEquals) ReconNodeManager(org.apache.hadoop.ozone.recon.scm.ReconNodeManager) NodeManager(org.apache.hadoop.hdds.scm.node.NodeManager) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) StorageContainerManager(org.apache.hadoop.hdds.scm.server.StorageContainerManager) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) ContainerManager(org.apache.hadoop.hdds.scm.container.ContainerManager) StorageContainerManager(org.apache.hadoop.hdds.scm.server.StorageContainerManager) PipelineManager(org.apache.hadoop.hdds.scm.pipeline.PipelineManager) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) Test(org.junit.Test)

Aggregations

XceiverClientGrpc (org.apache.hadoop.hdds.scm.XceiverClientGrpc)22 Test (org.junit.Test)20 DatanodeDetails (org.apache.hadoop.hdds.protocol.DatanodeDetails)12 Pipeline (org.apache.hadoop.hdds.scm.pipeline.Pipeline)12 OzoneConfiguration (org.apache.hadoop.hdds.conf.OzoneConfiguration)10 MockPipeline (org.apache.hadoop.hdds.scm.pipeline.MockPipeline)7 MiniOzoneCluster (org.apache.hadoop.ozone.MiniOzoneCluster)7 MockDatanodeDetails (org.apache.hadoop.hdds.protocol.MockDatanodeDetails)6 MockDatanodeDetails.randomDatanodeDetails (org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails)5 ContainerProtos (org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos)4 ContainerCommandResponseProto (org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto)4 XceiverServerGrpc (org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc)4 GenericTestUtils (org.apache.ozone.test.GenericTestUtils)4 IOException (java.io.IOException)3 ArrayList (java.util.ArrayList)3 ContainerCommandRequestProto (org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto)3 HddsDispatcher (org.apache.hadoop.ozone.container.common.impl.HddsDispatcher)3 DatanodeStateMachine (org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine)3 StateContext (org.apache.hadoop.ozone.container.common.statemachine.StateContext)3 Maps (com.google.common.collect.Maps)2