Search in sources :

Example 1 with OzoneConfiguration

use of org.apache.hadoop.hdds.conf.OzoneConfiguration in project ozone by apache.

the class TestReplicatedFileChecksumHelper method testOneBlock.

@Test
public void testOneBlock() throws IOException {
    // test the file checksum of a file with one block.
    OzoneConfiguration conf = new OzoneConfiguration();
    RpcClient mockRpcClient = Mockito.mock(RpcClient.class);
    List<DatanodeDetails> dns = Arrays.asList(DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).build());
    Pipeline pipeline;
    pipeline = Pipeline.newBuilder().setId(PipelineID.randomId()).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)).setState(Pipeline.PipelineState.CLOSED).setNodes(dns).build();
    XceiverClientGrpc xceiverClientGrpc = new XceiverClientGrpc(pipeline, conf) {

        @Override
        public XceiverClientReply sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request, DatanodeDetails dn) {
            return buildValidResponse();
        }
    };
    XceiverClientFactory factory = Mockito.mock(XceiverClientFactory.class);
    when(factory.acquireClientForReadData(ArgumentMatchers.any())).thenReturn(xceiverClientGrpc);
    when(mockRpcClient.getXceiverClientManager()).thenReturn(factory);
    OzoneManagerProtocol om = Mockito.mock(OzoneManagerProtocol.class);
    when(mockRpcClient.getOzoneManagerClient()).thenReturn(om);
    BlockID blockID = new BlockID(1, 1);
    OmKeyLocationInfo omKeyLocationInfo = new OmKeyLocationInfo.Builder().setPipeline(pipeline).setBlockID(blockID).build();
    List<OmKeyLocationInfo> omKeyLocationInfoList = Arrays.asList(omKeyLocationInfo);
    OmKeyInfo omKeyInfo = new OmKeyInfo.Builder().setVolumeName(null).setBucketName(null).setKeyName(null).setOmKeyLocationInfos(Collections.singletonList(new OmKeyLocationInfoGroup(0, omKeyLocationInfoList))).setCreationTime(Time.now()).setModificationTime(Time.now()).setDataSize(0).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setFileEncryptionInfo(null).setAcls(null).build();
    when(om.lookupKey(ArgumentMatchers.any())).thenReturn(omKeyInfo);
    OzoneVolume mockVolume = Mockito.mock(OzoneVolume.class);
    when(mockVolume.getName()).thenReturn("vol1");
    OzoneBucket bucket = Mockito.mock(OzoneBucket.class);
    when(bucket.getName()).thenReturn("bucket1");
    ReplicatedFileChecksumHelper helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", 10, mockRpcClient);
    helper.compute();
    FileChecksum fileChecksum = helper.getFileChecksum();
    assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
    assertEquals(1, helper.getKeyLocationInfoList().size());
}
Also used : OzoneManagerProtocol(org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) MD5MD5CRC32GzipFileChecksum(org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum) MockXceiverClientFactory(org.apache.hadoop.ozone.client.MockXceiverClientFactory) XceiverClientFactory(org.apache.hadoop.hdds.scm.XceiverClientFactory) OmKeyLocationInfo(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo) MD5MD5CRC32GzipFileChecksum(org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum) FileChecksum(org.apache.hadoop.fs.FileChecksum) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) OzoneVolume(org.apache.hadoop.ozone.client.OzoneVolume) OzoneBucket(org.apache.hadoop.ozone.client.OzoneBucket) OmKeyLocationInfoGroup(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) OmKeyInfo(org.apache.hadoop.ozone.om.helpers.OmKeyInfo) BlockID(org.apache.hadoop.hdds.client.BlockID) RpcClient(org.apache.hadoop.ozone.client.rpc.RpcClient) Test(org.junit.Test)

Example 2 with OzoneConfiguration

use of org.apache.hadoop.hdds.conf.OzoneConfiguration in project ozone by apache.

the class TestHddsClientUtils method testGetScmClientAddress.

/**
 * Verify that the client endpoint can be correctly parsed from
 * configuration.
 */
@Test
public void testGetScmClientAddress() {
    final OzoneConfiguration conf = new OzoneConfiguration();
    // First try a client address with just a host name. Verify it falls
    // back to the default port.
    conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4");
    checkAddr(conf, "1.2.3.4", OZONE_SCM_CLIENT_PORT_DEFAULT);
    // Next try a client address with a host name and port. Verify both
    // are used correctly.
    conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100");
    checkAddr(conf, "1.2.3.4", 100);
}
Also used : OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) Test(org.junit.Test)

Example 3 with OzoneConfiguration

use of org.apache.hadoop.hdds.conf.OzoneConfiguration in project ozone by apache.

the class TestHddsClientUtils method testClientFallbackToScmNamesWithPort.

@Test
@SuppressWarnings("StringSplitter")
public void testClientFallbackToScmNamesWithPort() {
    // When OZONE_SCM_CLIENT_ADDRESS_KEY is undefined, it should fallback
    // to OZONE_SCM_NAMES.
    // 
    // Verify that the OZONE_SCM_NAMES port number is ignored, if present.
    // Instead we should use OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT.
    final String scmHost = "host456:300";
    final OzoneConfiguration conf = new OzoneConfiguration();
    conf.set(OZONE_SCM_NAMES, scmHost);
    final Collection<InetSocketAddress> address = HddsUtils.getScmAddressForClients(conf);
    Assert.assertTrue(address.iterator().hasNext());
    InetSocketAddress socketAddress = address.iterator().next();
    assertEquals(scmHost.split(":")[0], socketAddress.getHostName());
    assertEquals(OZONE_SCM_CLIENT_PORT_DEFAULT, socketAddress.getPort());
}
Also used : InetSocketAddress(java.net.InetSocketAddress) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) Test(org.junit.Test)

Example 4 with OzoneConfiguration

use of org.apache.hadoop.hdds.conf.OzoneConfiguration in project ozone by apache.

the class TestHddsClientUtils method testBlockClientFallbackToClientNoPort.

@Test
public void testBlockClientFallbackToClientNoPort() {
    // When OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY is undefined it should
    // fallback to OZONE_SCM_CLIENT_ADDRESS_KEY.
    final String scmHost = "host123";
    final OzoneConfiguration conf = new OzoneConfiguration();
    conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmHost);
    final InetSocketAddress address = NetUtils.createSocketAddr(SCMNodeInfo.buildNodeInfo(conf).get(0).getBlockClientAddress());
    assertEquals(scmHost, address.getHostName());
    assertEquals(OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT, address.getPort());
}
Also used : InetSocketAddress(java.net.InetSocketAddress) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) Test(org.junit.Test)

Example 5 with OzoneConfiguration

use of org.apache.hadoop.hdds.conf.OzoneConfiguration in project ozone by apache.

the class TestHddsClientUtils method testMissingScmClientAddress.

/**
 * Verify client endpoint lookup failure if it is not configured.
 */
@Test
public void testMissingScmClientAddress() {
    final OzoneConfiguration conf = new OzoneConfiguration();
    thrown.expect(ConfigurationException.class);
    HddsUtils.getScmAddressForClients(conf);
}
Also used : OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) Test(org.junit.Test)

Aggregations

OzoneConfiguration (org.apache.hadoop.hdds.conf.OzoneConfiguration)595 Test (org.junit.Test)265 Before (org.junit.Before)133 BeforeClass (org.junit.BeforeClass)83 File (java.io.File)78 DatanodeDetails (org.apache.hadoop.hdds.protocol.DatanodeDetails)61 IOException (java.io.IOException)52 ArrayList (java.util.ArrayList)45 OmMetadataManagerImpl (org.apache.hadoop.ozone.om.OmMetadataManagerImpl)34 MockDatanodeDetails (org.apache.hadoop.hdds.protocol.MockDatanodeDetails)31 MockDatanodeDetails.randomDatanodeDetails (org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails)31 UUID (java.util.UUID)28 InetSocketAddress (java.net.InetSocketAddress)27 DatanodeStateMachine (org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine)22 MockDatanodeDetails.createDatanodeDetails (org.apache.hadoop.hdds.protocol.MockDatanodeDetails.createDatanodeDetails)21 Pipeline (org.apache.hadoop.hdds.scm.pipeline.Pipeline)20 HDDSLayoutVersionManager (org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager)20 Path (java.nio.file.Path)19 EventQueue (org.apache.hadoop.hdds.server.events.EventQueue)19 XceiverClientManager (org.apache.hadoop.hdds.scm.XceiverClientManager)18