Search in sources :

Example 1 with RpcClient

use of org.apache.hadoop.ozone.client.rpc.RpcClient in project ozone by apache.

the class TestOzoneClient method init.

@Before
public void init() throws IOException {
    ConfigurationSource config = new InMemoryConfiguration();
    client = new OzoneClient(config, new RpcClient(config, null) {

        @Override
        protected OmTransport createOmTransport(String omServiceId) throws IOException {
            return new MockOmTransport();
        }

        @NotNull
        @Override
        protected XceiverClientFactory createXceiverClientFactory(List<X509Certificate> x509Certificates) throws IOException {
            return new MockXceiverClientFactory();
        }
    });
    store = client.getObjectStore();
}
Also used : ConfigurationSource(org.apache.hadoop.hdds.conf.ConfigurationSource) InMemoryConfiguration(org.apache.hadoop.hdds.conf.InMemoryConfiguration) List(java.util.List) RpcClient(org.apache.hadoop.ozone.client.rpc.RpcClient) Before(org.junit.Before)

Example 2 with RpcClient

use of org.apache.hadoop.ozone.client.rpc.RpcClient in project ozone by apache.

the class TestReplicatedFileChecksumHelper method init.

@Before
public void init() throws IOException {
    ConfigurationSource config = new InMemoryConfiguration();
    rpcClient = new RpcClient(config, null) {

        @Override
        protected OmTransport createOmTransport(String omServiceId) throws IOException {
            return new MockOmTransport();
        }

        @NotNull
        @Override
        protected XceiverClientFactory createXceiverClientFactory(List<X509Certificate> x509Certificates) throws IOException {
            return new MockXceiverClientFactory();
        }
    };
    client = new OzoneClient(config, rpcClient);
    store = client.getObjectStore();
}
Also used : ConfigurationSource(org.apache.hadoop.hdds.conf.ConfigurationSource) MockOmTransport(org.apache.hadoop.ozone.client.MockOmTransport) MockOmTransport(org.apache.hadoop.ozone.client.MockOmTransport) OmTransport(org.apache.hadoop.ozone.om.protocolPB.OmTransport) ByteString(org.apache.ratis.thirdparty.com.google.protobuf.ByteString) IOException(java.io.IOException) OzoneClient(org.apache.hadoop.ozone.client.OzoneClient) MockXceiverClientFactory(org.apache.hadoop.ozone.client.MockXceiverClientFactory) XceiverClientFactory(org.apache.hadoop.hdds.scm.XceiverClientFactory) NotNull(org.jetbrains.annotations.NotNull) X509Certificate(java.security.cert.X509Certificate) MockXceiverClientFactory(org.apache.hadoop.ozone.client.MockXceiverClientFactory) InMemoryConfiguration(org.apache.hadoop.hdds.conf.InMemoryConfiguration) RpcClient(org.apache.hadoop.ozone.client.rpc.RpcClient) Before(org.junit.Before)

Example 3 with RpcClient

use of org.apache.hadoop.ozone.client.rpc.RpcClient in project ozone by apache.

the class TestReplicatedFileChecksumHelper method testOneBlock.

@Test
public void testOneBlock() throws IOException {
    // test the file checksum of a file with one block.
    OzoneConfiguration conf = new OzoneConfiguration();
    RpcClient mockRpcClient = Mockito.mock(RpcClient.class);
    List<DatanodeDetails> dns = Arrays.asList(DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).build());
    Pipeline pipeline;
    pipeline = Pipeline.newBuilder().setId(PipelineID.randomId()).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)).setState(Pipeline.PipelineState.CLOSED).setNodes(dns).build();
    XceiverClientGrpc xceiverClientGrpc = new XceiverClientGrpc(pipeline, conf) {

        @Override
        public XceiverClientReply sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request, DatanodeDetails dn) {
            return buildValidResponse();
        }
    };
    XceiverClientFactory factory = Mockito.mock(XceiverClientFactory.class);
    when(factory.acquireClientForReadData(ArgumentMatchers.any())).thenReturn(xceiverClientGrpc);
    when(mockRpcClient.getXceiverClientManager()).thenReturn(factory);
    OzoneManagerProtocol om = Mockito.mock(OzoneManagerProtocol.class);
    when(mockRpcClient.getOzoneManagerClient()).thenReturn(om);
    BlockID blockID = new BlockID(1, 1);
    OmKeyLocationInfo omKeyLocationInfo = new OmKeyLocationInfo.Builder().setPipeline(pipeline).setBlockID(blockID).build();
    List<OmKeyLocationInfo> omKeyLocationInfoList = Arrays.asList(omKeyLocationInfo);
    OmKeyInfo omKeyInfo = new OmKeyInfo.Builder().setVolumeName(null).setBucketName(null).setKeyName(null).setOmKeyLocationInfos(Collections.singletonList(new OmKeyLocationInfoGroup(0, omKeyLocationInfoList))).setCreationTime(Time.now()).setModificationTime(Time.now()).setDataSize(0).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setFileEncryptionInfo(null).setAcls(null).build();
    when(om.lookupKey(ArgumentMatchers.any())).thenReturn(omKeyInfo);
    OzoneVolume mockVolume = Mockito.mock(OzoneVolume.class);
    when(mockVolume.getName()).thenReturn("vol1");
    OzoneBucket bucket = Mockito.mock(OzoneBucket.class);
    when(bucket.getName()).thenReturn("bucket1");
    ReplicatedFileChecksumHelper helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", 10, mockRpcClient);
    helper.compute();
    FileChecksum fileChecksum = helper.getFileChecksum();
    assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
    assertEquals(1, helper.getKeyLocationInfoList().size());
}
Also used : OzoneManagerProtocol(org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) MD5MD5CRC32GzipFileChecksum(org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum) MockXceiverClientFactory(org.apache.hadoop.ozone.client.MockXceiverClientFactory) XceiverClientFactory(org.apache.hadoop.hdds.scm.XceiverClientFactory) OmKeyLocationInfo(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo) MD5MD5CRC32GzipFileChecksum(org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum) FileChecksum(org.apache.hadoop.fs.FileChecksum) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) OzoneVolume(org.apache.hadoop.ozone.client.OzoneVolume) OzoneBucket(org.apache.hadoop.ozone.client.OzoneBucket) OmKeyLocationInfoGroup(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) OmKeyInfo(org.apache.hadoop.ozone.om.helpers.OmKeyInfo) BlockID(org.apache.hadoop.hdds.client.BlockID) RpcClient(org.apache.hadoop.ozone.client.rpc.RpcClient) Test(org.junit.Test)

Example 4 with RpcClient

use of org.apache.hadoop.ozone.client.rpc.RpcClient in project ozone by apache.

the class TestReplicatedFileChecksumHelper method testEmptyBlock.

@Test
public void testEmptyBlock() throws IOException {
    // test the file checksum of a file with an empty block.
    RpcClient mockRpcClient = Mockito.mock(RpcClient.class);
    OzoneManagerProtocol om = Mockito.mock(OzoneManagerProtocol.class);
    when(mockRpcClient.getOzoneManagerClient()).thenReturn(om);
    OmKeyInfo omKeyInfo = new OmKeyInfo.Builder().setVolumeName(null).setBucketName(null).setKeyName(null).setOmKeyLocationInfos(Collections.singletonList(new OmKeyLocationInfoGroup(0, new ArrayList<>()))).setCreationTime(Time.now()).setModificationTime(Time.now()).setDataSize(0).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setFileEncryptionInfo(null).setAcls(null).build();
    when(om.lookupKey(ArgumentMatchers.any())).thenReturn(omKeyInfo);
    OzoneVolume mockVolume = Mockito.mock(OzoneVolume.class);
    when(mockVolume.getName()).thenReturn("vol1");
    OzoneBucket bucket = Mockito.mock(OzoneBucket.class);
    when(bucket.getName()).thenReturn("bucket1");
    ReplicatedFileChecksumHelper helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", 10, mockRpcClient);
    helper.compute();
    FileChecksum fileChecksum = helper.getFileChecksum();
    assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
    assertEquals(DataChecksum.Type.CRC32, ((MD5MD5CRC32GzipFileChecksum) fileChecksum).getCrcType());
    // test negative length
    helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", -1, mockRpcClient);
    helper.compute();
    assertNull(helper.getKeyLocationInfoList());
}
Also used : OzoneVolume(org.apache.hadoop.ozone.client.OzoneVolume) OzoneBucket(org.apache.hadoop.ozone.client.OzoneBucket) OmKeyLocationInfoGroup(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup) OzoneManagerProtocol(org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol) OmKeyInfo(org.apache.hadoop.ozone.om.helpers.OmKeyInfo) ArrayList(java.util.ArrayList) MD5MD5CRC32GzipFileChecksum(org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum) MD5MD5CRC32GzipFileChecksum(org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum) FileChecksum(org.apache.hadoop.fs.FileChecksum) RpcClient(org.apache.hadoop.ozone.client.rpc.RpcClient) Test(org.junit.Test)

Example 5 with RpcClient

use of org.apache.hadoop.ozone.client.rpc.RpcClient in project ozone by apache.

the class ReadReplicas method execute.

@Override
protected void execute(OzoneClient client, OzoneAddress address) throws IOException, OzoneClientException {
    boolean isChecksumVerifyEnabled = getConf().getBoolean("ozone.client.verify.checksum", true);
    OzoneConfiguration configuration = new OzoneConfiguration(getConf());
    configuration.setBoolean("ozone.client.verify.checksum", !isChecksumVerifyEnabled);
    if (isChecksumVerifyEnabled) {
        clientProtocol = client.getObjectStore().getClientProxy();
        clientProtocolWithoutChecksum = new RpcClient(configuration, null);
    } else {
        clientProtocol = new RpcClient(configuration, null);
        clientProtocolWithoutChecksum = client.getObjectStore().getClientProxy();
    }
    address.ensureKeyAddress();
    String volumeName = address.getVolumeName();
    String bucketName = address.getBucketName();
    String keyName = address.getKeyName();
    String directoryName = createDirectory(volumeName, bucketName, keyName);
    OzoneKeyDetails keyInfoDetails = clientProtocol.getKeyDetails(volumeName, bucketName, keyName);
    Map<OmKeyLocationInfo, Map<DatanodeDetails, OzoneInputStream>> replicas = clientProtocol.getKeysEveryReplicas(volumeName, bucketName, keyName);
    Map<OmKeyLocationInfo, Map<DatanodeDetails, OzoneInputStream>> replicasWithoutChecksum = clientProtocolWithoutChecksum.getKeysEveryReplicas(volumeName, bucketName, keyName);
    JsonObject result = new JsonObject();
    result.addProperty(JSON_PROPERTY_FILE_NAME, volumeName + "/" + bucketName + "/" + keyName);
    result.addProperty(JSON_PROPERTY_FILE_SIZE, keyInfoDetails.getDataSize());
    JsonArray blocks = new JsonArray();
    downloadReplicasAndCreateManifest(keyName, replicas, replicasWithoutChecksum, directoryName, blocks);
    result.add(JSON_PROPERTY_FILE_BLOCKS, blocks);
    Gson gson = new GsonBuilder().setPrettyPrinting().create();
    String prettyJson = gson.toJson(result);
    String manifestFileName = keyName + "_manifest";
    System.out.println("Writing manifest file : " + manifestFileName);
    File manifestFile = new File(outputDir + "/" + directoryName + "/" + manifestFileName);
    Files.write(manifestFile.toPath(), prettyJson.getBytes(StandardCharsets.UTF_8));
}
Also used : GsonBuilder(com.google.gson.GsonBuilder) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) JsonObject(com.google.gson.JsonObject) Gson(com.google.gson.Gson) OmKeyLocationInfo(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo) JsonArray(com.google.gson.JsonArray) OzoneKeyDetails(org.apache.hadoop.ozone.client.OzoneKeyDetails) Map(java.util.Map) File(java.io.File) RpcClient(org.apache.hadoop.ozone.client.rpc.RpcClient)

Aggregations

RpcClient (org.apache.hadoop.ozone.client.rpc.RpcClient)5 FileChecksum (org.apache.hadoop.fs.FileChecksum)2 MD5MD5CRC32GzipFileChecksum (org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum)2 ConfigurationSource (org.apache.hadoop.hdds.conf.ConfigurationSource)2 InMemoryConfiguration (org.apache.hadoop.hdds.conf.InMemoryConfiguration)2 OzoneConfiguration (org.apache.hadoop.hdds.conf.OzoneConfiguration)2 XceiverClientFactory (org.apache.hadoop.hdds.scm.XceiverClientFactory)2 MockXceiverClientFactory (org.apache.hadoop.ozone.client.MockXceiverClientFactory)2 OzoneBucket (org.apache.hadoop.ozone.client.OzoneBucket)2 OzoneVolume (org.apache.hadoop.ozone.client.OzoneVolume)2 OmKeyInfo (org.apache.hadoop.ozone.om.helpers.OmKeyInfo)2 OmKeyLocationInfo (org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo)2 OmKeyLocationInfoGroup (org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup)2 OzoneManagerProtocol (org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol)2 Before (org.junit.Before)2 Test (org.junit.Test)2 Gson (com.google.gson.Gson)1 GsonBuilder (com.google.gson.GsonBuilder)1 JsonArray (com.google.gson.JsonArray)1 JsonObject (com.google.gson.JsonObject)1