use of org.apache.hadoop.ozone.client.rpc.RpcClient in project ozone by apache.
the class TestOzoneClient method init.
@Before
public void init() throws IOException {
ConfigurationSource config = new InMemoryConfiguration();
client = new OzoneClient(config, new RpcClient(config, null) {
@Override
protected OmTransport createOmTransport(String omServiceId) throws IOException {
return new MockOmTransport();
}
@NotNull
@Override
protected XceiverClientFactory createXceiverClientFactory(List<X509Certificate> x509Certificates) throws IOException {
return new MockXceiverClientFactory();
}
});
store = client.getObjectStore();
}
use of org.apache.hadoop.ozone.client.rpc.RpcClient in project ozone by apache.
the class TestReplicatedFileChecksumHelper method init.
@Before
public void init() throws IOException {
ConfigurationSource config = new InMemoryConfiguration();
rpcClient = new RpcClient(config, null) {
@Override
protected OmTransport createOmTransport(String omServiceId) throws IOException {
return new MockOmTransport();
}
@NotNull
@Override
protected XceiverClientFactory createXceiverClientFactory(List<X509Certificate> x509Certificates) throws IOException {
return new MockXceiverClientFactory();
}
};
client = new OzoneClient(config, rpcClient);
store = client.getObjectStore();
}
use of org.apache.hadoop.ozone.client.rpc.RpcClient in project ozone by apache.
the class TestReplicatedFileChecksumHelper method testOneBlock.
@Test
public void testOneBlock() throws IOException {
// test the file checksum of a file with one block.
OzoneConfiguration conf = new OzoneConfiguration();
RpcClient mockRpcClient = Mockito.mock(RpcClient.class);
List<DatanodeDetails> dns = Arrays.asList(DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).build());
Pipeline pipeline;
pipeline = Pipeline.newBuilder().setId(PipelineID.randomId()).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)).setState(Pipeline.PipelineState.CLOSED).setNodes(dns).build();
XceiverClientGrpc xceiverClientGrpc = new XceiverClientGrpc(pipeline, conf) {
@Override
public XceiverClientReply sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request, DatanodeDetails dn) {
return buildValidResponse();
}
};
XceiverClientFactory factory = Mockito.mock(XceiverClientFactory.class);
when(factory.acquireClientForReadData(ArgumentMatchers.any())).thenReturn(xceiverClientGrpc);
when(mockRpcClient.getXceiverClientManager()).thenReturn(factory);
OzoneManagerProtocol om = Mockito.mock(OzoneManagerProtocol.class);
when(mockRpcClient.getOzoneManagerClient()).thenReturn(om);
BlockID blockID = new BlockID(1, 1);
OmKeyLocationInfo omKeyLocationInfo = new OmKeyLocationInfo.Builder().setPipeline(pipeline).setBlockID(blockID).build();
List<OmKeyLocationInfo> omKeyLocationInfoList = Arrays.asList(omKeyLocationInfo);
OmKeyInfo omKeyInfo = new OmKeyInfo.Builder().setVolumeName(null).setBucketName(null).setKeyName(null).setOmKeyLocationInfos(Collections.singletonList(new OmKeyLocationInfoGroup(0, omKeyLocationInfoList))).setCreationTime(Time.now()).setModificationTime(Time.now()).setDataSize(0).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setFileEncryptionInfo(null).setAcls(null).build();
when(om.lookupKey(ArgumentMatchers.any())).thenReturn(omKeyInfo);
OzoneVolume mockVolume = Mockito.mock(OzoneVolume.class);
when(mockVolume.getName()).thenReturn("vol1");
OzoneBucket bucket = Mockito.mock(OzoneBucket.class);
when(bucket.getName()).thenReturn("bucket1");
ReplicatedFileChecksumHelper helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", 10, mockRpcClient);
helper.compute();
FileChecksum fileChecksum = helper.getFileChecksum();
assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
assertEquals(1, helper.getKeyLocationInfoList().size());
}
use of org.apache.hadoop.ozone.client.rpc.RpcClient in project ozone by apache.
the class TestReplicatedFileChecksumHelper method testEmptyBlock.
@Test
public void testEmptyBlock() throws IOException {
// test the file checksum of a file with an empty block.
RpcClient mockRpcClient = Mockito.mock(RpcClient.class);
OzoneManagerProtocol om = Mockito.mock(OzoneManagerProtocol.class);
when(mockRpcClient.getOzoneManagerClient()).thenReturn(om);
OmKeyInfo omKeyInfo = new OmKeyInfo.Builder().setVolumeName(null).setBucketName(null).setKeyName(null).setOmKeyLocationInfos(Collections.singletonList(new OmKeyLocationInfoGroup(0, new ArrayList<>()))).setCreationTime(Time.now()).setModificationTime(Time.now()).setDataSize(0).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setFileEncryptionInfo(null).setAcls(null).build();
when(om.lookupKey(ArgumentMatchers.any())).thenReturn(omKeyInfo);
OzoneVolume mockVolume = Mockito.mock(OzoneVolume.class);
when(mockVolume.getName()).thenReturn("vol1");
OzoneBucket bucket = Mockito.mock(OzoneBucket.class);
when(bucket.getName()).thenReturn("bucket1");
ReplicatedFileChecksumHelper helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", 10, mockRpcClient);
helper.compute();
FileChecksum fileChecksum = helper.getFileChecksum();
assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
assertEquals(DataChecksum.Type.CRC32, ((MD5MD5CRC32GzipFileChecksum) fileChecksum).getCrcType());
// test negative length
helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", -1, mockRpcClient);
helper.compute();
assertNull(helper.getKeyLocationInfoList());
}
use of org.apache.hadoop.ozone.client.rpc.RpcClient in project ozone by apache.
the class ReadReplicas method execute.
@Override
protected void execute(OzoneClient client, OzoneAddress address) throws IOException, OzoneClientException {
boolean isChecksumVerifyEnabled = getConf().getBoolean("ozone.client.verify.checksum", true);
OzoneConfiguration configuration = new OzoneConfiguration(getConf());
configuration.setBoolean("ozone.client.verify.checksum", !isChecksumVerifyEnabled);
if (isChecksumVerifyEnabled) {
clientProtocol = client.getObjectStore().getClientProxy();
clientProtocolWithoutChecksum = new RpcClient(configuration, null);
} else {
clientProtocol = new RpcClient(configuration, null);
clientProtocolWithoutChecksum = client.getObjectStore().getClientProxy();
}
address.ensureKeyAddress();
String volumeName = address.getVolumeName();
String bucketName = address.getBucketName();
String keyName = address.getKeyName();
String directoryName = createDirectory(volumeName, bucketName, keyName);
OzoneKeyDetails keyInfoDetails = clientProtocol.getKeyDetails(volumeName, bucketName, keyName);
Map<OmKeyLocationInfo, Map<DatanodeDetails, OzoneInputStream>> replicas = clientProtocol.getKeysEveryReplicas(volumeName, bucketName, keyName);
Map<OmKeyLocationInfo, Map<DatanodeDetails, OzoneInputStream>> replicasWithoutChecksum = clientProtocolWithoutChecksum.getKeysEveryReplicas(volumeName, bucketName, keyName);
JsonObject result = new JsonObject();
result.addProperty(JSON_PROPERTY_FILE_NAME, volumeName + "/" + bucketName + "/" + keyName);
result.addProperty(JSON_PROPERTY_FILE_SIZE, keyInfoDetails.getDataSize());
JsonArray blocks = new JsonArray();
downloadReplicasAndCreateManifest(keyName, replicas, replicasWithoutChecksum, directoryName, blocks);
result.add(JSON_PROPERTY_FILE_BLOCKS, blocks);
Gson gson = new GsonBuilder().setPrettyPrinting().create();
String prettyJson = gson.toJson(result);
String manifestFileName = keyName + "_manifest";
System.out.println("Writing manifest file : " + manifestFileName);
File manifestFile = new File(outputDir + "/" + directoryName + "/" + manifestFileName);
Files.write(manifestFile.toPath(), prettyJson.getBytes(StandardCharsets.UTF_8));
}
Aggregations