use of org.apache.hadoop.ozone.om.helpers.OmKeyInfo in project ozone by apache.
the class RpcClient method getKey.
@Override
public OzoneInputStream getKey(String volumeName, String bucketName, String keyName) throws IOException {
verifyVolumeName(volumeName);
verifyBucketName(bucketName);
Preconditions.checkNotNull(keyName);
OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setRefreshPipeline(true).setSortDatanodesInPipeline(topologyAwareReadEnabled).setLatestVersionLocation(getLatestVersionLocation).build();
OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
return getInputStreamWithRetryFunction(keyInfo);
}
use of org.apache.hadoop.ozone.om.helpers.OmKeyInfo in project ozone by apache.
the class RpcClient method getKeyDetails.
@Override
public OzoneKeyDetails getKeyDetails(String volumeName, String bucketName, String keyName) throws IOException {
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
Preconditions.checkNotNull(keyName);
OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setRefreshPipeline(true).setSortDatanodesInPipeline(topologyAwareReadEnabled).setLatestVersionLocation(getLatestVersionLocation).build();
OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
List<OzoneKeyLocation> ozoneKeyLocations = new ArrayList<>();
long lastKeyOffset = 0L;
List<OmKeyLocationInfo> omKeyLocationInfos = keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly();
for (OmKeyLocationInfo info : omKeyLocationInfos) {
ozoneKeyLocations.add(new OzoneKeyLocation(info.getContainerID(), info.getLocalID(), info.getLength(), info.getOffset(), lastKeyOffset));
lastKeyOffset += info.getLength();
}
return new OzoneKeyDetails(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(), keyInfo.getModificationTime(), ozoneKeyLocations, keyInfo.getReplicationConfig(), keyInfo.getMetadata(), keyInfo.getFileEncryptionInfo());
}
use of org.apache.hadoop.ozone.om.helpers.OmKeyInfo in project ozone by apache.
the class RpcClient method getKeysEveryReplicas.
@Override
public Map<OmKeyLocationInfo, Map<DatanodeDetails, OzoneInputStream>> getKeysEveryReplicas(String volumeName, String bucketName, String keyName) throws IOException {
Map<OmKeyLocationInfo, Map<DatanodeDetails, OzoneInputStream>> result = new LinkedHashMap<>();
verifyVolumeName(volumeName);
verifyBucketName(bucketName);
Preconditions.checkNotNull(keyName);
OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setRefreshPipeline(true).setSortDatanodesInPipeline(topologyAwareReadEnabled).build();
OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
List<OmKeyLocationInfo> keyLocationInfos = keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly();
for (OmKeyLocationInfo keyLocationInfo : keyLocationInfos) {
Map<DatanodeDetails, OzoneInputStream> blocks = new HashMap<>();
Pipeline pipelineBefore = keyLocationInfo.getPipeline();
List<DatanodeDetails> datanodes = pipelineBefore.getNodes();
for (DatanodeDetails dn : datanodes) {
List<DatanodeDetails> nodes = new ArrayList<>();
nodes.add(dn);
Pipeline pipeline = new Pipeline.Builder(pipelineBefore).setNodes(nodes).setId(PipelineID.randomId()).build();
keyLocationInfo.setPipeline(pipeline);
List<OmKeyLocationInfo> keyLocationInfoList = new ArrayList<>();
keyLocationInfoList.add(keyLocationInfo);
OmKeyLocationInfoGroup keyLocationInfoGroup = new OmKeyLocationInfoGroup(0, keyLocationInfoList);
List<OmKeyLocationInfoGroup> keyLocationInfoGroups = new ArrayList<>();
keyLocationInfoGroups.add(keyLocationInfoGroup);
keyInfo.setKeyLocationVersions(keyLocationInfoGroups);
OzoneInputStream is = createInputStream(keyInfo, Function.identity());
blocks.put(dn, is);
}
result.put(keyLocationInfo, blocks);
}
return result;
}
use of org.apache.hadoop.ozone.om.helpers.OmKeyInfo in project ozone by apache.
the class RpcClient method headObject.
@Override
public OzoneKey headObject(String volumeName, String bucketName, String keyName) throws IOException {
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
Preconditions.checkNotNull(keyName);
OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setLatestVersionLocation(true).setHeadOp(true).build();
OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(), keyInfo.getModificationTime(), keyInfo.getReplicationConfig());
}
use of org.apache.hadoop.ozone.om.helpers.OmKeyInfo in project ozone by apache.
the class TestReplicatedFileChecksumHelper method testOneBlock.
@Test
public void testOneBlock() throws IOException {
// test the file checksum of a file with one block.
OzoneConfiguration conf = new OzoneConfiguration();
RpcClient mockRpcClient = Mockito.mock(RpcClient.class);
List<DatanodeDetails> dns = Arrays.asList(DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).build());
Pipeline pipeline;
pipeline = Pipeline.newBuilder().setId(PipelineID.randomId()).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)).setState(Pipeline.PipelineState.CLOSED).setNodes(dns).build();
XceiverClientGrpc xceiverClientGrpc = new XceiverClientGrpc(pipeline, conf) {
@Override
public XceiverClientReply sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request, DatanodeDetails dn) {
return buildValidResponse();
}
};
XceiverClientFactory factory = Mockito.mock(XceiverClientFactory.class);
when(factory.acquireClientForReadData(ArgumentMatchers.any())).thenReturn(xceiverClientGrpc);
when(mockRpcClient.getXceiverClientManager()).thenReturn(factory);
OzoneManagerProtocol om = Mockito.mock(OzoneManagerProtocol.class);
when(mockRpcClient.getOzoneManagerClient()).thenReturn(om);
BlockID blockID = new BlockID(1, 1);
OmKeyLocationInfo omKeyLocationInfo = new OmKeyLocationInfo.Builder().setPipeline(pipeline).setBlockID(blockID).build();
List<OmKeyLocationInfo> omKeyLocationInfoList = Arrays.asList(omKeyLocationInfo);
OmKeyInfo omKeyInfo = new OmKeyInfo.Builder().setVolumeName(null).setBucketName(null).setKeyName(null).setOmKeyLocationInfos(Collections.singletonList(new OmKeyLocationInfoGroup(0, omKeyLocationInfoList))).setCreationTime(Time.now()).setModificationTime(Time.now()).setDataSize(0).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setFileEncryptionInfo(null).setAcls(null).build();
when(om.lookupKey(ArgumentMatchers.any())).thenReturn(omKeyInfo);
OzoneVolume mockVolume = Mockito.mock(OzoneVolume.class);
when(mockVolume.getName()).thenReturn("vol1");
OzoneBucket bucket = Mockito.mock(OzoneBucket.class);
when(bucket.getName()).thenReturn("bucket1");
ReplicatedFileChecksumHelper helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", 10, mockRpcClient);
helper.compute();
FileChecksum fileChecksum = helper.getFileChecksum();
assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
assertEquals(1, helper.getKeyLocationInfoList().size());
}
Aggregations