Examples with OmKeyLocationInfoGroup org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup used on opensource projects

Example 1 with OmKeyLocationInfoGroup

use of org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup in project ozone by apache.

the class RpcClient method getKeysEveryReplicas.

@Override
public Map<OmKeyLocationInfo, Map<DatanodeDetails, OzoneInputStream>> getKeysEveryReplicas(String volumeName, String bucketName, String keyName) throws IOException {
    Map<OmKeyLocationInfo, Map<DatanodeDetails, OzoneInputStream>> result = new LinkedHashMap<>();
    verifyVolumeName(volumeName);
    verifyBucketName(bucketName);
    Preconditions.checkNotNull(keyName);
    OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setRefreshPipeline(true).setSortDatanodesInPipeline(topologyAwareReadEnabled).build();
    OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
    List<OmKeyLocationInfo> keyLocationInfos = keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly();
    for (OmKeyLocationInfo keyLocationInfo : keyLocationInfos) {
        Map<DatanodeDetails, OzoneInputStream> blocks = new HashMap<>();
        Pipeline pipelineBefore = keyLocationInfo.getPipeline();
        List<DatanodeDetails> datanodes = pipelineBefore.getNodes();
        for (DatanodeDetails dn : datanodes) {
            List<DatanodeDetails> nodes = new ArrayList<>();
            nodes.add(dn);
            Pipeline pipeline = new Pipeline.Builder(pipelineBefore).setNodes(nodes).setId(PipelineID.randomId()).build();
            keyLocationInfo.setPipeline(pipeline);
            List<OmKeyLocationInfo> keyLocationInfoList = new ArrayList<>();
            keyLocationInfoList.add(keyLocationInfo);
            OmKeyLocationInfoGroup keyLocationInfoGroup = new OmKeyLocationInfoGroup(0, keyLocationInfoList);
            List<OmKeyLocationInfoGroup> keyLocationInfoGroups = new ArrayList<>();
            keyLocationInfoGroups.add(keyLocationInfoGroup);
            keyInfo.setKeyLocationVersions(keyLocationInfoGroups);
            OzoneInputStream is = createInputStream(keyInfo, Function.identity());
            blocks.put(dn, is);
        }
        result.put(keyLocationInfo, blocks);
    }
    return result;
}

Example 2 with OmKeyLocationInfoGroup

use of org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup in project ozone by apache.

the class TestReplicatedFileChecksumHelper method testOneBlock.

@Test
public void testOneBlock() throws IOException {
    // test the file checksum of a file with one block.
    OzoneConfiguration conf = new OzoneConfiguration();
    RpcClient mockRpcClient = Mockito.mock(RpcClient.class);
    List<DatanodeDetails> dns = Arrays.asList(DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).build());
    Pipeline pipeline;
    pipeline = Pipeline.newBuilder().setId(PipelineID.randomId()).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)).setState(Pipeline.PipelineState.CLOSED).setNodes(dns).build();
    XceiverClientGrpc xceiverClientGrpc = new XceiverClientGrpc(pipeline, conf) {

        @Override
        public XceiverClientReply sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request, DatanodeDetails dn) {
            return buildValidResponse();
        }
    };
    XceiverClientFactory factory = Mockito.mock(XceiverClientFactory.class);
    when(factory.acquireClientForReadData(ArgumentMatchers.any())).thenReturn(xceiverClientGrpc);
    when(mockRpcClient.getXceiverClientManager()).thenReturn(factory);
    OzoneManagerProtocol om = Mockito.mock(OzoneManagerProtocol.class);
    when(mockRpcClient.getOzoneManagerClient()).thenReturn(om);
    BlockID blockID = new BlockID(1, 1);
    OmKeyLocationInfo omKeyLocationInfo = new OmKeyLocationInfo.Builder().setPipeline(pipeline).setBlockID(blockID).build();
    List<OmKeyLocationInfo> omKeyLocationInfoList = Arrays.asList(omKeyLocationInfo);
    OmKeyInfo omKeyInfo = new OmKeyInfo.Builder().setVolumeName(null).setBucketName(null).setKeyName(null).setOmKeyLocationInfos(Collections.singletonList(new OmKeyLocationInfoGroup(0, omKeyLocationInfoList))).setCreationTime(Time.now()).setModificationTime(Time.now()).setDataSize(0).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setFileEncryptionInfo(null).setAcls(null).build();
    when(om.lookupKey(ArgumentMatchers.any())).thenReturn(omKeyInfo);
    OzoneVolume mockVolume = Mockito.mock(OzoneVolume.class);
    when(mockVolume.getName()).thenReturn("vol1");
    OzoneBucket bucket = Mockito.mock(OzoneBucket.class);
    when(bucket.getName()).thenReturn("bucket1");
    ReplicatedFileChecksumHelper helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", 10, mockRpcClient);
    helper.compute();
    FileChecksum fileChecksum = helper.getFileChecksum();
    assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
    assertEquals(1, helper.getKeyLocationInfoList().size());
}

Example 3 with OmKeyLocationInfoGroup

use of org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup in project ozone by apache.

the class TestReconWithOzoneManager method addKeys.

/**
 * Helper function to add voli/bucketi/keyi to containeri to OM Metadata.
 * For test purpose each container will have only one key.
 */
private void addKeys(int start, int end) throws Exception {
    for (int i = start; i < end; i++) {
        Pipeline pipeline = HddsTestUtils.getRandomPipeline();
        List<OmKeyLocationInfo> omKeyLocationInfoList = new ArrayList<>();
        BlockID blockID = new BlockID(i, 1);
        OmKeyLocationInfo omKeyLocationInfo1 = getOmKeyLocationInfo(blockID, pipeline);
        omKeyLocationInfoList.add(omKeyLocationInfo1);
        OmKeyLocationInfoGroup omKeyLocationInfoGroup = new OmKeyLocationInfoGroup(0, omKeyLocationInfoList);
        writeDataToOm("key" + i, "bucket" + i, "vol" + i, Collections.singletonList(omKeyLocationInfoGroup));
    }
}

Example 4 with OmKeyLocationInfoGroup

use of org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup in project ozone by apache.

the class TestOmBlockVersioning method testAllocateCommit.

@Test
public void testAllocateCommit() throws Exception {
    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
    String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
    String keyName = "key" + RandomStringUtils.randomNumeric(5);
    OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster, volumeName, bucketName);
    // Versioning isn't supported currently, but just preserving old behaviour
    bucket.setVersioning(true);
    OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setDataSize(1000).setRefreshPipeline(true).setAcls(new ArrayList<>()).setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)).build();
    // 1st update, version 0
    OpenKeySession openKey = writeClient.openKey(keyArgs);
    // explicitly set the keyLocation list before committing the key.
    keyArgs.setLocationInfoList(openKey.getKeyInfo().getLatestVersionLocations().getBlocksLatestVersionOnly());
    writeClient.commitKey(keyArgs, openKey.getId());
    OmKeyInfo keyInfo = ozoneManager.lookupKey(keyArgs);
    OmKeyLocationInfoGroup highestVersion = checkVersions(keyInfo.getKeyLocationVersions());
    assertEquals(0, highestVersion.getVersion());
    assertEquals(1, highestVersion.getLocationList().size());
    // 2nd update, version 1
    openKey = writeClient.openKey(keyArgs);
    // OmKeyLocationInfo locationInfo =
    // writeClient.allocateBlock(keyArgs, openKey.getId());
    // explicitly set the keyLocation list before committing the key.
    keyArgs.setLocationInfoList(openKey.getKeyInfo().getLatestVersionLocations().getBlocksLatestVersionOnly());
    writeClient.commitKey(keyArgs, openKey.getId());
    keyInfo = ozoneManager.lookupKey(keyArgs);
    highestVersion = checkVersions(keyInfo.getKeyLocationVersions());
    assertEquals(1, highestVersion.getVersion());
    assertEquals(1, highestVersion.getLocationList().size());
    // 3rd update, version 2
    openKey = writeClient.openKey(keyArgs);
    // this block will be appended to the latest version of version 2.
    OmKeyLocationInfo locationInfo = writeClient.allocateBlock(keyArgs, openKey.getId(), new ExcludeList());
    List<OmKeyLocationInfo> locationInfoList = openKey.getKeyInfo().getLatestVersionLocations().getBlocksLatestVersionOnly();
    Assert.assertTrue(locationInfoList.size() == 1);
    locationInfoList.add(locationInfo);
    keyArgs.setLocationInfoList(locationInfoList);
    writeClient.commitKey(keyArgs, openKey.getId());
    keyInfo = ozoneManager.lookupKey(keyArgs);
    highestVersion = checkVersions(keyInfo.getKeyLocationVersions());
    assertEquals(2, highestVersion.getVersion());
    assertEquals(2, highestVersion.getLocationList().size());
}

Example 5 with OmKeyLocationInfoGroup

use of org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup in project ozone by apache.

the class TestKeyManagerUnit method listStatus.

@Test
public void listStatus() throws Exception {
    String volume = "vol";
    String bucket = "bucket";
    String keyPrefix = "key";
    String client = "client.host";
    OMRequestTestUtils.addVolumeToDB(volume, OzoneConsts.OZONE, metadataManager);
    OMRequestTestUtils.addBucketToDB(volume, bucket, metadataManager);
    final Pipeline pipeline = MockPipeline.createPipeline(3);
    final List<String> nodes = pipeline.getNodes().stream().map(DatanodeDetails::getUuidString).collect(toList());
    List<Long> containerIDs = new ArrayList<>();
    List<ContainerWithPipeline> containersWithPipeline = new ArrayList<>();
    for (long i = 1; i <= 10; i++) {
        final OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder().setBlockID(new BlockID(i, 1L)).setPipeline(pipeline).setOffset(0).setLength(256000).build();
        ContainerInfo containerInfo = new ContainerInfo.Builder().setContainerID(i).build();
        containersWithPipeline.add(new ContainerWithPipeline(containerInfo, pipeline));
        containerIDs.add(i);
        OmKeyInfo keyInfo = new OmKeyInfo.Builder().setVolumeName(volume).setBucketName(bucket).setCreationTime(Time.now()).setOmKeyLocationInfos(singletonList(new OmKeyLocationInfoGroup(0, new ArrayList<>()))).setReplicationConfig(RatisReplicationConfig.getInstance(ReplicationFactor.THREE)).setKeyName(keyPrefix + i).setObjectID(i).setUpdateID(i).build();
        keyInfo.appendNewBlocks(singletonList(keyLocationInfo), false);
        OMRequestTestUtils.addKeyToOM(metadataManager, keyInfo);
    }
    when(containerClient.getContainerWithPipelineBatch(containerIDs)).thenReturn(containersWithPipeline);
    OmKeyArgs.Builder builder = new OmKeyArgs.Builder().setVolumeName(volume).setBucketName(bucket).setKeyName("").setSortDatanodesInPipeline(true);
    List<OzoneFileStatus> fileStatusList = keyManager.listStatus(builder.build(), false, null, Long.MAX_VALUE, client);
    Assert.assertEquals(10, fileStatusList.size());
    verify(containerClient).getContainerWithPipelineBatch(containerIDs);
    verify(blockClient).sortDatanodes(nodes, client);
}