Examples with OmKeyLocationInfoGroup org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup used on opensource projects

Example 6 with OmKeyLocationInfoGroup

use of org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup in project ozone by apache.

the class TestKeyManagerUnit method testLookupFileWithDnFailure.

@Test
public void testLookupFileWithDnFailure() throws IOException {
    final DatanodeDetails dnOne = MockDatanodeDetails.randomDatanodeDetails();
    final DatanodeDetails dnTwo = MockDatanodeDetails.randomDatanodeDetails();
    final DatanodeDetails dnThree = MockDatanodeDetails.randomDatanodeDetails();
    final DatanodeDetails dnFour = MockDatanodeDetails.randomDatanodeDetails();
    final DatanodeDetails dnFive = MockDatanodeDetails.randomDatanodeDetails();
    final DatanodeDetails dnSix = MockDatanodeDetails.randomDatanodeDetails();
    final Pipeline pipelineOne = Pipeline.newBuilder().setId(PipelineID.randomId()).setReplicationConfig(RatisReplicationConfig.getInstance(ReplicationFactor.THREE)).setState(Pipeline.PipelineState.OPEN).setLeaderId(dnOne.getUuid()).setNodes(Arrays.asList(dnOne, dnTwo, dnThree)).build();
    final Pipeline pipelineTwo = Pipeline.newBuilder().setId(PipelineID.randomId()).setReplicationConfig(RatisReplicationConfig.getInstance(ReplicationFactor.THREE)).setState(Pipeline.PipelineState.OPEN).setLeaderId(dnFour.getUuid()).setNodes(Arrays.asList(dnFour, dnFive, dnSix)).build();
    List<Long> containerIDs = new ArrayList<>();
    containerIDs.add(1L);
    List<ContainerWithPipeline> cps = new ArrayList<>();
    ContainerInfo ci = Mockito.mock(ContainerInfo.class);
    when(ci.getContainerID()).thenReturn(1L);
    cps.add(new ContainerWithPipeline(ci, pipelineTwo));
    when(containerClient.getContainerWithPipelineBatch(containerIDs)).thenReturn(cps);
    final OmVolumeArgs volumeArgs = OmVolumeArgs.newBuilder().setVolume("volumeOne").setAdminName("admin").setOwnerName("admin").build();
    OMRequestTestUtils.addVolumeToOM(metadataManager, volumeArgs);
    final OmBucketInfo bucketInfo = OmBucketInfo.newBuilder().setVolumeName("volumeOne").setBucketName("bucketOne").build();
    OMRequestTestUtils.addBucketToOM(metadataManager, bucketInfo);
    final OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder().setBlockID(new BlockID(1L, 1L)).setPipeline(pipelineOne).setOffset(0).setLength(256000).build();
    final OmKeyInfo keyInfo = new OmKeyInfo.Builder().setVolumeName("volumeOne").setBucketName("bucketOne").setKeyName("keyOne").setOmKeyLocationInfos(singletonList(new OmKeyLocationInfoGroup(0, singletonList(keyLocationInfo)))).setCreationTime(Time.now()).setModificationTime(Time.now()).setDataSize(256000).setReplicationConfig(RatisReplicationConfig.getInstance(ReplicationFactor.THREE)).setAcls(Collections.emptyList()).build();
    OMRequestTestUtils.addKeyToOM(metadataManager, keyInfo);
    final OmKeyArgs.Builder keyArgs = new OmKeyArgs.Builder().setVolumeName("volumeOne").setBucketName("bucketOne").setKeyName("keyOne");
    final OmKeyInfo newKeyInfo = keyManager.lookupFile(keyArgs.build(), "test");
    final OmKeyLocationInfo newBlockLocation = newKeyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly().get(0);
    Assert.assertEquals(1L, newBlockLocation.getContainerID());
    Assert.assertEquals(1L, newBlockLocation.getBlockID().getLocalID());
    Assert.assertEquals(pipelineTwo.getId(), newBlockLocation.getPipeline().getId());
    Assert.assertTrue(newBlockLocation.getPipeline().getNodes().contains(dnFour));
    Assert.assertTrue(newBlockLocation.getPipeline().getNodes().contains(dnFive));
    Assert.assertTrue(newBlockLocation.getPipeline().getNodes().contains(dnSix));
}

Example 7 with OmKeyLocationInfoGroup

use of org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup in project ozone by apache.

the class ContainerMapper method parseOmDB.

/**
 * Generates Container Id to Blocks and BlockDetails mapping.
 * @param configuration @{@link OzoneConfiguration}
 * @return Map<Long, List<Map<Long, @BlockDetails>>>
 *   Map of ContainerId -> (Block, Block info)
 * @throws IOException
 */
public Map<Long, List<Map<Long, BlockIdDetails>>> parseOmDB(OzoneConfiguration configuration) throws IOException {
    String path = configuration.get(OZONE_OM_DB_DIRS);
    if (path == null || path.isEmpty()) {
        throw new IOException(OZONE_OM_DB_DIRS + "should be set ");
    } else {
        Table keyTable = getMetaTable(configuration);
        Map<Long, List<Map<Long, BlockIdDetails>>> dataMap = new HashMap<>();
        if (keyTable != null) {
            try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> keyValueTableIterator = keyTable.iterator()) {
                while (keyValueTableIterator.hasNext()) {
                    Table.KeyValue<String, OmKeyInfo> keyValue = keyValueTableIterator.next();
                    OmKeyInfo omKeyInfo = keyValue.getValue();
                    byte[] value = omKeyInfo.getProtobuf(true, CURRENT_VERSION).toByteArray();
                    OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(OzoneManagerProtocolProtos.KeyInfo.parseFrom(value));
                    for (OmKeyLocationInfoGroup keyLocationInfoGroup : keyInfo.getKeyLocationVersions()) {
                        List<OmKeyLocationInfo> keyLocationInfo = keyLocationInfoGroup.getLocationList();
                        for (OmKeyLocationInfo keyLocation : keyLocationInfo) {
                            BlockIdDetails blockIdDetails = new BlockIdDetails();
                            Map<Long, BlockIdDetails> innerMap = new HashMap<>();
                            long containerID = keyLocation.getBlockID().getContainerID();
                            long blockID = keyLocation.getBlockID().getLocalID();
                            blockIdDetails.setBucketName(keyInfo.getBucketName());
                            blockIdDetails.setBlockVol(keyInfo.getVolumeName());
                            blockIdDetails.setKeyName(keyInfo.getKeyName());
                            List<Map<Long, BlockIdDetails>> innerList = new ArrayList<>();
                            innerMap.put(blockID, blockIdDetails);
                            if (dataMap.containsKey(containerID)) {
                                innerList = dataMap.get(containerID);
                            }
                            innerList.add(innerMap);
                            dataMap.put(containerID, innerList);
                        }
                    }
                }
            }
        }
        return dataMap;
    }
}

Example 8 with OmKeyLocationInfoGroup

use of org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup in project ozone by apache.

the class TestOmKeyInfoCodec method getKeyInfo.

private OmKeyInfo getKeyInfo(int chunkNum) {
    List<OmKeyLocationInfo> omKeyLocationInfoList = new ArrayList<>();
    Pipeline pipeline = HddsTestUtils.getRandomPipeline();
    for (int i = 0; i < chunkNum; i++) {
        BlockID blockID = new BlockID(i, i);
        OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder().setBlockID(blockID).setPipeline(pipeline).build();
        omKeyLocationInfoList.add(keyLocationInfo);
    }
    OmKeyLocationInfoGroup omKeyLocationInfoGroup = new OmKeyLocationInfoGroup(0, omKeyLocationInfoList);
    return new OmKeyInfo.Builder().setCreationTime(Time.now()).setModificationTime(Time.now()).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)).setVolumeName(VOLUME).setBucketName(BUCKET).setKeyName(KEYNAME).setObjectID(Time.now()).setUpdateID(Time.now()).setDataSize(100).setOmKeyLocationInfos(Collections.singletonList(omKeyLocationInfoGroup)).build();
}

Example 9 with OmKeyLocationInfoGroup

use of org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup in project ozone by apache.

the class OzoneTestUtils method performOperationOnKeyContainers.

/**
 * Performs the provided consumer on containers which contain the blocks
 * listed in omKeyLocationInfoGroups.
 *
 * @param consumer Consumer which accepts BlockID as argument.
 * @param omKeyLocationInfoGroups locationInfos for a key.
 * @throws IOException
 */
public static void performOperationOnKeyContainers(CheckedConsumer<BlockID, Exception> consumer, List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups) throws Exception {
    for (OmKeyLocationInfoGroup omKeyLocationInfoGroup : omKeyLocationInfoGroups) {
        List<OmKeyLocationInfo> omKeyLocationInfos = omKeyLocationInfoGroup.getLocationList();
        for (OmKeyLocationInfo omKeyLocationInfo : omKeyLocationInfos) {
            BlockID blockID = omKeyLocationInfo.getBlockID();
            consumer.accept(blockID);
        }
    }
}

Example 10 with OmKeyLocationInfoGroup

use of org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup in project ozone by apache.

the class ContainerKeyMapperTask method writeOMKeyToContainerDB.

/**
 * Write an OM key to container DB and update containerID -> no. of keys
 * count.
 *
 * @param key key String
 * @param omKeyInfo omKeyInfo value
 * @throws IOException if unable to write to recon DB.
 */
private void writeOMKeyToContainerDB(String key, OmKeyInfo omKeyInfo) throws IOException {
    long containerCountToIncrement = 0;
    for (OmKeyLocationInfoGroup omKeyLocationInfoGroup : omKeyInfo.getKeyLocationVersions()) {
        long keyVersion = omKeyLocationInfoGroup.getVersion();
        for (OmKeyLocationInfo omKeyLocationInfo : omKeyLocationInfoGroup.getLocationList()) {
            long containerId = omKeyLocationInfo.getContainerID();
            ContainerKeyPrefix containerKeyPrefix = new ContainerKeyPrefix(containerId, key, keyVersion);
            if (reconContainerMetadataManager.getCountForContainerKeyPrefix(containerKeyPrefix) == 0) {
                // Save on writes. No need to save same container-key prefix
                // mapping again.
                reconContainerMetadataManager.storeContainerKeyMapping(containerKeyPrefix, 1);
                // increment the count of containers if it does not exist
                if (!reconContainerMetadataManager.doesContainerExists(containerId)) {
                    containerCountToIncrement++;
                }
                // update the count of keys for the given containerID
                long keyCount = reconContainerMetadataManager.getKeyCountForContainer(containerId);
                // increment the count and update containerKeyCount.
                // keyCount will be 0 if containerID is not found. So, there is no
                // need to initialize keyCount for the first time.
                reconContainerMetadataManager.storeContainerKeyCount(containerId, ++keyCount);
            }
        }
    }
    if (containerCountToIncrement > 0) {
        reconContainerMetadataManager.incrementContainerCountBy(containerCountToIncrement);
    }
}