Search in sources :

Example 1 with BlockIterator

use of org.apache.hadoop.ozone.container.common.interfaces.BlockIterator in project ozone by apache.

the class TestSchemaOneBackwardsCompatibility method testReadDeletingBlockData.

@Test
public void testReadDeletingBlockData() throws Exception {
    try (ReferenceCountedDB refCountedDB = BlockUtils.getDB(newKvData(), conf)) {
        Table<String, BlockData> blockDataTable = refCountedDB.getStore().getBlockDataTable();
        for (String blockID : TestDB.DELETING_BLOCK_IDS) {
            BlockData blockData = blockDataTable.get(OzoneConsts.DELETING_KEY_PREFIX + blockID);
            Assert.assertEquals(Long.toString(blockData.getLocalID()), blockID);
        }
        // Test decoding keys from the database.
        List<? extends Table.KeyValue<String, BlockData>> blockKeyValues = blockDataTable.getRangeKVs(null, 100, MetadataKeyFilters.getDeletingKeyFilter());
        List<String> decodedKeys = new ArrayList<>();
        for (Table.KeyValue<String, BlockData> blockDataKV : blockKeyValues) {
            decodedKeys.add(blockDataKV.getKey());
        }
        // Apply the deleting prefix to the saved block IDs so we can compare
        // them to the retrieved keys.
        List<String> expectedKeys = TestDB.DELETING_BLOCK_IDS.stream().map(key -> OzoneConsts.DELETING_KEY_PREFIX + key).collect(Collectors.toList());
        Assert.assertEquals(expectedKeys, decodedKeys);
        // Test reading deleting blocks with block iterator.
        MetadataKeyFilters.KeyPrefixFilter filter = MetadataKeyFilters.getDeletingKeyFilter();
        try (BlockIterator<BlockData> iter = refCountedDB.getStore().getBlockIterator(filter)) {
            List<String> iteratorBlockIDs = new ArrayList<>();
            while (iter.hasNext()) {
                long localID = iter.nextBlock().getBlockID().getLocalID();
                iteratorBlockIDs.add(Long.toString(localID));
            }
            Assert.assertEquals(TestDB.DELETING_BLOCK_IDS, iteratorBlockIDs);
        }
    }
}
Also used : ScmConfigKeys(org.apache.hadoop.hdds.scm.ScmConfigKeys) Arrays(java.util.Arrays) BlockUtils(org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils) URL(java.net.URL) ContainerSet(org.apache.hadoop.ozone.container.common.impl.ContainerSet) MutableVolumeSet(org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet) BlockIterator(org.apache.hadoop.ozone.container.common.interfaces.BlockIterator) KeyValueContainerData(org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData) DatanodeStore(org.apache.hadoop.ozone.container.metadata.DatanodeStore) ChunkInfoList(org.apache.hadoop.ozone.container.common.helpers.ChunkInfoList) HddsConfigKeys(org.apache.hadoop.hdds.HddsConfigKeys) ReferenceCountedDB(org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB) SchemaOneDeletedBlocksTable(org.apache.hadoop.ozone.container.metadata.SchemaOneDeletedBlocksTable) MetadataKeyFilters(org.apache.hadoop.hdds.utils.MetadataKeyFilters) KeyValueHandler(org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler) Set(java.util.Set) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) List(java.util.List) Assert.assertFalse(org.junit.Assert.assertFalse) OzoneConfigKeys(org.apache.hadoop.ozone.OzoneConfigKeys) GenericTestUtils(org.apache.ozone.test.GenericTestUtils) Mockito.mock(org.mockito.Mockito.mock) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) ContainerDataYaml(org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml) KeyValueContainerUtil(org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil) Yaml(org.yaml.snakeyaml.Yaml) ArrayList(java.util.ArrayList) BlockData(org.apache.hadoop.ozone.container.common.helpers.BlockData) HashSet(java.util.HashSet) BlockDeletingServiceTestImpl(org.apache.hadoop.ozone.container.testutils.BlockDeletingServiceTestImpl) OzoneContainer(org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer) ContainerDispatcher(org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher) Before(org.junit.Before) KeyValueContainer(org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer) OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL(org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL) ContainerMetrics(org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics) Assert.assertTrue(org.junit.Assert.assertTrue) FileUtils(org.apache.commons.io.FileUtils) Test(org.junit.Test) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) OzoneConsts(org.apache.hadoop.ozone.OzoneConsts) File(java.io.File) StorageVolume(org.apache.hadoop.ozone.container.common.volume.StorageVolume) VolumeSet(org.apache.hadoop.ozone.container.common.volume.VolumeSet) Rule(org.junit.Rule) Assert.assertNull(org.junit.Assert.assertNull) Table(org.apache.hadoop.hdds.utils.db.Table) Assert(org.junit.Assert) TemporaryFolder(org.junit.rules.TemporaryFolder) Assert.assertEquals(org.junit.Assert.assertEquals) MetadataKeyFilters(org.apache.hadoop.hdds.utils.MetadataKeyFilters) SchemaOneDeletedBlocksTable(org.apache.hadoop.ozone.container.metadata.SchemaOneDeletedBlocksTable) Table(org.apache.hadoop.hdds.utils.db.Table) ArrayList(java.util.ArrayList) ReferenceCountedDB(org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB) BlockData(org.apache.hadoop.ozone.container.common.helpers.BlockData) Test(org.junit.Test)

Aggregations

File (java.io.File)1 IOException (java.io.IOException)1 URL (java.net.URL)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Set (java.util.Set)1 UUID (java.util.UUID)1 Collectors (java.util.stream.Collectors)1 FileUtils (org.apache.commons.io.FileUtils)1 HddsConfigKeys (org.apache.hadoop.hdds.HddsConfigKeys)1 OzoneConfiguration (org.apache.hadoop.hdds.conf.OzoneConfiguration)1 ScmConfigKeys (org.apache.hadoop.hdds.scm.ScmConfigKeys)1 MetadataKeyFilters (org.apache.hadoop.hdds.utils.MetadataKeyFilters)1 Table (org.apache.hadoop.hdds.utils.db.Table)1 OzoneConfigKeys (org.apache.hadoop.ozone.OzoneConfigKeys)1 OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL (org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL)1 OzoneConsts (org.apache.hadoop.ozone.OzoneConsts)1 BlockData (org.apache.hadoop.ozone.container.common.helpers.BlockData)1