Examples with DBHandle org.apache.hadoop.ozone.container.common.interfaces.DBHandle used on opensource projects

Example 1 with DBHandle

use of org.apache.hadoop.ozone.container.common.interfaces.DBHandle in project ozone by apache.

the class TestContainerPersistence method testCreateContainer.

@Test
public void testCreateContainer() throws Exception {
    long testContainerID = getTestContainerID();
    addContainer(containerSet, testContainerID);
    Assert.assertTrue(containerSet.getContainerMapCopy().containsKey(testContainerID));
    KeyValueContainerData kvData = (KeyValueContainerData) containerSet.getContainer(testContainerID).getContainerData();
    Assert.assertNotNull(kvData);
    Assert.assertTrue(new File(kvData.getMetadataPath()).exists());
    Assert.assertTrue(new File(kvData.getChunksPath()).exists());
    Assert.assertTrue(kvData.getDbFile().exists());
    Path meta = kvData.getDbFile().toPath().getParent();
    Assert.assertTrue(meta != null && Files.exists(meta));
    DBHandle store = null;
    try {
        store = BlockUtils.getDB(kvData, conf);
        Assert.assertNotNull(store);
    } finally {
        if (store != null) {
            store.close();
        }
    }
}

Example 2 with DBHandle

use of org.apache.hadoop.ozone.container.common.interfaces.DBHandle in project ozone by apache.

the class TestBlockDeletingService method createPendingDeleteBlocksSchema1.

@SuppressWarnings("checkstyle:parameternumber")
private void createPendingDeleteBlocksSchema1(int numOfBlocksPerContainer, KeyValueContainerData data, long containerID, int numOfChunksPerBlock, ChunkBuffer buffer, ChunkManager chunkManager, KeyValueContainer container) {
    BlockID blockID = null;
    try (DBHandle metadata = BlockUtils.getDB(data, conf)) {
        for (int j = 0; j < numOfBlocksPerContainer; j++) {
            blockID = ContainerTestHelper.getTestBlockID(containerID);
            String deleteStateName = data.deletingBlockKey(blockID.getLocalID());
            BlockData kd = new BlockData(blockID);
            List<ContainerProtos.ChunkInfo> chunks = Lists.newArrayList();
            putChunksInBlock(numOfChunksPerBlock, j, chunks, buffer, chunkManager, container, blockID);
            kd.setChunks(chunks);
            metadata.getStore().getBlockDataTable().put(deleteStateName, kd);
            container.getContainerData().incrPendingDeletionBlocks(1);
        }
        updateMetaData(data, container, numOfBlocksPerContainer, numOfChunksPerBlock);
    } catch (IOException exception) {
        LOG.info("Exception " + exception);
        LOG.warn("Failed to put block: " + blockID + " in BlockDataTable.");
    }
}

Example 3 with DBHandle

use of org.apache.hadoop.ozone.container.common.interfaces.DBHandle in project ozone by apache.

the class TestBlockDeletingService method testBlockDeletion.

@Test
public void testBlockDeletion() throws Exception {
    DatanodeConfiguration dnConf = conf.getObject(DatanodeConfiguration.class);
    dnConf.setBlockDeletionLimit(2);
    this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
    conf.setFromObject(dnConf);
    ContainerSet containerSet = new ContainerSet();
    createToDeleteBlocks(containerSet, 1, 3, 1);
    ContainerMetrics metrics = ContainerMetrics.create(conf);
    KeyValueHandler keyValueHandler = new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet, metrics, c -> {
    });
    BlockDeletingServiceTestImpl svc = getBlockDeletingService(containerSet, conf, keyValueHandler);
    svc.start();
    GenericTestUtils.waitFor(svc::isStarted, 100, 3000);
    // Ensure 1 container was created
    List<ContainerData> containerData = Lists.newArrayList();
    containerSet.listContainer(0L, 1, containerData);
    Assert.assertEquals(1, containerData.size());
    KeyValueContainerData data = (KeyValueContainerData) containerData.get(0);
    try (DBHandle meta = BlockUtils.getDB(data, conf)) {
        Map<Long, Container<?>> containerMap = containerSet.getContainerMapCopy();
        // NOTE: this test assumes that all the container is KetValueContainer and
        // have DeleteTransactionId in KetValueContainerData. If other
        // types is going to be added, this test should be checked.
        long transactionId = ((KeyValueContainerData) containerMap.get(containerData.get(0).getContainerID()).getContainerData()).getDeleteTransactionId();
        long containerSpace = containerData.get(0).getBytesUsed();
        // Number of deleted blocks in container should be equal to 0 before
        // block delete
        Assert.assertEquals(0, transactionId);
        // Ensure there are 3 blocks under deletion and 0 deleted blocks
        Assert.assertEquals(3, getUnderDeletionBlocksCount(meta, data));
        Assert.assertEquals(3, meta.getStore().getMetadataTable().get(data.pendingDeleteBlockCountKey()).longValue());
        // Container contains 3 blocks. So, space used by the container
        // should be greater than zero.
        Assert.assertTrue(containerSpace > 0);
        // An interval will delete 1 * 2 blocks
        deleteAndWait(svc, 1);
        GenericTestUtils.waitFor(() -> containerData.get(0).getBytesUsed() == containerSpace / 3, 100, 3000);
        // After first interval 2 blocks will be deleted. Hence, current space
        // used by the container should be less than the space used by the
        // container initially(before running deletion services).
        Assert.assertTrue(containerData.get(0).getBytesUsed() < containerSpace);
        deleteAndWait(svc, 2);
        // After deletion of all 3 blocks, space used by the containers
        // should be zero.
        GenericTestUtils.waitFor(() -> containerData.get(0).getBytesUsed() == 0, 100, 3000);
        // Check finally DB counters.
        // Not checking bytes used, as handler is a mock call.
        Assert.assertEquals(0, meta.getStore().getMetadataTable().get(data.pendingDeleteBlockCountKey()).longValue());
        Assert.assertEquals(0, meta.getStore().getMetadataTable().get(data.blockCountKey()).longValue());
    }
    svc.shutdown();
}

Example 4 with DBHandle

use of org.apache.hadoop.ozone.container.common.interfaces.DBHandle in project ozone by apache.

the class TestBlockDeletingService method createPendingDeleteBlocksViaTxn.

@SuppressWarnings("checkstyle:parameternumber")
private void createPendingDeleteBlocksViaTxn(int numOfBlocksPerContainer, int txnID, long containerID, int numOfChunksPerBlock, ChunkBuffer buffer, ChunkManager chunkManager, KeyValueContainer container, KeyValueContainerData data) {
    List<Long> containerBlocks = new ArrayList<>();
    for (int i = 0; i < numOfBlocksPerContainer; i++) {
        txnID = txnID + 1;
        BlockID blockID = ContainerTestHelper.getTestBlockID(containerID);
        BlockData kd = new BlockData(blockID);
        List<ContainerProtos.ChunkInfo> chunks = Lists.newArrayList();
        putChunksInBlock(numOfChunksPerBlock, i, chunks, buffer, chunkManager, container, blockID);
        kd.setChunks(chunks);
        try (DBHandle metadata = BlockUtils.getDB(data, conf)) {
            String blockKey = data.blockKey(blockID.getLocalID());
            metadata.getStore().getBlockDataTable().put(blockKey, kd);
        } catch (IOException exception) {
            LOG.info("Exception = " + exception);
            LOG.warn("Failed to put block: " + blockID.getLocalID() + " in BlockDataTable.");
        }
        container.getContainerData().incrPendingDeletionBlocks(1);
        // Below we are creating one transaction per block just for
        // testing purpose
        containerBlocks.add(blockID.getLocalID());
        createTxn(data, containerBlocks, txnID, containerID);
        containerBlocks.clear();
    }
    updateMetaData(data, container, numOfBlocksPerContainer, numOfChunksPerBlock);
}

Example 5 with DBHandle

use of org.apache.hadoop.ozone.container.common.interfaces.DBHandle in project ozone by apache.

the class TestSchemaOneBackwardsCompatibility method testBlockIteration.

/**
 * Counts the number of deleted, pending delete, and regular blocks in the
 * database, and checks that they match the expected values.
 * Also makes sure that internal prefixes used to manage data in the schema
 * one deleted blocks table are removed from keys in iterator results.
 * @throws IOException
 */
@Test
public void testBlockIteration() throws IOException {
    KeyValueContainerData cData = newKvData();
    try (DBHandle refCountedDB = BlockUtils.getDB(cData, conf)) {
        assertEquals(TestDB.NUM_DELETED_BLOCKS, countDeletedBlocks(refCountedDB, cData));
        assertEquals(TestDB.NUM_PENDING_DELETION_BLOCKS, countDeletingBlocks(refCountedDB, cData));
        assertEquals(TestDB.KEY_COUNT - TestDB.NUM_PENDING_DELETION_BLOCKS, countUnprefixedBlocks(refCountedDB, cData));
        // Test that deleted block keys do not have a visible prefix when
        // iterating.
        final String prefix = SchemaOneDeletedBlocksTable.DELETED_KEY_PREFIX;
        Table<String, ChunkInfoList> deletedBlocksTable = refCountedDB.getStore().getDeletedBlocksTable();
        // Test rangeKVs.
        List<? extends Table.KeyValue<String, ChunkInfoList>> deletedBlocks = deletedBlocksTable.getRangeKVs(cData.startKeyEmpty(), 100, cData.containerPrefix());
        for (Table.KeyValue<String, ChunkInfoList> kv : deletedBlocks) {
            assertFalse(kv.getKey().contains(prefix));
        }
        // Test sequentialRangeKVs.
        deletedBlocks = deletedBlocksTable.getRangeKVs(cData.startKeyEmpty(), 100, cData.containerPrefix());
        for (Table.KeyValue<String, ChunkInfoList> kv : deletedBlocks) {
            assertFalse(kv.getKey().contains(prefix));
        }
    }
}