Examples with KeyValueHandler org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler used on opensource projects

Example 1 with KeyValueHandler

use of org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler in project ozone by apache.

the class TestHandler method testGetKeyValueHandler.

@Test
public void testGetKeyValueHandler() throws Exception {
    Handler kvHandler = dispatcher.getHandler(ContainerProtos.ContainerType.KeyValueContainer);
    Assert.assertTrue("getHandlerForContainerType returned incorrect handler", (kvHandler instanceof KeyValueHandler));
}

Example 2 with KeyValueHandler

use of org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler in project ozone by apache.

the class TestBlockDeletingService method testBlockDeletion.

@Test
public void testBlockDeletion() throws Exception {
    DatanodeConfiguration dnConf = conf.getObject(DatanodeConfiguration.class);
    dnConf.setBlockDeletionLimit(2);
    this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
    conf.setFromObject(dnConf);
    ContainerSet containerSet = new ContainerSet();
    createToDeleteBlocks(containerSet, 1, 3, 1);
    ContainerMetrics metrics = ContainerMetrics.create(conf);
    KeyValueHandler keyValueHandler = new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet, metrics, c -> {
    });
    BlockDeletingServiceTestImpl svc = getBlockDeletingService(containerSet, conf, keyValueHandler);
    svc.start();
    GenericTestUtils.waitFor(svc::isStarted, 100, 3000);
    // Ensure 1 container was created
    List<ContainerData> containerData = Lists.newArrayList();
    containerSet.listContainer(0L, 1, containerData);
    KeyValueContainerData data = (KeyValueContainerData) containerData.get(0);
    Assert.assertEquals(1, containerData.size());
    try (ReferenceCountedDB meta = BlockUtils.getDB((KeyValueContainerData) containerData.get(0), conf)) {
        Map<Long, Container<?>> containerMap = containerSet.getContainerMapCopy();
        // NOTE: this test assumes that all the container is KetValueContainer and
        // have DeleteTransactionId in KetValueContainerData. If other
        // types is going to be added, this test should be checked.
        long transactionId = ((KeyValueContainerData) containerMap.get(containerData.get(0).getContainerID()).getContainerData()).getDeleteTransactionId();
        long containerSpace = containerData.get(0).getBytesUsed();
        // Number of deleted blocks in container should be equal to 0 before
        // block delete
        Assert.assertEquals(0, transactionId);
        // Ensure there are 3 blocks under deletion and 0 deleted blocks
        Assert.assertEquals(3, getUnderDeletionBlocksCount(meta, data));
        Assert.assertEquals(3, meta.getStore().getMetadataTable().get(OzoneConsts.PENDING_DELETE_BLOCK_COUNT).longValue());
        // Container contains 3 blocks. So, space used by the container
        // should be greater than zero.
        Assert.assertTrue(containerSpace > 0);
        // An interval will delete 1 * 2 blocks
        deleteAndWait(svc, 1);
        GenericTestUtils.waitFor(() -> containerData.get(0).getBytesUsed() == containerSpace / 3, 100, 3000);
        // After first interval 2 blocks will be deleted. Hence, current space
        // used by the container should be less than the space used by the
        // container initially(before running deletion services).
        Assert.assertTrue(containerData.get(0).getBytesUsed() < containerSpace);
        deleteAndWait(svc, 2);
        // After deletion of all 3 blocks, space used by the containers
        // should be zero.
        GenericTestUtils.waitFor(() -> containerData.get(0).getBytesUsed() == 0, 100, 3000);
        // Check finally DB counters.
        // Not checking bytes used, as handler is a mock call.
        Assert.assertEquals(0, meta.getStore().getMetadataTable().get(OzoneConsts.PENDING_DELETE_BLOCK_COUNT).longValue());
        Assert.assertEquals(0, meta.getStore().getMetadataTable().get(OzoneConsts.BLOCK_COUNT).longValue());
    }
    svc.shutdown();
}

Example 3 with KeyValueHandler

use of org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler in project ozone by apache.

the class TestBlockDeletingService method testShutdownService.

@Test
// waitFor => assertion with timeout
@SuppressWarnings("java:S2699")
public void testShutdownService() throws Exception {
    conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 500, TimeUnit.MILLISECONDS);
    conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
    conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 10);
    ContainerSet containerSet = new ContainerSet();
    // Create 1 container with 100 blocks
    createToDeleteBlocks(containerSet, 1, 100, 1);
    ContainerMetrics metrics = ContainerMetrics.create(conf);
    KeyValueHandler keyValueHandler = new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet, metrics, c -> {
    });
    BlockDeletingServiceTestImpl service = getBlockDeletingService(containerSet, conf, keyValueHandler);
    service.start();
    GenericTestUtils.waitFor(service::isStarted, 100, 3000);
    // Run some deleting tasks and verify there are threads running
    service.runDeletingTasks();
    GenericTestUtils.waitFor(() -> service.getThreadCount() > 0, 100, 1000);
    // Shutdown service and verify all threads are stopped
    service.shutdown();
    GenericTestUtils.waitFor(() -> service.getThreadCount() == 0, 100, 1000);
}

Example 4 with KeyValueHandler

use of org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler in project ozone by apache.

the class TestBlockDeletingService method testBlockDeletionTimeout.

@Test
public void testBlockDeletionTimeout() throws Exception {
    DatanodeConfiguration dnConf = conf.getObject(DatanodeConfiguration.class);
    dnConf.setBlockDeletionLimit(3);
    blockLimitPerInterval = dnConf.getBlockDeletionLimit();
    conf.setFromObject(dnConf);
    ContainerSet containerSet = new ContainerSet();
    createToDeleteBlocks(containerSet, 1, 3, 1);
    ContainerMetrics metrics = ContainerMetrics.create(conf);
    KeyValueHandler keyValueHandler = new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet, metrics, c -> {
    });
    // set timeout value as 1ns to trigger timeout behavior
    long timeout = 1;
    OzoneContainer ozoneContainer = mockDependencies(containerSet, keyValueHandler);
    BlockDeletingService svc = new BlockDeletingService(ozoneContainer, TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.NANOSECONDS, 10, conf);
    svc.start();
    LogCapturer log = LogCapturer.captureLogs(BackgroundService.LOG);
    GenericTestUtils.waitFor(() -> {
        if (log.getOutput().contains("Background task execution took")) {
            log.stopCapturing();
            return true;
        }
        return false;
    }, 100, 1000);
    log.stopCapturing();
    svc.shutdown();
    // test for normal case that doesn't have timeout limitation
    createToDeleteBlocks(containerSet, 1, 3, 1);
    timeout = 0;
    svc = new BlockDeletingService(ozoneContainer, TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.MILLISECONDS, 10, conf);
    svc.start();
    // get container meta data
    KeyValueContainer container = (KeyValueContainer) containerSet.getContainerIterator().next();
    KeyValueContainerData data = container.getContainerData();
    try (ReferenceCountedDB meta = BlockUtils.getDB(data, conf)) {
        LogCapturer newLog = LogCapturer.captureLogs(BackgroundService.LOG);
        GenericTestUtils.waitFor(() -> {
            try {
                return getUnderDeletionBlocksCount(meta, data) == 0;
            } catch (IOException ignored) {
            }
            return false;
        }, 100, 1000);
        newLog.stopCapturing();
        // The block deleting successfully and shouldn't catch timed
        // out warning log.
        Assert.assertFalse(newLog.getOutput().contains("Background task executes timed out, retrying in next interval"));
    }
    svc.shutdown();
}

Example 5 with KeyValueHandler

use of org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler in project ozone by apache.

the class TestSchemaOneBackwardsCompatibility method testDelete.

/**
 * Tests reading blocks marked for deletion from a container written in
 * schema version 1. Because the block deleting service both reads for
 * deleted blocks and deletes them, this test will modify its copy of the
 * database.
 */
@Test
public void testDelete() throws Exception {
    final long numBlocksToDelete = TestDB.NUM_PENDING_DELETION_BLOCKS;
    String datanodeUuid = UUID.randomUUID().toString();
    ContainerSet containerSet = makeContainerSet();
    VolumeSet volumeSet = new MutableVolumeSet(datanodeUuid, conf, null, StorageVolume.VolumeType.DATA_VOLUME, null);
    ContainerMetrics metrics = ContainerMetrics.create(conf);
    KeyValueHandler keyValueHandler = new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet, metrics, c -> {
    });
    long initialTotalSpace = newKvData().getBytesUsed();
    long blockSpace = initialTotalSpace / TestDB.KEY_COUNT;
    runBlockDeletingService(keyValueHandler);
    GenericTestUtils.waitFor(() -> {
        try {
            return (newKvData().getBytesUsed() != initialTotalSpace);
        } catch (IOException ex) {
        }
        return false;
    }, 100, 3000);
    long currentTotalSpace = newKvData().getBytesUsed();
    long numberOfBlocksDeleted = (initialTotalSpace - currentTotalSpace) / blockSpace;
    // Expected values after blocks with #deleting# prefix in original DB are
    // deleted.
    final long expectedDeletingBlocks = TestDB.NUM_PENDING_DELETION_BLOCKS - numBlocksToDelete;
    final long expectedDeletedBlocks = TestDB.NUM_DELETED_BLOCKS + numBlocksToDelete;
    final long expectedRegularBlocks = TestDB.KEY_COUNT - numBlocksToDelete;
    try (ReferenceCountedDB refCountedDB = BlockUtils.getDB(newKvData(), conf)) {
        // Test results via block iteration.
        assertEquals(expectedDeletingBlocks, countDeletingBlocks(refCountedDB));
        assertEquals(expectedDeletedBlocks, TestDB.NUM_DELETED_BLOCKS + numberOfBlocksDeleted);
        assertEquals(expectedRegularBlocks, countUnprefixedBlocks(refCountedDB));
        // Test table metadata.
        // Because the KeyValueHandler used for the block deleting service is
        // mocked, the bytes used will not be updated by a ChunkManager in the
        // KeyValueHandler. Therefore, this value is not checked.
        Table<String, Long> metadataTable = refCountedDB.getStore().getMetadataTable();
        assertEquals(expectedRegularBlocks + expectedDeletingBlocks, (long) metadataTable.get(OzoneConsts.BLOCK_COUNT));
    }
}