Examples with ReferenceCountedDB org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB used on opensource projects

Example 1 with ReferenceCountedDB

use of org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB in project ozone by apache.

the class DeleteBlocksCommandHandler method markBlocksForDeletionSchemaV1.

private void markBlocksForDeletionSchemaV1(KeyValueContainerData containerData, DeletedBlocksTransaction delTX) throws IOException {
    long containerId = delTX.getContainerID();
    if (!isTxnIdValid(containerId, containerData, delTX)) {
        return;
    }
    int newDeletionBlocks = 0;
    try (ReferenceCountedDB containerDB = BlockUtils.getDB(containerData, conf)) {
        Table<String, BlockData> blockDataTable = containerDB.getStore().getBlockDataTable();
        Table<String, ChunkInfoList> deletedBlocksTable = containerDB.getStore().getDeletedBlocksTable();
        try (BatchOperation batch = containerDB.getStore().getBatchHandler().initBatchOperation()) {
            for (Long blkLong : delTX.getLocalIDList()) {
                String blk = blkLong.toString();
                BlockData blkInfo = blockDataTable.get(blk);
                if (blkInfo != null) {
                    String deletingKey = OzoneConsts.DELETING_KEY_PREFIX + blk;
                    if (blockDataTable.get(deletingKey) != null || deletedBlocksTable.get(blk) != null) {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug(String.format("Ignoring delete for block %s in container %d." + " Entry already added.", blk, containerId));
                        }
                        continue;
                    }
                    // Found the block in container db,
                    // use an atomic update to change its state to deleting.
                    blockDataTable.putWithBatch(batch, deletingKey, blkInfo);
                    blockDataTable.deleteWithBatch(batch, blk);
                    newDeletionBlocks++;
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Transited Block {} to DELETING state in container {}", blk, containerId);
                    }
                } else {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Block {} not found or already under deletion in" + " container {}, skip deleting it.", blk, containerId);
                    }
                }
            }
            updateMetaData(containerData, delTX, newDeletionBlocks, containerDB, batch);
            containerDB.getStore().getBatchHandler().commitBatchOperation(batch);
        } catch (IOException e) {
            // with a certain number of retries.
            throw new IOException("Failed to delete blocks for TXID = " + delTX.getTxID(), e);
        }
    }
}

Example 2 with ReferenceCountedDB

use of org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB in project ozone by apache.

the class KeyValueContainerUtil method createContainerMetaData.

/**
 * @param containerMetaDataPath
 * @throws IOException
 */
/**
 * creates metadata path, chunks path and metadata DB for the specified
 * container.
 *
 * @param containerMetaDataPath Path to the container's metadata directory.
 * @param chunksPath Path were chunks for this container should be stored.
 * @param dbFile Path to the container's .db file.
 * @param schemaVersion The schema version of the container. If this method
 * has not been updated after a schema version addition
 * and does not recognize the latest SchemaVersion, an
 * {@link IllegalArgumentException} is thrown.
 * @param conf The configuration to use for this container.
 * @throws IOException
 */
public static void createContainerMetaData(long containerID, File containerMetaDataPath, File chunksPath, File dbFile, String schemaVersion, ConfigurationSource conf) throws IOException {
    Preconditions.checkNotNull(containerMetaDataPath);
    Preconditions.checkNotNull(conf);
    if (!containerMetaDataPath.mkdirs()) {
        LOG.error("Unable to create directory for metadata storage. Path: {}", containerMetaDataPath);
        throw new IOException("Unable to create directory for metadata storage." + " Path: " + containerMetaDataPath);
    }
    if (!chunksPath.mkdirs()) {
        LOG.error("Unable to create chunks directory Container {}", chunksPath);
        // clean up container metadata path and metadata db
        FileUtils.deleteDirectory(containerMetaDataPath);
        FileUtils.deleteDirectory(containerMetaDataPath.getParentFile());
        throw new IOException("Unable to create directory for data storage." + " Path: " + chunksPath);
    }
    DatanodeStore store;
    if (schemaVersion.equals(OzoneConsts.SCHEMA_V1)) {
        store = new DatanodeStoreSchemaOneImpl(conf, containerID, dbFile.getAbsolutePath(), false);
    } else if (schemaVersion.equals(OzoneConsts.SCHEMA_V2)) {
        store = new DatanodeStoreSchemaTwoImpl(conf, containerID, dbFile.getAbsolutePath(), false);
    } else {
        throw new IllegalArgumentException("Unrecognized schema version for container: " + schemaVersion);
    }
    ReferenceCountedDB db = new ReferenceCountedDB(store, dbFile.getAbsolutePath());
    // add db handler into cache
    BlockUtils.addDB(db, dbFile.getAbsolutePath(), conf);
}

Example 3 with ReferenceCountedDB

use of org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB in project ozone by apache.

the class KeyValueContainerUtil method parseKVContainerData.

/**
 * Parse KeyValueContainerData and verify checksum. Set block related
 * metadata like block commit sequence id, block count, bytes used and
 * pending delete block count and delete transaction id.
 * @param kvContainerData
 * @param config
 * @throws IOException
 */
public static void parseKVContainerData(KeyValueContainerData kvContainerData, ConfigurationSource config) throws IOException {
    long containerID = kvContainerData.getContainerID();
    File metadataPath = new File(kvContainerData.getMetadataPath());
    // Verify Checksum
    ContainerUtils.verifyChecksum(kvContainerData, config);
    File dbFile = KeyValueContainerLocationUtil.getContainerDBFile(metadataPath, containerID);
    if (!dbFile.exists()) {
        LOG.error("Container DB file is missing for ContainerID {}. " + "Skipping loading of this container.", containerID);
        // Don't further process this container, as it is missing db file.
        return;
    }
    kvContainerData.setDbFile(dbFile);
    if (kvContainerData.getSchemaVersion() == null) {
        // If this container has not specified a schema version, it is in the old
        // format with one default column family.
        kvContainerData.setSchemaVersion(OzoneConsts.SCHEMA_V1);
    }
    boolean isBlockMetadataSet = false;
    ReferenceCountedDB cachedDB = null;
    DatanodeStore store = null;
    try {
        try {
            boolean readOnly = ContainerInspectorUtil.isReadOnly(ContainerProtos.ContainerType.KeyValueContainer);
            store = BlockUtils.getUncachedDatanodeStore(kvContainerData, config, readOnly);
        } catch (IOException e) {
            // If an exception is thrown, then it may indicate the RocksDB is
            // already open in the container cache. As this code is only executed at
            // DN startup, this should only happen in the tests.
            cachedDB = BlockUtils.getDB(kvContainerData, config);
            store = cachedDB.getStore();
            LOG.warn("Attempt to get an uncached RocksDB handle failed and an " + "instance was retrieved from the cache. This should only happen " + "in tests");
        }
        Table<String, Long> metadataTable = store.getMetadataTable();
        // Set pending deleted block count.
        Long pendingDeleteBlockCount = metadataTable.get(OzoneConsts.PENDING_DELETE_BLOCK_COUNT);
        if (pendingDeleteBlockCount != null) {
            kvContainerData.incrPendingDeletionBlocks(pendingDeleteBlockCount);
        } else {
            // Set pending deleted block count.
            MetadataKeyFilters.KeyPrefixFilter filter = MetadataKeyFilters.getDeletingKeyFilter();
            int numPendingDeletionBlocks = store.getBlockDataTable().getSequentialRangeKVs(null, Integer.MAX_VALUE, filter).size();
            kvContainerData.incrPendingDeletionBlocks(numPendingDeletionBlocks);
        }
        // Set delete transaction id.
        Long delTxnId = metadataTable.get(OzoneConsts.DELETE_TRANSACTION_KEY);
        if (delTxnId != null) {
            kvContainerData.updateDeleteTransactionId(delTxnId);
        }
        // Set BlockCommitSequenceId.
        Long bcsId = metadataTable.get(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID);
        if (bcsId != null) {
            kvContainerData.updateBlockCommitSequenceId(bcsId);
        }
        // Set bytes used.
        // commitSpace for Open Containers relies on usedBytes
        Long bytesUsed = metadataTable.get(OzoneConsts.CONTAINER_BYTES_USED);
        if (bytesUsed != null) {
            isBlockMetadataSet = true;
            kvContainerData.setBytesUsed(bytesUsed);
        }
        // Set block count.
        Long blockCount = metadataTable.get(OzoneConsts.BLOCK_COUNT);
        if (blockCount != null) {
            isBlockMetadataSet = true;
            kvContainerData.setBlockCount(blockCount);
        }
        if (!isBlockMetadataSet) {
            initializeUsedBytesAndBlockCount(store, kvContainerData);
        }
        // If the container is missing a chunks directory, possibly due to the
        // bug fixed by HDDS-6235, create it here.
        File chunksDir = new File(kvContainerData.getChunksPath());
        if (!chunksDir.exists()) {
            Files.createDirectories(chunksDir.toPath());
        }
        // Run advanced container inspection/repair operations if specified on
        // startup. If this method is called but not as a part of startup,
        // The inspectors will be unloaded and this will be a no-op.
        ContainerInspectorUtil.process(kvContainerData, store);
    } finally {
        if (cachedDB != null) {
            // If we get a cached instance, calling close simply decrements the
            // reference count.
            cachedDB.close();
        } else if (store != null) {
            // close the rocksDB handle in the cache and the next reader would fail
            try {
                store.stop();
            } catch (IOException e) {
                throw e;
            } catch (Exception e) {
                throw new RuntimeException("Unexpected exception closing the " + "RocksDB when loading containers", e);
            }
        }
    }
}

Example 4 with ReferenceCountedDB

use of org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB in project ozone by apache.

the class BlockManagerImpl method persistPutBlock.

public static long persistPutBlock(KeyValueContainer container, BlockData data, ConfigurationSource config, boolean endOfBlock) throws IOException {
    Preconditions.checkNotNull(data, "BlockData cannot be null for put " + "operation.");
    Preconditions.checkState(data.getContainerID() >= 0, "Container Id " + "cannot be negative");
    // against a single DB. We rely on DB level locking to avoid conflicts.
    try (ReferenceCountedDB db = BlockUtils.getDB(container.getContainerData(), config)) {
        // This is a post condition that acts as a hint to the user.
        // Should never fail.
        Preconditions.checkNotNull(db, DB_NULL_ERR_MSG);
        long bcsId = data.getBlockCommitSequenceId();
        long containerBCSId = container.getContainerData().getBlockCommitSequenceId();
        // In such cases, we should overwrite the block as well
        if ((bcsId != 0) && (bcsId <= containerBCSId)) {
            // Since the blockCommitSequenceId stored in the db is greater than
            // equal to blockCommitSequenceId to be updated, it means the putBlock
            // transaction is reapplied in the ContainerStateMachine on restart.
            // It also implies that the given block must already exist in the db.
            // just log and return
            LOG.debug("blockCommitSequenceId {} in the Container Db is greater" + " than the supplied value {}. Ignoring it", containerBCSId, bcsId);
            return data.getSize();
        }
        // Check if the block is present in the pendingPutBlockCache for the
        // container to determine whether the blockCount is already incremented
        // for this block in the DB or not.
        long localID = data.getLocalID();
        boolean isBlockInCache = container.isBlockInPendingPutBlockCache(localID);
        boolean incrBlockCount = false;
        // update the blockData as well as BlockCommitSequenceId here
        try (BatchOperation batch = db.getStore().getBatchHandler().initBatchOperation()) {
            // If block exists in cache, blockCount should not be incremented.
            if (!isBlockInCache) {
                if (db.getStore().getBlockDataTable().get(Long.toString(localID)) == null) {
                    // Block does not exist in DB => blockCount needs to be
                    // incremented when the block is added into DB.
                    incrBlockCount = true;
                }
            }
            db.getStore().getBlockDataTable().putWithBatch(batch, Long.toString(localID), data);
            if (bcsId != 0) {
                db.getStore().getMetadataTable().putWithBatch(batch, OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID, bcsId);
            }
            // Set Bytes used, this bytes used will be updated for every write and
            // only get committed for every put block. In this way, when datanode
            // is up, for computation of disk space by container only committed
            // block length is used, And also on restart the blocks committed to DB
            // is only used to compute the bytes used. This is done to keep the
            // current behavior and avoid DB write during write chunk operation.
            db.getStore().getMetadataTable().putWithBatch(batch, OzoneConsts.CONTAINER_BYTES_USED, container.getContainerData().getBytesUsed());
            // Set Block Count for a container.
            if (incrBlockCount) {
                db.getStore().getMetadataTable().putWithBatch(batch, OzoneConsts.BLOCK_COUNT, container.getContainerData().getBlockCount() + 1);
            }
            db.getStore().getBatchHandler().commitBatchOperation(batch);
        }
        if (bcsId != 0) {
            container.updateBlockCommitSequenceId(bcsId);
        }
        // in-memory after the DB update.
        if (incrBlockCount) {
            container.getContainerData().incrBlockCount();
        }
        // have to read the DB to check for block existence
        if (!isBlockInCache && !endOfBlock) {
            container.addToPendingPutBlockCache(localID);
        } else if (isBlockInCache && endOfBlock) {
            // Remove the block from the PendingPutBlockCache as there would not
            // be any more writes to this block
            container.removeFromPendingPutBlockCache(localID);
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("Block " + data.getBlockID() + " successfully committed with bcsId " + bcsId + " chunk size " + data.getChunks().size());
        }
        return data.getSize();
    }
}

Example 5 with ReferenceCountedDB

use of org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB in project ozone by apache.

the class KeyValueContainer method flushAndSyncDB.

private void flushAndSyncDB() throws StorageContainerException {
    try {
        try (ReferenceCountedDB db = BlockUtils.getDB(containerData, config)) {
            db.getStore().flushLog(true);
            LOG.info("Container {} is synced with bcsId {}.", containerData.getContainerID(), containerData.getBlockCommitSequenceId());
        }
    } catch (StorageContainerException ex) {
        throw ex;
    } catch (IOException ex) {
        LOG.error("Error in DB sync while closing container", ex);
        onFailure(containerData.getVolume());
        throw new StorageContainerException(ex, ERROR_IN_DB_SYNC);
    }
}