Examples with DatanodeStoreSchemaTwoImpl org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl used on opensource projects

Example 1 with DatanodeStoreSchemaTwoImpl

use of org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl in project ozone by apache.

the class KeyValueContainerUtil method createContainerMetaData.

/**
 * @param containerMetaDataPath
 * @throws IOException
 */
/**
 * creates metadata path, chunks path and metadata DB for the specified
 * container.
 *
 * @param containerMetaDataPath Path to the container's metadata directory.
 * @param chunksPath Path were chunks for this container should be stored.
 * @param dbFile Path to the container's .db file.
 * @param schemaVersion The schema version of the container. If this method
 * has not been updated after a schema version addition
 * and does not recognize the latest SchemaVersion, an
 * {@link IllegalArgumentException} is thrown.
 * @param conf The configuration to use for this container.
 * @throws IOException
 */
public static void createContainerMetaData(long containerID, File containerMetaDataPath, File chunksPath, File dbFile, String schemaVersion, ConfigurationSource conf) throws IOException {
    Preconditions.checkNotNull(containerMetaDataPath);
    Preconditions.checkNotNull(conf);
    if (!containerMetaDataPath.mkdirs()) {
        LOG.error("Unable to create directory for metadata storage. Path: {}", containerMetaDataPath);
        throw new IOException("Unable to create directory for metadata storage." + " Path: " + containerMetaDataPath);
    }
    if (!chunksPath.mkdirs()) {
        LOG.error("Unable to create chunks directory Container {}", chunksPath);
        // clean up container metadata path and metadata db
        FileUtils.deleteDirectory(containerMetaDataPath);
        FileUtils.deleteDirectory(containerMetaDataPath.getParentFile());
        throw new IOException("Unable to create directory for data storage." + " Path: " + chunksPath);
    }
    DatanodeStore store;
    if (schemaVersion.equals(OzoneConsts.SCHEMA_V1)) {
        store = new DatanodeStoreSchemaOneImpl(conf, containerID, dbFile.getAbsolutePath(), false);
    } else if (schemaVersion.equals(OzoneConsts.SCHEMA_V2)) {
        store = new DatanodeStoreSchemaTwoImpl(conf, containerID, dbFile.getAbsolutePath(), false);
    } else {
        throw new IllegalArgumentException("Unrecognized schema version for container: " + schemaVersion);
    }
    ReferenceCountedDB db = new ReferenceCountedDB(store, dbFile.getAbsolutePath());
    // add db handler into cache
    BlockUtils.addDB(db, dbFile.getAbsolutePath(), conf);
}

Example 2 with DatanodeStoreSchemaTwoImpl

use of org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl in project ozone by apache.

the class TestStorageContainerManagerHelper method verifyBlocksWithTxnTable.

public boolean verifyBlocksWithTxnTable(Map<Long, List<Long>> containerBlocks) throws IOException {
    for (Map.Entry<Long, List<Long>> entry : containerBlocks.entrySet()) {
        ReferenceCountedDB meta = getContainerMetadata(entry.getKey());
        DatanodeStore ds = meta.getStore();
        DatanodeStoreSchemaTwoImpl dnStoreTwoImpl = (DatanodeStoreSchemaTwoImpl) ds;
        List<? extends Table.KeyValue<Long, DeletedBlocksTransaction>> txnsInTxnTable = dnStoreTwoImpl.getDeleteTransactionTable().getRangeKVs(null, Integer.MAX_VALUE, null);
        List<Long> conID = new ArrayList<>();
        for (Table.KeyValue<Long, DeletedBlocksTransaction> txn : txnsInTxnTable) {
            conID.addAll(txn.getValue().getLocalIDList());
        }
        if (!conID.equals(containerBlocks.get(entry.getKey()))) {
            return false;
        }
        meta.close();
    }
    return true;
}

Example 3 with DatanodeStoreSchemaTwoImpl

use of org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl in project ozone by apache.

the class TestContainerCache method createContainerDB.

private void createContainerDB(OzoneConfiguration conf, File dbFile) throws Exception {
    DatanodeStore store = new DatanodeStoreSchemaTwoImpl(conf, 1, dbFile.getAbsolutePath(), false);
    // we close since the SCM pre-creates containers.
    // we will open and put Db handle into a cache when keys are being created
    // in a container.
    store.stop();
}

Example 4 with DatanodeStoreSchemaTwoImpl

use of org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl in project ozone by apache.

the class DeleteBlocksCommandHandler method markBlocksForDeletionSchemaV2.

/**
 * Move a bunch of blocks from a container to deleting state. This is a meta
 * update, the actual deletes happen in async mode.
 *
 * @param containerData - KeyValueContainerData
 * @param delTX a block deletion transaction.
 * @throws IOException if I/O error occurs.
 */
private void markBlocksForDeletionSchemaV2(KeyValueContainerData containerData, DeletedBlocksTransaction delTX, int newDeletionBlocks, long txnID) throws IOException {
    long containerId = delTX.getContainerID();
    if (!isTxnIdValid(containerId, containerData, delTX)) {
        return;
    }
    try (ReferenceCountedDB containerDB = BlockUtils.getDB(containerData, conf)) {
        DatanodeStore ds = containerDB.getStore();
        DatanodeStoreSchemaTwoImpl dnStoreTwoImpl = (DatanodeStoreSchemaTwoImpl) ds;
        Table<Long, DeletedBlocksTransaction> delTxTable = dnStoreTwoImpl.getDeleteTransactionTable();
        try (BatchOperation batch = containerDB.getStore().getBatchHandler().initBatchOperation()) {
            delTxTable.putWithBatch(batch, txnID, delTX);
            newDeletionBlocks += delTX.getLocalIDList().size();
            updateMetaData(containerData, delTX, newDeletionBlocks, containerDB, batch);
            containerDB.getStore().getBatchHandler().commitBatchOperation(batch);
        }
    }
}

Example 5 with DatanodeStoreSchemaTwoImpl

use of org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl in project ozone by apache.

the class KeyValueContainerMetadataInspector method getAggregateValues.

private JsonObject getAggregateValues(DatanodeStore store, String schemaVersion) throws IOException {
    JsonObject aggregates = new JsonObject();
    long usedBytesTotal = 0;
    long blockCountTotal = 0;
    long pendingDeleteBlockCountTotal = 0;
    // Count normal blocks.
    try (BlockIterator<BlockData> blockIter = store.getBlockIterator(MetadataKeyFilters.getUnprefixedKeyFilter())) {
        while (blockIter.hasNext()) {
            blockCountTotal++;
            usedBytesTotal += getBlockLength(blockIter.nextBlock());
        }
    }
    // Count pending delete blocks.
    if (schemaVersion.equals(OzoneConsts.SCHEMA_V1)) {
        try (BlockIterator<BlockData> blockIter = store.getBlockIterator(MetadataKeyFilters.getDeletingKeyFilter())) {
            while (blockIter.hasNext()) {
                blockCountTotal++;
                pendingDeleteBlockCountTotal++;
                usedBytesTotal += getBlockLength(blockIter.nextBlock());
            }
        }
    } else if (schemaVersion.equals(OzoneConsts.SCHEMA_V2)) {
        DatanodeStoreSchemaTwoImpl schemaTwoStore = (DatanodeStoreSchemaTwoImpl) store;
        pendingDeleteBlockCountTotal = countPendingDeletesSchemaV2(schemaTwoStore);
    } else {
        throw new IOException("Failed to process deleted blocks for unknown " + "container schema " + schemaVersion);
    }
    aggregates.addProperty("blockCount", blockCountTotal);
    aggregates.addProperty("usedBytes", usedBytesTotal);
    aggregates.addProperty("pendingDeleteBlocks", pendingDeleteBlockCountTotal);
    return aggregates;
}