Examples with DeletedBlocksTransaction org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction used on opensource projects

Example 1 with DeletedBlocksTransaction

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction in project ozone by apache.

the class SequenceIdGenerator method upgradeToSequenceId.

/**
 * TODO
 *  Relocate the code after upgrade framework is ready.
 *
 * Upgrade localID, delTxnId, containerId from legacy solution
 * to SequenceIdGenerator.
 */
public static void upgradeToSequenceId(SCMMetadataStore scmMetadataStore) throws IOException {
    Table<String, Long> sequenceIdTable = scmMetadataStore.getSequenceIdTable();
    // operations can take effect exactly once in a SCM HA cluster.
    if (sequenceIdTable.get(LOCAL_ID) == null) {
        long millisSinceEpoch = TimeUnit.DAYS.toMillis(LocalDate.of(LocalDate.now().getYear() + 1, 1, 1).toEpochDay());
        long localId = millisSinceEpoch << Short.SIZE;
        Preconditions.checkArgument(localId > UniqueId.next());
        sequenceIdTable.put(LOCAL_ID, localId);
        LOG.info("upgrade {} to {}", LOCAL_ID, sequenceIdTable.get(LOCAL_ID));
    }
    // upgrade delTxnId
    if (sequenceIdTable.get(DEL_TXN_ID) == null) {
        // fetch delTxnId from DeletedBlocksTXTable
        // check HDDS-4477 for details.
        DeletedBlocksTransaction txn = scmMetadataStore.getDeletedBlocksTXTable().get(0L);
        sequenceIdTable.put(DEL_TXN_ID, txn != null ? txn.getTxID() : 0L);
        LOG.info("upgrade {} to {}", DEL_TXN_ID, sequenceIdTable.get(DEL_TXN_ID));
    }
    // upgrade containerId
    if (sequenceIdTable.get(CONTAINER_ID) == null) {
        long largestContainerId = 0;
        TableIterator<ContainerID, ? extends KeyValue<ContainerID, ContainerInfo>> iterator = scmMetadataStore.getContainerTable().iterator();
        while (iterator.hasNext()) {
            ContainerInfo containerInfo = iterator.next().getValue();
            largestContainerId = Long.max(containerInfo.getContainerID(), largestContainerId);
        }
        sequenceIdTable.put(CONTAINER_ID, largestContainerId);
        LOG.info("upgrade {} to {}", CONTAINER_ID, sequenceIdTable.get(CONTAINER_ID));
    }
}

Example 2 with DeletedBlocksTransaction

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction in project ozone by apache.

the class DeletedBlockLogImpl method getTransactions.

@Override
public DatanodeDeletedBlockTransactions getTransactions(int blockDeletionLimit) throws IOException {
    lock.lock();
    try {
        DatanodeDeletedBlockTransactions transactions = new DatanodeDeletedBlockTransactions();
        try (TableIterator<Long, ? extends Table.KeyValue<Long, DeletedBlocksTransaction>> iter = deletedBlockLogStateManager.getReadOnlyIterator()) {
            int numBlocksAdded = 0;
            ArrayList<Long> txIDs = new ArrayList<>();
            while (iter.hasNext() && numBlocksAdded < blockDeletionLimit) {
                Table.KeyValue<Long, DeletedBlocksTransaction> keyValue = iter.next();
                DeletedBlocksTransaction txn = keyValue.getValue();
                final ContainerID id = ContainerID.valueOf(txn.getContainerID());
                try {
                    if (txn.getCount() > -1 && txn.getCount() <= maxRetry && !containerManager.getContainer(id).isOpen()) {
                        numBlocksAdded += txn.getLocalIDCount();
                        getTransaction(txn, transactions);
                        transactionToDNsCommitMap.putIfAbsent(txn.getTxID(), new LinkedHashSet<>());
                    }
                } catch (ContainerNotFoundException ex) {
                    LOG.warn("Container: " + id + " was not found for the transaction: " + txn);
                    txIDs.add(txn.getTxID());
                }
            }
            if (!txIDs.isEmpty()) {
                deletedBlockLogStateManager.removeTransactionsFromDB(txIDs);
                metrics.incrBlockDeletionTransactionCompleted(txIDs.size());
            }
        }
        return transactions;
    } finally {
        lock.unlock();
    }
}

Example 3 with DeletedBlocksTransaction

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction in project ozone by apache.

the class DeletedBlockLogImpl method addTransactions.

/**
 * {@inheritDoc}
 *
 * @param containerBlocksMap a map of containerBlocks.
 * @throws IOException
 */
@Override
public void addTransactions(Map<Long, List<Long>> containerBlocksMap) throws IOException {
    lock.lock();
    try {
        ArrayList<DeletedBlocksTransaction> txsToBeAdded = new ArrayList<>();
        for (Map.Entry<Long, List<Long>> entry : containerBlocksMap.entrySet()) {
            long nextTXID = sequenceIdGen.getNextId(DEL_TXN_ID);
            DeletedBlocksTransaction tx = constructNewTransaction(nextTXID, entry.getKey(), entry.getValue());
            txsToBeAdded.add(tx);
        }
        deletedBlockLogStateManager.addTransactionsToDB(txsToBeAdded);
        metrics.incrBlockDeletionTransactionCreated(txsToBeAdded.size());
    } finally {
        lock.unlock();
    }
}

Example 4 with DeletedBlocksTransaction

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction in project ozone by apache.

the class KeyValueContainerMetadataInspector method countPendingDeletesSchemaV2.

private long countPendingDeletesSchemaV2(DatanodeStoreSchemaTwoImpl schemaTwoStore) throws IOException {
    long pendingDeleteBlockCountTotal = 0;
    Table<Long, DeletedBlocksTransaction> delTxTable = schemaTwoStore.getDeleteTransactionTable();
    try (TableIterator<Long, ? extends Table.KeyValue<Long, DeletedBlocksTransaction>> iterator = delTxTable.iterator()) {
        while (iterator.hasNext()) {
            DeletedBlocksTransaction txn = iterator.next().getValue();
            // In schema 2, pending delete blocks are stored in the
            // transaction object. Since the actual blocks still exist in the
            // block data table with no prefix, they have already been
            // counted towards bytes used and total block count above.
            pendingDeleteBlockCountTotal += txn.getLocalIDList().size();
        }
    }
    return pendingDeleteBlockCountTotal;
}

Example 5 with DeletedBlocksTransaction

use of org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction in project ozone by apache.

the class TestStorageContainerManagerHelper method verifyBlocksWithTxnTable.

public boolean verifyBlocksWithTxnTable(Map<Long, List<Long>> containerBlocks) throws IOException {
    for (Map.Entry<Long, List<Long>> entry : containerBlocks.entrySet()) {
        ReferenceCountedDB meta = getContainerMetadata(entry.getKey());
        DatanodeStore ds = meta.getStore();
        DatanodeStoreSchemaTwoImpl dnStoreTwoImpl = (DatanodeStoreSchemaTwoImpl) ds;
        List<? extends Table.KeyValue<Long, DeletedBlocksTransaction>> txnsInTxnTable = dnStoreTwoImpl.getDeleteTransactionTable().getRangeKVs(null, Integer.MAX_VALUE, null);
        List<Long> conID = new ArrayList<>();
        for (Table.KeyValue<Long, DeletedBlocksTransaction> txn : txnsInTxnTable) {
            conID.addAll(txn.getValue().getLocalIDList());
        }
        if (!conID.equals(containerBlocks.get(entry.getKey()))) {
            return false;
        }
        meta.close();
    }
    return true;
}