Examples with BlockDoesNotExistException alluxio.exception.BlockDoesNotExistException used on opensource projects

Example 26 with BlockDoesNotExistException

use of alluxio.exception.BlockDoesNotExistException in project alluxio by Alluxio.

the class DefaultBlockWorker method commitBlock.

@Override
public void commitBlock(long sessionId, long blockId, boolean pinOnCreate) throws BlockAlreadyExistsException, BlockDoesNotExistException, InvalidWorkerStateException, IOException, WorkerOutOfSpaceException {
    long lockId = BlockWorker.INVALID_LOCK_ID;
    try {
        lockId = mLocalBlockStore.commitBlockLocked(sessionId, blockId, pinOnCreate);
    } catch (BlockAlreadyExistsException e) {
        LOG.debug("Block {} has been in block store, this could be a retry due to master-side RPC " + "failure, therefore ignore the exception", blockId, e);
    }
    // Block successfully committed, update master with new block metadata
    if (lockId == BlockWorker.INVALID_LOCK_ID) {
        lockId = mLocalBlockStore.lockBlock(sessionId, blockId);
    }
    BlockMasterClient blockMasterClient = mBlockMasterClientPool.acquire();
    try {
        BlockMeta meta = mLocalBlockStore.getBlockMeta(sessionId, blockId, lockId);
        BlockStoreLocation loc = meta.getBlockLocation();
        String mediumType = loc.mediumType();
        Long length = meta.getBlockSize();
        BlockStoreMeta storeMeta = mLocalBlockStore.getBlockStoreMeta();
        Long bytesUsedOnTier = storeMeta.getUsedBytesOnTiers().get(loc.tierAlias());
        blockMasterClient.commitBlock(mWorkerId.get(), bytesUsedOnTier, loc.tierAlias(), mediumType, blockId, length);
    } catch (Exception e) {
        throw new IOException(ExceptionMessage.FAILED_COMMIT_BLOCK_TO_MASTER.getMessage(blockId), e);
    } finally {
        mBlockMasterClientPool.release(blockMasterClient);
        mLocalBlockStore.unlockBlock(lockId);
        Metrics.WORKER_ACTIVE_CLIENTS.dec();
    }
}

Example 27 with BlockDoesNotExistException

use of alluxio.exception.BlockDoesNotExistException in project alluxio by Alluxio.

the class DefaultBlockWorker method createBlockReader.

@Override
public BlockReader createBlockReader(BlockReadRequest request) throws BlockDoesNotExistException, IOException {
    long sessionId = request.getSessionId();
    long blockId = request.getId();
    RetryPolicy retryPolicy = new TimeoutRetry(UFS_BLOCK_OPEN_TIMEOUT_MS, Constants.SECOND_MS);
    while (retryPolicy.attempt()) {
        BlockReader reader = createLocalBlockReader(sessionId, blockId, request.getStart());
        if (reader != null) {
            Metrics.WORKER_ACTIVE_CLIENTS.inc();
            return reader;
        }
        boolean checkUfs = request.isPersisted() || (request.getOpenUfsBlockOptions() != null && request.getOpenUfsBlockOptions().hasBlockInUfsTier() && request.getOpenUfsBlockOptions().getBlockInUfsTier());
        if (!checkUfs) {
            throw new BlockDoesNotExistException(ExceptionMessage.NO_BLOCK_ID_FOUND, blockId);
        }
        // When the block does not exist in Alluxio but exists in UFS, try to open the UFS block.
        try {
            Metrics.WORKER_ACTIVE_CLIENTS.inc();
            return createUfsBlockReader(request.getSessionId(), request.getId(), request.getStart(), request.isPositionShort(), request.getOpenUfsBlockOptions());
        } catch (Exception e) {
            throw new UnavailableException(String.format("Failed to read block ID=%s from tiered storage and UFS tier: %s", request.getId(), e.toString()));
        }
    }
    throw new UnavailableException(ExceptionMessage.UFS_BLOCK_ACCESS_TOKEN_UNAVAILABLE.getMessage(request.getId(), request.getOpenUfsBlockOptions().getUfsPath()));
}

Example 28 with BlockDoesNotExistException

use of alluxio.exception.BlockDoesNotExistException in project alluxio by Alluxio.

the class AbstractEvictor method cascadingEvict.

/**
 * A recursive implementation of cascading eviction.
 *
 * This method uses a specific eviction strategy to find blocks to evict in the requested
 * location. After eviction, one {@link alluxio.worker.block.meta.StorageDir} in the location has
 * the specific amount of free space. It then uses an allocation strategy to allocate space in the
 * next tier to move each evicted blocks. If the next tier fails to allocate space for the evicted
 * blocks, the next tier will continue to evict its blocks to free space.
 *
 * This method is only used in
 * {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataEvictorView)}.
 *
 * @param bytesToBeAvailable bytes to be available after eviction
 * @param location target location to evict blocks from
 * @param plan the plan to be recursively updated, is empty when first called in
 *        {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataEvictorView)}
 * @param mode the eviction mode
 * @return the first {@link StorageDirEvictorView} in the range of location
 *         to evict/move bytes from, or null if there is no plan
 */
protected StorageDirEvictorView cascadingEvict(long bytesToBeAvailable, BlockStoreLocation location, EvictionPlan plan, Mode mode) {
    location = updateBlockStoreLocation(bytesToBeAvailable, location);
    // 1. If bytesToBeAvailable can already be satisfied without eviction, return the eligible
    // StorageDirView
    StorageDirEvictorView candidateDirView = (StorageDirEvictorView) EvictorUtils.selectDirWithRequestedSpace(bytesToBeAvailable, location, mMetadataView);
    if (candidateDirView != null) {
        return candidateDirView;
    }
    // 2. Iterate over blocks in order until we find a StorageDirEvictorView that is
    // in the range of location and can satisfy bytesToBeAvailable
    // after evicting its blocks iterated so far
    EvictionDirCandidates dirCandidates = new EvictionDirCandidates();
    Iterator<Long> it = getBlockIterator();
    while (it.hasNext() && dirCandidates.candidateSize() < bytesToBeAvailable) {
        long blockId = it.next();
        try {
            BlockMeta block = mMetadataView.getBlockMeta(blockId);
            if (block != null) {
                // might not present in this view
                if (block.getBlockLocation().belongsTo(location)) {
                    String tierAlias = block.getParentDir().getParentTier().getTierAlias();
                    int dirIndex = block.getParentDir().getDirIndex();
                    StorageDirView dirView = mMetadataView.getTierView(tierAlias).getDirView(dirIndex);
                    if (dirView != null) {
                        dirCandidates.add((StorageDirEvictorView) dirView, blockId, block.getBlockSize());
                    }
                }
            }
        } catch (BlockDoesNotExistException e) {
            LOG.warn("Remove block {} from evictor cache because {}", blockId, e);
            it.remove();
            onRemoveBlockFromIterator(blockId);
        }
    }
    // 3. If there is no eligible StorageDirEvictorView, return null
    if (mode == Mode.GUARANTEED && dirCandidates.candidateSize() < bytesToBeAvailable) {
        return null;
    }
    // 4. cascading eviction: try to allocate space in the next tier to move candidate blocks
    // there. If allocation fails, the next tier will continue to evict its blocks to free space.
    // Blocks are only evicted from the last tier or it can not be moved to the next tier.
    candidateDirView = dirCandidates.candidateDir();
    if (candidateDirView == null) {
        return null;
    }
    List<Long> candidateBlocks = dirCandidates.candidateBlocks();
    StorageTierView nextTierView = mMetadataView.getNextTier(candidateDirView.getParentTierView());
    if (nextTierView == null) {
        // This is the last tier, evict all the blocks.
        for (Long blockId : candidateBlocks) {
            try {
                BlockMeta block = mMetadataView.getBlockMeta(blockId);
                if (block != null) {
                    candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                    plan.toEvict().add(new Pair<>(blockId, candidateDirView.toBlockStoreLocation()));
                }
            } catch (BlockDoesNotExistException e) {
                continue;
            }
        }
    } else {
        for (Long blockId : candidateBlocks) {
            try {
                BlockMeta block = mMetadataView.getBlockMeta(blockId);
                if (block == null) {
                    continue;
                }
                StorageDirEvictorView nextDirView = (StorageDirEvictorView) mAllocator.allocateBlockWithView(Sessions.MIGRATE_DATA_SESSION_ID, block.getBlockSize(), BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), mMetadataView, true);
                if (nextDirView == null) {
                    nextDirView = cascadingEvict(block.getBlockSize(), BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), plan, mode);
                }
                if (nextDirView == null) {
                    // If we failed to find a dir in the next tier to move this block, evict it and
                    // continue. Normally this should not happen.
                    plan.toEvict().add(new Pair<>(blockId, block.getBlockLocation()));
                    candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                    continue;
                }
                plan.toMove().add(BlockTransferInfo.createMove(block.getBlockLocation(), blockId, nextDirView.toBlockStoreLocation()));
                candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                nextDirView.markBlockMoveIn(blockId, block.getBlockSize());
            } catch (BlockDoesNotExistException e) {
                continue;
            }
        }
    }
    return candidateDirView;
}

Example 29 with BlockDoesNotExistException

use of alluxio.exception.BlockDoesNotExistException in project alluxio by Alluxio.

the class PromoteTask method getTransferInfos.

/**
 * @return list of block transfers
 */
private List<BlockTransferInfo> getTransferInfos(Iterator<Long> iterator, BlockStoreLocation tierUpLocation, BlockStoreLocation tierDownLocation) {
    // Acquire promotion range from the configuration.
    // This will limit promotions in single task run.
    final int promoteRange = ServerConfiguration.getInt(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_RANGE);
    // Tier for where promotions are going to.
    StorageTier tierUp = mMetadataManager.getTier(tierUpLocation.tierAlias());
    // Get quota for promotions.
    int promotionQuota = ServerConfiguration.getInt(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_QUOTA_PERCENT);
    Preconditions.checkArgument(promotionQuota >= 0 && promotionQuota <= 100, "Invalid promotion quota percent");
    double quotaRatio = (double) promotionQuota / 100;
    // List to store transfer infos for selected blocks.
    List<BlockTransferInfo> transferInfos = new LinkedList<>();
    // Projected allocation for selected blocks.
    long bytesToAllocate = 0;
    // Gather blocks from iterator upto configured free space limit.
    while (iterator.hasNext() && transferInfos.size() < promoteRange) {
        // Stop moving if reached promotion quota on higher tier.
        double projectedUsedRatio = 1.0 - ((double) (tierUp.getAvailableBytes() - bytesToAllocate) / tierUp.getCapacityBytes());
        if (projectedUsedRatio >= quotaRatio) {
            break;
        }
        long blockId = iterator.next();
        // Read block info and store it.
        try {
            BlockMeta blockMeta = mEvictorView.getBlockMeta(blockId);
            if (blockMeta == null) {
                LOG.debug("Block:{} exist but not available for promotion.", blockId);
                continue;
            }
            bytesToAllocate += blockMeta.getBlockSize();
            transferInfos.add(BlockTransferInfo.createMove(blockMeta.getBlockLocation(), blockId, tierUpLocation));
        } catch (BlockDoesNotExistException e) {
            LOG.warn("Failed to find location of a block:{}. Error: {}", blockId, e);
            continue;
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated {} promotions from {} to {}.\n" + "Promotions transfers:\n ->{}", transferInfos.size(), tierDownLocation.tierAlias(), tierUpLocation.tierAlias(), transferInfos.stream().map(Objects::toString).collect(Collectors.joining("\n ->")));
    }
    return transferInfos;
}

Example 30 with BlockDoesNotExistException

use of alluxio.exception.BlockDoesNotExistException in project alluxio by Alluxio.

the class SwapRestoreTask method getSwapRestorePlan.

/**
 * @return the pair of <blocks-to-remove, blocks-to-transfer> for swap-restore plan
 */
private Pair<List<Long>, List<BlockTransferInfo>> getSwapRestorePlan() {
    StorageTierAssoc storageTierAssoc = mMetadataManager.getStorageTierAssoc();
    List<Long> blocksToRemove = new LinkedList<>();
    List<BlockTransferInfo> blocksToTransfer = new LinkedList<>();
    long cascadingFromAbove = 0;
    for (StorageTierView tierView : mEvictorView.getTierViews()) {
        int currentTierOrdinal = tierView.getTierViewOrdinal();
        String tierAlias = storageTierAssoc.getAlias(currentTierOrdinal);
        BlockStoreLocation destLocation = BlockStoreLocation.anyDirInTier(tierAlias);
        if (currentTierOrdinal < storageTierAssoc.size() - 1) {
            destLocation = BlockStoreLocation.anyDirInTier(storageTierAssoc.getAlias(currentTierOrdinal + 1));
        }
        boolean lastTier = currentTierOrdinal == storageTierAssoc.size() - 1;
        long tierReservedBytes = 0;
        long tierCapacityBytes = 0;
        long tierCommittedBytes = 0;
        for (StorageDirView dirView : tierView.getDirViews()) {
            tierReservedBytes += dirView.getReservedBytes();
            tierCapacityBytes += dirView.getCapacityBytes();
            tierCommittedBytes += dirView.getCommittedBytes();
        }
        long bytesBeyondReserve = (tierCommittedBytes + cascadingFromAbove) - (tierCapacityBytes - tierReservedBytes);
        long moveOutBytes = Math.max(0, bytesBeyondReserve);
        // Store for the next tier.
        cascadingFromAbove = moveOutBytes;
        Iterator<Long> tierIterator = mMetadataManager.getBlockIterator().getIterator(BlockStoreLocation.anyDirInTier(tierAlias), BlockOrder.NATURAL);
        while (tierIterator.hasNext() && moveOutBytes > 0) {
            long blockId = tierIterator.next();
            try {
                BlockMeta nextBlockFromTier = mEvictorView.getBlockMeta(blockId);
                if (nextBlockFromTier == null) {
                    LOG.debug("Block:{} exist but not available for moving.", blockId);
                    continue;
                }
                moveOutBytes -= nextBlockFromTier.getBlockSize();
                if (lastTier) {
                    blocksToRemove.add(nextBlockFromTier.getBlockId());
                } else {
                    blocksToTransfer.add(BlockTransferInfo.createMove(nextBlockFromTier.getBlockLocation(), nextBlockFromTier.getBlockId(), destLocation));
                }
            } catch (BlockDoesNotExistException e) {
                LOG.warn("Failed to find block:{} during cascading calculation.", blockId);
            }
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated swap-restore plan with {} deletions and {} transfers.\n" + "Block deletions:\n ->{}\nBlock transfers:\n ->{}", blocksToRemove.size(), blocksToTransfer.size(), blocksToRemove.stream().map(Object::toString).collect(Collectors.joining(",\n ->")), blocksToTransfer.stream().map(Object::toString).collect(Collectors.joining(",\n ->")));
    }
    return new Pair<>(blocksToRemove, blocksToTransfer);
}