Examples with BlockTransferInfo alluxio.worker.block.evictor.BlockTransferInfo used on opensource projects

Example 6 with BlockTransferInfo

use of alluxio.worker.block.evictor.BlockTransferInfo in project alluxio by Alluxio.

the class BlockTransferPartitionerTest method generateTransfers.

/**
 * Generates transfers with controls to specify if src/dst is allocated.
 * It also acceptslocation-group parameter to specify how many transfers will
 * each unique location contain.
 */
List<BlockTransferInfo> generateTransfers(int count, boolean srcAllocated, boolean dstAllocated, int[] srcLocGroups, int[] dstLocGroups) {
    Assert.assertEquals(count, Arrays.stream(srcLocGroups).sum());
    Assert.assertEquals(count, Arrays.stream(dstLocGroups).sum());
    List<BlockTransferInfo> transferInfos = new ArrayList<>(count);
    int[] srcLocGroupsCopy = Arrays.copyOf(srcLocGroups, srcLocGroups.length);
    int[] dstLocGroupsCopy = Arrays.copyOf(dstLocGroups, dstLocGroups.length);
    int srcTierOrdinal = 0;
    int dstTierOrdinal = 0;
    int srcDirIndex = 0;
    int dstDirIndex = 0;
    int currLocationGroupSrc = 0;
    int currLocationGroupDst = 0;
    for (int blockId = 0; blockId < count; blockId++) {
        // Generate the src location.
        BlockStoreLocation srcLoc;
        if (!srcAllocated) {
            srcLoc = BlockStoreLocation.anyDirInTier(Integer.toString(srcTierOrdinal));
        } else {
            srcLoc = new BlockStoreLocation(Integer.toString(srcTierOrdinal), srcDirIndex);
        }
        // Generate the dst location.
        BlockStoreLocation dstLoc;
        if (!dstAllocated) {
            dstLoc = BlockStoreLocation.anyDirInTier(Integer.toString(dstTierOrdinal));
        } else {
            dstLoc = new BlockStoreLocation(Integer.toString(dstTierOrdinal), dstDirIndex);
        }
        // Create the transfer info.
        transferInfos.add(BlockTransferInfo.createMove(srcLoc, blockId, dstLoc));
        // Advance counters for the next location based on requested location groups.
        if (--srcLocGroupsCopy[currLocationGroupSrc] == 0) {
            currLocationGroupSrc++;
            srcTierOrdinal++;
            srcDirIndex++;
        }
        if (--dstLocGroupsCopy[currLocationGroupDst] == 0) {
            currLocationGroupDst++;
            dstTierOrdinal++;
            dstDirIndex++;
        }
    }
    return transferInfos;
}

Example 7 with BlockTransferInfo

use of alluxio.worker.block.evictor.BlockTransferInfo in project alluxio by Alluxio.

the class PromoteTask method getTransferInfos.

/**
 * @return list of block transfers
 */
private List<BlockTransferInfo> getTransferInfos(Iterator<Long> iterator, BlockStoreLocation tierUpLocation, BlockStoreLocation tierDownLocation) {
    // Acquire promotion range from the configuration.
    // This will limit promotions in single task run.
    final int promoteRange = ServerConfiguration.getInt(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_RANGE);
    // Tier for where promotions are going to.
    StorageTier tierUp = mMetadataManager.getTier(tierUpLocation.tierAlias());
    // Get quota for promotions.
    int promotionQuota = ServerConfiguration.getInt(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_QUOTA_PERCENT);
    Preconditions.checkArgument(promotionQuota >= 0 && promotionQuota <= 100, "Invalid promotion quota percent");
    double quotaRatio = (double) promotionQuota / 100;
    // List to store transfer infos for selected blocks.
    List<BlockTransferInfo> transferInfos = new LinkedList<>();
    // Projected allocation for selected blocks.
    long bytesToAllocate = 0;
    // Gather blocks from iterator upto configured free space limit.
    while (iterator.hasNext() && transferInfos.size() < promoteRange) {
        // Stop moving if reached promotion quota on higher tier.
        double projectedUsedRatio = 1.0 - ((double) (tierUp.getAvailableBytes() - bytesToAllocate) / tierUp.getCapacityBytes());
        if (projectedUsedRatio >= quotaRatio) {
            break;
        }
        long blockId = iterator.next();
        // Read block info and store it.
        try {
            BlockMeta blockMeta = mEvictorView.getBlockMeta(blockId);
            if (blockMeta == null) {
                LOG.debug("Block:{} exist but not available for promotion.", blockId);
                continue;
            }
            bytesToAllocate += blockMeta.getBlockSize();
            transferInfos.add(BlockTransferInfo.createMove(blockMeta.getBlockLocation(), blockId, tierUpLocation));
        } catch (BlockDoesNotExistException e) {
            LOG.warn("Failed to find location of a block:{}. Error: {}", blockId, e);
            continue;
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated {} promotions from {} to {}.\n" + "Promotions transfers:\n ->{}", transferInfos.size(), tierDownLocation.tierAlias(), tierUpLocation.tierAlias(), transferInfos.stream().map(Objects::toString).collect(Collectors.joining("\n ->")));
    }
    return transferInfos;
}

Example 8 with BlockTransferInfo

use of alluxio.worker.block.evictor.BlockTransferInfo in project alluxio by Alluxio.

the class BlockTransferExecutor method executeTransferPartition.

/**
 * Used as entry point for executing a single transfer partition.
 */
private BlockOperationResult executeTransferPartition(List<BlockTransferInfo> transferInfos, Consumer<Exception> exceptionHandler) {
    LOG.debug("Executing transfer partition of size {}", transferInfos.size());
    // Counters for failure and back-offs.
    int failCount = 0;
    int backOffCount = 0;
    // Execute transfers in order.
    for (BlockTransferInfo transferInfo : transferInfos) {
        try {
            if (mLoadTracker.loadDetected(transferInfo.getSrcLocation(), transferInfo.getDstLocation())) {
                LOG.debug("Skipping transfer-order: {} due to user activity.", transferInfo);
                backOffCount++;
                continue;
            }
            boolean useReservedSpace = transferInfo.isSwap();
            mBlockStore.moveBlock(Sessions.createInternalSessionId(), transferInfo.getSrcBlockId(), AllocateOptions.forTierMove(transferInfo.getDstLocation()).setUseReservedSpace(useReservedSpace));
            if (transferInfo.isSwap()) {
                // TODO(ggezer): Implement external allocations to guarantee a swap.
                mBlockStore.moveBlock(Sessions.createInternalSessionId(), transferInfo.getDstBlockId(), AllocateOptions.forTierMove(transferInfo.getSrcLocation()).setUseReservedSpace(useReservedSpace));
            }
        } catch (Exception e) {
            LOG.warn("Transfer-order: {} failed. {} ", transferInfo, e.toString());
            failCount++;
            if (exceptionHandler != null) {
                exceptionHandler.accept(e);
            }
        }
    }
    return new BlockOperationResult(transferInfos.size(), failCount, backOffCount);
}

Example 9 with BlockTransferInfo

use of alluxio.worker.block.evictor.BlockTransferInfo in project alluxio by Alluxio.

the class SwapRestoreTask method getSwapRestorePlan.

/**
 * @return the pair of <blocks-to-remove, blocks-to-transfer> for swap-restore plan
 */
private Pair<List<Long>, List<BlockTransferInfo>> getSwapRestorePlan() {
    StorageTierAssoc storageTierAssoc = mMetadataManager.getStorageTierAssoc();
    List<Long> blocksToRemove = new LinkedList<>();
    List<BlockTransferInfo> blocksToTransfer = new LinkedList<>();
    long cascadingFromAbove = 0;
    for (StorageTierView tierView : mEvictorView.getTierViews()) {
        int currentTierOrdinal = tierView.getTierViewOrdinal();
        String tierAlias = storageTierAssoc.getAlias(currentTierOrdinal);
        BlockStoreLocation destLocation = BlockStoreLocation.anyDirInTier(tierAlias);
        if (currentTierOrdinal < storageTierAssoc.size() - 1) {
            destLocation = BlockStoreLocation.anyDirInTier(storageTierAssoc.getAlias(currentTierOrdinal + 1));
        }
        boolean lastTier = currentTierOrdinal == storageTierAssoc.size() - 1;
        long tierReservedBytes = 0;
        long tierCapacityBytes = 0;
        long tierCommittedBytes = 0;
        for (StorageDirView dirView : tierView.getDirViews()) {
            tierReservedBytes += dirView.getReservedBytes();
            tierCapacityBytes += dirView.getCapacityBytes();
            tierCommittedBytes += dirView.getCommittedBytes();
        }
        long bytesBeyondReserve = (tierCommittedBytes + cascadingFromAbove) - (tierCapacityBytes - tierReservedBytes);
        long moveOutBytes = Math.max(0, bytesBeyondReserve);
        // Store for the next tier.
        cascadingFromAbove = moveOutBytes;
        Iterator<Long> tierIterator = mMetadataManager.getBlockIterator().getIterator(BlockStoreLocation.anyDirInTier(tierAlias), BlockOrder.NATURAL);
        while (tierIterator.hasNext() && moveOutBytes > 0) {
            long blockId = tierIterator.next();
            try {
                BlockMeta nextBlockFromTier = mEvictorView.getBlockMeta(blockId);
                if (nextBlockFromTier == null) {
                    LOG.debug("Block:{} exist but not available for moving.", blockId);
                    continue;
                }
                moveOutBytes -= nextBlockFromTier.getBlockSize();
                if (lastTier) {
                    blocksToRemove.add(nextBlockFromTier.getBlockId());
                } else {
                    blocksToTransfer.add(BlockTransferInfo.createMove(nextBlockFromTier.getBlockLocation(), nextBlockFromTier.getBlockId(), destLocation));
                }
            } catch (BlockDoesNotExistException e) {
                LOG.warn("Failed to find block:{} during cascading calculation.", blockId);
            }
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated swap-restore plan with {} deletions and {} transfers.\n" + "Block deletions:\n ->{}\nBlock transfers:\n ->{}", blocksToRemove.size(), blocksToTransfer.size(), blocksToRemove.stream().map(Object::toString).collect(Collectors.joining(",\n ->")), blocksToTransfer.stream().map(Object::toString).collect(Collectors.joining(",\n ->")));
    }
    return new Pair<>(blocksToRemove, blocksToTransfer);
}

Example 10 with BlockTransferInfo

use of alluxio.worker.block.evictor.BlockTransferInfo in project alluxio by Alluxio.

the class SwapRestoreTask method getBalancingTransfersList.

/**
 * @return the list of transfer in order to balance swap-space within tier
 */
private List<BlockTransferInfo> getBalancingTransfersList() {
    List<BlockTransferInfo> transferInfos = new LinkedList<>();
    for (StorageTierView tierView : mEvictorView.getTierViews()) {
        for (StorageDirView dirView : tierView.getDirViews()) {
            Iterator<Long> dirBlockIter = mMetadataManager.getBlockIterator().getIterator(new BlockStoreLocation(tierView.getTierViewAlias(), dirView.getDirViewIndex()), BlockOrder.NATURAL);
            while (dirBlockIter.hasNext() && dirView.getAvailableBytes() < dirView.getReservedBytes()) {
                long blockId = dirBlockIter.next();
                try {
                    BlockMeta movingOutBlock = mEvictorView.getBlockMeta(blockId);
                    if (movingOutBlock == null) {
                        LOG.debug("Block:{} exist but not available for balancing.", blockId);
                        continue;
                    }
                    // Find where to move the block.
                    StorageDirView dstDirView = null;
                    for (StorageDirView candidateDirView : tierView.getDirViews()) {
                        if (candidateDirView.getDirViewIndex() == dirView.getDirViewIndex()) {
                            continue;
                        }
                        if (candidateDirView.getAvailableBytes() - candidateDirView.getReservedBytes() >= movingOutBlock.getBlockSize()) {
                            dstDirView = candidateDirView;
                            break;
                        }
                    }
                    if (dstDirView == null) {
                        LOG.warn("Could not balance swap-restore space for location: {}", dirView.toBlockStoreLocation());
                        break;
                    }
                    // TODO(ggezer): Consider allowing evictions for this move.
                    transferInfos.add(BlockTransferInfo.createMove(movingOutBlock.getBlockLocation(), blockId, dstDirView.toBlockStoreLocation()));
                    // Account for moving-out blocks.
                    ((StorageDirEvictorView) dirView).markBlockMoveOut(blockId, movingOutBlock.getBlockSize());
                    // Account for moving-in blocks.
                    ((StorageDirEvictorView) dstDirView).markBlockMoveIn(blockId, movingOutBlock.getBlockSize());
                } catch (BlockDoesNotExistException e) {
                    LOG.warn("Failed to find metadata for block:{} during swap-restore balancing.", blockId);
                }
            }
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated {} balance transfers:\n ->{}", transferInfos.size(), transferInfos.stream().map(Object::toString).collect(Collectors.joining(",\n ->")));
    }
    return transferInfos;
}