Examples with BlockTransferInfo alluxio.worker.block.evictor.BlockTransferInfo used on opensource projects

Example 1 with BlockTransferInfo

use of alluxio.worker.block.evictor.BlockTransferInfo in project alluxio by Alluxio.

the class BlockTransferPartitioner method findTransferBucketKey.

/**
 * Used to determine right partitioning key by inspecting list of transfers.
 */
private TransferPartitionKey findTransferBucketKey(List<BlockTransferInfo> transferInfos) {
    // How many src/dst locations are fully identified.
    int srcAllocatedCount = 0;
    int dstAllocatedCount = 0;
    // How many unique src/dst locations are seen.
    Set<BlockStoreLocation> srcLocations = new HashSet<>();
    Set<BlockStoreLocation> dstLocations = new HashSet<>();
    // Iterate and process all transfers.
    for (BlockTransferInfo transferInfo : transferInfos) {
        if (transferInfo.getSrcLocation().dir() != BlockStoreLocation.ANY_DIR) {
            srcAllocatedCount++;
        }
        if (transferInfo.getDstLocation().dir() != BlockStoreLocation.ANY_DIR) {
            dstAllocatedCount++;
        }
        srcLocations.add(transferInfo.getSrcLocation());
        dstLocations.add(transferInfo.getDstLocation());
    }
    // The case for when optimization is not possible.
    if (srcAllocatedCount == 0 && dstAllocatedCount == 0) {
        // Partitioning not possible. (This is not expected).
        return TransferPartitionKey.NONE;
    }
    // Choose the key by masses.
    if (srcAllocatedCount > dstAllocatedCount) {
        return TransferPartitionKey.SRC;
    } else if (dstAllocatedCount > srcAllocatedCount) {
        return TransferPartitionKey.DST;
    } else {
        // This will later be capped by configured parallelism.
        if (srcLocations.size() >= dstLocations.size()) {
            return TransferPartitionKey.SRC;
        } else {
            return TransferPartitionKey.DST;
        }
    }
}

Example 2 with BlockTransferInfo

use of alluxio.worker.block.evictor.BlockTransferInfo in project alluxio by Alluxio.

the class BlockTransferPartitioner method partitionTransfers.

/**
 * It greedily partitions given transfers into sub-lists.
 *
 * @param transferInfos list of transfers to partition
 * @param maxPartitionCount max partition count
 * @return transfers partitioned into sub-lists
 */
public List<List<BlockTransferInfo>> partitionTransfers(List<BlockTransferInfo> transferInfos, int maxPartitionCount) {
    // Bucketing is possible if source or destination has exact location.
    // Those allocated locations will be bucket key[s].
    TransferPartitionKey key = findTransferBucketKey(transferInfos);
    // Can't bucketize transfers.
    if (key == TransferPartitionKey.NONE) {
        LOG.debug("Un-optimizable transfer list encountered.");
        return new ArrayList<List<BlockTransferInfo>>() {

            {
                add(transferInfos);
            }
        };
    }
    Map<BlockStoreLocation, List<BlockTransferInfo>> transferBuckets = new HashMap<>();
    for (BlockTransferInfo transferInfo : transferInfos) {
        BlockStoreLocation keyLoc;
        switch(key) {
            case SRC:
                keyLoc = transferInfo.getSrcLocation();
                break;
            case DST:
                keyLoc = transferInfo.getDstLocation();
                break;
            default:
                throw new IllegalStateException(String.format("Unsupported key type for bucketing transfer infos: %s", key.name()));
        }
        if (!transferBuckets.containsKey(keyLoc)) {
            transferBuckets.put(keyLoc, new LinkedList<>());
        }
        transferBuckets.get(keyLoc).add(transferInfo);
    }
    List<List<BlockTransferInfo>> balancedPartitions = balancePartitions(transferBuckets.values().stream().collect(Collectors.toList()), maxPartitionCount);
    // Log partition details.
    if (LOG.isDebugEnabled()) {
        StringBuilder partitionDbgStr = new StringBuilder();
        partitionDbgStr.append(String.format("Bucketed %d transfers into %d partitions using key:%s.%n", transferInfos.size(), balancedPartitions.size(), key.name()));
        // List each partition content.
        for (int i = 0; i < balancedPartitions.size(); i++) {
            partitionDbgStr.append(String.format("Partition-%d:%n ->%s%n", i, balancedPartitions.get(i).stream().map(Objects::toString).collect(Collectors.joining("\n ->"))));
        }
        LOG.debug("{}", partitionDbgStr);
    }
    return balancedPartitions;
}

Example 3 with BlockTransferInfo

use of alluxio.worker.block.evictor.BlockTransferInfo in project alluxio by Alluxio.

the class BlockTransferExecutor method executeTransferList.

/**
 * Executes given list of {@link BlockTransferInfo}s.
 *
 * @param transferInfos the list of transfers
 * @param exceptionHandler exception handler for when a transfer fails
 * @return the result of transfers
 */
public BlockOperationResult executeTransferList(List<BlockTransferInfo> transferInfos, Consumer<Exception> exceptionHandler) {
    LOG.debug("Executing transfer list of size: {}. Concurrency limit: {}", transferInfos.size(), mConcurrencyLimit);
    // Return immediately for an empty transfer list.
    if (transferInfos.isEmpty()) {
        return new BlockOperationResult();
    }
    // Partition executions into sub-lists.
    List<List<BlockTransferInfo>> executionPartitions = mPartitioner.partitionTransfers(transferInfos, mConcurrencyLimit);
    // Counters for ops/failures/backoffs.
    AtomicInteger opCount = new AtomicInteger(0);
    AtomicInteger failCount = new AtomicInteger(0);
    AtomicInteger backOffCount = new AtomicInteger(0);
    // Execute to-be-transferred blocks from the plan.
    Collection<Callable<Void>> executionTasks = new LinkedList<>();
    for (List<BlockTransferInfo> executionPartition : executionPartitions) {
        executionTasks.add(() -> {
            // TODO(ggezer): Prevent collisions by locking on locations.
            // Above to-do requires both source and destination locations to be allocated.
            BlockOperationResult res = executeTransferPartition(executionPartition, exceptionHandler);
            // Accumulate partition results.
            opCount.addAndGet(res.opCount());
            failCount.addAndGet(res.failCount());
            backOffCount.addAndGet(res.backOffCount());
            return null;
        });
    }
    LOG.debug("Executing {} concurrent transfer partitions.", executionTasks.size());
    try {
        mExecutor.invokeAll(executionTasks);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
    return new BlockOperationResult(opCount.get(), failCount.get(), backOffCount.get());
}

Example 4 with BlockTransferInfo

use of alluxio.worker.block.evictor.BlockTransferInfo in project alluxio by Alluxio.

the class AlignTask method generateSwapTransferInfos.

private List<BlockTransferInfo> generateSwapTransferInfos(Pair<List<Long>, List<Long>> swapLists) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generating transfer infos from swap lists.\n" + "Source list of size:{} : {}\nDestination list of size:{} : {}", swapLists.getFirst().size(), swapLists.getFirst().stream().map(Object::toString).collect(Collectors.joining(",")), swapLists.getSecond().size(), swapLists.getSecond().stream().map(Object::toString).collect(Collectors.joining(",")));
    }
    // Function that is used to map blockId to <blockId,location> pair.
    Function<Long, Pair<Long, BlockStoreLocation>> blockToPairFunc = (blockId) -> {
        try {
            return new Pair(blockId, mEvictorView.getBlockMeta(blockId).getBlockLocation());
        } catch (BlockDoesNotExistException e) {
            LOG.warn("Failed to find location of a block:{}. Error: {}", blockId, e);
            return new Pair(blockId, BlockStoreLocation.anyTier());
        }
    };
    // Generate an augmented block lists with locations.
    List<Pair<Long, BlockStoreLocation>> blockLocPairListSrc = swapLists.getFirst().stream().map(blockToPairFunc).collect(Collectors.toList());
    List<Pair<Long, BlockStoreLocation>> blockLocPairListDst = swapLists.getSecond().stream().map(blockToPairFunc).collect(Collectors.toList());
    // Sort augmented lists by location.
    // This will help to generate the buckets by a linear sweep.
    Comparator<Pair<Long, BlockStoreLocation>> comparator = (o1, o2) -> {
        BlockStoreLocation loc1 = o1.getSecond();
        BlockStoreLocation loc2 = o2.getSecond();
        int tierComp = loc1.tierAlias().compareTo(loc2.tierAlias());
        if (tierComp != 0) {
            return tierComp;
        } else {
            return loc1.dir() - loc2.dir();
        }
    };
    Collections.sort(blockLocPairListSrc, comparator);
    Collections.sort(blockLocPairListDst, comparator);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated and sorted augmented swap lists.\n" + "Source list of size:{} :\n ->{}\nDestination list of size:{} :\n ->{}", blockLocPairListSrc.size(), blockLocPairListSrc.stream().map(Object::toString).collect(Collectors.joining("\n ->")), blockLocPairListDst.size(), blockLocPairListDst.stream().map(Object::toString).collect(Collectors.joining("\n ->")));
    }
    // Build transfer infos using the sorted locations.
    List<BlockTransferInfo> transferInfos = new ArrayList<>(blockLocPairListSrc.size());
    for (int i = 0; i < blockLocPairListSrc.size(); i++) {
        transferInfos.add(BlockTransferInfo.createSwap(blockLocPairListSrc.get(i).getSecond(), blockLocPairListSrc.get(i).getFirst(), blockLocPairListDst.get(i).getSecond(), blockLocPairListDst.get(i).getFirst()));
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated {} swap transfers: \n ->{}", transferInfos.size(), transferInfos.stream().map(Object::toString).collect(Collectors.joining(",\n ->")));
    }
    return transferInfos;
}

Example 5 with BlockTransferInfo

use of alluxio.worker.block.evictor.BlockTransferInfo in project alluxio by Alluxio.

the class TieredBlockStore method freeSpaceInternal.

/**
   * Tries to get an eviction plan to free a certain amount of space in the given location, and
   * carries out this plan with the best effort.
   *
   * @param sessionId the session Id
   * @param availableBytes amount of space in bytes to free
   * @param location location of space
   * @throws WorkerOutOfSpaceException if it is impossible to achieve the free requirement
   * @throws IOException if I/O errors occur when removing or moving block files
   */
private void freeSpaceInternal(long sessionId, long availableBytes, BlockStoreLocation location) throws WorkerOutOfSpaceException, IOException {
    EvictionPlan plan;
    try (LockResource r = new LockResource(mMetadataReadLock)) {
        plan = mEvictor.freeSpaceWithView(availableBytes, location, getUpdatedView());
        // Absent plan means failed to evict enough space.
        if (plan == null) {
            throw new WorkerOutOfSpaceException(ExceptionMessage.NO_EVICTION_PLAN_TO_FREE_SPACE);
        }
    }
    // 1. remove blocks to make room.
    for (Pair<Long, BlockStoreLocation> blockInfo : plan.toEvict()) {
        try {
            removeBlockInternal(sessionId, blockInfo.getFirst(), blockInfo.getSecond());
        } catch (InvalidWorkerStateException e) {
            // Evictor is not working properly
            LOG.error("Failed to evict blockId {}, this is temp block", blockInfo.getFirst());
            continue;
        } catch (BlockDoesNotExistException e) {
            LOG.info("Failed to evict blockId {}, it could be already deleted", blockInfo.getFirst());
            continue;
        }
        synchronized (mBlockStoreEventListeners) {
            for (BlockStoreEventListener listener : mBlockStoreEventListeners) {
                listener.onRemoveBlockByWorker(sessionId, blockInfo.getFirst());
            }
        }
    }
    // 2. transfer blocks among tiers.
    // 2.1. group blocks move plan by the destination tier.
    Map<String, Set<BlockTransferInfo>> blocksGroupedByDestTier = new HashMap<>();
    for (BlockTransferInfo entry : plan.toMove()) {
        String alias = entry.getDstLocation().tierAlias();
        if (!blocksGroupedByDestTier.containsKey(alias)) {
            blocksGroupedByDestTier.put(alias, new HashSet<BlockTransferInfo>());
        }
        blocksGroupedByDestTier.get(alias).add(entry);
    }
    // 2.2. move blocks in the order of their dst tiers, from bottom to top
    for (int tierOrdinal = mStorageTierAssoc.size() - 1; tierOrdinal >= 0; --tierOrdinal) {
        Set<BlockTransferInfo> toMove = blocksGroupedByDestTier.get(mStorageTierAssoc.getAlias(tierOrdinal));
        if (toMove == null) {
            toMove = new HashSet<>();
        }
        for (BlockTransferInfo entry : toMove) {
            long blockId = entry.getBlockId();
            BlockStoreLocation oldLocation = entry.getSrcLocation();
            BlockStoreLocation newLocation = entry.getDstLocation();
            MoveBlockResult moveResult;
            try {
                moveResult = moveBlockInternal(sessionId, blockId, oldLocation, newLocation);
            } catch (InvalidWorkerStateException e) {
                // Evictor is not working properly
                LOG.error("Failed to evict blockId {}, this is temp block", blockId);
                continue;
            } catch (BlockAlreadyExistsException e) {
                continue;
            } catch (BlockDoesNotExistException e) {
                LOG.info("Failed to move blockId {}, it could be already deleted", blockId);
                continue;
            }
            if (moveResult.getSuccess()) {
                synchronized (mBlockStoreEventListeners) {
                    for (BlockStoreEventListener listener : mBlockStoreEventListeners) {
                        listener.onMoveBlockByWorker(sessionId, blockId, moveResult.getSrcLocation(), newLocation);
                    }
                }
            }
        }
    }
}