Examples with BlockMeta alluxio.worker.block.meta.BlockMeta used on opensource projects

Example 1 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class AbstractEvictor method cascadingEvict.

/**
   * A recursive implementation of cascading eviction.
   *
   * This method uses a specific eviction strategy to find blocks to evict in the requested
   * location. After eviction, one {@link alluxio.worker.block.meta.StorageDir} in the location has
   * the specific amount of free space. It then uses an allocation strategy to allocate space in the
   * next tier to move each evicted blocks. If the next tier fails to allocate space for the evicted
   * blocks, the next tier will continue to evict its blocks to free space.
   *
   * This method is only used in
   * {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataManagerView)}.
   *
   * @param bytesToBeAvailable bytes to be available after eviction
   * @param location target location to evict blocks from
   * @param plan the plan to be recursively updated, is empty when first called in
   *        {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataManagerView)}
   * @return the first {@link StorageDirView} in the range of location to evict/move bytes from, or
   *         null if there is no plan
   */
protected StorageDirView cascadingEvict(long bytesToBeAvailable, BlockStoreLocation location, EvictionPlan plan) {
    location = updateBlockStoreLocation(bytesToBeAvailable, location);
    // 1. If bytesToBeAvailable can already be satisfied without eviction, return the eligible
    // StoargeDirView
    StorageDirView candidateDirView = EvictorUtils.selectDirWithRequestedSpace(bytesToBeAvailable, location, mManagerView);
    if (candidateDirView != null) {
        return candidateDirView;
    }
    // 2. Iterate over blocks in order until we find a StorageDirView that is in the range of
    // location and can satisfy bytesToBeAvailable after evicting its blocks iterated so far
    EvictionDirCandidates dirCandidates = new EvictionDirCandidates();
    Iterator<Long> it = getBlockIterator();
    while (it.hasNext() && dirCandidates.candidateSize() < bytesToBeAvailable) {
        long blockId = it.next();
        try {
            BlockMeta block = mManagerView.getBlockMeta(blockId);
            if (block != null) {
                // might not present in this view
                if (block.getBlockLocation().belongsTo(location)) {
                    String tierAlias = block.getParentDir().getParentTier().getTierAlias();
                    int dirIndex = block.getParentDir().getDirIndex();
                    dirCandidates.add(mManagerView.getTierView(tierAlias).getDirView(dirIndex), blockId, block.getBlockSize());
                }
            }
        } catch (BlockDoesNotExistException e) {
            LOG.warn("Remove block {} from evictor cache because {}", blockId, e);
            it.remove();
            onRemoveBlockFromIterator(blockId);
        }
    }
    // 3. If there is no eligible StorageDirView, return null
    if (dirCandidates.candidateSize() < bytesToBeAvailable) {
        return null;
    }
    // 4. cascading eviction: try to allocate space in the next tier to move candidate blocks
    // there. If allocation fails, the next tier will continue to evict its blocks to free space.
    // Blocks are only evicted from the last tier or it can not be moved to the next tier.
    candidateDirView = dirCandidates.candidateDir();
    List<Long> candidateBlocks = dirCandidates.candidateBlocks();
    StorageTierView nextTierView = mManagerView.getNextTier(candidateDirView.getParentTierView());
    if (nextTierView == null) {
        // This is the last tier, evict all the blocks.
        for (Long blockId : candidateBlocks) {
            try {
                BlockMeta block = mManagerView.getBlockMeta(blockId);
                if (block != null) {
                    candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                    plan.toEvict().add(new Pair<>(blockId, candidateDirView.toBlockStoreLocation()));
                }
            } catch (BlockDoesNotExistException e) {
                continue;
            }
        }
    } else {
        for (Long blockId : candidateBlocks) {
            try {
                BlockMeta block = mManagerView.getBlockMeta(blockId);
                if (block == null) {
                    continue;
                }
                StorageDirView nextDirView = mAllocator.allocateBlockWithView(Sessions.MIGRATE_DATA_SESSION_ID, block.getBlockSize(), BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), mManagerView);
                if (nextDirView == null) {
                    nextDirView = cascadingEvict(block.getBlockSize(), BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), plan);
                }
                if (nextDirView == null) {
                    // If we failed to find a dir in the next tier to move this block, evict it and
                    // continue. Normally this should not happen.
                    plan.toEvict().add(new Pair<>(blockId, block.getBlockLocation()));
                    candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                    continue;
                }
                plan.toMove().add(new BlockTransferInfo(blockId, block.getBlockLocation(), nextDirView.toBlockStoreLocation()));
                candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                nextDirView.markBlockMoveIn(blockId, block.getBlockSize());
            } catch (BlockDoesNotExistException e) {
                continue;
            }
        }
    }
    return candidateDirView;
}

Example 2 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class BlockMetadataManager method moveBlockMeta.

/**
   * Moves an existing block to another location currently hold by a temp block.
   *
   * @param blockMeta the metadata of the block to move
   * @param tempBlockMeta a placeholder in the destination directory
   * @return the new block metadata if success, absent otherwise
   * @throws BlockDoesNotExistException when the block to move is not found
   * @throws BlockAlreadyExistsException when the block to move already exists in the destination
   * @throws WorkerOutOfSpaceException when destination have no extra space to hold the block to
   *         move
   */
public BlockMeta moveBlockMeta(BlockMeta blockMeta, TempBlockMeta tempBlockMeta) throws BlockDoesNotExistException, WorkerOutOfSpaceException, BlockAlreadyExistsException {
    StorageDir srcDir = blockMeta.getParentDir();
    StorageDir dstDir = tempBlockMeta.getParentDir();
    srcDir.removeBlockMeta(blockMeta);
    BlockMeta newBlockMeta = new BlockMeta(blockMeta.getBlockId(), blockMeta.getBlockSize(), dstDir);
    dstDir.removeTempBlockMeta(tempBlockMeta);
    dstDir.addBlockMeta(newBlockMeta);
    return newBlockMeta;
}

Example 3 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class WebInterfaceWorkerBlockInfoServlet method getUiFileInfo.

/**
   * Gets the {@link UIFileInfo} object based on URI status.
   *
   * @param status the URI status to use
   * @return the {@link UIFileInfo} object of the file
   * @throws BlockDoesNotExistException if the block does not exist
   * @throws FileDoesNotExistException if the file does not exist
   */
private UIFileInfo getUiFileInfo(URIStatus status) throws BlockDoesNotExistException, FileDoesNotExistException {
    UIFileInfo uiFileInfo = new UIFileInfo(status);
    boolean blockExistOnWorker = false;
    for (long blockId : status.getBlockIds()) {
        if (mBlockWorker.hasBlockMeta(blockId)) {
            blockExistOnWorker = true;
            BlockMeta blockMeta = mBlockWorker.getVolatileBlockMeta(blockId);
            long blockSize = blockMeta.getBlockSize();
            // The block last access time is not available. Use -1 for now.
            // It's not necessary to show location information here since
            // we are viewing at the context of this worker.
            uiFileInfo.addBlock(blockMeta.getBlockLocation().tierAlias(), blockId, blockSize, -1);
        }
    }
    if (!blockExistOnWorker) {
        throw new FileDoesNotExistException(status.getPath());
    }
    return uiFileInfo;
}

Example 4 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class FileDataManager method persistFile.

/**
   * Persists the blocks of a file into the under file system.
   *
   * @param fileId the id of the file
   * @param blockIds the list of block ids
   * @throws AlluxioException if an unexpected Alluxio exception is thrown
   * @throws IOException if the file persistence fails
   */
public void persistFile(long fileId, List<Long> blockIds) throws AlluxioException, IOException {
    Map<Long, Long> blockIdToLockId;
    synchronized (mLock) {
        blockIdToLockId = mPersistingInProgressFiles.get(fileId);
        if (blockIdToLockId == null || !blockIdToLockId.keySet().equals(new HashSet<>(blockIds))) {
            throw new IOException("Not all the blocks of file " + fileId + " are locked");
        }
    }
    String dstPath = prepareUfsFilePath(fileId);
    UnderFileSystem ufs = UnderFileSystem.Factory.get(dstPath);
    FileInfo fileInfo = mBlockWorker.getFileInfo(fileId);
    OutputStream outputStream = ufs.create(dstPath, CreateOptions.defaults().setOwner(fileInfo.getOwner()).setGroup(fileInfo.getGroup()).setMode(new Mode((short) fileInfo.getMode())));
    final WritableByteChannel outputChannel = Channels.newChannel(outputStream);
    List<Throwable> errors = new ArrayList<>();
    try {
        for (long blockId : blockIds) {
            long lockId = blockIdToLockId.get(blockId);
            if (Configuration.getBoolean(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT_ENABLED)) {
                BlockMeta blockMeta = mBlockWorker.getBlockMeta(Sessions.CHECKPOINT_SESSION_ID, blockId, lockId);
                mPersistenceRateLimiter.acquire((int) blockMeta.getBlockSize());
            }
            // obtain block reader
            BlockReader reader = mBlockWorker.readBlockRemote(Sessions.CHECKPOINT_SESSION_ID, blockId, lockId);
            // write content out
            ReadableByteChannel inputChannel = reader.getChannel();
            BufferUtils.fastCopy(inputChannel, outputChannel);
            reader.close();
        }
    } catch (BlockDoesNotExistException | InvalidWorkerStateException e) {
        errors.add(e);
    } finally {
        // make sure all the locks are released
        for (long lockId : blockIdToLockId.values()) {
            try {
                mBlockWorker.unlockBlock(lockId);
            } catch (BlockDoesNotExistException e) {
                errors.add(e);
            }
        }
        // Process any errors
        if (!errors.isEmpty()) {
            StringBuilder errorStr = new StringBuilder();
            errorStr.append("the blocks of file").append(fileId).append(" are failed to persist\n");
            for (Throwable e : errors) {
                errorStr.append(e).append('\n');
            }
            throw new IOException(errorStr.toString());
        }
    }
    outputStream.flush();
    outputChannel.close();
    outputStream.close();
    synchronized (mLock) {
        mPersistingInProgressFiles.remove(fileId);
        mPersistedFiles.add(fileId);
    }
}

Example 5 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class DefaultBlockWorker method commitBlock.

@Override
public void commitBlock(long sessionId, long blockId) throws BlockAlreadyExistsException, BlockDoesNotExistException, InvalidWorkerStateException, IOException, WorkerOutOfSpaceException {
    // TODO(binfan): find a better way to handle retry logic
    try {
        mBlockStore.commitBlock(sessionId, blockId);
    } catch (BlockAlreadyExistsException e) {
        LOG.debug("Block {} has been in block store, this could be a retry due to master-side RPC " + "failure, therefore ignore the exception", blockId, e);
    }
    // TODO(calvin): Reconsider how to do this without heavy locking.
    // Block successfully committed, update master with new block metadata
    Long lockId = mBlockStore.lockBlock(sessionId, blockId);
    try {
        BlockMeta meta = mBlockStore.getBlockMeta(sessionId, blockId, lockId);
        BlockStoreLocation loc = meta.getBlockLocation();
        Long length = meta.getBlockSize();
        BlockStoreMeta storeMeta = mBlockStore.getBlockStoreMeta();
        Long bytesUsedOnTier = storeMeta.getUsedBytesOnTiers().get(loc.tierAlias());
        mBlockMasterClient.commitBlock(mWorkerId.get(), bytesUsedOnTier, loc.tierAlias(), blockId, length);
    } catch (AlluxioTException | IOException | ConnectionFailedException e) {
        throw new IOException(ExceptionMessage.FAILED_COMMIT_BLOCK_TO_MASTER.getMessage(blockId), e);
    } finally {
        mBlockStore.unlockBlock(lockId);
    }
}