Examples with BlockMeta alluxio.worker.block.meta.BlockMeta used on opensource projects

Example 11 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class TieredBlockStore method removeBlockInternal.

/**
   * Removes a block.
   *
   * @param sessionId session Id
   * @param blockId block Id
   * @param location the source location of the block
   * @throws InvalidWorkerStateException if the block to remove is a temp block
   * @throws BlockDoesNotExistException if this block can not be found
   * @throws IOException if I/O errors occur when removing this block file
   */
private void removeBlockInternal(long sessionId, long blockId, BlockStoreLocation location) throws InvalidWorkerStateException, BlockDoesNotExistException, IOException {
    long lockId = mLockManager.lockBlock(sessionId, blockId, BlockLockType.WRITE);
    try {
        String filePath;
        BlockMeta blockMeta;
        try (LockResource r = new LockResource(mMetadataReadLock)) {
            if (mMetaManager.hasTempBlockMeta(blockId)) {
                throw new InvalidWorkerStateException(ExceptionMessage.REMOVE_UNCOMMITTED_BLOCK, blockId);
            }
            blockMeta = mMetaManager.getBlockMeta(blockId);
            filePath = blockMeta.getPath();
        }
        if (!blockMeta.getBlockLocation().belongsTo(location)) {
            throw new BlockDoesNotExistException(ExceptionMessage.BLOCK_NOT_FOUND_AT_LOCATION, blockId, location);
        }
        // Heavy IO is guarded by block lock but not metadata lock. This may throw IOException.
        Files.delete(Paths.get(filePath));
        try (LockResource r = new LockResource(mMetadataWriteLock)) {
            mMetaManager.removeBlockMeta(blockMeta);
        } catch (BlockDoesNotExistException e) {
            // we shall never reach here
            throw Throwables.propagate(e);
        }
    } finally {
        mLockManager.unlockBlock(lockId);
    }
}

Example 12 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class TieredBlockStore method moveBlockInternal.

/**
   * Moves a block to new location only if allocator finds available space in newLocation. This
   * method will not trigger any eviction. Returns {@link MoveBlockResult}.
   *
   * @param sessionId session Id
   * @param blockId block Id
   * @param oldLocation the source location of the block
   * @param newLocation new location to move this block
   * @return the resulting information about the move operation
   * @throws BlockDoesNotExistException if block is not found
   * @throws BlockAlreadyExistsException if a block with same Id already exists in new location
   * @throws InvalidWorkerStateException if the block to move is a temp block
   * @throws IOException if I/O errors occur when moving block file
   */
private MoveBlockResult moveBlockInternal(long sessionId, long blockId, BlockStoreLocation oldLocation, BlockStoreLocation newLocation) throws BlockDoesNotExistException, BlockAlreadyExistsException, InvalidWorkerStateException, IOException {
    long lockId = mLockManager.lockBlock(sessionId, blockId, BlockLockType.WRITE);
    try {
        long blockSize;
        String srcFilePath;
        String dstFilePath;
        BlockMeta srcBlockMeta;
        BlockStoreLocation srcLocation;
        BlockStoreLocation dstLocation;
        try (LockResource r = new LockResource(mMetadataReadLock)) {
            if (mMetaManager.hasTempBlockMeta(blockId)) {
                throw new InvalidWorkerStateException(ExceptionMessage.MOVE_UNCOMMITTED_BLOCK, blockId);
            }
            srcBlockMeta = mMetaManager.getBlockMeta(blockId);
            srcLocation = srcBlockMeta.getBlockLocation();
            srcFilePath = srcBlockMeta.getPath();
            blockSize = srcBlockMeta.getBlockSize();
        }
        if (!srcLocation.belongsTo(oldLocation)) {
            throw new BlockDoesNotExistException(ExceptionMessage.BLOCK_NOT_FOUND_AT_LOCATION, blockId, oldLocation);
        }
        TempBlockMeta dstTempBlock = createBlockMetaInternal(sessionId, blockId, newLocation, blockSize, false);
        if (dstTempBlock == null) {
            return new MoveBlockResult(false, blockSize, null, null);
        }
        // When `newLocation` is some specific location, the `newLocation` and the `dstLocation` are
        // just the same; while for `newLocation` with a wildcard significance, the `dstLocation`
        // is a specific one with specific tier and dir which belongs to newLocation.
        dstLocation = dstTempBlock.getBlockLocation();
        // internally from the newLocation and return success with specific block location.
        if (dstLocation.belongsTo(srcLocation)) {
            mMetaManager.abortTempBlockMeta(dstTempBlock);
            return new MoveBlockResult(true, blockSize, srcLocation, dstLocation);
        }
        dstFilePath = dstTempBlock.getCommitPath();
        // Heavy IO is guarded by block lock but not metadata lock. This may throw IOException.
        FileUtils.move(srcFilePath, dstFilePath);
        try (LockResource r = new LockResource(mMetadataWriteLock)) {
            // If this metadata update fails, we panic for now.
            // TODO(bin): Implement rollback scheme to recover from IO failures.
            mMetaManager.moveBlockMeta(srcBlockMeta, dstTempBlock);
        } catch (BlockAlreadyExistsException | BlockDoesNotExistException | WorkerOutOfSpaceException e) {
            // we shall never reach here
            throw Throwables.propagate(e);
        }
        return new MoveBlockResult(true, blockSize, srcLocation, dstLocation);
    } finally {
        mLockManager.unlockBlock(lockId);
    }
}

Example 13 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class BlockMetadataManager method commitTempBlockMeta.

/**
   * Commits a temp block.
   *
   * @param tempBlockMeta the metadata of the temp block to commit
   * @throws WorkerOutOfSpaceException when no more space left to hold the block
   * @throws BlockAlreadyExistsException when the block already exists in committed blocks
   * @throws BlockDoesNotExistException when temp block can not be found
   */
public void commitTempBlockMeta(TempBlockMeta tempBlockMeta) throws WorkerOutOfSpaceException, BlockAlreadyExistsException, BlockDoesNotExistException {
    long blockId = tempBlockMeta.getBlockId();
    if (hasBlockMeta(blockId)) {
        BlockMeta blockMeta = getBlockMeta(blockId);
        throw new BlockAlreadyExistsException(ExceptionMessage.ADD_EXISTING_BLOCK.getMessage(blockId, blockMeta.getBlockLocation().tierAlias()));
    }
    BlockMeta block = new BlockMeta(Preconditions.checkNotNull(tempBlockMeta));
    StorageDir dir = tempBlockMeta.getParentDir();
    dir.removeTempBlockMeta(tempBlockMeta);
    dir.addBlockMeta(block);
}

Example 14 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class BlockMetadataManager method moveBlockMeta.

/**
   * Moves the metadata of an existing block to another location or throws IOExceptions. Throws an
   * {@link IllegalArgumentException} if the newLocation is not in the tiered storage.
   *
   * @param blockMeta the metadata of the block to move
   * @param newLocation new location of the block
   * @return the new block metadata if success, absent otherwise
   * @throws BlockDoesNotExistException when the block to move is not found
   * @throws BlockAlreadyExistsException when the block to move already exists in the destination
   * @throws WorkerOutOfSpaceException when destination have no extra space to hold the block to
   *         move
   * @deprecated As of version 0.8. Use {@link #moveBlockMeta(BlockMeta, TempBlockMeta)} instead.
   */
@Deprecated
public BlockMeta moveBlockMeta(BlockMeta blockMeta, BlockStoreLocation newLocation) throws BlockDoesNotExistException, BlockAlreadyExistsException, WorkerOutOfSpaceException {
    // If existing location belongs to the target location, simply return the current block meta.
    BlockStoreLocation oldLocation = blockMeta.getBlockLocation();
    if (oldLocation.belongsTo(newLocation)) {
        LOG.info("moveBlockMeta: moving {} to {} is a noop", oldLocation, newLocation);
        return blockMeta;
    }
    long blockSize = blockMeta.getBlockSize();
    String newTierAlias = newLocation.tierAlias();
    StorageTier newTier = getTier(newTierAlias);
    StorageDir newDir = null;
    if (newLocation.equals(BlockStoreLocation.anyDirInTier(newTierAlias))) {
        for (StorageDir dir : newTier.getStorageDirs()) {
            if (dir.getAvailableBytes() >= blockSize) {
                newDir = dir;
                break;
            }
        }
    } else {
        StorageDir dir = newTier.getDir(newLocation.dir());
        if (dir.getAvailableBytes() >= blockSize) {
            newDir = dir;
        }
    }
    if (newDir == null) {
        throw new WorkerOutOfSpaceException("Failed to move BlockMeta: newLocation " + newLocation + " does not have enough space for " + blockSize + " bytes");
    }
    StorageDir oldDir = blockMeta.getParentDir();
    oldDir.removeBlockMeta(blockMeta);
    BlockMeta newBlockMeta = new BlockMeta(blockMeta.getBlockId(), blockSize, newDir);
    newDir.addBlockMeta(newBlockMeta);
    return newBlockMeta;
}

Example 15 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class GreedyEvictor method freeSpaceWithView.

@Override
public EvictionPlan freeSpaceWithView(long availableBytes, BlockStoreLocation location, BlockMetadataManagerView view) {
    Preconditions.checkNotNull(location);
    Preconditions.checkNotNull(view);
    // 1. Select a StorageDirView that has enough capacity for required bytes.
    StorageDirView selectedDirView = null;
    if (location.equals(BlockStoreLocation.anyTier())) {
        selectedDirView = selectEvictableDirFromAnyTier(view, availableBytes);
    } else {
        String tierAlias = location.tierAlias();
        StorageTierView tierView = view.getTierView(tierAlias);
        if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) {
            selectedDirView = selectEvictableDirFromTier(tierView, availableBytes);
        } else {
            int dirIndex = location.dir();
            StorageDirView dir = tierView.getDirView(dirIndex);
            if (canEvictBlocksFromDir(dir, availableBytes)) {
                selectedDirView = dir;
            }
        }
    }
    if (selectedDirView == null) {
        LOG.error("Failed to freeSpace: No StorageDirView has enough capacity of {} bytes", availableBytes);
        return null;
    }
    // 2. Check if the selected StorageDirView already has enough space.
    List<BlockTransferInfo> toTransfer = new ArrayList<>();
    List<Pair<Long, BlockStoreLocation>> toEvict = new ArrayList<>();
    long bytesAvailableInDir = selectedDirView.getAvailableBytes();
    if (bytesAvailableInDir >= availableBytes) {
        // No need to evict anything, return an eviction plan with empty instructions.
        return new EvictionPlan(toTransfer, toEvict);
    }
    // 3. Collect victim blocks from the selected StorageDirView. They could either be evicted or
    // moved.
    List<BlockMeta> victimBlocks = new ArrayList<>();
    for (BlockMeta block : selectedDirView.getEvictableBlocks()) {
        victimBlocks.add(block);
        bytesAvailableInDir += block.getBlockSize();
        if (bytesAvailableInDir >= availableBytes) {
            break;
        }
    }
    // 4. Make best effort to transfer victim blocks to lower tiers rather than evict them.
    Map<StorageDirView, Long> pendingBytesInDir = new HashMap<>();
    for (BlockMeta block : victimBlocks) {
        // TODO(qifan): Should avoid calling getParentDir.
        String fromTierAlias = block.getParentDir().getParentTier().getTierAlias();
        List<StorageTierView> candidateTiers = view.getTierViewsBelow(fromTierAlias);
        StorageDirView dstDir = selectAvailableDir(block, candidateTiers, pendingBytesInDir);
        if (dstDir == null) {
            // Not possible to transfer
            toEvict.add(new Pair<>(block.getBlockId(), block.getBlockLocation()));
        } else {
            StorageTierView dstTier = dstDir.getParentTierView();
            toTransfer.add(new BlockTransferInfo(block.getBlockId(), block.getBlockLocation(), new BlockStoreLocation(dstTier.getTierViewAlias(), dstDir.getDirViewIndex())));
            if (pendingBytesInDir.containsKey(dstDir)) {
                pendingBytesInDir.put(dstDir, pendingBytesInDir.get(dstDir) + block.getBlockSize());
            } else {
                pendingBytesInDir.put(dstDir, block.getBlockSize());
            }
        }
    }
    return new EvictionPlan(toTransfer, toEvict);
}