Examples with StorageDir alluxio.worker.block.meta.StorageDir used on opensource projects

Example 41 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class TieredBlockStore method checkStorage.

@Override
public boolean checkStorage() {
    try (LockResource r = new LockResource(mMetadataWriteLock)) {
        List<StorageDir> dirsToRemove = new ArrayList<>();
        for (StorageTier tier : mMetaManager.getTiers()) {
            for (StorageDir dir : tier.getStorageDirs()) {
                String path = dir.getDirPath();
                if (!FileUtils.isStorageDirAccessible(path)) {
                    LOG.error("Storage check failed for path {}. The directory will be excluded.", path);
                    dirsToRemove.add(dir);
                }
            }
        }
        dirsToRemove.forEach(this::removeDir);
        return !dirsToRemove.isEmpty();
    }
}

Example 42 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class BlockMetadataManager method swapBlocks.

/**
 * Swaps location of two blocks in metadata.
 *
 * @param blockMeta1 the first block meta
 * @param blockMeta2 the second block meta
 * @throws BlockDoesNotExistException
 * @throws BlockAlreadyExistsException
 * @throws WorkerOutOfSpaceException
 */
public void swapBlocks(BlockMeta blockMeta1, BlockMeta blockMeta2) throws BlockDoesNotExistException, BlockAlreadyExistsException, WorkerOutOfSpaceException {
    StorageDir blockDir1 = blockMeta1.getParentDir();
    StorageDir blockDir2 = blockMeta2.getParentDir();
    // Remove existing metas from dirs.
    blockDir1.removeBlockMeta(blockMeta1);
    blockDir2.removeBlockMeta(blockMeta2);
    // Add new block metas with new block id and sizes.
    blockDir1.addBlockMeta(new DefaultBlockMeta(blockMeta2.getBlockId(), blockMeta2.getBlockSize(), blockDir1));
    blockDir2.addBlockMeta(new DefaultBlockMeta(blockMeta1.getBlockId(), blockMeta1.getBlockSize(), blockDir2));
}

Example 43 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class BlockMetadataManager method commitTempBlockMeta.

/**
 * Commits a temp block.
 *
 * @param tempBlockMeta the metadata of the temp block to commit
 * @throws WorkerOutOfSpaceException when no more space left to hold the block
 * @throws BlockAlreadyExistsException when the block already exists in committed blocks
 * @throws BlockDoesNotExistException when temp block can not be found
 */
public void commitTempBlockMeta(TempBlockMeta tempBlockMeta) throws WorkerOutOfSpaceException, BlockAlreadyExistsException, BlockDoesNotExistException {
    long blockId = tempBlockMeta.getBlockId();
    if (hasBlockMeta(blockId)) {
        BlockMeta blockMeta = getBlockMeta(blockId);
        throw new BlockAlreadyExistsException(ExceptionMessage.ADD_EXISTING_BLOCK.getMessage(blockId, blockMeta.getBlockLocation().tierAlias()));
    }
    BlockMeta block = new DefaultBlockMeta(Preconditions.checkNotNull(tempBlockMeta));
    StorageDir dir = tempBlockMeta.getParentDir();
    dir.removeTempBlockMeta(tempBlockMeta);
    dir.addBlockMeta(block);
}

Example 44 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class BlockMetadataManager method moveBlockMeta.

/**
 * Moves the metadata of an existing block to another location or throws IOExceptions. Throws an
 * {@link IllegalArgumentException} if the newLocation is not in the tiered storage.
 *
 * @param blockMeta the metadata of the block to move
 * @param newLocation new location of the block
 * @return the new block metadata if success, absent otherwise
 * @throws BlockDoesNotExistException when the block to move is not found
 * @throws BlockAlreadyExistsException when the block to move already exists in the destination
 * @throws WorkerOutOfSpaceException when destination have no extra space to hold the block to
 *         move
 * @deprecated As of version 0.8. Use {@link #moveBlockMeta(BlockMeta, TempBlockMeta)} instead.
 */
@Deprecated
public BlockMeta moveBlockMeta(BlockMeta blockMeta, BlockStoreLocation newLocation) throws BlockDoesNotExistException, BlockAlreadyExistsException, WorkerOutOfSpaceException {
    // If existing location belongs to the target location, simply return the current block meta.
    BlockStoreLocation oldLocation = blockMeta.getBlockLocation();
    if (oldLocation.belongsTo(newLocation)) {
        LOG.info("moveBlockMeta: moving {} to {} is a noop", oldLocation, newLocation);
        return blockMeta;
    }
    long blockSize = blockMeta.getBlockSize();
    String newTierAlias = newLocation.tierAlias();
    StorageTier newTier = getTier(newTierAlias);
    StorageDir newDir = null;
    if (newLocation.equals(BlockStoreLocation.anyDirInTier(newTierAlias))) {
        for (StorageDir dir : newTier.getStorageDirs()) {
            if (dir.getAvailableBytes() >= blockSize) {
                newDir = dir;
                break;
            }
        }
    } else {
        StorageDir dir = newTier.getDir(newLocation.dir());
        if (dir != null && dir.getAvailableBytes() >= blockSize) {
            newDir = dir;
        }
    }
    if (newDir == null) {
        throw new WorkerOutOfSpaceException("Failed to move BlockMeta: newLocation " + newLocation + " does not have enough space for " + blockSize + " bytes");
    }
    StorageDir oldDir = blockMeta.getParentDir();
    oldDir.removeBlockMeta(blockMeta);
    BlockMeta newBlockMeta = new DefaultBlockMeta(blockMeta.getBlockId(), blockSize, newDir);
    newDir.addBlockMeta(newBlockMeta);
    return newBlockMeta;
}

Example 45 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class BlockMetadataManagerTest method cleanupSession.

/**
 * Tests the {@link BlockMetadataManager#cleanupSessionTempBlocks(long, List)} method.
 */
@Test
public void cleanupSession() throws Exception {
    StorageDir dir = mMetaManager.getTier(Constants.MEDIUM_MEM).getDir(0);
    final long tempBlockId1 = 1;
    final long tempBlockId2 = 2;
    final long tempBlockId3 = 3;
    final long sessionId1 = 100;
    final long sessionId2 = 200;
    TempBlockMeta tempBlockMeta1 = new DefaultTempBlockMeta(sessionId1, tempBlockId1, TEST_BLOCK_SIZE, dir);
    TempBlockMeta tempBlockMeta2 = new DefaultTempBlockMeta(sessionId1, tempBlockId2, TEST_BLOCK_SIZE, dir);
    TempBlockMeta tempBlockMeta3 = new DefaultTempBlockMeta(sessionId2, tempBlockId3, TEST_BLOCK_SIZE, dir);
    BlockMeta blockMeta = new DefaultBlockMeta(TEST_BLOCK_ID, TEST_BLOCK_SIZE, dir);
    dir.addTempBlockMeta(tempBlockMeta1);
    dir.addTempBlockMeta(tempBlockMeta2);
    dir.addTempBlockMeta(tempBlockMeta3);
    dir.addBlockMeta(blockMeta);
    // Get temp blocks for sessionId1, expect to get tempBlock1 and tempBlock2
    List<TempBlockMeta> toRemove = mMetaManager.getSessionTempBlocks(sessionId1);
    List<Long> toRemoveBlockIds = new ArrayList<>(toRemove.size());
    for (TempBlockMeta tempBlockMeta : toRemove) {
        toRemoveBlockIds.add(tempBlockMeta.getBlockId());
    }
    assertEquals(Sets.newHashSet(tempBlockMeta1, tempBlockMeta2), new HashSet<>(toRemove));
    assertTrue(dir.hasTempBlockMeta(tempBlockId1));
    assertTrue(dir.hasTempBlockMeta(tempBlockId2));
    // Clean up sessionId1, expect tempBlock1 and tempBlock2 to be removed.
    mMetaManager.cleanupSessionTempBlocks(sessionId1, toRemoveBlockIds);
    assertFalse(dir.hasTempBlockMeta(tempBlockId1));
    assertFalse(dir.hasTempBlockMeta(tempBlockId2));
    assertTrue(dir.hasTempBlockMeta(tempBlockId3));
    assertTrue(dir.hasBlockMeta(TEST_BLOCK_ID));
    // Get temp blocks for sessionId1 again, expect to get nothing
    toRemove = mMetaManager.getSessionTempBlocks(sessionId1);
    toRemoveBlockIds = new ArrayList<>(toRemove.size());
    for (TempBlockMeta tempBlockMeta : toRemove) {
        toRemoveBlockIds.add(tempBlockMeta.getBlockId());
    }
    assertTrue(toRemove.isEmpty());
    // Clean up sessionId1 again, expect nothing to happen
    mMetaManager.cleanupSessionTempBlocks(sessionId1, toRemoveBlockIds);
    assertFalse(dir.hasTempBlockMeta(tempBlockId1));
    assertFalse(dir.hasTempBlockMeta(tempBlockId2));
    assertTrue(dir.hasTempBlockMeta(tempBlockId3));
    assertTrue(dir.hasBlockMeta(TEST_BLOCK_ID));
    // Get temp blocks for sessionId2, expect to get tempBlock3
    toRemove = mMetaManager.getSessionTempBlocks(sessionId2);
    toRemoveBlockIds = new ArrayList<>(toRemove.size());
    for (TempBlockMeta tempBlockMeta : toRemove) {
        toRemoveBlockIds.add(tempBlockMeta.getBlockId());
    }
    assertEquals(Sets.newHashSet(tempBlockMeta3), new HashSet<>(toRemove));
    assertTrue(dir.hasTempBlockMeta(tempBlockId3));
    // Clean up sessionId2, expect tempBlock3 to be removed
    mMetaManager.cleanupSessionTempBlocks(sessionId2, toRemoveBlockIds);
    assertFalse(dir.hasTempBlockMeta(tempBlockId1));
    assertFalse(dir.hasTempBlockMeta(tempBlockId2));
    assertFalse(dir.hasTempBlockMeta(tempBlockId3));
    assertTrue(dir.hasBlockMeta(TEST_BLOCK_ID));
}