Examples with BlockInfo alluxio.wire.BlockInfo used on opensource projects

Example 21 with BlockInfo

use of alluxio.wire.BlockInfo in project alluxio by Alluxio.

the class BlockMasterTest method getBlockInfo.

@Test
public void getBlockInfo() throws Exception {
    // Create a worker with a block.
    long worker1 = mMaster.getWorkerId(NET_ADDRESS_1);
    long blockId = 1L;
    long blockLength = 20L;
    mMaster.workerRegister(worker1, Arrays.asList("MEM"), ImmutableMap.of("MEM", 100L), ImmutableMap.of("MEM", 0L), NO_BLOCKS_ON_TIERS);
    mMaster.commitBlock(worker1, 50L, "MEM", blockId, blockLength);
    BlockLocation blockLocation = new BlockLocation().setTierAlias("MEM").setWorkerAddress(NET_ADDRESS_1).setWorkerId(worker1);
    BlockInfo expectedBlockInfo = new BlockInfo().setBlockId(1L).setLength(20L).setLocations(ImmutableList.of(blockLocation));
    Assert.assertEquals(expectedBlockInfo, mMaster.getBlockInfo(blockId));
}

Example 22 with BlockInfo

use of alluxio.wire.BlockInfo in project alluxio by Alluxio.

the class FileSystemMaster method completeFileAndJournal.

/**
   * Completes a file. After a file is completed, it cannot be written to.
   * <p>
   * Writes to the journal.
   *
   * @param inodePath the {@link LockedInodePath} to complete
   * @param options the method options
   * @param journalContext the journal context
   * @throws InvalidPathException if an invalid path is encountered
   * @throws FileDoesNotExistException if the file does not exist
   * @throws BlockInfoException if a block information exception is encountered
   * @throws FileAlreadyCompletedException if the file is already completed
   * @throws InvalidFileSizeException if an invalid file size is encountered
   */
private void completeFileAndJournal(LockedInodePath inodePath, CompleteFileOptions options, JournalContext journalContext) throws InvalidPathException, FileDoesNotExistException, BlockInfoException, FileAlreadyCompletedException, InvalidFileSizeException {
    Inode<?> inode = inodePath.getInode();
    if (!inode.isFile()) {
        throw new FileDoesNotExistException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(inodePath.getUri()));
    }
    InodeFile fileInode = (InodeFile) inode;
    List<Long> blockIdList = fileInode.getBlockIds();
    List<BlockInfo> blockInfoList = mBlockMaster.getBlockInfoList(blockIdList);
    if (!fileInode.isPersisted() && blockInfoList.size() != blockIdList.size()) {
        throw new BlockInfoException("Cannot complete a file without all the blocks committed");
    }
    // Iterate over all file blocks committed to Alluxio, computing the length and verify that all
    // the blocks (except the last one) is the same size as the file block size.
    long inMemoryLength = 0;
    long fileBlockSize = fileInode.getBlockSizeBytes();
    for (int i = 0; i < blockInfoList.size(); i++) {
        BlockInfo blockInfo = blockInfoList.get(i);
        inMemoryLength += blockInfo.getLength();
        if (i < blockInfoList.size() - 1 && blockInfo.getLength() != fileBlockSize) {
            throw new BlockInfoException("Block index " + i + " has a block size smaller than the file block size (" + fileInode.getBlockSizeBytes() + ")");
        }
    }
    // If the file is persisted, its length is determined by UFS. Otherwise, its length is
    // determined by its memory footprint.
    long length = fileInode.isPersisted() ? options.getUfsLength() : inMemoryLength;
    completeFileInternal(fileInode.getBlockIds(), inodePath, length, options.getOperationTimeMs());
    CompleteFileEntry completeFileEntry = CompleteFileEntry.newBuilder().addAllBlockIds(fileInode.getBlockIds()).setId(inode.getId()).setLength(length).setOpTimeMs(options.getOperationTimeMs()).build();
    appendJournalEntry(JournalEntry.newBuilder().setCompleteFile(completeFileEntry).build(), journalContext);
}

Example 23 with BlockInfo

use of alluxio.wire.BlockInfo in project alluxio by Alluxio.

the class FileSystemMaster method getInMemoryPercentage.

/**
   * Gets the in-memory percentage of an Inode. For a file that has all blocks in memory, it returns
   * 100; for a file that has no block in memory, it returns 0. Returns 0 for a directory.
   *
   * @param inode the inode
   * @return the in memory percentage
   */
private int getInMemoryPercentage(Inode<?> inode) {
    if (!inode.isFile()) {
        return 0;
    }
    InodeFile inodeFile = (InodeFile) inode;
    long length = inodeFile.getLength();
    if (length == 0) {
        return 100;
    }
    long inMemoryLength = 0;
    for (BlockInfo info : mBlockMaster.getBlockInfoList(inodeFile.getBlockIds())) {
        if (isInTopStorageTier(info)) {
            inMemoryLength += info.getLength();
        }
    }
    return (int) (inMemoryLength * 100 / length);
}

Example 24 with BlockInfo

use of alluxio.wire.BlockInfo in project alluxio by Alluxio.

the class BlockMaster method generateBlockInfo.

/**
   * Creates a {@link BlockInfo} form a given {@link MasterBlockInfo}, by populating worker
   * locations.
   *
   * @param masterBlockInfo the {@link MasterBlockInfo}
   * @return a {@link BlockInfo} from a {@link MasterBlockInfo}. Populates worker locations
   */
@GuardedBy("masterBlockInfo")
private BlockInfo generateBlockInfo(MasterBlockInfo masterBlockInfo) {
    // "Join" to get all the addresses of the workers.
    List<BlockLocation> locations = new ArrayList<>();
    List<MasterBlockLocation> blockLocations = masterBlockInfo.getBlockLocations();
    // Sort the block locations by their alias ordinal in the master storage tier mapping
    Collections.sort(blockLocations, new Comparator<MasterBlockLocation>() {

        @Override
        public int compare(MasterBlockLocation o1, MasterBlockLocation o2) {
            return mGlobalStorageTierAssoc.getOrdinal(o1.getTierAlias()) - mGlobalStorageTierAssoc.getOrdinal(o2.getTierAlias());
        }
    });
    for (MasterBlockLocation masterBlockLocation : blockLocations) {
        MasterWorkerInfo workerInfo = mWorkers.getFirstByField(ID_INDEX, masterBlockLocation.getWorkerId());
        if (workerInfo != null) {
            // worker metadata is intentionally not locked here because:
            // - it would be an incorrect order (correct order is lock worker first, then block)
            // - only uses getters of final variables
            locations.add(new BlockLocation().setWorkerId(masterBlockLocation.getWorkerId()).setWorkerAddress(workerInfo.getWorkerAddress()).setTierAlias(masterBlockLocation.getTierAlias()));
        }
    }
    return new BlockInfo().setBlockId(masterBlockInfo.getBlockId()).setLength(masterBlockInfo.getLength()).setLocations(locations);
}

Example 25 with BlockInfo

use of alluxio.wire.BlockInfo in project alluxio by Alluxio.

the class DataServerIntegrationTest method readThroughClient.

@Test
public void readThroughClient() throws Exception {
    final int length = 10;
    FileSystemTestUtils.createByteFile(mFileSystem, "/file", WriteType.MUST_CACHE, length);
    BlockInfo block = getFirstBlockInfo(new AlluxioURI("/file"));
    RemoteBlockReader client = RemoteBlockReader.Factory.create(FileSystemContext.INSTANCE);
    ByteBuffer result = readRemotely(client, block, length);
    Assert.assertEquals(BufferUtils.getIncreasingByteBuffer(length), result);
}