Examples with BlockMeta alluxio.proto.meta.Block.BlockMeta used on opensource projects

Example 1 with BlockMeta

use of alluxio.proto.meta.Block.BlockMeta in project alluxio by Alluxio.

the class DefaultBlockMaster method processWorkerRemovedBlocks.

/**
 * Updates the worker and block metadata for blocks removed from a worker.
 *
 * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)}
 * with {@link WorkerMetaLockSection#BLOCKS} specified.
 * An exclusive lock is required.
 *
 * @param workerInfo The worker metadata object
 * @param removedBlockIds A list of block ids removed from the worker
 */
private void processWorkerRemovedBlocks(MasterWorkerInfo workerInfo, Collection<Long> removedBlockIds, boolean sendCommand) {
    for (long removedBlockId : removedBlockIds) {
        try (LockResource r = lockBlock(removedBlockId)) {
            Optional<BlockMeta> block = mBlockStore.getBlock(removedBlockId);
            if (block.isPresent()) {
                LOG.debug("Block {} is removed on worker {}.", removedBlockId, workerInfo.getId());
                mBlockStore.removeLocation(removedBlockId, workerInfo.getId());
                if (mBlockStore.getLocations(removedBlockId).size() == 0) {
                    mLostBlocks.add(removedBlockId);
                }
            }
            // Remove the block even if its metadata has been deleted already.
            if (sendCommand) {
                workerInfo.scheduleRemoveFromWorker(removedBlockId);
            } else {
                workerInfo.removeBlockFromWorkerMeta(removedBlockId);
            }
        }
    }
}

Example 2 with BlockMeta

use of alluxio.proto.meta.Block.BlockMeta in project alluxio by Alluxio.

the class DefaultBlockMaster method processJournalEntry.

@Override
public boolean processJournalEntry(JournalEntry entry) {
    // TODO(gene): A better way to process entries besides a huge switch?
    if (entry.hasBlockContainerIdGenerator()) {
        mJournaledNextContainerId = (entry.getBlockContainerIdGenerator()).getNextContainerId();
        mBlockContainerIdGenerator.setNextContainerId((mJournaledNextContainerId));
    } else if (entry.hasDeleteBlock()) {
        mBlockStore.removeBlock(entry.getDeleteBlock().getBlockId());
    } else if (entry.hasBlockInfo()) {
        BlockInfoEntry blockInfoEntry = entry.getBlockInfo();
        long length = blockInfoEntry.getLength();
        Optional<BlockMeta> block = mBlockStore.getBlock(blockInfoEntry.getBlockId());
        if (block.isPresent()) {
            long oldLen = block.get().getLength();
            if (oldLen != Constants.UNKNOWN_SIZE) {
                LOG.warn("Attempting to update block length ({}) to a different length ({}).", oldLen, length);
                return true;
            }
        }
        mBlockStore.putBlock(blockInfoEntry.getBlockId(), BlockMeta.newBuilder().setLength(blockInfoEntry.getLength()).build());
    } else {
        return false;
    }
    return true;
}

Example 3 with BlockMeta

use of alluxio.proto.meta.Block.BlockMeta in project alluxio by Alluxio.

the class DefaultBlockMaster method generateBlockInfo.

/**
 * Generates block info, including worker locations, for a block id.
 * This requires no locks on the {@link MasterWorkerInfo} because it is only reading
 * final fields.
 *
 * @param blockId a block id
 * @return optional block info, empty if the block does not exist
 */
private Optional<BlockInfo> generateBlockInfo(long blockId) throws UnavailableException {
    if (mSafeModeManager.isInSafeMode()) {
        throw new UnavailableException(ExceptionMessage.MASTER_IN_SAFEMODE.getMessage());
    }
    BlockMeta block;
    List<BlockLocation> blockLocations;
    try (LockResource r = lockBlock(blockId)) {
        Optional<BlockMeta> blockOpt = mBlockStore.getBlock(blockId);
        if (!blockOpt.isPresent()) {
            return Optional.empty();
        }
        block = blockOpt.get();
        blockLocations = new ArrayList<>(mBlockStore.getLocations(blockId));
    }
    // Sort the block locations by their alias ordinal in the master storage tier mapping
    Collections.sort(blockLocations, Comparator.comparingInt(o -> mGlobalStorageTierAssoc.getOrdinal(o.getTier())));
    List<alluxio.wire.BlockLocation> locations = new ArrayList<>();
    for (BlockLocation location : blockLocations) {
        MasterWorkerInfo workerInfo = mWorkers.getFirstByField(ID_INDEX, location.getWorkerId());
        if (workerInfo != null) {
            // worker metadata is intentionally not locked here because:
            // - it would be an incorrect order (correct order is lock worker first, then block)
            // - only uses getters of final variables
            locations.add(new alluxio.wire.BlockLocation().setWorkerId(location.getWorkerId()).setWorkerAddress(workerInfo.getWorkerAddress()).setTierAlias(location.getTier()).setMediumType(location.getMediumType()));
        }
    }
    return Optional.of(new BlockInfo().setBlockId(blockId).setLength(block.getLength()).setLocations(locations));
}

Example 4 with BlockMeta

use of alluxio.proto.meta.Block.BlockMeta in project alluxio by Alluxio.

the class DefaultBlockMaster method processWorkerAddedBlocks.

/**
 * Updates the worker and block metadata for blocks added to a worker.
 *
 * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)}
 * with {@link WorkerMetaLockSection#BLOCKS} specified.
 * An exclusive lock is required.
 *
 * @param workerInfo The worker metadata object
 * @param addedBlockIds A mapping from storage tier alias to a list of block ids added
 */
private void processWorkerAddedBlocks(MasterWorkerInfo workerInfo, Map<BlockLocation, List<Long>> addedBlockIds) {
    long invalidBlockCount = 0;
    for (Map.Entry<BlockLocation, List<Long>> entry : addedBlockIds.entrySet()) {
        for (long blockId : entry.getValue()) {
            try (LockResource r = lockBlock(blockId)) {
                Optional<BlockMeta> block = mBlockStore.getBlock(blockId);
                if (block.isPresent()) {
                    workerInfo.addBlock(blockId);
                    BlockLocation location = entry.getKey();
                    Preconditions.checkState(location.getWorkerId() == workerInfo.getId(), "BlockLocation has a different workerId %s from the request sender's workerId %s", location.getWorkerId(), workerInfo.getId());
                    mBlockStore.addLocation(blockId, location);
                    mLostBlocks.remove(blockId);
                } else {
                    invalidBlockCount++;
                    // The block is not recognized and should therefore be purged from the worker
                    // The file may have been removed when the worker was lost
                    workerInfo.scheduleRemoveFromWorker(blockId);
                    LOG.debug("Invalid block: {} from worker {}.", blockId, workerInfo.getWorkerAddress().getHost());
                }
            }
        }
    }
    if (invalidBlockCount > 0) {
        LOG.warn("{} invalid blocks found on worker {} in total", invalidBlockCount, workerInfo.getWorkerAddress().getHost());
    }
}

Example 5 with BlockMeta

use of alluxio.proto.meta.Block.BlockMeta in project alluxio by Alluxio.

the class DefaultBlockMaster method commitBlock.

// TODO(binfan): check the logic is correct or not when commitBlock is a retry
@Override
public void commitBlock(long workerId, long usedBytesOnTier, String tierAlias, String mediumType, long blockId, long length) throws NotFoundException, UnavailableException {
    LOG.debug("Commit block from workerId: {}, usedBytesOnTier: {}, blockId: {}, length: {}", workerId, usedBytesOnTier, blockId, length);
    MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
    // TODO(peis): Check lost workers as well.
    if (worker == null) {
        throw new NotFoundException(ExceptionMessage.NO_WORKER_FOUND.getMessage(workerId));
    }
    try (JournalContext journalContext = createJournalContext()) {
        // The worker metadata must be locked before the blocks
        try (LockResource lr = worker.lockWorkerMeta(EnumSet.of(WorkerMetaLockSection.USAGE, WorkerMetaLockSection.BLOCKS), false)) {
            try (LockResource r = lockBlock(blockId)) {
                Optional<BlockMeta> block = mBlockStore.getBlock(blockId);
                if (!block.isPresent() || block.get().getLength() != length) {
                    if (block.isPresent() && block.get().getLength() != Constants.UNKNOWN_SIZE) {
                        LOG.warn("Rejecting attempt to change block length from {} to {}", block.get().getLength(), length);
                    } else {
                        mBlockStore.putBlock(blockId, BlockMeta.newBuilder().setLength(length).build());
                        BlockInfoEntry blockInfo = BlockInfoEntry.newBuilder().setBlockId(blockId).setLength(length).build();
                        journalContext.append(JournalEntry.newBuilder().setBlockInfo(blockInfo).build());
                    }
                }
                // Update the block metadata with the new worker location.
                mBlockStore.addLocation(blockId, BlockLocation.newBuilder().setWorkerId(workerId).setTier(tierAlias).setMediumType(mediumType).build());
                // This worker has this block, so it is no longer lost.
                mLostBlocks.remove(blockId);
                // Update the worker information for this new block.
                // TODO(binfan): when retry commitBlock on master is expected, make sure metrics are not
                // double counted.
                worker.addBlock(blockId);
                worker.updateUsedBytes(tierAlias, usedBytesOnTier);
            }
        }
        worker.updateLastUpdatedTimeMs();
    }
}