Examples with BlockLocation alluxio.proto.meta.Block.BlockLocation used on opensource projects

Example 1 with BlockLocation

use of alluxio.proto.meta.Block.BlockLocation in project alluxio by Alluxio.

the class DefaultBlockMaster method generateBlockInfo.

/**
 * Generates block info, including worker locations, for a block id.
 * This requires no locks on the {@link MasterWorkerInfo} because it is only reading
 * final fields.
 *
 * @param blockId a block id
 * @return optional block info, empty if the block does not exist
 */
private Optional<BlockInfo> generateBlockInfo(long blockId) throws UnavailableException {
    if (mSafeModeManager.isInSafeMode()) {
        throw new UnavailableException(ExceptionMessage.MASTER_IN_SAFEMODE.getMessage());
    }
    BlockMeta block;
    List<BlockLocation> blockLocations;
    try (LockResource r = lockBlock(blockId)) {
        Optional<BlockMeta> blockOpt = mBlockStore.getBlock(blockId);
        if (!blockOpt.isPresent()) {
            return Optional.empty();
        }
        block = blockOpt.get();
        blockLocations = new ArrayList<>(mBlockStore.getLocations(blockId));
    }
    // Sort the block locations by their alias ordinal in the master storage tier mapping
    Collections.sort(blockLocations, Comparator.comparingInt(o -> mGlobalStorageTierAssoc.getOrdinal(o.getTier())));
    List<alluxio.wire.BlockLocation> locations = new ArrayList<>();
    for (BlockLocation location : blockLocations) {
        MasterWorkerInfo workerInfo = mWorkers.getFirstByField(ID_INDEX, location.getWorkerId());
        if (workerInfo != null) {
            // worker metadata is intentionally not locked here because:
            // - it would be an incorrect order (correct order is lock worker first, then block)
            // - only uses getters of final variables
            locations.add(new alluxio.wire.BlockLocation().setWorkerId(location.getWorkerId()).setWorkerAddress(workerInfo.getWorkerAddress()).setTierAlias(location.getTier()).setMediumType(location.getMediumType()));
        }
    }
    return Optional.of(new BlockInfo().setBlockId(blockId).setLength(block.getLength()).setLocations(locations));
}

Example 2 with BlockLocation

use of alluxio.proto.meta.Block.BlockLocation in project alluxio by Alluxio.

the class DefaultBlockMaster method processWorkerAddedBlocks.

/**
 * Updates the worker and block metadata for blocks added to a worker.
 *
 * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)}
 * with {@link WorkerMetaLockSection#BLOCKS} specified.
 * An exclusive lock is required.
 *
 * @param workerInfo The worker metadata object
 * @param addedBlockIds A mapping from storage tier alias to a list of block ids added
 */
private void processWorkerAddedBlocks(MasterWorkerInfo workerInfo, Map<BlockLocation, List<Long>> addedBlockIds) {
    long invalidBlockCount = 0;
    for (Map.Entry<BlockLocation, List<Long>> entry : addedBlockIds.entrySet()) {
        for (long blockId : entry.getValue()) {
            try (LockResource r = lockBlock(blockId)) {
                Optional<BlockMeta> block = mBlockStore.getBlock(blockId);
                if (block.isPresent()) {
                    workerInfo.addBlock(blockId);
                    BlockLocation location = entry.getKey();
                    Preconditions.checkState(location.getWorkerId() == workerInfo.getId(), "BlockLocation has a different workerId %s from the request sender's workerId %s", location.getWorkerId(), workerInfo.getId());
                    mBlockStore.addLocation(blockId, location);
                    mLostBlocks.remove(blockId);
                } else {
                    invalidBlockCount++;
                    // The block is not recognized and should therefore be purged from the worker
                    // The file may have been removed when the worker was lost
                    workerInfo.scheduleRemoveFromWorker(blockId);
                    LOG.debug("Invalid block: {} from worker {}.", blockId, workerInfo.getWorkerAddress().getHost());
                }
            }
        }
    }
    if (invalidBlockCount > 0) {
        LOG.warn("{} invalid blocks found on worker {} in total", invalidBlockCount, workerInfo.getWorkerAddress().getHost());
    }
}

Example 3 with BlockLocation

use of alluxio.proto.meta.Block.BlockLocation in project alluxio by Alluxio.

the class DefaultBlockMaster method workerRegisterBatch.

protected void workerRegisterBatch(WorkerRegisterContext context, RegisterWorkerPRequest chunk) {
    final Map<alluxio.proto.meta.Block.BlockLocation, List<Long>> currentBlocksOnLocation = BlockMasterWorkerServiceHandler.reconstructBlocksOnLocationMap(chunk.getCurrentBlocksList(), context.getWorkerId());
    MasterWorkerInfo worker = context.mWorker;
    Preconditions.checkState(worker != null, "No worker metadata found in the WorkerRegisterContext!");
    // Even if we add the BlockLocation before the worker is fully registered,
    // it should be fine because the block can be read on this worker.
    // If the stream fails in the middle, the blocks recorded on the MasterWorkerInfo
    // will be removed by processLostWorker()
    processWorkerAddedBlocks(worker, currentBlocksOnLocation);
    processWorkerOrphanedBlocks(worker);
    // Update the TS at the end of the process
    worker.updateLastUpdatedTimeMs();
}

Example 4 with BlockLocation

use of alluxio.proto.meta.Block.BlockLocation in project alluxio by Alluxio.

the class DefaultBlockMaster method removeBlocks.

@Override
public void removeBlocks(Collection<Long> blockIds, boolean delete) throws UnavailableException {
    try (JournalContext journalContext = createJournalContext()) {
        for (long blockId : blockIds) {
            Set<Long> workerIds;
            try (LockResource r = lockBlock(blockId)) {
                Optional<BlockMeta> block = mBlockStore.getBlock(blockId);
                if (!block.isPresent()) {
                    continue;
                }
                List<BlockLocation> locations = mBlockStore.getLocations(blockId);
                workerIds = new HashSet<>(locations.size());
                for (BlockLocation loc : locations) {
                    workerIds.add(loc.getWorkerId());
                }
                // processWorkerRemovedBlocks
                if (delete) {
                    // Make sure blockId is removed from mLostBlocks when the block metadata is deleted.
                    // Otherwise blockId in mLostBlock can be dangling index if the metadata is gone.
                    mLostBlocks.remove(blockId);
                    mBlockStore.removeBlock(blockId);
                    JournalEntry entry = JournalEntry.newBuilder().setDeleteBlock(DeleteBlockEntry.newBuilder().setBlockId(blockId)).build();
                    journalContext.append(entry);
                }
            }
            // workerRegister should be changed to address this race condition.
            for (long workerId : workerIds) {
                MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
                if (worker != null) {
                    try (LockResource r = worker.lockWorkerMeta(EnumSet.of(WorkerMetaLockSection.BLOCKS), false)) {
                        worker.updateToRemovedBlock(true, blockId);
                    }
                }
            }
        }
    }
}

Example 5 with BlockLocation

use of alluxio.proto.meta.Block.BlockLocation in project alluxio by Alluxio.

the class RocksBlockStore method getLocations.

@Override
public List<BlockLocation> getLocations(long id) {
    byte[] startKey = RocksUtils.toByteArray(id, 0);
    byte[] endKey = RocksUtils.toByteArray(id, Long.MAX_VALUE);
    // Ref: https://docs.oracle.com/javase/tutorial/essential/exceptions/tryResourceClose.html
    try (final Slice slice = new Slice(endKey);
        final ReadOptions readOptions = new ReadOptions().setIterateUpperBound(slice);
        final RocksIterator iter = db().newIterator(mBlockLocationsColumn.get(), readOptions)) {
        iter.seek(startKey);
        List<BlockLocation> locations = new ArrayList<>();
        for (; iter.isValid(); iter.next()) {
            try {
                locations.add(BlockLocation.parseFrom(iter.value()));
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
        return locations;
    }
}