Examples with MasterWorkerInfo alluxio.master.block.meta.MasterWorkerInfo used on opensource projects

Example 16 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class BlockMaster method generateBlockInfo.

/**
   * Creates a {@link BlockInfo} form a given {@link MasterBlockInfo}, by populating worker
   * locations.
   *
   * @param masterBlockInfo the {@link MasterBlockInfo}
   * @return a {@link BlockInfo} from a {@link MasterBlockInfo}. Populates worker locations
   */
@GuardedBy("masterBlockInfo")
private BlockInfo generateBlockInfo(MasterBlockInfo masterBlockInfo) {
    // "Join" to get all the addresses of the workers.
    List<BlockLocation> locations = new ArrayList<>();
    List<MasterBlockLocation> blockLocations = masterBlockInfo.getBlockLocations();
    // Sort the block locations by their alias ordinal in the master storage tier mapping
    Collections.sort(blockLocations, new Comparator<MasterBlockLocation>() {

        @Override
        public int compare(MasterBlockLocation o1, MasterBlockLocation o2) {
            return mGlobalStorageTierAssoc.getOrdinal(o1.getTierAlias()) - mGlobalStorageTierAssoc.getOrdinal(o2.getTierAlias());
        }
    });
    for (MasterBlockLocation masterBlockLocation : blockLocations) {
        MasterWorkerInfo workerInfo = mWorkers.getFirstByField(ID_INDEX, masterBlockLocation.getWorkerId());
        if (workerInfo != null) {
            // worker metadata is intentionally not locked here because:
            // - it would be an incorrect order (correct order is lock worker first, then block)
            // - only uses getters of final variables
            locations.add(new BlockLocation().setWorkerId(masterBlockLocation.getWorkerId()).setWorkerAddress(workerInfo.getWorkerAddress()).setTierAlias(masterBlockLocation.getTierAlias()));
        }
    }
    return new BlockInfo().setBlockId(masterBlockInfo.getBlockId()).setLength(masterBlockInfo.getLength()).setLocations(locations);
}

Example 17 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class BlockMaster method removeBlocks.

/**
   * Removes blocks from workers.
   *
   * @param blockIds a list of block ids to remove from Alluxio space
   * @param delete whether to delete blocks' metadata in Master
   */
public void removeBlocks(List<Long> blockIds, boolean delete) {
    for (long blockId : blockIds) {
        MasterBlockInfo block = mBlocks.get(blockId);
        if (block == null) {
            continue;
        }
        HashSet<Long> workerIds = new HashSet<>();
        synchronized (block) {
            // Technically, 'block' should be confirmed to still be in the data structure. A
            // concurrent removeBlock call can remove it. However, we are intentionally ignoring this
            // race, since deleting the same block again is a noop.
            workerIds.addAll(block.getWorkers());
            // processWorkerRemovedBlocks
            if (delete) {
                // Make sure blockId is removed from mLostBlocks when the block metadata is deleted.
                // Otherwise blockId in mLostBlock can be dangling index if the metadata is gone.
                mLostBlocks.remove(blockId);
                mBlocks.remove(blockId);
            }
        }
        // metadata, since it is essentially an asynchronous signal to the worker to remove the block.
        for (long workerId : workerIds) {
            MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
            if (worker != null) {
                synchronized (worker) {
                    worker.updateToRemovedBlock(true, blockId);
                }
            }
        }
    }
}

Example 18 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class BlockMaster method workerRegister.

/**
   * Updates metadata when a worker registers with the master.
   *
   * @param workerId the worker id of the worker registering
   * @param storageTiers a list of storage tier aliases in order of their position in the worker's
   *        hierarchy
   * @param totalBytesOnTiers a mapping from storage tier alias to total bytes
   * @param usedBytesOnTiers a mapping from storage tier alias to the used byes
   * @param currentBlocksOnTiers a mapping from storage tier alias to a list of blocks
   * @throws NoWorkerException if workerId cannot be found
   */
public void workerRegister(long workerId, List<String> storageTiers, Map<String, Long> totalBytesOnTiers, Map<String, Long> usedBytesOnTiers, Map<String, List<Long>> currentBlocksOnTiers) throws NoWorkerException {
    MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
    if (worker == null) {
        throw new NoWorkerException(ExceptionMessage.NO_WORKER_FOUND.getMessage(workerId));
    }
    // Gather all blocks on this worker.
    HashSet<Long> blocks = new HashSet<>();
    for (List<Long> blockIds : currentBlocksOnTiers.values()) {
        blocks.addAll(blockIds);
    }
    synchronized (worker) {
        worker.updateLastUpdatedTimeMs();
        // Detect any lost blocks on this worker.
        Set<Long> removedBlocks = worker.register(mGlobalStorageTierAssoc, storageTiers, totalBytesOnTiers, usedBytesOnTiers, blocks);
        processWorkerRemovedBlocks(worker, removedBlocks);
        processWorkerAddedBlocks(worker, currentBlocksOnTiers);
    }
    LOG.info("registerWorker(): {}", worker);
}

Example 19 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class DefaultBlockMaster method generateBlockInfo.

/**
 * Generates block info, including worker locations, for a block id.
 * This requires no locks on the {@link MasterWorkerInfo} because it is only reading
 * final fields.
 *
 * @param blockId a block id
 * @return optional block info, empty if the block does not exist
 */
private Optional<BlockInfo> generateBlockInfo(long blockId) throws UnavailableException {
    if (mSafeModeManager.isInSafeMode()) {
        throw new UnavailableException(ExceptionMessage.MASTER_IN_SAFEMODE.getMessage());
    }
    BlockMeta block;
    List<BlockLocation> blockLocations;
    try (LockResource r = lockBlock(blockId)) {
        Optional<BlockMeta> blockOpt = mBlockStore.getBlock(blockId);
        if (!blockOpt.isPresent()) {
            return Optional.empty();
        }
        block = blockOpt.get();
        blockLocations = new ArrayList<>(mBlockStore.getLocations(blockId));
    }
    // Sort the block locations by their alias ordinal in the master storage tier mapping
    Collections.sort(blockLocations, Comparator.comparingInt(o -> mGlobalStorageTierAssoc.getOrdinal(o.getTier())));
    List<alluxio.wire.BlockLocation> locations = new ArrayList<>();
    for (BlockLocation location : blockLocations) {
        MasterWorkerInfo workerInfo = mWorkers.getFirstByField(ID_INDEX, location.getWorkerId());
        if (workerInfo != null) {
            // worker metadata is intentionally not locked here because:
            // - it would be an incorrect order (correct order is lock worker first, then block)
            // - only uses getters of final variables
            locations.add(new alluxio.wire.BlockLocation().setWorkerId(location.getWorkerId()).setWorkerAddress(workerInfo.getWorkerAddress()).setTierAlias(location.getTier()).setMediumType(location.getMediumType()));
        }
    }
    return Optional.of(new BlockInfo().setBlockId(blockId).setLength(block.getLength()).setLocations(locations));
}

Example 20 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class DefaultBlockMaster method getWorkerId.

@Override
public long getWorkerId(WorkerNetAddress workerNetAddress) {
    MasterWorkerInfo existingWorker = mWorkers.getFirstByField(ADDRESS_INDEX, workerNetAddress);
    if (existingWorker != null) {
        // This worker address is already mapped to a worker id.
        long oldWorkerId = existingWorker.getId();
        LOG.warn("The worker {} already exists as id {}.", workerNetAddress, oldWorkerId);
        return oldWorkerId;
    }
    existingWorker = findUnregisteredWorker(workerNetAddress);
    if (existingWorker != null) {
        return existingWorker.getId();
    }
    // Generate a new worker id.
    long workerId = IdUtils.getRandomNonNegativeLong();
    while (!mTempWorkers.add(new MasterWorkerInfo(workerId, workerNetAddress))) {
        workerId = IdUtils.getRandomNonNegativeLong();
    }
    LOG.info("getWorkerId(): WorkerNetAddress: {} id: {}", workerNetAddress, workerId);
    return workerId;
}