Examples with MasterWorkerInfo alluxio.master.block.meta.MasterWorkerInfo used on opensource projects

Example 21 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class DefaultBlockMaster method workerRegisterBatch.

protected void workerRegisterBatch(WorkerRegisterContext context, RegisterWorkerPRequest chunk) {
    final Map<alluxio.proto.meta.Block.BlockLocation, List<Long>> currentBlocksOnLocation = BlockMasterWorkerServiceHandler.reconstructBlocksOnLocationMap(chunk.getCurrentBlocksList(), context.getWorkerId());
    MasterWorkerInfo worker = context.mWorker;
    Preconditions.checkState(worker != null, "No worker metadata found in the WorkerRegisterContext!");
    // Even if we add the BlockLocation before the worker is fully registered,
    // it should be fine because the block can be read on this worker.
    // If the stream fails in the middle, the blocks recorded on the MasterWorkerInfo
    // will be removed by processLostWorker()
    processWorkerAddedBlocks(worker, currentBlocksOnLocation);
    processWorkerOrphanedBlocks(worker);
    // Update the TS at the end of the process
    worker.updateLastUpdatedTimeMs();
}

Example 22 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class DefaultBlockMaster method getLostWorkersInfoList.

@Override
public List<WorkerInfo> getLostWorkersInfoList() throws UnavailableException {
    if (mSafeModeManager.isInSafeMode()) {
        throw new UnavailableException(ExceptionMessage.MASTER_IN_SAFEMODE.getMessage());
    }
    List<WorkerInfo> workerInfoList = new ArrayList<>(mLostWorkers.size());
    for (MasterWorkerInfo worker : mLostWorkers) {
        // extractWorkerInfo handles the locking internally
        workerInfoList.add(extractWorkerInfo(worker, null, false));
    }
    Collections.sort(workerInfoList, new WorkerInfo.LastContactSecComparator());
    return workerInfoList;
}

Example 23 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class DefaultBlockMaster method getWorkerReport.

@Override
public List<WorkerInfo> getWorkerReport(GetWorkerReportOptions options) throws UnavailableException, InvalidArgumentException {
    if (mSafeModeManager.isInSafeMode()) {
        throw new UnavailableException(ExceptionMessage.MASTER_IN_SAFEMODE.getMessage());
    }
    Set<MasterWorkerInfo> selectedLiveWorkers = new HashSet<>();
    Set<MasterWorkerInfo> selectedLostWorkers = new HashSet<>();
    WorkerRange workerRange = options.getWorkerRange();
    switch(workerRange) {
        case ALL:
            selectedLiveWorkers.addAll(mWorkers);
            selectedLostWorkers.addAll(mLostWorkers);
            break;
        case LIVE:
            selectedLiveWorkers.addAll(mWorkers);
            break;
        case LOST:
            selectedLostWorkers.addAll(mLostWorkers);
            break;
        case SPECIFIED:
            Set<String> addresses = options.getAddresses();
            Set<String> workerNames = new HashSet<>();
            selectedLiveWorkers = selectInfoByAddress(addresses, mWorkers, workerNames);
            selectedLostWorkers = selectInfoByAddress(addresses, mLostWorkers, workerNames);
            if (!addresses.isEmpty()) {
                String info = String.format("Unrecognized worker names: %s%n" + "Supported worker names: %s%n", addresses.toString(), workerNames.toString());
                throw new InvalidArgumentException(info);
            }
            break;
        default:
            throw new InvalidArgumentException("Unrecognized worker range: " + workerRange);
    }
    List<WorkerInfo> workerInfoList = new ArrayList<>(selectedLiveWorkers.size() + selectedLostWorkers.size());
    for (MasterWorkerInfo worker : selectedLiveWorkers) {
        // extractWorkerInfo handles the locking internally
        workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), true));
    }
    for (MasterWorkerInfo worker : selectedLostWorkers) {
        // extractWorkerInfo handles the locking internally
        workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), false));
    }
    return workerInfoList;
}

Example 24 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class DefaultBlockMaster method commitBlock.

// TODO(binfan): check the logic is correct or not when commitBlock is a retry
@Override
public void commitBlock(long workerId, long usedBytesOnTier, String tierAlias, String mediumType, long blockId, long length) throws NotFoundException, UnavailableException {
    LOG.debug("Commit block from workerId: {}, usedBytesOnTier: {}, blockId: {}, length: {}", workerId, usedBytesOnTier, blockId, length);
    MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
    // TODO(peis): Check lost workers as well.
    if (worker == null) {
        throw new NotFoundException(ExceptionMessage.NO_WORKER_FOUND.getMessage(workerId));
    }
    try (JournalContext journalContext = createJournalContext()) {
        // The worker metadata must be locked before the blocks
        try (LockResource lr = worker.lockWorkerMeta(EnumSet.of(WorkerMetaLockSection.USAGE, WorkerMetaLockSection.BLOCKS), false)) {
            try (LockResource r = lockBlock(blockId)) {
                Optional<BlockMeta> block = mBlockStore.getBlock(blockId);
                if (!block.isPresent() || block.get().getLength() != length) {
                    if (block.isPresent() && block.get().getLength() != Constants.UNKNOWN_SIZE) {
                        LOG.warn("Rejecting attempt to change block length from {} to {}", block.get().getLength(), length);
                    } else {
                        mBlockStore.putBlock(blockId, BlockMeta.newBuilder().setLength(length).build());
                        BlockInfoEntry blockInfo = BlockInfoEntry.newBuilder().setBlockId(blockId).setLength(length).build();
                        journalContext.append(JournalEntry.newBuilder().setBlockInfo(blockInfo).build());
                    }
                }
                // Update the block metadata with the new worker location.
                mBlockStore.addLocation(blockId, BlockLocation.newBuilder().setWorkerId(workerId).setTier(tierAlias).setMediumType(mediumType).build());
                // This worker has this block, so it is no longer lost.
                mLostBlocks.remove(blockId);
                // Update the worker information for this new block.
                // TODO(binfan): when retry commitBlock on master is expected, make sure metrics are not
                // double counted.
                worker.addBlock(blockId);
                worker.updateUsedBytes(tierAlias, usedBytesOnTier);
            }
        }
        worker.updateLastUpdatedTimeMs();
    }
}

Example 25 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class DefaultBlockMaster method getUsedBytesOnTiers.

@Override
public Map<String, Long> getUsedBytesOnTiers() {
    Map<String, Long> ret = new HashMap<>();
    for (MasterWorkerInfo worker : mWorkers) {
        try (LockResource r = worker.lockWorkerMeta(EnumSet.of(WorkerMetaLockSection.USAGE), true)) {
            for (Map.Entry<String, Long> entry : worker.getUsedBytesOnTiers().entrySet()) {
                Long used = ret.get(entry.getKey());
                ret.put(entry.getKey(), (used == null ? 0L : used) + entry.getValue());
            }
        }
    }
    return ret;
}