Examples with MasterWorkerInfo alluxio.master.block.meta.MasterWorkerInfo used on opensource projects

Example 1 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class BlockMasterPrivateAccess method isWorkerRegistered.

/**
   * Checks whether a worker with the given workerId is registered with the given block master.
   *
   * @param master the block master
   * @param workerId the workerId
   * @return true if the worker has registered, false otherwise
   */
public static boolean isWorkerRegistered(BlockMaster master, long workerId) {
    IndexedSet<MasterWorkerInfo> workers = Whitebox.getInternalState(master, "mWorkers");
    IndexDefinition<MasterWorkerInfo> idIndex = Whitebox.getInternalState(BlockMaster.class, "ID_INDEX");
    synchronized (workers) {
        MasterWorkerInfo workerInfo = workers.getFirstByField(idIndex, workerId);
        return workerInfo != null && workerInfo.isRegistered();
    }
}

Example 2 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class BlockMaster method getUsedBytesOnTiers.

/**
   * @return the used bytes on each storage tier
   */
public Map<String, Long> getUsedBytesOnTiers() {
    Map<String, Long> ret = new HashMap<>();
    for (MasterWorkerInfo worker : mWorkers) {
        synchronized (worker) {
            for (Map.Entry<String, Long> entry : worker.getUsedBytesOnTiers().entrySet()) {
                Long used = ret.get(entry.getKey());
                ret.put(entry.getKey(), (used == null ? 0L : used) + entry.getValue());
            }
        }
    }
    return ret;
}

Example 3 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class BlockMaster method getWorkerId.

/**
   * Returns a worker id for the given worker, creating one if the worker is new.
   *
   * @param workerNetAddress the worker {@link WorkerNetAddress}
   * @return the worker id for this worker
   */
public long getWorkerId(WorkerNetAddress workerNetAddress) {
    // TODO(gpang): Clone WorkerNetAddress in case thrift re-uses the object. Does thrift re-use it?
    MasterWorkerInfo existingWorker = mWorkers.getFirstByField(ADDRESS_INDEX, workerNetAddress);
    if (existingWorker != null) {
        // This worker address is already mapped to a worker id.
        long oldWorkerId = existingWorker.getId();
        LOG.warn("The worker {} already exists as id {}.", workerNetAddress, oldWorkerId);
        return oldWorkerId;
    }
    MasterWorkerInfo lostWorker = mLostWorkers.getFirstByField(ADDRESS_INDEX, workerNetAddress);
    if (lostWorker != null) {
        // this is one of the lost workers
        synchronized (lostWorker) {
            final long lostWorkerId = lostWorker.getId();
            LOG.warn("A lost worker {} has requested its old id {}.", workerNetAddress, lostWorkerId);
            // Update the timestamp of the worker before it is considered an active worker.
            lostWorker.updateLastUpdatedTimeMs();
            mWorkers.add(lostWorker);
            mLostWorkers.remove(lostWorker);
            return lostWorkerId;
        }
    }
    // Generate a new worker id.
    long workerId = IdUtils.getRandomNonNegativeLong();
    while (!mWorkers.add(new MasterWorkerInfo(workerId, workerNetAddress))) {
        workerId = IdUtils.getRandomNonNegativeLong();
    }
    LOG.info("getWorkerId(): WorkerNetAddress: {} id: {}", workerNetAddress, workerId);
    return workerId;
}

Example 4 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class DefaultBlockMaster method workerRegisterStart.

protected void workerRegisterStart(WorkerRegisterContext context, RegisterWorkerPRequest chunk) {
    final List<String> storageTiers = chunk.getStorageTiersList();
    final Map<String, Long> totalBytesOnTiers = chunk.getTotalBytesOnTiersMap();
    final Map<String, Long> usedBytesOnTiers = chunk.getUsedBytesOnTiersMap();
    final Map<String, StorageList> lostStorage = chunk.getLostStorageMap();
    final Map<alluxio.proto.meta.Block.BlockLocation, List<Long>> currentBlocksOnLocation = BlockMasterWorkerServiceHandler.reconstructBlocksOnLocationMap(chunk.getCurrentBlocksList(), context.getWorkerId());
    RegisterWorkerPOptions options = chunk.getOptions();
    MasterWorkerInfo worker = context.mWorker;
    Preconditions.checkState(worker != null, "No worker metadata found in the WorkerRegisterContext!");
    mActiveRegisterContexts.put(worker.getId(), context);
    // The worker is locked so we can operate on its blocks without race conditions
    // We start with assuming all blocks in (mBlocks + mToRemoveBlocks) do not exist.
    // With each batch we receive, we mark them not-to-be-removed.
    // Eventually what's left in the mToRemove will be the ones that do not exist anymore.
    worker.markAllBlocksToRemove();
    worker.updateUsage(mGlobalStorageTierAssoc, storageTiers, totalBytesOnTiers, usedBytesOnTiers);
    processWorkerAddedBlocks(worker, currentBlocksOnLocation);
    processWorkerOrphanedBlocks(worker);
    worker.addLostStorage(lostStorage);
    // TODO(jiacheng): This block can be moved to a non-locked section
    if (options.getConfigsCount() > 0) {
        for (BiConsumer<Address, List<ConfigProperty>> function : mWorkerRegisteredListeners) {
            WorkerNetAddress workerAddress = worker.getWorkerAddress();
            function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort()), options.getConfigsList());
        }
    }
}

Example 5 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class DefaultBlockMaster method recordWorkerRegistration.

/**
 * Re-register a lost worker or complete registration after getting a worker id.
 * This method requires no locking on {@link MasterWorkerInfo} because it is only
 * reading final fields.
 *
 * @param workerId the worker id to register
 */
@Nullable
private MasterWorkerInfo recordWorkerRegistration(long workerId) {
    for (IndexedSet<MasterWorkerInfo> workers : Arrays.asList(mTempWorkers, mLostWorkers)) {
        MasterWorkerInfo worker = workers.getFirstByField(ID_INDEX, workerId);
        if (worker == null) {
            continue;
        }
        mWorkers.add(worker);
        workers.remove(worker);
        if (workers == mLostWorkers) {
            for (Consumer<Address> function : mLostWorkerFoundListeners) {
                // The worker address is final, no need for locking here
                function.accept(new Address(worker.getWorkerAddress().getHost(), worker.getWorkerAddress().getRpcPort()));
            }
            LOG.warn("A lost worker {} has requested its old id {}.", worker.getWorkerAddress(), worker.getId());
        }
        return worker;
    }
    return null;
}