Examples with RegisterWorkerPOptions alluxio.grpc.RegisterWorkerPOptions used on opensource projects

Example 1 with RegisterWorkerPOptions

use of alluxio.grpc.RegisterWorkerPOptions in project alluxio by Alluxio.

the class DefaultBlockMaster method workerRegisterStart.

protected void workerRegisterStart(WorkerRegisterContext context, RegisterWorkerPRequest chunk) {
    final List<String> storageTiers = chunk.getStorageTiersList();
    final Map<String, Long> totalBytesOnTiers = chunk.getTotalBytesOnTiersMap();
    final Map<String, Long> usedBytesOnTiers = chunk.getUsedBytesOnTiersMap();
    final Map<String, StorageList> lostStorage = chunk.getLostStorageMap();
    final Map<alluxio.proto.meta.Block.BlockLocation, List<Long>> currentBlocksOnLocation = BlockMasterWorkerServiceHandler.reconstructBlocksOnLocationMap(chunk.getCurrentBlocksList(), context.getWorkerId());
    RegisterWorkerPOptions options = chunk.getOptions();
    MasterWorkerInfo worker = context.mWorker;
    Preconditions.checkState(worker != null, "No worker metadata found in the WorkerRegisterContext!");
    mActiveRegisterContexts.put(worker.getId(), context);
    // The worker is locked so we can operate on its blocks without race conditions
    // We start with assuming all blocks in (mBlocks + mToRemoveBlocks) do not exist.
    // With each batch we receive, we mark them not-to-be-removed.
    // Eventually what's left in the mToRemove will be the ones that do not exist anymore.
    worker.markAllBlocksToRemove();
    worker.updateUsage(mGlobalStorageTierAssoc, storageTiers, totalBytesOnTiers, usedBytesOnTiers);
    processWorkerAddedBlocks(worker, currentBlocksOnLocation);
    processWorkerOrphanedBlocks(worker);
    worker.addLostStorage(lostStorage);
    // TODO(jiacheng): This block can be moved to a non-locked section
    if (options.getConfigsCount() > 0) {
        for (BiConsumer<Address, List<ConfigProperty>> function : mWorkerRegisteredListeners) {
            WorkerNetAddress workerAddress = worker.getWorkerAddress();
            function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort()), options.getConfigsList());
        }
    }
}

Example 2 with RegisterWorkerPOptions

use of alluxio.grpc.RegisterWorkerPOptions in project alluxio by Alluxio.

the class BlockMasterWorkerServiceHandler method registerWorker.

@Override
public void registerWorker(RegisterWorkerPRequest request, StreamObserver<RegisterWorkerPResponse> responseObserver) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Register worker request is {} bytes, containing {} blocks", request.getSerializedSize(), request.getCurrentBlocksCount());
    }
    final long workerId = request.getWorkerId();
    RegisterWorkerPOptions options = request.getOptions();
    RpcUtils.call(LOG, (RpcUtils.RpcCallableThrowsIOException<RegisterWorkerPResponse>) () -> {
        // The exception will be propagated to the worker side and the worker should retry.
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_WORKER_REGISTER_LEASE_ENABLED) && !mBlockMaster.hasRegisterLease(workerId)) {
            String errorMsg = String.format("Worker %s does not have a lease or the lease " + "has expired. The worker should acquire a new lease and retry to register.", workerId);
            LOG.warn(errorMsg);
            throw new RegisterLeaseNotFoundException(errorMsg);
        }
        LOG.debug("Worker {} proceeding to register...", workerId);
        final List<String> storageTiers = request.getStorageTiersList();
        final Map<String, Long> totalBytesOnTiers = request.getTotalBytesOnTiersMap();
        final Map<String, Long> usedBytesOnTiers = request.getUsedBytesOnTiersMap();
        final Map<String, StorageList> lostStorageMap = request.getLostStorageMap();
        final Map<Block.BlockLocation, List<Long>> currBlocksOnLocationMap = reconstructBlocksOnLocationMap(request.getCurrentBlocksList(), workerId);
        // If the register is unsuccessful, the lease will be kept around until the expiry.
        // The worker can retry and use the existing lease.
        mBlockMaster.workerRegister(workerId, storageTiers, totalBytesOnTiers, usedBytesOnTiers, currBlocksOnLocationMap, lostStorageMap, options);
        LOG.info("Worker {} finished registering, releasing its lease.", workerId);
        mBlockMaster.releaseRegisterLease(workerId);
        return RegisterWorkerPResponse.getDefaultInstance();
    }, "registerWorker", true, "request=%s", responseObserver, workerId);
}

Example 3 with RegisterWorkerPOptions

use of alluxio.grpc.RegisterWorkerPOptions in project alluxio by Alluxio.

the class BlockMasterClient method register.

/**
 * The method the worker should execute to register with the block master.
 *
 * @param workerId the worker id of the worker registering
 * @param storageTierAliases a list of storage tier aliases in ordinal order
 * @param totalBytesOnTiers mapping from storage tier alias to total bytes
 * @param usedBytesOnTiers mapping from storage tier alias to used bytes
 * @param currentBlocksOnLocation mapping from storage tier alias to the list of list of blocks
 * @param lostStorage mapping from storage tier alias to the list of lost storage paths
 * @param configList a list of configurations
 */
// TODO(yupeng): rename to workerBlockReport or workerInitialize?
public void register(final long workerId, final List<String> storageTierAliases, final Map<String, Long> totalBytesOnTiers, final Map<String, Long> usedBytesOnTiers, final Map<BlockStoreLocation, List<Long>> currentBlocksOnLocation, final Map<String, List<String>> lostStorage, final List<ConfigProperty> configList) throws IOException {
    final RegisterWorkerPOptions options = RegisterWorkerPOptions.newBuilder().addAllConfigs(configList).build();
    final List<LocationBlockIdListEntry> currentBlocks = convertBlockListMapToProto(currentBlocksOnLocation);
    final Map<String, StorageList> lostStorageMap = lostStorage.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> StorageList.newBuilder().addAllStorage(e.getValue()).build()));
    final RegisterWorkerPRequest request = RegisterWorkerPRequest.newBuilder().setWorkerId(workerId).addAllStorageTiers(storageTierAliases).putAllTotalBytesOnTiers(totalBytesOnTiers).putAllUsedBytesOnTiers(usedBytesOnTiers).addAllCurrentBlocks(currentBlocks).putAllLostStorage(lostStorageMap).setOptions(options).build();
    retryRPC(() -> {
        mClient.registerWorker(request);
        return null;
    }, LOG, "Register", "workerId=%d", workerId);
}