Examples with Address alluxio.wire.Address used on opensource projects

Example 1 with Address

use of alluxio.wire.Address in project alluxio by Alluxio.

the class DefaultBlockMaster method workerRegisterStart.

protected void workerRegisterStart(WorkerRegisterContext context, RegisterWorkerPRequest chunk) {
    final List<String> storageTiers = chunk.getStorageTiersList();
    final Map<String, Long> totalBytesOnTiers = chunk.getTotalBytesOnTiersMap();
    final Map<String, Long> usedBytesOnTiers = chunk.getUsedBytesOnTiersMap();
    final Map<String, StorageList> lostStorage = chunk.getLostStorageMap();
    final Map<alluxio.proto.meta.Block.BlockLocation, List<Long>> currentBlocksOnLocation = BlockMasterWorkerServiceHandler.reconstructBlocksOnLocationMap(chunk.getCurrentBlocksList(), context.getWorkerId());
    RegisterWorkerPOptions options = chunk.getOptions();
    MasterWorkerInfo worker = context.mWorker;
    Preconditions.checkState(worker != null, "No worker metadata found in the WorkerRegisterContext!");
    mActiveRegisterContexts.put(worker.getId(), context);
    // The worker is locked so we can operate on its blocks without race conditions
    // We start with assuming all blocks in (mBlocks + mToRemoveBlocks) do not exist.
    // With each batch we receive, we mark them not-to-be-removed.
    // Eventually what's left in the mToRemove will be the ones that do not exist anymore.
    worker.markAllBlocksToRemove();
    worker.updateUsage(mGlobalStorageTierAssoc, storageTiers, totalBytesOnTiers, usedBytesOnTiers);
    processWorkerAddedBlocks(worker, currentBlocksOnLocation);
    processWorkerOrphanedBlocks(worker);
    worker.addLostStorage(lostStorage);
    // TODO(jiacheng): This block can be moved to a non-locked section
    if (options.getConfigsCount() > 0) {
        for (BiConsumer<Address, List<ConfigProperty>> function : mWorkerRegisteredListeners) {
            WorkerNetAddress workerAddress = worker.getWorkerAddress();
            function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort()), options.getConfigsList());
        }
    }
}

Example 2 with Address

use of alluxio.wire.Address in project alluxio by Alluxio.

the class DefaultBlockMaster method recordWorkerRegistration.

/**
 * Re-register a lost worker or complete registration after getting a worker id.
 * This method requires no locking on {@link MasterWorkerInfo} because it is only
 * reading final fields.
 *
 * @param workerId the worker id to register
 */
@Nullable
private MasterWorkerInfo recordWorkerRegistration(long workerId) {
    for (IndexedSet<MasterWorkerInfo> workers : Arrays.asList(mTempWorkers, mLostWorkers)) {
        MasterWorkerInfo worker = workers.getFirstByField(ID_INDEX, workerId);
        if (worker == null) {
            continue;
        }
        mWorkers.add(worker);
        workers.remove(worker);
        if (workers == mLostWorkers) {
            for (Consumer<Address> function : mLostWorkerFoundListeners) {
                // The worker address is final, no need for locking here
                function.accept(new Address(worker.getWorkerAddress().getHost(), worker.getWorkerAddress().getRpcPort()));
            }
            LOG.warn("A lost worker {} has requested its old id {}.", worker.getWorkerAddress(), worker.getId());
        }
        return worker;
    }
    return null;
}

Example 3 with Address

use of alluxio.wire.Address in project alluxio by Alluxio.

the class DefaultBlockMaster method workerRegister.

@Override
public void workerRegister(long workerId, List<String> storageTiers, Map<String, Long> totalBytesOnTiers, Map<String, Long> usedBytesOnTiers, Map<BlockLocation, List<Long>> currentBlocksOnLocation, Map<String, StorageList> lostStorage, RegisterWorkerPOptions options) throws NotFoundException {
    MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
    if (worker == null) {
        worker = findUnregisteredWorker(workerId);
    }
    if (worker == null) {
        throw new NotFoundException(ExceptionMessage.NO_WORKER_FOUND.getMessage(workerId));
    }
    // Gather all blocks on this worker.
    int totalSize = currentBlocksOnLocation.values().stream().mapToInt(List::size).sum();
    HashSet<Long> blocks = new HashSet<>(totalSize);
    for (List<Long> blockIds : currentBlocksOnLocation.values()) {
        blocks.addAll(blockIds);
    }
    // Lock all the locks
    try (LockResource r = worker.lockWorkerMeta(EnumSet.of(WorkerMetaLockSection.STATUS, WorkerMetaLockSection.USAGE, WorkerMetaLockSection.BLOCKS), false)) {
        // Detect any lost blocks on this worker.
        Set<Long> removedBlocks = worker.register(mGlobalStorageTierAssoc, storageTiers, totalBytesOnTiers, usedBytesOnTiers, blocks);
        processWorkerRemovedBlocks(worker, removedBlocks, false);
        processWorkerAddedBlocks(worker, currentBlocksOnLocation);
        processWorkerOrphanedBlocks(worker);
        worker.addLostStorage(lostStorage);
    }
    if (options.getConfigsCount() > 0) {
        for (BiConsumer<Address, List<ConfigProperty>> function : mWorkerRegisteredListeners) {
            WorkerNetAddress workerAddress = worker.getWorkerAddress();
            function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort()), options.getConfigsList());
        }
    }
    recordWorkerRegistration(workerId);
    // Update the TS at the end of the process
    worker.updateLastUpdatedTimeMs();
    // Invalidate cache to trigger new build of worker info list
    mWorkerInfoCache.invalidate(WORKER_INFO_CACHE_KEY);
    LOG.info("registerWorker(): {}", worker);
}

Example 4 with Address

use of alluxio.wire.Address in project alluxio by Alluxio.

the class DefaultBlockMaster method processLostWorker.

/**
 * Updates the metadata for the specified lost worker.
 *
 * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)}
 * with {@link WorkerMetaLockSection#BLOCKS} specified.
 * An exclusive lock is required.
 *
 * @param worker the worker metadata
 */
private void processLostWorker(MasterWorkerInfo worker) {
    mLostWorkers.add(worker);
    mWorkers.remove(worker);
    WorkerNetAddress workerAddress = worker.getWorkerAddress();
    for (Consumer<Address> function : mWorkerLostListeners) {
        function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort()));
    }
    // We only remove the blocks from master locations but do not
    // mark these blocks to-remove from the worker.
    // So if the worker comes back again the blocks are kept.
    processWorkerRemovedBlocks(worker, worker.getBlocks(), false);
}

Example 5 with Address

use of alluxio.wire.Address in project alluxio by Alluxio.

the class ServerConfigurationChecker method fillConfMap.

/**
 * Fills the configuration map.
 *
 * @param targetMap the map to fill
 * @param recordMap the map to get data from
 */
private void fillConfMap(Map<PropertyKey, Map<Optional<String>, List<String>>> targetMap, Map<Address, List<ConfigRecord>> recordMap) {
    for (Map.Entry<Address, List<ConfigRecord>> record : recordMap.entrySet()) {
        Address address = record.getKey();
        String addressStr = String.format("%s:%s", address.getHost(), address.getRpcPort());
        for (ConfigRecord conf : record.getValue()) {
            PropertyKey key = conf.getKey();
            if (key.getConsistencyLevel() == ConsistencyCheckLevel.IGNORE) {
                continue;
            }
            Optional<String> value = conf.getValue();
            targetMap.putIfAbsent(key, new HashMap<>());
            Map<Optional<String>, List<String>> values = targetMap.get(key);
            values.putIfAbsent(value, new ArrayList<>());
            values.get(value).add(addressStr);
        }
    }
}