Examples with StorageList alluxio.grpc.StorageList used on opensource projects

Example 1 with StorageList

use of alluxio.grpc.StorageList in project alluxio by Alluxio.

the class DefaultBlockMaster method workerRegisterStart.

protected void workerRegisterStart(WorkerRegisterContext context, RegisterWorkerPRequest chunk) {
    final List<String> storageTiers = chunk.getStorageTiersList();
    final Map<String, Long> totalBytesOnTiers = chunk.getTotalBytesOnTiersMap();
    final Map<String, Long> usedBytesOnTiers = chunk.getUsedBytesOnTiersMap();
    final Map<String, StorageList> lostStorage = chunk.getLostStorageMap();
    final Map<alluxio.proto.meta.Block.BlockLocation, List<Long>> currentBlocksOnLocation = BlockMasterWorkerServiceHandler.reconstructBlocksOnLocationMap(chunk.getCurrentBlocksList(), context.getWorkerId());
    RegisterWorkerPOptions options = chunk.getOptions();
    MasterWorkerInfo worker = context.mWorker;
    Preconditions.checkState(worker != null, "No worker metadata found in the WorkerRegisterContext!");
    mActiveRegisterContexts.put(worker.getId(), context);
    // The worker is locked so we can operate on its blocks without race conditions
    // We start with assuming all blocks in (mBlocks + mToRemoveBlocks) do not exist.
    // With each batch we receive, we mark them not-to-be-removed.
    // Eventually what's left in the mToRemove will be the ones that do not exist anymore.
    worker.markAllBlocksToRemove();
    worker.updateUsage(mGlobalStorageTierAssoc, storageTiers, totalBytesOnTiers, usedBytesOnTiers);
    processWorkerAddedBlocks(worker, currentBlocksOnLocation);
    processWorkerOrphanedBlocks(worker);
    worker.addLostStorage(lostStorage);
    // TODO(jiacheng): This block can be moved to a non-locked section
    if (options.getConfigsCount() > 0) {
        for (BiConsumer<Address, List<ConfigProperty>> function : mWorkerRegisteredListeners) {
            WorkerNetAddress workerAddress = worker.getWorkerAddress();
            function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort()), options.getConfigsList());
        }
    }
}

Example 2 with StorageList

use of alluxio.grpc.StorageList in project alluxio by Alluxio.

the class DefaultBlockMaster method workerRegister.

@Override
public void workerRegister(long workerId, List<String> storageTiers, Map<String, Long> totalBytesOnTiers, Map<String, Long> usedBytesOnTiers, Map<BlockLocation, List<Long>> currentBlocksOnLocation, Map<String, StorageList> lostStorage, RegisterWorkerPOptions options) throws NotFoundException {
    MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
    if (worker == null) {
        worker = findUnregisteredWorker(workerId);
    }
    if (worker == null) {
        throw new NotFoundException(ExceptionMessage.NO_WORKER_FOUND.getMessage(workerId));
    }
    // Gather all blocks on this worker.
    int totalSize = currentBlocksOnLocation.values().stream().mapToInt(List::size).sum();
    HashSet<Long> blocks = new HashSet<>(totalSize);
    for (List<Long> blockIds : currentBlocksOnLocation.values()) {
        blocks.addAll(blockIds);
    }
    // Lock all the locks
    try (LockResource r = worker.lockWorkerMeta(EnumSet.of(WorkerMetaLockSection.STATUS, WorkerMetaLockSection.USAGE, WorkerMetaLockSection.BLOCKS), false)) {
        // Detect any lost blocks on this worker.
        Set<Long> removedBlocks = worker.register(mGlobalStorageTierAssoc, storageTiers, totalBytesOnTiers, usedBytesOnTiers, blocks);
        processWorkerRemovedBlocks(worker, removedBlocks, false);
        processWorkerAddedBlocks(worker, currentBlocksOnLocation);
        processWorkerOrphanedBlocks(worker);
        worker.addLostStorage(lostStorage);
    }
    if (options.getConfigsCount() > 0) {
        for (BiConsumer<Address, List<ConfigProperty>> function : mWorkerRegisteredListeners) {
            WorkerNetAddress workerAddress = worker.getWorkerAddress();
            function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort()), options.getConfigsList());
        }
    }
    recordWorkerRegistration(workerId);
    // Update the TS at the end of the process
    worker.updateLastUpdatedTimeMs();
    // Invalidate cache to trigger new build of worker info list
    mWorkerInfoCache.invalidate(WORKER_INFO_CACHE_KEY);
    LOG.info("registerWorker(): {}", worker);
}

Example 3 with StorageList

use of alluxio.grpc.StorageList in project alluxio by Alluxio.

the class DefaultBlockMaster method getWorkerLostStorage.

@Override
public List<WorkerLostStorageInfo> getWorkerLostStorage() {
    List<WorkerLostStorageInfo> workerLostStorageList = new ArrayList<>();
    for (MasterWorkerInfo worker : mWorkers) {
        try (LockResource r = worker.lockWorkerMeta(EnumSet.of(WorkerMetaLockSection.USAGE), true)) {
            if (worker.hasLostStorage()) {
                Map<String, StorageList> lostStorage = worker.getLostStorage().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> StorageList.newBuilder().addAllStorage(e.getValue()).build()));
                workerLostStorageList.add(WorkerLostStorageInfo.newBuilder().setAddress(GrpcUtils.toProto(worker.getWorkerAddress())).putAllLostStorage(lostStorage).build());
            }
        }
    }
    return workerLostStorageList;
}

Example 4 with StorageList

use of alluxio.grpc.StorageList in project alluxio by Alluxio.

the class BlockMasterClient method heartbeat.

/**
 * The method the worker should periodically execute to heartbeat back to the master.
 *
 * @param workerId the worker id
 * @param capacityBytesOnTiers a mapping from storage tier alias to capacity bytes
 * @param usedBytesOnTiers a mapping from storage tier alias to used bytes
 * @param removedBlocks a list of block removed from this worker
 * @param addedBlocks a mapping from storage tier alias to added blocks
 * @param lostStorage a mapping from storage tier alias to a list of lost storage paths
 * @param metrics a list of worker metrics
 * @return an optional command for the worker to execute
 */
public synchronized Command heartbeat(final long workerId, final Map<String, Long> capacityBytesOnTiers, final Map<String, Long> usedBytesOnTiers, final List<Long> removedBlocks, final Map<BlockStoreLocation, List<Long>> addedBlocks, final Map<String, List<String>> lostStorage, final List<Metric> metrics) throws IOException {
    final BlockHeartbeatPOptions options = BlockHeartbeatPOptions.newBuilder().addAllMetrics(metrics).putAllCapacityBytesOnTiers(capacityBytesOnTiers).build();
    final List<LocationBlockIdListEntry> entryList = convertBlockListMapToProto(addedBlocks);
    final Map<String, StorageList> lostStorageMap = lostStorage.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> StorageList.newBuilder().addAllStorage(e.getValue()).build()));
    final BlockHeartbeatPRequest request = BlockHeartbeatPRequest.newBuilder().setWorkerId(workerId).putAllUsedBytesOnTiers(usedBytesOnTiers).addAllRemovedBlockIds(removedBlocks).addAllAddedBlocks(entryList).setOptions(options).putAllLostStorage(lostStorageMap).build();
    return retryRPC(() -> mClient.withDeadlineAfter(mContext.getClusterConf().getMs(PropertyKey.WORKER_MASTER_PERIODICAL_RPC_TIMEOUT), TimeUnit.MILLISECONDS).blockHeartbeat(request).getCommand(), LOG, "Heartbeat", "workerId=%d", workerId);
}

Example 5 with StorageList

use of alluxio.grpc.StorageList in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method requestsForEmptyWorker.

/**
 * Tests below cover the most normal cases.
 */
@Test
public void requestsForEmptyWorker() throws Exception {
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForEmptyWorker(WORKER_ID);
    // Verify the size and content of the requests
    assertEquals(1, requestChunks.size());
    RegisterWorkerPRequest request = requestChunks.get(0);
    assertEquals(WORKER_ID, request.getWorkerId());
    assertEquals(MEM_USAGE_EMPTY, request.getUsedBytesOnTiersMap());
    assertEquals(MEM_CAPACITY, request.getTotalBytesOnTiersMap());
    Map<String, StorageList> lostMap = request.getLostStorageMap();
    assertEquals(1, lostMap.size());
    assertEquals(StorageList.newBuilder().build(), lostMap.get("MEM"));
    assertEquals(ImmutableList.of("MEM"), request.getStorageTiersList());
    List<LocationBlockIdListEntry> entries = request.getCurrentBlocksList();
    assertEquals(0, entries.size());
}