Examples with StorageList alluxio.grpc.StorageList used on opensource projects

Example 6 with StorageList

use of alluxio.grpc.StorageList in project alluxio by Alluxio.

the class FileSystemMasterTest method startServices.

private void startServices() throws Exception {
    mRegistry = new MasterRegistry();
    mJournalSystem = JournalTestUtils.createJournalSystem(mJournalFolder);
    CoreMasterContext masterContext = MasterTestUtils.testMasterContext(mJournalSystem, new TestUserState(TEST_USER, ServerConfiguration.global()));
    mMetricsMaster = new MetricsMasterFactory().create(mRegistry, masterContext);
    mRegistry.add(MetricsMaster.class, mMetricsMaster);
    mMetrics = Lists.newArrayList();
    mBlockMaster = new BlockMasterFactory().create(mRegistry, masterContext);
    mExecutorService = Executors.newFixedThreadPool(4, ThreadFactoryUtils.build("DefaultFileSystemMasterTest-%d", true));
    mFileSystemMaster = new DefaultFileSystemMaster(mBlockMaster, masterContext, ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService));
    mInodeStore = mFileSystemMaster.getInodeStore();
    mInodeTree = mFileSystemMaster.getInodeTree();
    mRegistry.add(FileSystemMaster.class, mFileSystemMaster);
    mJournalSystem.start();
    mJournalSystem.gainPrimacy();
    mRegistry.start(true);
    // set up workers
    mWorkerId1 = mBlockMaster.getWorkerId(new WorkerNetAddress().setHost("localhost").setRpcPort(80).setDataPort(81).setWebPort(82));
    mBlockMaster.workerRegister(mWorkerId1, Arrays.asList(Constants.MEDIUM_MEM, Constants.MEDIUM_SSD), ImmutableMap.of(Constants.MEDIUM_MEM, (long) Constants.MB, Constants.MEDIUM_SSD, (long) Constants.MB), ImmutableMap.of(Constants.MEDIUM_MEM, (long) Constants.KB, Constants.MEDIUM_SSD, (long) Constants.KB), ImmutableMap.of(), new HashMap<String, StorageList>(), RegisterWorkerPOptions.getDefaultInstance());
    mWorkerId2 = mBlockMaster.getWorkerId(new WorkerNetAddress().setHost("remote").setRpcPort(80).setDataPort(81).setWebPort(82));
    mBlockMaster.workerRegister(mWorkerId2, Arrays.asList(Constants.MEDIUM_MEM, Constants.MEDIUM_SSD), ImmutableMap.of(Constants.MEDIUM_MEM, (long) Constants.MB, Constants.MEDIUM_SSD, (long) Constants.MB), ImmutableMap.of(Constants.MEDIUM_MEM, (long) Constants.KB, Constants.MEDIUM_SSD, (long) Constants.KB), ImmutableMap.of(), new HashMap<String, StorageList>(), RegisterWorkerPOptions.getDefaultInstance());
}

Example 7 with StorageList

use of alluxio.grpc.StorageList in project alluxio by Alluxio.

the class BlockMasterWorkerServiceHandler method registerWorker.

@Override
public void registerWorker(RegisterWorkerPRequest request, StreamObserver<RegisterWorkerPResponse> responseObserver) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Register worker request is {} bytes, containing {} blocks", request.getSerializedSize(), request.getCurrentBlocksCount());
    }
    final long workerId = request.getWorkerId();
    RegisterWorkerPOptions options = request.getOptions();
    RpcUtils.call(LOG, (RpcUtils.RpcCallableThrowsIOException<RegisterWorkerPResponse>) () -> {
        // The exception will be propagated to the worker side and the worker should retry.
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_WORKER_REGISTER_LEASE_ENABLED) && !mBlockMaster.hasRegisterLease(workerId)) {
            String errorMsg = String.format("Worker %s does not have a lease or the lease " + "has expired. The worker should acquire a new lease and retry to register.", workerId);
            LOG.warn(errorMsg);
            throw new RegisterLeaseNotFoundException(errorMsg);
        }
        LOG.debug("Worker {} proceeding to register...", workerId);
        final List<String> storageTiers = request.getStorageTiersList();
        final Map<String, Long> totalBytesOnTiers = request.getTotalBytesOnTiersMap();
        final Map<String, Long> usedBytesOnTiers = request.getUsedBytesOnTiersMap();
        final Map<String, StorageList> lostStorageMap = request.getLostStorageMap();
        final Map<Block.BlockLocation, List<Long>> currBlocksOnLocationMap = reconstructBlocksOnLocationMap(request.getCurrentBlocksList(), workerId);
        // If the register is unsuccessful, the lease will be kept around until the expiry.
        // The worker can retry and use the existing lease.
        mBlockMaster.workerRegister(workerId, storageTiers, totalBytesOnTiers, usedBytesOnTiers, currBlocksOnLocationMap, lostStorageMap, options);
        LOG.info("Worker {} finished registering, releasing its lease.", workerId);
        mBlockMaster.releaseRegisterLease(workerId);
        return RegisterWorkerPResponse.getDefaultInstance();
    }, "registerWorker", true, "request=%s", responseObserver, workerId);
}

Example 8 with StorageList

use of alluxio.grpc.StorageList in project alluxio by Alluxio.

the class BlockMasterClient method register.

/**
 * The method the worker should execute to register with the block master.
 *
 * @param workerId the worker id of the worker registering
 * @param storageTierAliases a list of storage tier aliases in ordinal order
 * @param totalBytesOnTiers mapping from storage tier alias to total bytes
 * @param usedBytesOnTiers mapping from storage tier alias to used bytes
 * @param currentBlocksOnLocation mapping from storage tier alias to the list of list of blocks
 * @param lostStorage mapping from storage tier alias to the list of lost storage paths
 * @param configList a list of configurations
 */
// TODO(yupeng): rename to workerBlockReport or workerInitialize?
public void register(final long workerId, final List<String> storageTierAliases, final Map<String, Long> totalBytesOnTiers, final Map<String, Long> usedBytesOnTiers, final Map<BlockStoreLocation, List<Long>> currentBlocksOnLocation, final Map<String, List<String>> lostStorage, final List<ConfigProperty> configList) throws IOException {
    final RegisterWorkerPOptions options = RegisterWorkerPOptions.newBuilder().addAllConfigs(configList).build();
    final List<LocationBlockIdListEntry> currentBlocks = convertBlockListMapToProto(currentBlocksOnLocation);
    final Map<String, StorageList> lostStorageMap = lostStorage.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> StorageList.newBuilder().addAllStorage(e.getValue()).build()));
    final RegisterWorkerPRequest request = RegisterWorkerPRequest.newBuilder().setWorkerId(workerId).addAllStorageTiers(storageTierAliases).putAllTotalBytesOnTiers(totalBytesOnTiers).putAllUsedBytesOnTiers(usedBytesOnTiers).addAllCurrentBlocks(currentBlocks).putAllLostStorage(lostStorageMap).setOptions(options).build();
    retryRPC(() -> {
        mClient.registerWorker(request);
        return null;
    }, LOG, "Register", "workerId=%d", workerId);
}

Example 9 with StorageList

use of alluxio.grpc.StorageList in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method requestsForWorker.

@Test
public void requestsForWorker() throws Exception {
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForWorker(WORKER_ID);
    // Verify the size and content of the requests
    int expectedBatchCount = (int) Math.ceil((TIER_BLOCK_TOTAL) / (double) BATCH_SIZE);
    Set<Long> containedBlockIds = new HashSet<>();
    assertEquals(expectedBatchCount, requestChunks.size());
    for (int i = 0; i < expectedBatchCount; i++) {
        RegisterWorkerPRequest request = requestChunks.get(i);
        assertEquals(WORKER_ID, request.getWorkerId());
        List<LocationBlockIdListEntry> entries = request.getCurrentBlocksList();
        int totalSize = 0;
        for (LocationBlockIdListEntry entry : entries) {
            totalSize += entry.getValue().getBlockIdCount();
            containedBlockIds.addAll(entry.getValue().getBlockIdList());
        }
        if (i != expectedBatchCount - 1) {
            assertEquals(BATCH_SIZE, totalSize);
        }
        // The 1st request contains metadata but the following do not
        if (i == 0) {
            assertEquals(USAGE_MAP, request.getUsedBytesOnTiersMap());
            assertEquals(CAPACITY_MAP, request.getTotalBytesOnTiersMap());
            Map<String, StorageList> lostMap = request.getLostStorageMap();
            assertEquals(1, lostMap.size());
            assertEquals(StorageList.newBuilder().build(), lostMap.get("MEM"));
            assertEquals(ImmutableList.of("MEM", "SSD", "HDD"), request.getStorageTiersList());
        } else {
            assertEquals(0, request.getStorageTiersCount());
            assertEquals(ImmutableMap.of(), request.getUsedBytesOnTiersMap());
            assertEquals(ImmutableMap.of(), request.getTotalBytesOnTiersMap());
            Map<String, StorageList> lostMap = request.getLostStorageMap();
            assertEquals(0, lostMap.size());
        }
    }
    assertEquals(containedBlockIds.size(), TIER_BLOCK_TOTAL);
}

Example 10 with StorageList

use of alluxio.grpc.StorageList in project alluxio by Alluxio.

the class BlockMasterTest method workerHeartbeatUpdatesLostStorage.

@Test
public void workerHeartbeatUpdatesLostStorage() throws Exception {
    // Create two workers.
    long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1);
    mBlockMaster.workerRegister(worker1, Arrays.asList(Constants.MEDIUM_MEM, Constants.MEDIUM_SSD), ImmutableMap.of(Constants.MEDIUM_MEM, 100L, Constants.MEDIUM_SSD, 200L), ImmutableMap.of(Constants.MEDIUM_MEM, 0L, Constants.MEDIUM_SSD, 0L), NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, RegisterWorkerPOptions.getDefaultInstance());
    long worker2 = mBlockMaster.getWorkerId(NET_ADDRESS_2);
    mBlockMaster.workerRegister(worker2, Arrays.asList(Constants.MEDIUM_MEM, Constants.MEDIUM_HDD), ImmutableMap.of(Constants.MEDIUM_MEM, 100L, Constants.MEDIUM_HDD, 300L), ImmutableMap.of(Constants.MEDIUM_MEM, 0L, Constants.MEDIUM_HDD, 0L), NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, RegisterWorkerPOptions.getDefaultInstance());
    Map<String, StorageList> lostStorageOnWorker1 = new HashMap<>();
    lostStorageOnWorker1.put(Constants.MEDIUM_SSD, StorageList.newBuilder().addAllStorage(Arrays.asList("/ssd/one", "/ssd/two")).build());
    Map<String, StorageList> lostStorageOnWorker2 = new HashMap<>();
    lostStorageOnWorker2.put(Constants.MEDIUM_HDD, StorageList.newBuilder().addStorage("/hdd/one").build());
    mBlockMaster.workerHeartbeat(worker1, ImmutableMap.of(Constants.MEDIUM_MEM, 100L, Constants.MEDIUM_SSD, 0L), ImmutableMap.of(Constants.MEDIUM_MEM, 0L, Constants.MEDIUM_SSD, 0L), NO_BLOCKS, NO_BLOCKS_ON_LOCATION, lostStorageOnWorker1, mMetrics);
    mBlockMaster.workerHeartbeat(worker2, ImmutableMap.of(Constants.MEDIUM_MEM, 100L, Constants.MEDIUM_HDD, 200L), ImmutableMap.of(Constants.MEDIUM_MEM, 0L, Constants.MEDIUM_HDD, 0L), NO_BLOCKS, NO_BLOCKS_ON_LOCATION, lostStorageOnWorker2, mMetrics);
    // Two workers have lost storage paths
    assertEquals(2, mBlockMaster.getWorkerLostStorage().size());
    int lostStorageNum = 0;
    for (WorkerLostStorageInfo info : mBlockMaster.getWorkerLostStorage()) {
        for (StorageList list : info.getLostStorageMap().values()) {
            lostStorageNum += list.getStorageList().size();
        }
    }
    assertEquals(3, lostStorageNum);
}