Examples with WorkerInfo alluxio.wire.WorkerInfo used on opensource projects

Example 36 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class WebUtils method generateOrderedNodeInfos.

/**
 * Order the nodes by hostName and generate {@link NodeInfo} list for UI display.
 *
 * @param workerInfos the list of {@link WorkerInfo} objects
 * @return the list of {@link NodeInfo} objects
 */
public static NodeInfo[] generateOrderedNodeInfos(Collection<WorkerInfo> workerInfos) {
    NodeInfo[] ret = new NodeInfo[workerInfos.size()];
    int index = 0;
    for (WorkerInfo workerInfo : workerInfos) {
        ret[index++] = new NodeInfo(workerInfo);
    }
    Arrays.sort(ret);
    return ret;
}

Example 37 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class AlluxioMasterRestServiceHandlerTest method getMasterInfo.

@Test
public void getMasterInfo() {
    // Mock for rpc address
    when(mMasterProcess.getRpcAddress()).thenReturn(new InetSocketAddress("localhost", 8080));
    // Mock for metrics
    final int FILES_PINNED_TEST_VALUE = 100;
    String filesPinnedProperty = MetricKey.MASTER_FILES_PINNED.getName();
    Gauge<Integer> filesPinnedGauge = () -> FILES_PINNED_TEST_VALUE;
    MetricSet mockMetricsSet = mock(MetricSet.class);
    Map<String, Metric> map = new HashMap<>();
    map.put(filesPinnedProperty, filesPinnedGauge);
    when(mockMetricsSet.getMetrics()).thenReturn(map);
    MetricsSystem.METRIC_REGISTRY.registerAll(mockMetricsSet);
    // Mock for start time
    when(mMasterProcess.getStartTimeMs()).thenReturn(101L);
    // Mock for up time
    when(mMasterProcess.getUptimeMs()).thenReturn(102L);
    Response response = mHandler.getInfo(false);
    try {
        assertNotNull("Response must be not null!", response);
        assertNotNull("Response must have a entry!", response.getEntity());
        assertTrue("Entry must be an AlluxioMasterInfo!", (response.getEntity() instanceof AlluxioMasterInfo));
        AlluxioMasterInfo info = (AlluxioMasterInfo) response.getEntity();
        // Validate configuration
        assertNotNull("Configuration must be not null", info.getConfiguration());
        assertFalse("Properties Map must be not empty!", (info.getConfiguration().isEmpty()));
        // Validate rpc address
        assertEquals("localhost/127.0.0.1:8080", info.getRpcAddress());
        // Validate metrics
        Map<String, Long> metricsMap = info.getMetrics();
        assertFalse("Metrics Map must be not empty!", (metricsMap.isEmpty()));
        assertTrue("Map must contain key " + filesPinnedProperty + "!", metricsMap.containsKey(filesPinnedProperty));
        assertEquals(FILES_PINNED_TEST_VALUE, metricsMap.get(filesPinnedProperty).longValue());
        // Validate StartTimeMs
        assertEquals(101L, info.getStartTimeMs());
        // Validate UptimeMs
        assertEquals(102L, info.getUptimeMs());
        // Validate version
        assertEquals(RuntimeConstants.VERSION, info.getVersion());
        // Validate capacity bytes
        Capacity cap = info.getCapacity();
        long sumCapacityBytes = 0;
        for (Map.Entry<String, Long> entry1 : WORKER1_TOTAL_BYTES_ON_TIERS.entrySet()) {
            Long totalBytes = entry1.getValue();
            sumCapacityBytes += totalBytes;
        }
        for (Map.Entry<String, Long> entry1 : WORKER2_TOTAL_BYTES_ON_TIERS.entrySet()) {
            Long totalBytes = entry1.getValue();
            sumCapacityBytes += totalBytes;
        }
        assertEquals(sumCapacityBytes, cap.getTotal());
        // Validate used bytes
        long sumUsedBytes = 0;
        for (Map.Entry<String, Long> entry1 : WORKER1_USED_BYTES_ON_TIERS.entrySet()) {
            Long totalBytes = entry1.getValue();
            sumUsedBytes += totalBytes;
        }
        for (Map.Entry<String, Long> entry1 : WORKER2_USED_BYTES_ON_TIERS.entrySet()) {
            Long totalBytes = entry1.getValue();
            sumUsedBytes += totalBytes;
        }
        assertEquals(sumUsedBytes, cap.getUsed());
        // Validate UFS capacity
        Capacity ufsCapacity = info.getUfsCapacity();
        assertEquals(UFS_SPACE_TOTAL, ufsCapacity.getTotal());
        assertEquals(UFS_SPACE_USED, ufsCapacity.getUsed());
        // Validate workers
        List<WorkerInfo> workers = info.getWorkers();
        assertEquals(2, workers.size());
        long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1);
        long worker2 = mBlockMaster.getWorkerId(NET_ADDRESS_2);
        Set<Long> expectedWorkers = new HashSet<>();
        expectedWorkers.add(worker1);
        expectedWorkers.add(worker2);
        Set<Long> actualWorkers = new HashSet<>();
        for (WorkerInfo w : workers) {
            actualWorkers.add(w.getId());
        }
        assertEquals(expectedWorkers, actualWorkers);
    } finally {
        response.close();
    }
}

Example 38 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class MigrateDefinition method getBestJobWorker.

private WorkerInfo getBestJobWorker(URIStatus status, List<BlockWorkerInfo> alluxioWorkerInfoList, List<WorkerInfo> jobWorkerInfoList, Map<String, WorkerInfo> hostnameToJobWorker) {
    BlockWorkerInfo bestWorker = JobUtils.getWorkerWithMostBlocks(alluxioWorkerInfoList, status.getFileBlockInfos());
    if (bestWorker == null) {
        // Nobody has blocks, choose a random worker.
        bestWorker = alluxioWorkerInfoList.get(mRandom.nextInt(jobWorkerInfoList.size()));
    }
    // Map the best Alluxio worker to a job worker.
    WorkerInfo worker = hostnameToJobWorker.get(bestWorker.getNetAddress().getHost());
    if (worker == null) {
        return jobWorkerInfoList.get(new Random().nextInt(jobWorkerInfoList.size()));
    }
    return worker;
}

Example 39 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class MigrateDefinition method selectExecutors.

/**
 * {@inheritDoc}
 *
 * Assigns each worker to migrate whichever files it has the most blocks for.
 * If no worker has blocks for a file, a random worker is chosen.
 * @return
 */
@Override
public Set<Pair<WorkerInfo, MigrateCommand>> selectExecutors(MigrateConfig config, List<WorkerInfo> jobWorkerInfoList, SelectExecutorsContext context) throws Exception {
    AlluxioURI source = new AlluxioURI(config.getSource());
    AlluxioURI destination = new AlluxioURI(config.getDestination());
    if (source.equals(destination)) {
        return Sets.newHashSet();
    }
    checkMigrateValid(config, context.getFileSystem());
    Preconditions.checkState(!jobWorkerInfoList.isEmpty(), "No workers are available");
    URIStatus status = context.getFileSystem().getStatus(source);
    ConcurrentMap<String, WorkerInfo> hostnameToWorker = Maps.newConcurrentMap();
    for (WorkerInfo workerInfo : jobWorkerInfoList) {
        hostnameToWorker.put(workerInfo.getAddress().getHost(), workerInfo);
    }
    List<BlockWorkerInfo> alluxioWorkerInfoList = context.getFsContext().getCachedWorkers();
    if (status.isFolder()) {
        throw new RuntimeException(ExceptionMessage.MIGRATE_DIRECTORY.getMessage());
    } else {
        WorkerInfo bestJobWorker = getBestJobWorker(status, alluxioWorkerInfoList, jobWorkerInfoList, hostnameToWorker);
        Set<Pair<WorkerInfo, MigrateCommand>> result = Sets.newHashSet();
        result.add(new Pair<>(bestJobWorker, new MigrateCommand(status.getPath(), destination.getPath())));
        return result;
    }
}

Example 40 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class EvictDefinition method selectExecutors.

@Override
public Set<Pair<WorkerInfo, SerializableVoid>> selectExecutors(EvictConfig config, List<WorkerInfo> jobWorkerInfoList, SelectExecutorsContext context) throws Exception {
    Preconditions.checkArgument(!jobWorkerInfoList.isEmpty(), "No worker is available");
    long blockId = config.getBlockId();
    int numReplicas = config.getReplicas();
    AlluxioBlockStore blockStore = AlluxioBlockStore.create(context.getFsContext());
    BlockInfo blockInfo = blockStore.getInfo(blockId);
    Set<String> hosts = new HashSet<>();
    for (BlockLocation blockLocation : blockInfo.getLocations()) {
        hosts.add(blockLocation.getWorkerAddress().getHost());
    }
    Set<Pair<WorkerInfo, SerializableVoid>> result = Sets.newHashSet();
    Collections.shuffle(jobWorkerInfoList);
    for (WorkerInfo workerInfo : jobWorkerInfoList) {
        // Select job workers that have this block locally to evict
        if (hosts.contains(workerInfo.getAddress().getHost())) {
            result.add(new Pair<>(workerInfo, null));
            if (result.size() >= numReplicas) {
                break;
            }
        }
    }
    return result;
}