Examples with WorkerInfo alluxio.wire.WorkerInfo used on opensource projects

Example 21 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class CapacityCommandTest method prepareLongInfoList.

/**
 * @return long worker info list to test
 */
private List<WorkerInfo> prepareLongInfoList() {
    List<WorkerInfo> infoList = new ArrayList<>();
    Map<String, Long> capacityBytesOnTiersOne = new HashMap<>();
    capacityBytesOnTiersOne.put(Constants.MEDIUM_MEM, 4000000000L);
    capacityBytesOnTiersOne.put("RAM", 6000000000L);
    capacityBytesOnTiersOne.put(Constants.MEDIUM_HDD, 2000000000L);
    Map<String, Long> usedBytesOnTiersOne = new HashMap<>();
    usedBytesOnTiersOne.put(Constants.MEDIUM_MEM, 3000000000L);
    usedBytesOnTiersOne.put("RAM", 5000000000L);
    usedBytesOnTiersOne.put(Constants.MEDIUM_HDD, 2000000000L);
    WorkerInfo firstInfo = new WorkerInfo().setAddress(new WorkerNetAddress().setHost("64.68.90.1")).setCapacityBytes(12000000000L).setCapacityBytesOnTiers(capacityBytesOnTiersOne).setId(1).setLastContactSec(3123).setStartTimeMs(1331231121212L).setState("In Service").setUsedBytes(10000000000L).setUsedBytesOnTiers(usedBytesOnTiersOne);
    Map<String, Long> capacityBytesOnTiersSec = new HashMap<>();
    capacityBytesOnTiersSec.put(Constants.MEDIUM_MEM, 5000000000L);
    capacityBytesOnTiersSec.put(Constants.MEDIUM_SSD, 5000000000L);
    capacityBytesOnTiersSec.put("DOM", 10000000000L);
    Map<String, Long> usedBytesOnTiersSec = new HashMap<>();
    usedBytesOnTiersSec.put(Constants.MEDIUM_MEM, 200000000L);
    usedBytesOnTiersSec.put(Constants.MEDIUM_SSD, 300000000L);
    usedBytesOnTiersSec.put("DOM", 500000000L);
    WorkerInfo secondInfo = new WorkerInfo().setAddress(new WorkerNetAddress().setHost("216.239.33.96")).setCapacityBytes(20000000000L).setCapacityBytesOnTiers(capacityBytesOnTiersSec).setId(2).setLastContactSec(542).setStartTimeMs(1131231121212L).setState("In Service").setUsedBytes(1000000000L).setUsedBytesOnTiers(usedBytesOnTiersSec);
    infoList.add(firstInfo);
    infoList.add(secondInfo);
    return infoList;
}

Example 22 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class CapacityCommandTest method shortCapacity.

@Test
public void shortCapacity() throws IOException {
    List<WorkerInfo> shortInfoList = prepareShortInfoList();
    Mockito.when(mBlockMasterClient.getWorkerReport(Mockito.any())).thenReturn(shortInfoList);
    try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
        PrintStream printStream = new PrintStream(outputStream, true, "utf-8")) {
        CapacityCommand capacityCommand = new CapacityCommand(mBlockMasterClient, printStream);
        capacityCommand.generateCapacityReport(GetWorkerReportOptions.defaults());
        String output = new String(outputStream.toByteArray(), StandardCharsets.UTF_8);
        List<String> expectedOutput = Arrays.asList("Capacity information for all workers: ", "    Total Capacity: 14.90GB", "        Tier: RAM  Size: 14.90GB", "    Used Capacity: 5.12GB", "        Tier: RAM  Size: 5.12GB", "    Used Percentage: 34%", "    Free Percentage: 66%", "", "Worker Name      Last Heartbeat   Storage       RAM", "215.42.95.24     953              capacity      9.31GB", "                                  used          476.84MB (5%)", "29.53.5.124      6424122          capacity      5.59GB", "                                  used          4768.37MB (83%)");
        List<String> testOutput = Arrays.asList(output.split("\n"));
        Assert.assertThat(testOutput, IsIterableContainingInOrder.contains(expectedOutput.toArray()));
    }
}

Example 23 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class FileOutStreamIntegrationTest method cancelWrite.

/**
 * Tests canceling after multiple blocks have been written correctly cleans up temporary worker
 * resources.
 */
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.MASTER_LOST_WORKER_FILE_DETECTION_INTERVAL, "250ms", PropertyKey.Name.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS, "250ms" })
@Test
public void cancelWrite() throws Exception {
    AlluxioURI path = new AlluxioURI(PathUtils.uniqPath());
    try (FileOutStream os = mFileSystem.createFile(path, CreateFilePOptions.newBuilder().setWriteType(mWriteType.toProto()).setRecursive(true).build())) {
        os.write(BufferUtils.getIncreasingByteArray(0, BLOCK_SIZE_BYTES * 3 + 1));
        os.cancel();
    }
    long gracePeriod = ServerConfiguration.getMs(PropertyKey.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS) * 2;
    CommonUtils.sleepMs(gracePeriod);
    List<WorkerInfo> workers = mLocalAlluxioClusterResource.get().getLocalAlluxioMaster().getMasterProcess().getMaster(FileSystemMaster.class).getFileSystemMasterView().getWorkerInfoList();
    for (WorkerInfo worker : workers) {
        Assert.assertEquals(0, worker.getUsedBytes());
    }
}

Example 24 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class StressBenchDefinition method selectExecutors.

@Override
public Set<Pair<WorkerInfo, ArrayList<String>>> selectExecutors(StressBenchConfig config, List<WorkerInfo> jobWorkerInfoList, SelectExecutorsContext context) {
    Set<Pair<WorkerInfo, ArrayList<String>>> result = Sets.newHashSet();
    // sort copy of workers by hashcode
    List<WorkerInfo> workerList = Lists.newArrayList(jobWorkerInfoList);
    workerList.sort(Comparator.comparing(w -> w.getAddress().getHost()));
    // take the first subset, according to cluster limit
    int clusterLimit = config.getClusterLimit();
    if (clusterLimit == 0) {
        clusterLimit = workerList.size();
    }
    if (clusterLimit < 0) {
        // if negative, reverse the list
        clusterLimit = -clusterLimit;
        Collections.reverse(workerList);
    }
    workerList = workerList.subList(0, clusterLimit);
    for (WorkerInfo worker : workerList) {
        LOG.info("Generating job for worker {}", worker.getId());
        ArrayList<String> args = new ArrayList<>(2);
        // Add the worker hostname + worker id as the unique task id for each distributed task.
        // The worker id is used since there may be multiple workers on a single host.
        args.add(BaseParameters.ID_FLAG);
        args.add(worker.getAddress().getHost() + "-" + worker.getId());
        result.add(new Pair<>(worker, args));
    }
    return result;
}

Example 25 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class StressBenchDefinition method join.

@Override
public String join(StressBenchConfig config, Map<WorkerInfo, String> taskResults) throws Exception {
    if (taskResults.isEmpty()) {
        throw new IOException("No results from any workers.");
    }
    AtomicReference<IOException> error = new AtomicReference<>(null);
    List<TaskResult> results = taskResults.entrySet().stream().map(entry -> {
        try {
            return JsonSerializable.fromJson(entry.getValue().trim(), new TaskResult[0]);
        } catch (IOException | ClassNotFoundException e) {
            // add log here because the exception details are lost at the client side
            LOG.warn("Failed to parse result into class {}", TaskResult.class, e);
            error.set(new IOException(String.format("Failed to parse task output from %s into result class %s: %s", entry.getKey().getAddress().getHost(), TaskResult.class, entry.getValue().trim()), e));
        }
        return null;
    }).collect(Collectors.toList());
    if (error.get() != null) {
        throw error.get();
    }
    return results.get(0).aggregator().aggregate(results).toJson();
}