Examples with WorkerInfo alluxio.wire.WorkerInfo used on opensource projects

Example 16 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class PlanCoordinator method join.

/**
 * Joins the task results and produces a final result.
 *
 * @param taskInfoList the list of task information
 * @return the aggregated result as a String
 * @throws Exception if any error occurs
 */
private String join(List<TaskInfo> taskInfoList) throws Exception {
    // get the job definition
    PlanDefinition<JobConfig, Serializable, Serializable> definition = PlanDefinitionRegistry.INSTANCE.getJobDefinition(mPlanInfo.getJobConfig());
    Map<WorkerInfo, Serializable> taskResults = Maps.newHashMap();
    for (TaskInfo taskInfo : taskInfoList) {
        taskResults.put(mTaskIdToWorkerInfo.get(taskInfo.getTaskId()), taskInfo.getResult());
    }
    return definition.join(mPlanInfo.getJobConfig(), taskResults);
}

Example 17 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class PlanTrackerTest method before.

@Before
public void before() {
    mMockWorkflowTracker = mock(WorkflowTracker.class);
    mTracker = new PlanTracker(CAPACITY, RETENTION_TIME, PURGE_CONUT, mMockWorkflowTracker);
    mCommandManager = new CommandManager();
    mMockJobServerContext = mock(JobServerContext.class);
    mWorkers = Lists.newArrayList(new WorkerInfo());
    mJobIdGenerator = new JobIdGenerator();
}

Example 18 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class CapacityCommand method collectWorkerInfo.

/**
 * Collects worker capacity information.
 *
 * @param workerInfoList the worker info list to collect info from
 */
private void collectWorkerInfo(List<WorkerInfo> workerInfoList) {
    initVariables();
    for (WorkerInfo workerInfo : workerInfoList) {
        long usedBytes = workerInfo.getUsedBytes();
        long capacityBytes = workerInfo.getCapacityBytes();
        mSumCapacityBytes += capacityBytes;
        mSumUsedBytes += usedBytes;
        String workerName = workerInfo.getAddress().getHost();
        Map<String, Long> totalBytesOnTiers = workerInfo.getCapacityBytesOnTiers();
        for (Map.Entry<String, Long> totalBytesTier : totalBytesOnTiers.entrySet()) {
            String tier = totalBytesTier.getKey();
            long value = totalBytesTier.getValue();
            mSumCapacityBytesOnTierMap.put(tier, value + mSumCapacityBytesOnTierMap.getOrDefault(tier, 0L));
            Map<String, String> map = mCapacityTierInfoMap.getOrDefault(tier, new HashMap<>());
            map.put(workerName, FormatUtils.getSizeFromBytes(value));
            mCapacityTierInfoMap.put(tier, map);
        }
        Map<String, Long> usedBytesOnTiers = workerInfo.getUsedBytesOnTiers();
        for (Map.Entry<String, Long> usedBytesTier : usedBytesOnTiers.entrySet()) {
            String tier = usedBytesTier.getKey();
            long value = usedBytesTier.getValue();
            mSumUsedBytesOnTierMap.put(tier, value + mSumUsedBytesOnTierMap.getOrDefault(tier, 0L));
            Map<String, String> map = mUsedTierInfoMap.getOrDefault(tier, new HashMap<>());
            map.put(workerName, FormatUtils.getSizeFromBytes(value));
            mUsedTierInfoMap.put(tier, map);
        }
    }
}

Example 19 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class CapacityCommand method printWorkerInfo.

/**
 * Prints worker capacity information.
 *
 * @param workerInfoList the worker info list to get info from
 */
private void printWorkerInfo(List<WorkerInfo> workerInfoList) {
    mIndentationLevel = 0;
    if (mCapacityTierInfoMap.size() == 0) {
        return;
    } else if (mCapacityTierInfoMap.size() == 1) {
        // Do not print Total value when only one tier exists
        printShortWorkerInfo(workerInfoList);
        return;
    }
    Set<String> tiers = mCapacityTierInfoMap.keySet();
    String tiersInfo = String.format(Strings.repeat("%-14s", tiers.size()), tiers.toArray());
    String longInfoFormat = getInfoFormat(workerInfoList, false);
    print(String.format("%n" + longInfoFormat, "Worker Name", "Last Heartbeat", "Storage", "Total", tiersInfo));
    for (WorkerInfo info : workerInfoList) {
        String workerName = info.getAddress().getHost();
        long usedBytes = info.getUsedBytes();
        long capacityBytes = info.getCapacityBytes();
        String usedPercentageInfo = "";
        if (capacityBytes != 0) {
            int usedPercentage = (int) (100L * usedBytes / capacityBytes);
            usedPercentageInfo = String.format(" (%s%%)", usedPercentage);
        }
        String capacityTierInfo = getWorkerFormattedTierValues(mCapacityTierInfoMap, workerName);
        String usedTierInfo = getWorkerFormattedTierValues(mUsedTierInfoMap, workerName);
        print(String.format(longInfoFormat, workerName, info.getLastContactSec(), "capacity", FormatUtils.getSizeFromBytes(capacityBytes), capacityTierInfo));
        print(String.format(longInfoFormat, "", "", "used", FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo, usedTierInfo));
    }
}

Example 20 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class CapacityCommandTest method prepareShortInfoList.

/**
 * @return short worker info list that only one tier exists
 */
private List<WorkerInfo> prepareShortInfoList() {
    List<WorkerInfo> infoList = new ArrayList<>();
    Map<String, Long> capacityBytesOnTiersOne = new HashMap<>();
    capacityBytesOnTiersOne.put("RAM", 6000000000L);
    Map<String, Long> usedBytesOnTiersOne = new HashMap<>();
    usedBytesOnTiersOne.put("RAM", 5000000000L);
    WorkerInfo firstInfo = new WorkerInfo().setAddress(new WorkerNetAddress().setHost("29.53.5.124")).setCapacityBytes(6000000000L).setCapacityBytesOnTiers(capacityBytesOnTiersOne).setId(1).setLastContactSec(6424122).setStartTimeMs(19365332L).setState("Out of Service").setUsedBytes(5000000000L).setUsedBytesOnTiers(usedBytesOnTiersOne);
    Map<String, Long> capacityBytesOnTiersSec = new HashMap<>();
    capacityBytesOnTiersSec.put("RAM", 10000000000L);
    Map<String, Long> usedBytesOnTiersSec = new HashMap<>();
    usedBytesOnTiersSec.put("RAM", 500000000L);
    WorkerInfo secondInfo = new WorkerInfo().setAddress(new WorkerNetAddress().setHost("215.42.95.24")).setCapacityBytes(10000000000L).setCapacityBytesOnTiers(capacityBytesOnTiersSec).setId(2).setLastContactSec(953).setStartTimeMs(112495222L).setState("In Service").setUsedBytes(500000000L).setUsedBytesOnTiers(usedBytesOnTiersSec);
    infoList.add(firstInfo);
    infoList.add(secondInfo);
    return infoList;
}