Examples with OperatorProfile org.apache.drill.exec.proto.UserBitShared.OperatorProfile used on opensource projects

Example 1 with OperatorProfile

use of org.apache.drill.exec.proto.UserBitShared.OperatorProfile in project drill by apache.

the class OperatorWrapper method addSummary.

public void addSummary(TableBuilder tb, HashMap<String, Long> majorFragmentBusyTally, long majorFragmentBusyTallyTotal) {
    //Select background color from palette
    String opTblBgColor = OPERATOR_OVERVIEW_BGCOLOR_PALETTE[major % OPERATOR_OVERVIEW_BGCOLOR_PALETTE.length];
    String path = new OperatorPathBuilder().setMajor(major).setOperator(firstProfile).build();
    tb.appendCell(path, null, null, opTblBgColor);
    tb.appendCell(operatorName);
    //Get MajorFragment Busy+Wait Time Tally
    long majorBusyNanos = majorFragmentBusyTally.get(new OperatorPathBuilder().setMajor(major).build());
    double setupSum = 0.0;
    double processSum = 0.0;
    double waitSum = 0.0;
    double memSum = 0.0;
    long recordSum = 0L;
    for (ImmutablePair<OperatorProfile, Integer> ip : ops) {
        OperatorProfile profile = ip.getLeft();
        setupSum += profile.getSetupNanos();
        processSum += profile.getProcessNanos();
        waitSum += profile.getWaitNanos();
        memSum += profile.getPeakLocalMemoryAllocated();
        for (final StreamProfile sp : profile.getInputProfileList()) {
            recordSum += sp.getRecords();
        }
    }
    final ImmutablePair<OperatorProfile, Integer> longSetup = Collections.max(ops, Comparators.setupTime);
    tb.appendNanos(Math.round(setupSum / size));
    tb.appendNanos(longSetup.getLeft().getSetupNanos());
    final ImmutablePair<OperatorProfile, Integer> longProcess = Collections.max(ops, Comparators.processTime);
    tb.appendNanos(Math.round(processSum / size));
    tb.appendNanos(longProcess.getLeft().getProcessNanos());
    final ImmutablePair<OperatorProfile, Integer> shortWait = Collections.min(ops, Comparators.waitTime);
    final ImmutablePair<OperatorProfile, Integer> longWait = Collections.max(ops, Comparators.waitTime);
    tb.appendNanos(shortWait.getLeft().getWaitNanos());
    tb.appendNanos(Math.round(waitSum / size));
    tb.appendNanos(longWait.getLeft().getWaitNanos());
    tb.appendPercent(processSum / majorBusyNanos);
    tb.appendPercent(processSum / majorFragmentBusyTallyTotal);
    tb.appendFormattedInteger(recordSum);
    final ImmutablePair<OperatorProfile, Integer> peakMem = Collections.max(ops, Comparators.operatorPeakMemory);
    tb.appendBytes(Math.round(memSum / size));
    tb.appendBytes(peakMem.getLeft().getPeakLocalMemoryAllocated());
}

Example 2 with OperatorProfile

use of org.apache.drill.exec.proto.UserBitShared.OperatorProfile in project drill by axbaretto.

the class FragmentWrapper method addFinalSummary.

public void addFinalSummary(TableBuilder tb) {
    // Use only minor fragments that have complete profiles
    // Complete iff the fragment profile has at least one operator profile, and start and end times.
    final List<MinorFragmentProfile> complete = new ArrayList<>(Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes));
    tb.appendCell(new OperatorPathBuilder().setMajor(major).build());
    tb.appendCell(complete.size() + " / " + major.getMinorFragmentProfileCount());
    // If there are no stats to aggregate, create an empty row
    if (complete.size() < 1) {
        tb.appendRepeated("", null, NUM_NULLABLE_COMPLETED_OVERVIEW_COLUMNS);
        return;
    }
    final MinorFragmentProfile firstStart = Collections.min(complete, Comparators.startTime);
    final MinorFragmentProfile lastStart = Collections.max(complete, Comparators.startTime);
    tb.appendMillis(firstStart.getStartTime() - start);
    tb.appendMillis(lastStart.getStartTime() - start);
    final MinorFragmentProfile firstEnd = Collections.min(complete, Comparators.endTime);
    final MinorFragmentProfile lastEnd = Collections.max(complete, Comparators.endTime);
    tb.appendMillis(firstEnd.getEndTime() - start);
    tb.appendMillis(lastEnd.getEndTime() - start);
    long totalDuration = 0L;
    double totalProcessInMillis = 0.0d;
    double totalWaitInMillis = 0.0d;
    for (final MinorFragmentProfile p : complete) {
        totalDuration += p.getEndTime() - p.getStartTime();
        // Capture Busy & Wait Time
        List<OperatorProfile> opProfileList = p.getOperatorProfileList();
        for (OperatorProfile operatorProfile : opProfileList) {
            totalProcessInMillis += operatorProfile.getProcessNanos() / 1E6;
            totalWaitInMillis += operatorProfile.getWaitNanos() / 1E6;
        }
    }
    final MinorFragmentProfile shortRun = Collections.min(complete, Comparators.runTime);
    final MinorFragmentProfile longRun = Collections.max(complete, Comparators.runTime);
    tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime());
    tb.appendMillis(totalDuration / complete.size());
    tb.appendMillis(longRun.getEndTime() - longRun.getStartTime());
    tb.appendPercent(totalProcessInMillis / (totalProcessInMillis + totalWaitInMillis), null, // #8721 is the summation sign: sum(Busy): ## + sum(Wait): ##
    String.format("&#8721;Busy: %,.2fs + &#8721;Wait: %,.2fs", totalProcessInMillis / 1E3, totalWaitInMillis / 1E3));
    // TODO(DRILL-3494): Names (maxMem, getMaxMemoryUsed) are misleading; the value is peak memory allocated to fragment
    final MinorFragmentProfile maxMem = Collections.max(complete, Comparators.fragmentPeakMemory);
    tb.appendBytes(maxMem.getMaxMemoryUsed());
}

Example 3 with OperatorProfile

use of org.apache.drill.exec.proto.UserBitShared.OperatorProfile in project drill by axbaretto.

the class FragmentWrapper method addSummary.

public void addSummary(TableBuilder tb) {
    // Use only minor fragments that have complete profiles
    // Complete iff the fragment profile has at least one operator profile, and start and end times.
    final List<MinorFragmentProfile> complete = new ArrayList<>(Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes));
    tb.appendCell(new OperatorPathBuilder().setMajor(major).build());
    tb.appendCell(complete.size() + " / " + major.getMinorFragmentProfileCount());
    // If there are no stats to aggregate, create an empty row
    if (complete.size() < 1) {
        tb.appendRepeated("", null, NUM_NULLABLE_ACTIVE_OVERVIEW_COLUMNS);
        return;
    }
    final MinorFragmentProfile firstStart = Collections.min(complete, Comparators.startTime);
    final MinorFragmentProfile lastStart = Collections.max(complete, Comparators.startTime);
    tb.appendMillis(firstStart.getStartTime() - start);
    tb.appendMillis(lastStart.getStartTime() - start);
    final MinorFragmentProfile firstEnd = Collections.min(complete, Comparators.endTime);
    final MinorFragmentProfile lastEnd = Collections.max(complete, Comparators.endTime);
    tb.appendMillis(firstEnd.getEndTime() - start);
    tb.appendMillis(lastEnd.getEndTime() - start);
    long cumulativeFragmentDurationInMillis = 0L;
    long cumulativeProcessInNanos = 0L;
    long cumulativeWaitInNanos = 0L;
    for (final MinorFragmentProfile p : complete) {
        cumulativeFragmentDurationInMillis += p.getEndTime() - p.getStartTime();
        // Capture Busy & Wait Time
        List<OperatorProfile> opProfileList = p.getOperatorProfileList();
        for (OperatorProfile operatorProfile : opProfileList) {
            cumulativeProcessInNanos += operatorProfile.getProcessNanos();
            cumulativeWaitInNanos += operatorProfile.getWaitNanos();
        }
    }
    double totalProcessInMillis = Math.round(cumulativeProcessInNanos / 1E6);
    double totalWaitInMillis = Math.round(cumulativeWaitInNanos / 1E6);
    final MinorFragmentProfile shortRun = Collections.min(complete, Comparators.runTime);
    final MinorFragmentProfile longRun = Collections.max(complete, Comparators.runTime);
    tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime());
    tb.appendMillis(cumulativeFragmentDurationInMillis / complete.size());
    tb.appendMillis(longRun.getEndTime() - longRun.getStartTime());
    tb.appendPercent(totalProcessInMillis / (totalProcessInMillis + totalWaitInMillis), null, // #8721 is the summation sign: sum(Busy): ## + sum(Wait): ##
    String.format("&#8721;Busy: %,.2fs + &#8721;Wait: %,.2fs", totalProcessInMillis / 1E3, totalWaitInMillis / 1E3));
    final MinorFragmentProfile lastUpdate = Collections.max(complete, Comparators.lastUpdate);
    tb.appendMillis(System.currentTimeMillis() - lastUpdate.getLastUpdate());
    final MinorFragmentProfile lastProgress = Collections.max(complete, Comparators.lastProgress);
    tb.appendMillis(System.currentTimeMillis() - lastProgress.getLastProgress());
    // TODO(DRILL-3494): Names (maxMem, getMaxMemoryUsed) are misleading; the value is peak memory allocated to fragment
    final MinorFragmentProfile maxMem = Collections.max(complete, Comparators.fragmentPeakMemory);
    tb.appendBytes(maxMem.getMaxMemoryUsed());
}

Example 4 with OperatorProfile

use of org.apache.drill.exec.proto.UserBitShared.OperatorProfile in project drill by axbaretto.

the class OperatorWrapper method addSummary.

public void addSummary(TableBuilder tb, HashMap<String, Long> majorFragmentBusyTally, long majorFragmentBusyTallyTotal) {
    // Select background color from palette
    String opTblBgColor = OPERATOR_OVERVIEW_BGCOLOR_PALETTE[major % OPERATOR_OVERVIEW_BGCOLOR_PALETTE.length];
    String path = new OperatorPathBuilder().setMajor(major).setOperator(firstProfile).build();
    tb.appendCell(path, null, null, opTblBgColor);
    tb.appendCell(operatorName);
    // Get MajorFragment Busy+Wait Time Tally
    long majorBusyNanos = majorFragmentBusyTally.get(new OperatorPathBuilder().setMajor(major).build());
    double setupSum = 0.0;
    double processSum = 0.0;
    double waitSum = 0.0;
    double memSum = 0.0;
    long recordSum = 0L;
    // Construct list for sorting purposes (using legacy Comparators)
    final List<ImmutablePair<OperatorProfile, Integer>> opList = new ArrayList<>();
    for (ImmutablePair<ImmutablePair<OperatorProfile, Integer>, String> ip : opsAndHosts) {
        OperatorProfile profile = ip.getLeft().getLeft();
        setupSum += profile.getSetupNanos();
        processSum += profile.getProcessNanos();
        waitSum += profile.getWaitNanos();
        memSum += profile.getPeakLocalMemoryAllocated();
        for (final StreamProfile sp : profile.getInputProfileList()) {
            recordSum += sp.getRecords();
        }
        opList.add(ip.getLeft());
    }
    final ImmutablePair<OperatorProfile, Integer> longSetup = Collections.max(opList, Comparators.setupTime);
    tb.appendNanos(Math.round(setupSum / size));
    tb.appendNanos(longSetup.getLeft().getSetupNanos());
    final ImmutablePair<OperatorProfile, Integer> longProcess = Collections.max(opList, Comparators.processTime);
    tb.appendNanos(Math.round(processSum / size));
    tb.appendNanos(longProcess.getLeft().getProcessNanos());
    final ImmutablePair<OperatorProfile, Integer> shortWait = Collections.min(opList, Comparators.waitTime);
    final ImmutablePair<OperatorProfile, Integer> longWait = Collections.max(opList, Comparators.waitTime);
    tb.appendNanos(shortWait.getLeft().getWaitNanos());
    tb.appendNanos(Math.round(waitSum / size));
    tb.appendNanos(longWait.getLeft().getWaitNanos());
    tb.appendPercent(processSum / majorBusyNanos);
    tb.appendPercent(processSum / majorFragmentBusyTallyTotal);
    tb.appendFormattedInteger(recordSum);
    final ImmutablePair<OperatorProfile, Integer> peakMem = Collections.max(opList, Comparators.operatorPeakMemory);
    tb.appendBytes(Math.round(memSum / size));
    tb.appendBytes(peakMem.getLeft().getPeakLocalMemoryAllocated());
}

Example 5 with OperatorProfile

use of org.apache.drill.exec.proto.UserBitShared.OperatorProfile in project drill by axbaretto.

the class OperatorWrapper method getMetricsTable.

public String getMetricsTable() {
    if (operatorType == null) {
        return "";
    }
    final String[] metricNames = OperatorMetricRegistry.getMetricNames(operatorType.getNumber());
    if (metricNames == null) {
        return "";
    }
    final String[] metricsTableColumnNames = new String[metricNames.length + 1];
    metricsTableColumnNames[0] = "Minor Fragment";
    int i = 1;
    for (final String metricName : metricNames) {
        metricsTableColumnNames[i++] = metricName;
    }
    final TableBuilder builder = new TableBuilder(metricsTableColumnNames, null);
    for (final ImmutablePair<ImmutablePair<OperatorProfile, Integer>, String> ip : opsAndHosts) {
        final OperatorProfile op = ip.getLeft().getLeft();
        builder.appendCell(new OperatorPathBuilder().setMajor(major).setMinor(ip.getLeft().getRight()).setOperator(op).build());
        final Number[] values = new Number[metricNames.length];
        // Track new/Unknown Metrics
        final Set<Integer> unknownMetrics = new TreeSet<Integer>();
        for (final MetricValue metric : op.getMetricList()) {
            if (metric.getMetricId() < metricNames.length) {
                if (metric.hasLongValue()) {
                    values[metric.getMetricId()] = metric.getLongValue();
                } else if (metric.hasDoubleValue()) {
                    values[metric.getMetricId()] = metric.getDoubleValue();
                }
            } else {
                // Tracking unknown metric IDs
                unknownMetrics.add(metric.getMetricId());
            }
        }
        for (final Number value : values) {
            if (value != null) {
                builder.appendFormattedNumber(value);
            } else {
                builder.appendCell("");
            }
        }
    }
    return builder.build();
}