Examples with OperatorProfile org.apache.drill.exec.proto.UserBitShared.OperatorProfile used on opensource projects

Example 6 with OperatorProfile

use of org.apache.drill.exec.proto.UserBitShared.OperatorProfile in project drill by apache.

the class OperatorStats method getProfile.

@SuppressWarnings("deprecation")
public OperatorProfile getProfile() {
    final OperatorProfile.Builder b = OperatorProfile.newBuilder().setOperatorTypeName(operatorType).setOperatorId(operatorId).setSetupNanos(setupNanos).setProcessNanos(processingNanos).setWaitNanos(waitNanos);
    CoreOperatorType coreOperatorType = CoreOperatorType.forName(operatorType);
    if (coreOperatorType != null) {
        b.setOperatorType(coreOperatorType.getId());
    }
    if (allocator != null) {
        b.setPeakLocalMemoryAllocated(allocator.getPeakMemoryAllocation());
    }
    addAllMetrics(b);
    return b.build();
}

Example 7 with OperatorProfile

use of org.apache.drill.exec.proto.UserBitShared.OperatorProfile in project drill by apache.

the class FragmentWrapper method addFinalSummary.

public void addFinalSummary(TableBuilder tb) {
    // Use only minor fragments that have complete profiles
    // Complete iff the fragment profile has at least one operator profile, and start and end times.
    final List<MinorFragmentProfile> complete = new ArrayList<>(Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes));
    tb.appendCell(new OperatorPathBuilder().setMajor(major).build());
    tb.appendCell(complete.size() + " / " + major.getMinorFragmentProfileCount());
    // If there are no stats to aggregate, create an empty row
    if (complete.size() < 1) {
        tb.appendRepeated("", NUM_NULLABLE_COMPLETED_OVERVIEW_COLUMNS);
        return;
    }
    final MinorFragmentProfile firstStart = Collections.min(complete, Comparators.startTime);
    final MinorFragmentProfile lastStart = Collections.max(complete, Comparators.startTime);
    tb.appendMillis(firstStart.getStartTime() - start);
    tb.appendMillis(lastStart.getStartTime() - start);
    final MinorFragmentProfile firstEnd = Collections.min(complete, Comparators.endTime);
    final MinorFragmentProfile lastEnd = Collections.max(complete, Comparators.endTime);
    tb.appendMillis(firstEnd.getEndTime() - start);
    tb.appendMillis(lastEnd.getEndTime() - start);
    long totalDuration = 0L;
    double totalProcessInMillis = 0.0d;
    double totalWaitInMillis = 0.0d;
    for (final MinorFragmentProfile p : complete) {
        totalDuration += p.getEndTime() - p.getStartTime();
        // Capture Busy & Wait Time
        List<OperatorProfile> opProfileList = p.getOperatorProfileList();
        for (OperatorProfile operatorProfile : opProfileList) {
            totalProcessInMillis += operatorProfile.getProcessNanos() / 1E6;
            totalWaitInMillis += operatorProfile.getWaitNanos() / 1E6;
        }
    }
    final MinorFragmentProfile shortRun = Collections.min(complete, Comparators.runTime);
    final MinorFragmentProfile longRun = Collections.max(complete, Comparators.runTime);
    tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime());
    tb.appendMillis(totalDuration / complete.size());
    tb.appendMillis(longRun.getEndTime() - longRun.getStartTime());
    Map<String, String> percBusyAttrMap = new HashMap<>();
    // #8721 is the summation sign: sum(Busy): ## + sum(Wait): ##
    percBusyAttrMap.put(HtmlAttribute.TITLE, String.format("&#8721;Busy: %,.2fs + &#8721;Wait: %,.2fs", totalProcessInMillis / 1E3, totalWaitInMillis / 1E3));
    tb.appendPercent(totalProcessInMillis / (totalProcessInMillis + totalWaitInMillis), percBusyAttrMap);
    // TODO(DRILL-3494): Names (maxMem, getMaxMemoryUsed) are misleading; the value is peak memory allocated to fragment
    final MinorFragmentProfile maxMem = Collections.max(complete, Comparators.fragmentPeakMemory);
    tb.appendBytes(maxMem.getMaxMemoryUsed());
}

Example 8 with OperatorProfile

use of org.apache.drill.exec.proto.UserBitShared.OperatorProfile in project drill by apache.

the class FragmentWrapper method addSummary.

public void addSummary(TableBuilder tb) {
    // Use only minor fragments that have complete profiles
    // Complete iff the fragment profile has at least one operator profile, and start and end times.
    final List<MinorFragmentProfile> complete = new ArrayList<>(Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes));
    tb.appendCell(new OperatorPathBuilder().setMajor(major).build());
    tb.appendCell(complete.size() + " / " + major.getMinorFragmentProfileCount());
    // If there are no stats to aggregate, create an empty row
    if (complete.size() < 1) {
        tb.appendRepeated("", NUM_NULLABLE_ACTIVE_OVERVIEW_COLUMNS);
        return;
    }
    final MinorFragmentProfile firstStart = Collections.min(complete, Comparators.startTime);
    final MinorFragmentProfile lastStart = Collections.max(complete, Comparators.startTime);
    tb.appendMillis(firstStart.getStartTime() - start);
    tb.appendMillis(lastStart.getStartTime() - start);
    final MinorFragmentProfile firstEnd = Collections.min(complete, Comparators.endTime);
    final MinorFragmentProfile lastEnd = Collections.max(complete, Comparators.endTime);
    tb.appendMillis(firstEnd.getEndTime() - start);
    tb.appendMillis(lastEnd.getEndTime() - start);
    long cumulativeFragmentDurationInMillis = 0L;
    long cumulativeProcessInNanos = 0L;
    long cumulativeWaitInNanos = 0L;
    for (final MinorFragmentProfile p : complete) {
        cumulativeFragmentDurationInMillis += p.getEndTime() - p.getStartTime();
        // Capture Busy & Wait Time
        List<OperatorProfile> opProfileList = p.getOperatorProfileList();
        for (OperatorProfile operatorProfile : opProfileList) {
            cumulativeProcessInNanos += operatorProfile.getProcessNanos();
            cumulativeWaitInNanos += operatorProfile.getWaitNanos();
        }
    }
    double totalProcessInMillis = Math.round(cumulativeProcessInNanos / 1E6);
    double totalWaitInMillis = Math.round(cumulativeWaitInNanos / 1E6);
    final MinorFragmentProfile shortRun = Collections.min(complete, Comparators.runTime);
    final MinorFragmentProfile longRun = Collections.max(complete, Comparators.runTime);
    tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime());
    tb.appendMillis(cumulativeFragmentDurationInMillis / complete.size());
    tb.appendMillis(longRun.getEndTime() - longRun.getStartTime());
    Map<String, String> percBusyAttrMap = new HashMap<>();
    // #8721 is the summation sign: sum(Busy): ## + sum(Wait): ##
    percBusyAttrMap.put(HtmlAttribute.TITLE, String.format("&#8721;Busy: %,.2fs + &#8721;Wait: %,.2fs", totalProcessInMillis / 1E3, totalWaitInMillis / 1E3));
    tb.appendPercent(totalProcessInMillis / (totalProcessInMillis + totalWaitInMillis), percBusyAttrMap);
    final MinorFragmentProfile lastUpdate = Collections.max(complete, Comparators.lastUpdate);
    tb.appendMillis(System.currentTimeMillis() - lastUpdate.getLastUpdate());
    final MinorFragmentProfile lastProgress = Collections.max(complete, Comparators.lastProgress);
    long elapsedSinceLastProgress = System.currentTimeMillis() - lastProgress.getLastProgress();
    Map<String, String> lastProgressAttrMap = null;
    if (elapsedSinceLastProgress > TimeUnit.SECONDS.toMillis(runningProfileProgressThreshold)) {
        lastProgressAttrMap = new HashMap<>();
        lastProgressAttrMap.put(HtmlAttribute.CLASS, HtmlAttribute.CLASS_VALUE_NO_PROGRESS_TAG);
    }
    tb.appendMillis(elapsedSinceLastProgress, lastProgressAttrMap);
    // TODO(DRILL-3494): Names (maxMem, getMaxMemoryUsed) are misleading; the value is peak memory allocated to fragment
    final MinorFragmentProfile maxMem = Collections.max(complete, Comparators.fragmentPeakMemory);
    tb.appendBytes(maxMem.getMaxMemoryUsed());
}

Example 9 with OperatorProfile

use of org.apache.drill.exec.proto.UserBitShared.OperatorProfile in project drill by apache.

the class FragmentWrapper method getContent.

public String getContent() {
    final TableBuilder builder = new TableBuilder(FRAGMENT_COLUMNS, FRAGMENT_COLUMNS_TOOLTIP, true);
    // Use only minor fragments that have complete profiles
    // Complete iff the fragment profile has at least one operator profile, and start and end times.
    final List<MinorFragmentProfile> complete = new ArrayList<>(Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes));
    final List<MinorFragmentProfile> incomplete = new ArrayList<>(Collections2.filter(major.getMinorFragmentProfileList(), Filters.missingOperatorsOrTimes));
    Collections.sort(complete, Comparators.minorId);
    // Reusing for different fragments
    Map<String, String> attributeMap = new HashMap<>();
    for (final MinorFragmentProfile minor : complete) {
        final List<OperatorProfile> ops = new ArrayList<>(minor.getOperatorProfileList());
        long biggestIncomingRecords = 0;
        long biggestBatches = 0;
        for (final OperatorProfile op : ops) {
            long incomingRecords = 0;
            long batches = 0;
            for (final StreamProfile sp : op.getInputProfileList()) {
                incomingRecords += sp.getRecords();
                batches += sp.getBatches();
            }
            biggestIncomingRecords = Math.max(biggestIncomingRecords, incomingRecords);
            biggestBatches = Math.max(biggestBatches, batches);
        }
        // Overwrite values from previous fragments
        attributeMap.put("data-order", String.valueOf(minor.getMinorFragmentId()));
        builder.appendCell(new OperatorPathBuilder().setMajor(major).setMinor(minor).build(), attributeMap);
        builder.appendCell(minor.getEndpoint().getAddress());
        builder.appendMillis(minor.getStartTime() - start);
        builder.appendMillis(minor.getEndTime() - start);
        builder.appendMillis(minor.getEndTime() - minor.getStartTime());
        builder.appendFormattedInteger(biggestIncomingRecords);
        builder.appendFormattedInteger(biggestBatches);
        builder.appendTime(minor.getLastUpdate());
        builder.appendTime(minor.getLastProgress());
        builder.appendBytes(minor.getMaxMemoryUsed());
        builder.appendCell(minor.getState().name());
    }
    for (final MinorFragmentProfile m : incomplete) {
        builder.appendCell(major.getMajorFragmentId() + "-" + m.getMinorFragmentId());
        builder.appendRepeated(m.getState().toString(), NUM_NULLABLE_FRAGMENTS_COLUMNS);
    }
    return builder.build();
}

Example 10 with OperatorProfile

use of org.apache.drill.exec.proto.UserBitShared.OperatorProfile in project drill by apache.

the class OperatorWrapper method getMetricsTable.

public String getMetricsTable() {
    if (operatorType == null) {
        return "";
    }
    final String[] metricNames = OperatorMetricRegistry.getMetricNames(operatorType);
    if (metricNames == null) {
        return "";
    }
    final String[] metricsTableColumnNames = new String[metricNames.length + 1];
    metricsTableColumnNames[0] = "Minor Fragment";
    int i = 1;
    for (final String metricName : metricNames) {
        metricsTableColumnNames[i++] = metricName;
    }
    final TableBuilder builder = new TableBuilder(metricsTableColumnNames, null);
    for (final ImmutablePair<ImmutablePair<OperatorProfile, Integer>, String> ip : opsAndHosts) {
        final OperatorProfile op = ip.getLeft().getLeft();
        builder.appendCell(new OperatorPathBuilder().setMajor(major).setMinor(ip.getLeft().getRight()).setOperator(op).build());
        final Number[] values = new Number[metricNames.length];
        // Track new/Unknown Metrics
        final Set<Integer> unknownMetrics = new TreeSet<>();
        for (final MetricValue metric : op.getMetricList()) {
            if (metric.getMetricId() < metricNames.length) {
                if (metric.hasLongValue()) {
                    values[metric.getMetricId()] = metric.getLongValue();
                } else if (metric.hasDoubleValue()) {
                    values[metric.getMetricId()] = metric.getDoubleValue();
                }
            } else {
                // Tracking unknown metric IDs
                unknownMetrics.add(metric.getMetricId());
            }
        }
        for (final Number value : values) {
            if (value != null) {
                builder.appendFormattedNumber(value);
            } else {
                builder.appendCell("");
            }
        }
    }
    return builder.build();
}