Examples with MinorFragmentProfile org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile used on opensource projects

Example 1 with MinorFragmentProfile

use of org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile in project drill by axbaretto.

the class FragmentWrapper method addFinalSummary.

public void addFinalSummary(TableBuilder tb) {
    // Use only minor fragments that have complete profiles
    // Complete iff the fragment profile has at least one operator profile, and start and end times.
    final List<MinorFragmentProfile> complete = new ArrayList<>(Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes));
    tb.appendCell(new OperatorPathBuilder().setMajor(major).build());
    tb.appendCell(complete.size() + " / " + major.getMinorFragmentProfileCount());
    // If there are no stats to aggregate, create an empty row
    if (complete.size() < 1) {
        tb.appendRepeated("", null, NUM_NULLABLE_COMPLETED_OVERVIEW_COLUMNS);
        return;
    }
    final MinorFragmentProfile firstStart = Collections.min(complete, Comparators.startTime);
    final MinorFragmentProfile lastStart = Collections.max(complete, Comparators.startTime);
    tb.appendMillis(firstStart.getStartTime() - start);
    tb.appendMillis(lastStart.getStartTime() - start);
    final MinorFragmentProfile firstEnd = Collections.min(complete, Comparators.endTime);
    final MinorFragmentProfile lastEnd = Collections.max(complete, Comparators.endTime);
    tb.appendMillis(firstEnd.getEndTime() - start);
    tb.appendMillis(lastEnd.getEndTime() - start);
    long totalDuration = 0L;
    double totalProcessInMillis = 0.0d;
    double totalWaitInMillis = 0.0d;
    for (final MinorFragmentProfile p : complete) {
        totalDuration += p.getEndTime() - p.getStartTime();
        // Capture Busy & Wait Time
        List<OperatorProfile> opProfileList = p.getOperatorProfileList();
        for (OperatorProfile operatorProfile : opProfileList) {
            totalProcessInMillis += operatorProfile.getProcessNanos() / 1E6;
            totalWaitInMillis += operatorProfile.getWaitNanos() / 1E6;
        }
    }
    final MinorFragmentProfile shortRun = Collections.min(complete, Comparators.runTime);
    final MinorFragmentProfile longRun = Collections.max(complete, Comparators.runTime);
    tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime());
    tb.appendMillis(totalDuration / complete.size());
    tb.appendMillis(longRun.getEndTime() - longRun.getStartTime());
    tb.appendPercent(totalProcessInMillis / (totalProcessInMillis + totalWaitInMillis), null, // #8721 is the summation sign: sum(Busy): ## + sum(Wait): ##
    String.format("&#8721;Busy: %,.2fs + &#8721;Wait: %,.2fs", totalProcessInMillis / 1E3, totalWaitInMillis / 1E3));
    // TODO(DRILL-3494): Names (maxMem, getMaxMemoryUsed) are misleading; the value is peak memory allocated to fragment
    final MinorFragmentProfile maxMem = Collections.max(complete, Comparators.fragmentPeakMemory);
    tb.appendBytes(maxMem.getMaxMemoryUsed());
}

Example 2 with MinorFragmentProfile

use of org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile in project drill by axbaretto.

the class FragmentWrapper method addSummary.

public void addSummary(TableBuilder tb) {
    // Use only minor fragments that have complete profiles
    // Complete iff the fragment profile has at least one operator profile, and start and end times.
    final List<MinorFragmentProfile> complete = new ArrayList<>(Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes));
    tb.appendCell(new OperatorPathBuilder().setMajor(major).build());
    tb.appendCell(complete.size() + " / " + major.getMinorFragmentProfileCount());
    // If there are no stats to aggregate, create an empty row
    if (complete.size() < 1) {
        tb.appendRepeated("", null, NUM_NULLABLE_ACTIVE_OVERVIEW_COLUMNS);
        return;
    }
    final MinorFragmentProfile firstStart = Collections.min(complete, Comparators.startTime);
    final MinorFragmentProfile lastStart = Collections.max(complete, Comparators.startTime);
    tb.appendMillis(firstStart.getStartTime() - start);
    tb.appendMillis(lastStart.getStartTime() - start);
    final MinorFragmentProfile firstEnd = Collections.min(complete, Comparators.endTime);
    final MinorFragmentProfile lastEnd = Collections.max(complete, Comparators.endTime);
    tb.appendMillis(firstEnd.getEndTime() - start);
    tb.appendMillis(lastEnd.getEndTime() - start);
    long cumulativeFragmentDurationInMillis = 0L;
    long cumulativeProcessInNanos = 0L;
    long cumulativeWaitInNanos = 0L;
    for (final MinorFragmentProfile p : complete) {
        cumulativeFragmentDurationInMillis += p.getEndTime() - p.getStartTime();
        // Capture Busy & Wait Time
        List<OperatorProfile> opProfileList = p.getOperatorProfileList();
        for (OperatorProfile operatorProfile : opProfileList) {
            cumulativeProcessInNanos += operatorProfile.getProcessNanos();
            cumulativeWaitInNanos += operatorProfile.getWaitNanos();
        }
    }
    double totalProcessInMillis = Math.round(cumulativeProcessInNanos / 1E6);
    double totalWaitInMillis = Math.round(cumulativeWaitInNanos / 1E6);
    final MinorFragmentProfile shortRun = Collections.min(complete, Comparators.runTime);
    final MinorFragmentProfile longRun = Collections.max(complete, Comparators.runTime);
    tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime());
    tb.appendMillis(cumulativeFragmentDurationInMillis / complete.size());
    tb.appendMillis(longRun.getEndTime() - longRun.getStartTime());
    tb.appendPercent(totalProcessInMillis / (totalProcessInMillis + totalWaitInMillis), null, // #8721 is the summation sign: sum(Busy): ## + sum(Wait): ##
    String.format("&#8721;Busy: %,.2fs + &#8721;Wait: %,.2fs", totalProcessInMillis / 1E3, totalWaitInMillis / 1E3));
    final MinorFragmentProfile lastUpdate = Collections.max(complete, Comparators.lastUpdate);
    tb.appendMillis(System.currentTimeMillis() - lastUpdate.getLastUpdate());
    final MinorFragmentProfile lastProgress = Collections.max(complete, Comparators.lastProgress);
    tb.appendMillis(System.currentTimeMillis() - lastProgress.getLastProgress());
    // TODO(DRILL-3494): Names (maxMem, getMaxMemoryUsed) are misleading; the value is peak memory allocated to fragment
    final MinorFragmentProfile maxMem = Collections.max(complete, Comparators.fragmentPeakMemory);
    tb.appendBytes(maxMem.getMaxMemoryUsed());
}

Example 3 with MinorFragmentProfile

use of org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile in project drill by axbaretto.

the class ProfileWrapper method getExecutionDuration.

public String getExecutionDuration() {
    // Check if State is PREPARING, PLANNING, STARTING or ENQUEUED
    if (profile.getState() == QueryState.PREPARING || profile.getState() == QueryState.PLANNING || profile.getState() == QueryState.STARTING || profile.getState() == QueryState.ENQUEUED) {
        return NOT_AVAILABLE_LABEL;
    }
    long queryEndTime;
    // Check if State is RUNNING, set end time to current time
    if (profile.getState() == QueryState.RUNNING) {
        queryEndTime = System.currentTimeMillis();
    } else {
        queryEndTime = profile.getEnd();
    }
    // Check if QueueEnd is known
    if (profile.getQueueWaitEnd() > 0L) {
        // Execution time [end(QueueWait) - endTime(Query)]
        return (new SimpleDurationFormat(profile.getQueueWaitEnd(), queryEndTime)).verbose();
    }
    // Check if Plan End is known
    if (profile.getPlanEnd() > 0L) {
        // Execution time [end(Planning) - endTime(Query)]
        return (new SimpleDurationFormat(profile.getPlanEnd(), queryEndTime)).verbose();
    }
    // Check if any fragments have started
    if (profile.getFragmentProfileCount() > 0) {
        // Providing Invalid Planning End Time (Will update later)
        long estimatedPlanEnd = Long.MAX_VALUE;
        // Using Screen MajorFragment as reference
        MajorFragmentProfile majorFrag0 = profile.getFragmentProfile(0);
        // Searching for earliest starting fragment
        for (MinorFragmentProfile fragmentWrapper : majorFrag0.getMinorFragmentProfileList()) {
            long minorFragmentStart = fragmentWrapper.getStartTime();
            if (minorFragmentStart > 0 && minorFragmentStart < estimatedPlanEnd) {
                estimatedPlanEnd = minorFragmentStart;
            }
        }
        // Execution time [start(rootFragment) - endTime(Query)]
        return (new SimpleDurationFormat(estimatedPlanEnd, queryEndTime)).verbose() + ESTIMATED_LABEL;
    }
    // Unable to  estimate/calculate Specific Execution Time
    return NOT_AVAILABLE_LABEL;
}

Example 4 with MinorFragmentProfile

use of org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile in project drill by apache.

the class FragmentData method madeProgress.

private boolean madeProgress(final FragmentStatus prev, final FragmentStatus cur) {
    final MinorFragmentProfile previous = prev.getProfile();
    final MinorFragmentProfile current = cur.getProfile();
    if (previous.getState() != current.getState()) {
        return true;
    }
    if (previous.getOperatorProfileCount() != current.getOperatorProfileCount()) {
        return true;
    }
    for (int i = 0; i < current.getOperatorProfileCount(); i++) {
        if (madeProgress(previous.getOperatorProfile(i), current.getOperatorProfile(i))) {
            return true;
        }
    }
    return false;
}

Example 5 with MinorFragmentProfile

use of org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile in project drill by apache.

the class FragmentWrapper method addFinalSummary.

public void addFinalSummary(TableBuilder tb) {
    // Use only minor fragments that have complete profiles
    // Complete iff the fragment profile has at least one operator profile, and start and end times.
    final List<MinorFragmentProfile> complete = new ArrayList<>(Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes));
    tb.appendCell(new OperatorPathBuilder().setMajor(major).build());
    tb.appendCell(complete.size() + " / " + major.getMinorFragmentProfileCount());
    // If there are no stats to aggregate, create an empty row
    if (complete.size() < 1) {
        tb.appendRepeated("", NUM_NULLABLE_COMPLETED_OVERVIEW_COLUMNS);
        return;
    }
    final MinorFragmentProfile firstStart = Collections.min(complete, Comparators.startTime);
    final MinorFragmentProfile lastStart = Collections.max(complete, Comparators.startTime);
    tb.appendMillis(firstStart.getStartTime() - start);
    tb.appendMillis(lastStart.getStartTime() - start);
    final MinorFragmentProfile firstEnd = Collections.min(complete, Comparators.endTime);
    final MinorFragmentProfile lastEnd = Collections.max(complete, Comparators.endTime);
    tb.appendMillis(firstEnd.getEndTime() - start);
    tb.appendMillis(lastEnd.getEndTime() - start);
    long totalDuration = 0L;
    double totalProcessInMillis = 0.0d;
    double totalWaitInMillis = 0.0d;
    for (final MinorFragmentProfile p : complete) {
        totalDuration += p.getEndTime() - p.getStartTime();
        // Capture Busy & Wait Time
        List<OperatorProfile> opProfileList = p.getOperatorProfileList();
        for (OperatorProfile operatorProfile : opProfileList) {
            totalProcessInMillis += operatorProfile.getProcessNanos() / 1E6;
            totalWaitInMillis += operatorProfile.getWaitNanos() / 1E6;
        }
    }
    final MinorFragmentProfile shortRun = Collections.min(complete, Comparators.runTime);
    final MinorFragmentProfile longRun = Collections.max(complete, Comparators.runTime);
    tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime());
    tb.appendMillis(totalDuration / complete.size());
    tb.appendMillis(longRun.getEndTime() - longRun.getStartTime());
    Map<String, String> percBusyAttrMap = new HashMap<>();
    // #8721 is the summation sign: sum(Busy): ## + sum(Wait): ##
    percBusyAttrMap.put(HtmlAttribute.TITLE, String.format("&#8721;Busy: %,.2fs + &#8721;Wait: %,.2fs", totalProcessInMillis / 1E3, totalWaitInMillis / 1E3));
    tb.appendPercent(totalProcessInMillis / (totalProcessInMillis + totalWaitInMillis), percBusyAttrMap);
    // TODO(DRILL-3494): Names (maxMem, getMaxMemoryUsed) are misleading; the value is peak memory allocated to fragment
    final MinorFragmentProfile maxMem = Collections.max(complete, Comparators.fragmentPeakMemory);
    tb.appendBytes(maxMem.getMaxMemoryUsed());
}