Examples with MinorFragmentProfile org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile used on opensource projects

Example 11 with MinorFragmentProfile

use of org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile in project drill by axbaretto.

the class FragmentWrapper method getContent.

public String getContent() {
    final TableBuilder builder = new TableBuilder(FRAGMENT_COLUMNS, FRAGMENT_COLUMNS_TOOLTIP, true);
    // Use only minor fragments that have complete profiles
    // Complete iff the fragment profile has at least one operator profile, and start and end times.
    final List<MinorFragmentProfile> complete = new ArrayList<>(Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes));
    final List<MinorFragmentProfile> incomplete = new ArrayList<>(Collections2.filter(major.getMinorFragmentProfileList(), Filters.missingOperatorsOrTimes));
    Collections.sort(complete, Comparators.minorId);
    // Reusing for different fragments
    Map<String, String> attributeMap = new HashMap<String, String>();
    for (final MinorFragmentProfile minor : complete) {
        final ArrayList<OperatorProfile> ops = new ArrayList<>(minor.getOperatorProfileList());
        long biggestIncomingRecords = 0;
        long biggestBatches = 0;
        for (final OperatorProfile op : ops) {
            long incomingRecords = 0;
            long batches = 0;
            for (final StreamProfile sp : op.getInputProfileList()) {
                incomingRecords += sp.getRecords();
                batches += sp.getBatches();
            }
            biggestIncomingRecords = Math.max(biggestIncomingRecords, incomingRecords);
            biggestBatches = Math.max(biggestBatches, batches);
        }
        // Overwrite values from previous fragments
        attributeMap.put("data-order", String.valueOf(minor.getMinorFragmentId()));
        builder.appendCell(new OperatorPathBuilder().setMajor(major).setMinor(minor).build(), attributeMap);
        builder.appendCell(minor.getEndpoint().getAddress());
        builder.appendMillis(minor.getStartTime() - start);
        builder.appendMillis(minor.getEndTime() - start);
        builder.appendMillis(minor.getEndTime() - minor.getStartTime());
        builder.appendFormattedInteger(biggestIncomingRecords);
        builder.appendFormattedInteger(biggestBatches);
        builder.appendTime(minor.getLastUpdate());
        builder.appendTime(minor.getLastProgress());
        builder.appendBytes(minor.getMaxMemoryUsed());
        builder.appendCell(minor.getState().name());
    }
    for (final MinorFragmentProfile m : incomplete) {
        builder.appendCell(major.getMajorFragmentId() + "-" + m.getMinorFragmentId());
        builder.appendRepeated(m.getState().toString(), null, NUM_NULLABLE_FRAGMENTS_COLUMNS);
    }
    return builder.build();
}

Example 12 with MinorFragmentProfile

use of org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile in project drill by axbaretto.

the class ProfileWrapper method tallyMajorFragmentCost.

private long tallyMajorFragmentCost(List<MajorFragmentProfile> majorFragments) {
    long globalProcessNanos = 0L;
    for (MajorFragmentProfile majorFP : majorFragments) {
        String majorFragmentId = new OperatorPathBuilder().setMajor(majorFP).build();
        long processNanos = 0L;
        for (MinorFragmentProfile minorFP : majorFP.getMinorFragmentProfileList()) {
            for (OperatorProfile op : minorFP.getOperatorProfileList()) {
                processNanos += op.getProcessNanos();
            }
        }
        majorFragmentTallyMap.put(majorFragmentId, processNanos);
        globalProcessNanos += processNanos;
    }
    return globalProcessNanos;
}

Example 13 with MinorFragmentProfile

use of org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile in project drill by axbaretto.

the class ProfileWrapper method getPlanningDuration.

public String getPlanningDuration() {
    // Check if Planning End is known
    if (profile.getPlanEnd() > 0L) {
        return (new SimpleDurationFormat(profile.getStart(), profile.getPlanEnd())).verbose();
    }
    // Check if any fragments have started
    if (profile.getFragmentProfileCount() > 0) {
        // Init Planning End Time
        long estimatedPlanEnd = Long.MAX_VALUE;
        // Using Screen MajorFragment as reference
        MajorFragmentProfile majorFrag0 = profile.getFragmentProfile(0);
        // Searching for earliest starting fragment
        for (MinorFragmentProfile fragmentWrapper : majorFrag0.getMinorFragmentProfileList()) {
            long minorFragmentStart = fragmentWrapper.getStartTime();
            if (minorFragmentStart > 0 && minorFragmentStart < estimatedPlanEnd) {
                estimatedPlanEnd = minorFragmentStart;
            }
        }
        // Provide estimated plan time
        return (new SimpleDurationFormat(profile.getStart(), estimatedPlanEnd)).verbose() + ESTIMATED_LABEL;
    }
    // Unable to  estimate/calculate Specific Time spent in Planning
    return NOT_AVAILABLE_LABEL;
}

Example 14 with MinorFragmentProfile

use of org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile in project drill by axbaretto.

the class FragmentData method madeProgress.

private boolean madeProgress(final FragmentStatus prev, final FragmentStatus cur) {
    final MinorFragmentProfile previous = prev.getProfile();
    final MinorFragmentProfile current = cur.getProfile();
    if (previous.getState() != current.getState()) {
        return true;
    }
    if (previous.getOperatorProfileCount() != current.getOperatorProfileCount()) {
        return true;
    }
    for (int i = 0; i < current.getOperatorProfileCount(); i++) {
        if (madeProgress(previous.getOperatorProfile(i), current.getOperatorProfile(i))) {
            return true;
        }
    }
    return false;
}