Examples with StreamProfile org.apache.drill.exec.proto.UserBitShared.StreamProfile used on opensource projects

Example 1 with StreamProfile

use of org.apache.drill.exec.proto.UserBitShared.StreamProfile in project drill by apache.

the class FragmentWrapper method getContent.

public String getContent() {
    final TableBuilder builder = new TableBuilder(FRAGMENT_COLUMNS, null);
    // Use only minor fragments that have complete profiles
    // Complete iff the fragment profile has at least one operator profile, and start and end times.
    final List<MinorFragmentProfile> complete = new ArrayList<>(Collections2.filter(major.getMinorFragmentProfileList(), Filters.hasOperatorsAndTimes));
    final List<MinorFragmentProfile> incomplete = new ArrayList<>(Collections2.filter(major.getMinorFragmentProfileList(), Filters.missingOperatorsOrTimes));
    Collections.sort(complete, Comparators.minorId);
    for (final MinorFragmentProfile minor : complete) {
        final ArrayList<OperatorProfile> ops = new ArrayList<>(minor.getOperatorProfileList());
        long biggestIncomingRecords = 0;
        long biggestBatches = 0;
        for (final OperatorProfile op : ops) {
            long incomingRecords = 0;
            long batches = 0;
            for (final StreamProfile sp : op.getInputProfileList()) {
                incomingRecords += sp.getRecords();
                batches += sp.getBatches();
            }
            biggestIncomingRecords = Math.max(biggestIncomingRecords, incomingRecords);
            biggestBatches = Math.max(biggestBatches, batches);
        }
        builder.appendCell(new OperatorPathBuilder().setMajor(major).setMinor(minor).build());
        builder.appendCell(minor.getEndpoint().getAddress());
        builder.appendMillis(minor.getStartTime() - start);
        builder.appendMillis(minor.getEndTime() - start);
        builder.appendMillis(minor.getEndTime() - minor.getStartTime());
        builder.appendFormattedInteger(biggestIncomingRecords);
        builder.appendFormattedInteger(biggestBatches);
        builder.appendTime(minor.getLastUpdate());
        builder.appendTime(minor.getLastProgress());
        builder.appendBytes(minor.getMaxMemoryUsed());
        builder.appendCell(minor.getState().name());
    }
    for (final MinorFragmentProfile m : incomplete) {
        builder.appendCell(major.getMajorFragmentId() + "-" + m.getMinorFragmentId());
        builder.appendRepeated(m.getState().toString(), null, NUM_NULLABLE_FRAGMENTS_COLUMNS);
    }
    return builder.build();
}

Example 2 with StreamProfile

use of org.apache.drill.exec.proto.UserBitShared.StreamProfile in project drill by apache.

the class OperatorWrapper method addSummary.

public void addSummary(TableBuilder tb, HashMap<String, Long> majorFragmentBusyTally, long majorFragmentBusyTallyTotal) {
    //Select background color from palette
    String opTblBgColor = OPERATOR_OVERVIEW_BGCOLOR_PALETTE[major % OPERATOR_OVERVIEW_BGCOLOR_PALETTE.length];
    String path = new OperatorPathBuilder().setMajor(major).setOperator(firstProfile).build();
    tb.appendCell(path, null, null, opTblBgColor);
    tb.appendCell(operatorName);
    //Get MajorFragment Busy+Wait Time Tally
    long majorBusyNanos = majorFragmentBusyTally.get(new OperatorPathBuilder().setMajor(major).build());
    double setupSum = 0.0;
    double processSum = 0.0;
    double waitSum = 0.0;
    double memSum = 0.0;
    long recordSum = 0L;
    for (ImmutablePair<OperatorProfile, Integer> ip : ops) {
        OperatorProfile profile = ip.getLeft();
        setupSum += profile.getSetupNanos();
        processSum += profile.getProcessNanos();
        waitSum += profile.getWaitNanos();
        memSum += profile.getPeakLocalMemoryAllocated();
        for (final StreamProfile sp : profile.getInputProfileList()) {
            recordSum += sp.getRecords();
        }
    }
    final ImmutablePair<OperatorProfile, Integer> longSetup = Collections.max(ops, Comparators.setupTime);
    tb.appendNanos(Math.round(setupSum / size));
    tb.appendNanos(longSetup.getLeft().getSetupNanos());
    final ImmutablePair<OperatorProfile, Integer> longProcess = Collections.max(ops, Comparators.processTime);
    tb.appendNanos(Math.round(processSum / size));
    tb.appendNanos(longProcess.getLeft().getProcessNanos());
    final ImmutablePair<OperatorProfile, Integer> shortWait = Collections.min(ops, Comparators.waitTime);
    final ImmutablePair<OperatorProfile, Integer> longWait = Collections.max(ops, Comparators.waitTime);
    tb.appendNanos(shortWait.getLeft().getWaitNanos());
    tb.appendNanos(Math.round(waitSum / size));
    tb.appendNanos(longWait.getLeft().getWaitNanos());
    tb.appendPercent(processSum / majorBusyNanos);
    tb.appendPercent(processSum / majorFragmentBusyTallyTotal);
    tb.appendFormattedInteger(recordSum);
    final ImmutablePair<OperatorProfile, Integer> peakMem = Collections.max(ops, Comparators.operatorPeakMemory);
    tb.appendBytes(Math.round(memSum / size));
    tb.appendBytes(peakMem.getLeft().getPeakLocalMemoryAllocated());
}

Example 3 with StreamProfile

use of org.apache.drill.exec.proto.UserBitShared.StreamProfile in project drill by apache.

the class OperatorWrapper method getContent.

public String getContent() {
    TableBuilder builder = new TableBuilder(OPERATOR_COLUMNS, null);
    for (ImmutablePair<OperatorProfile, Integer> ip : ops) {
        int minor = ip.getRight();
        OperatorProfile op = ip.getLeft();
        String path = new OperatorPathBuilder().setMajor(major).setMinor(minor).setOperator(op).build();
        builder.appendCell(path);
        builder.appendNanos(op.getSetupNanos());
        builder.appendNanos(op.getProcessNanos());
        builder.appendNanos(op.getWaitNanos());
        long maxBatches = Long.MIN_VALUE;
        long maxRecords = Long.MIN_VALUE;
        for (StreamProfile sp : op.getInputProfileList()) {
            maxBatches = Math.max(sp.getBatches(), maxBatches);
            maxRecords = Math.max(sp.getRecords(), maxRecords);
        }
        builder.appendFormattedInteger(maxBatches);
        builder.appendFormattedInteger(maxRecords);
        builder.appendBytes(op.getPeakLocalMemoryAllocated());
    }
    return builder.build();
}