Examples with StepPerformanceSnapShot org.pentaho.di.trans.performance.StepPerformanceSnapShot used on opensource projects

Example 1 with StepPerformanceSnapShot

use of org.pentaho.di.trans.performance.StepPerformanceSnapShot in project pentaho-kettle by pentaho.

the class StepPerformanceSnapShotDialog method updateCanvas.

private void updateCanvas() {
    Rectangle bounds = canvas.getBounds();
    if (bounds.width <= 0 || bounds.height <= 0) {
        return;
    }
    // The list of snapshots : convert to JFreeChart dataset
    // 
    DefaultCategoryDataset dataset = new DefaultCategoryDataset();
    String[] selectedSteps = stepsList.getSelection();
    if (selectedSteps == null || selectedSteps.length == 0) {
        // first step
        selectedSteps = new String[] { steps[0] };
        stepsList.select(0);
    }
    int[] dataIndices = dataList.getSelectionIndices();
    if (dataIndices == null || dataIndices.length == 0) {
        dataIndices = new int[] { DATA_CHOICE_WRITTEN };
        dataList.select(0);
    }
    boolean multiStep = stepsList.getSelectionCount() > 1;
    boolean multiData = dataList.getSelectionCount() > 1;
    // A single metric shown for a single step
    boolean calcMoving = !multiStep && !multiData;
    List<Double> movingList = new ArrayList<Double>();
    int movingSize = 10;
    double movingTotal = 0;
    int totalTimeInSeconds = 0;
    for (int t = 0; t < selectedSteps.length; t++) {
        String stepNameCopy = selectedSteps[t];
        List<StepPerformanceSnapShot> snapShotList = stepPerformanceSnapShots.get(stepNameCopy);
        if (snapShotList != null && snapShotList.size() > 1) {
            totalTimeInSeconds = (int) Math.round(((double) (snapShotList.get(snapShotList.size() - 1).getDate().getTime() - snapShotList.get(0).getDate().getTime())) / 1000);
            for (int i = 0; i < snapShotList.size(); i++) {
                StepPerformanceSnapShot snapShot = snapShotList.get(i);
                if (snapShot.getTimeDifference() != 0) {
                    double factor = (double) 1000 / (double) snapShot.getTimeDifference();
                    for (int d = 0; d < dataIndices.length; d++) {
                        String dataType;
                        if (multiStep) {
                            dataType = stepNameCopy;
                        } else {
                            dataType = dataChoices[dataIndices[d]];
                        }
                        String xLabel = Integer.toString(Math.round(i * timeDifference / 1000));
                        Double metric = null;
                        switch(dataIndices[d]) {
                            case DATA_CHOICE_INPUT:
                                metric = snapShot.getLinesInput() * factor;
                                break;
                            case DATA_CHOICE_OUTPUT:
                                metric = snapShot.getLinesOutput() * factor;
                                break;
                            case DATA_CHOICE_READ:
                                metric = snapShot.getLinesRead() * factor;
                                break;
                            case DATA_CHOICE_WRITTEN:
                                metric = snapShot.getLinesWritten() * factor;
                                break;
                            case DATA_CHOICE_UPDATED:
                                metric = snapShot.getLinesUpdated() * factor;
                                break;
                            case DATA_CHOICE_REJECTED:
                                metric = snapShot.getLinesRejected() * factor;
                                break;
                            case DATA_CHOICE_INPUT_BUFFER_SIZE:
                                metric = (double) snapShot.getInputBufferSize();
                                break;
                            case DATA_CHOICE_OUTPUT_BUFFER_SIZE:
                                metric = (double) snapShot.getOutputBufferSize();
                                break;
                            default:
                                break;
                        }
                        if (metric != null) {
                            dataset.addValue(metric, dataType, xLabel);
                            if (calcMoving) {
                                movingTotal += metric;
                                movingList.add(metric);
                                if (movingList.size() > movingSize) {
                                    movingTotal -= movingList.get(0);
                                    movingList.remove(0);
                                }
                                double movingAverage = movingTotal / movingList.size();
                                dataset.addValue(movingAverage, dataType + "(Avg)", xLabel);
                            // System.out.println("moving average = "+movingAverage+", movingTotal="+movingTotal+", m");
                            }
                        }
                    }
                }
            }
        }
    }
    String chartTitle = title;
    if (multiStep) {
        chartTitle += " (" + dataChoices[dataIndices[0]] + ")";
    } else {
        chartTitle += " (" + selectedSteps[0] + ")";
    }
    final JFreeChart chart = // chart title
    ChartFactory.createLineChart(// chart title
    chartTitle, BaseMessages.getString(PKG, "StepPerformanceSnapShotDialog.TimeInSeconds.Label", Integer.toString(totalTimeInSeconds), // domain axis label
    Long.toString(timeDifference)), // range axis label
    BaseMessages.getString(PKG, "StepPerformanceSnapShotDialog.RowsPerSecond.Label"), // data
    dataset, // orientation
    PlotOrientation.VERTICAL, // include legend
    true, // tooltips
    true, // urls
    false);
    chart.setBackgroundPaint(Color.white);
    CategoryPlot plot = (CategoryPlot) chart.getPlot();
    plot.setBackgroundPaint(Color.white);
    plot.setForegroundAlpha(0.5f);
    plot.setRangeGridlinesVisible(true);
    NumberAxis rangeAxis = (NumberAxis) plot.getRangeAxis();
    rangeAxis.setStandardTickUnits(NumberAxis.createIntegerTickUnits());
    CategoryAxis domainAxis = plot.getDomainAxis();
    domainAxis.setTickLabelsVisible(false);
    // Customize the renderer...
    // 
    LineAndShapeRenderer renderer = (LineAndShapeRenderer) plot.getRenderer();
    renderer.setBaseShapesVisible(true);
    renderer.setDrawOutlines(true);
    renderer.setUseFillPaint(true);
    renderer.setBaseFillPaint(Color.white);
    renderer.setSeriesStroke(0, new BasicStroke(1.5f));
    renderer.setSeriesOutlineStroke(0, new BasicStroke(1.5f));
    renderer.setSeriesStroke(1, new BasicStroke(2.5f));
    renderer.setSeriesOutlineStroke(1, new BasicStroke(2.5f));
    renderer.setSeriesShape(0, new Ellipse2D.Double(-3.0, -3.0, 6.0, 6.0));
    BufferedImage bufferedImage = chart.createBufferedImage(bounds.width, bounds.height);
    ImageData imageData = ImageUtil.convertToSWT(bufferedImage);
    // 
    if (image != null) {
        image.dispose();
    }
    image = new Image(display, imageData);
    // Draw the image on the canvas...
    // 
    canvas.redraw();
}

Example 2 with StepPerformanceSnapShot

use of org.pentaho.di.trans.performance.StepPerformanceSnapShot in project pentaho-kettle by pentaho.

the class Trans method writeStepPerformanceLogRecords.

/**
 * Write step performance log records.
 *
 * @param startSequenceNr
 *          the start sequence numberr
 * @param status
 *          the logging status. If this is End, perform cleanup
 * @return the new sequence number
 * @throws KettleException
 *           if any errors occur during logging
 */
private int writeStepPerformanceLogRecords(int startSequenceNr, LogStatus status) throws KettleException {
    int lastSeqNr = 0;
    Database ldb = null;
    PerformanceLogTable performanceLogTable = transMeta.getPerformanceLogTable();
    if (!performanceLogTable.isDefined() || !transMeta.isCapturingStepPerformanceSnapShots() || stepPerformanceSnapShots == null || stepPerformanceSnapShots.isEmpty()) {
        // nothing to do here!
        return 0;
    }
    try {
        ldb = new Database(this, performanceLogTable.getDatabaseMeta());
        ldb.shareVariablesWith(this);
        ldb.connect();
        ldb.setCommit(logCommitSize);
        // Write to the step performance log table...
        // 
        RowMetaInterface rowMeta = performanceLogTable.getLogRecord(LogStatus.START, null, null).getRowMeta();
        ldb.prepareInsert(rowMeta, performanceLogTable.getActualSchemaName(), performanceLogTable.getActualTableName());
        synchronized (stepPerformanceSnapShots) {
            Iterator<List<StepPerformanceSnapShot>> iterator = stepPerformanceSnapShots.values().iterator();
            while (iterator.hasNext()) {
                List<StepPerformanceSnapShot> snapshots = iterator.next();
                synchronized (snapshots) {
                    Iterator<StepPerformanceSnapShot> snapshotsIterator = snapshots.iterator();
                    while (snapshotsIterator.hasNext()) {
                        StepPerformanceSnapShot snapshot = snapshotsIterator.next();
                        if (snapshot.getSeqNr() >= startSequenceNr && snapshot.getSeqNr() <= lastStepPerformanceSnapshotSeqNrAdded) {
                            RowMetaAndData row = performanceLogTable.getLogRecord(LogStatus.START, snapshot, null);
                            ldb.setValuesInsert(row.getRowMeta(), row.getData());
                            ldb.insertRow(true);
                        }
                        lastSeqNr = snapshot.getSeqNr();
                    }
                }
            }
        }
        ldb.insertFinished(true);
        // 
        if (status.equals(LogStatus.END)) {
            ldb.cleanupLogRecords(performanceLogTable);
        }
    } catch (Exception e) {
        throw new KettleException(BaseMessages.getString(PKG, "Trans.Exception.ErrorWritingStepPerformanceLogRecordToTable"), e);
    } finally {
        if (ldb != null) {
            ldb.disconnect();
        }
    }
    return lastSeqNr + 1;
}

Example 3 with StepPerformanceSnapShot

use of org.pentaho.di.trans.performance.StepPerformanceSnapShot in project pentaho-kettle by pentaho.

the class Trans method addStepPerformanceSnapShot.

/**
 * Adds a step performance snapshot.
 */
protected void addStepPerformanceSnapShot() {
    if (stepPerformanceSnapShots == null) {
        // Race condition somewhere?
        return;
    }
    boolean pausedAndNotEmpty = isPaused() && !stepPerformanceSnapShots.isEmpty();
    boolean stoppedAndNotEmpty = isStopped() && !stepPerformanceSnapShots.isEmpty();
    if (transMeta.isCapturingStepPerformanceSnapShots() && !pausedAndNotEmpty && !stoppedAndNotEmpty) {
        // get the statistics from the steps and keep them...
        // 
        int seqNr = stepPerformanceSnapshotSeqNr.incrementAndGet();
        for (int i = 0; i < steps.size(); i++) {
            StepMeta stepMeta = steps.get(i).stepMeta;
            StepInterface step = steps.get(i).step;
            StepPerformanceSnapShot snapShot = new StepPerformanceSnapShot(seqNr, getBatchId(), new Date(), getName(), stepMeta.getName(), step.getCopy(), step.getLinesRead(), step.getLinesWritten(), step.getLinesInput(), step.getLinesOutput(), step.getLinesUpdated(), step.getLinesRejected(), step.getErrors());
            List<StepPerformanceSnapShot> snapShotList = stepPerformanceSnapShots.get(step.toString());
            StepPerformanceSnapShot previous;
            if (snapShotList == null) {
                snapShotList = new ArrayList<>();
                stepPerformanceSnapShots.put(step.toString(), snapShotList);
                previous = null;
            } else {
                // the last one...
                previous = snapShotList.get(snapShotList.size() - 1);
            }
            // Make the difference...
            // 
            snapShot.diff(previous, step.rowsetInputSize(), step.rowsetOutputSize());
            synchronized (stepPerformanceSnapShots) {
                snapShotList.add(snapShot);
                if (stepPerformanceSnapshotSizeLimit > 0 && snapShotList.size() > stepPerformanceSnapshotSizeLimit) {
                    snapShotList.remove(0);
                }
            }
        }
        lastStepPerformanceSnapshotSeqNrAdded = stepPerformanceSnapshotSeqNr.get();
    }
}

Example 4 with StepPerformanceSnapShot

use of org.pentaho.di.trans.performance.StepPerformanceSnapShot in project pentaho-kettle by pentaho.

the class TransPerfDelegate method updateCanvas.

private void updateCanvas() {
    Rectangle bounds = canvas.getBounds();
    if (bounds.width <= 0 || bounds.height <= 0) {
        return;
    }
    // The list of snapshots : convert to JFreeChart dataset
    // 
    DefaultCategoryDataset dataset = new DefaultCategoryDataset();
    String[] selectedSteps = stepsList.getSelection();
    if (selectedSteps == null || selectedSteps.length == 0) {
        // first step
        selectedSteps = new String[] { steps[0] };
        stepsList.select(0);
    }
    int[] dataIndices = dataList.getSelectionIndices();
    if (dataIndices == null || dataIndices.length == 0) {
        dataIndices = new int[] { DATA_CHOICE_WRITTEN };
        dataList.select(0);
    }
    boolean multiStep = stepsList.getSelectionCount() > 1;
    boolean multiData = dataList.getSelectionCount() > 1;
    // A single metric shown for a single step
    boolean calcMoving = !multiStep && !multiData;
    List<Double> movingList = new ArrayList<Double>();
    int movingSize = 10;
    double movingTotal = 0;
    int totalTimeInSeconds = 0;
    for (int t = 0; t < selectedSteps.length; t++) {
        String stepNameCopy = selectedSteps[t];
        List<StepPerformanceSnapShot> snapShotList = stepPerformanceSnapShots.get(stepNameCopy);
        if (snapShotList != null && snapShotList.size() > 1) {
            totalTimeInSeconds = (int) Math.round(((double) (snapShotList.get(snapShotList.size() - 1).getDate().getTime() - snapShotList.get(0).getDate().getTime())) / 1000);
            for (int i = 0; i < snapShotList.size(); i++) {
                StepPerformanceSnapShot snapShot = snapShotList.get(i);
                if (snapShot.getTimeDifference() != 0) {
                    double factor = (double) 1000 / (double) snapShot.getTimeDifference();
                    for (int d = 0; d < dataIndices.length; d++) {
                        String dataType;
                        if (multiStep) {
                            dataType = stepNameCopy;
                        } else {
                            dataType = dataChoices[dataIndices[d]];
                        }
                        String xLabel = Integer.toString(Math.round(i * timeDifference / 1000));
                        Double metric = null;
                        switch(dataIndices[d]) {
                            case DATA_CHOICE_INPUT:
                                metric = snapShot.getLinesInput() * factor;
                                break;
                            case DATA_CHOICE_OUTPUT:
                                metric = snapShot.getLinesOutput() * factor;
                                break;
                            case DATA_CHOICE_READ:
                                metric = snapShot.getLinesRead() * factor;
                                break;
                            case DATA_CHOICE_WRITTEN:
                                metric = snapShot.getLinesWritten() * factor;
                                break;
                            case DATA_CHOICE_UPDATED:
                                metric = snapShot.getLinesUpdated() * factor;
                                break;
                            case DATA_CHOICE_REJECTED:
                                metric = snapShot.getLinesRejected() * factor;
                                break;
                            case DATA_CHOICE_INPUT_BUFFER_SIZE:
                                metric = (double) snapShot.getInputBufferSize();
                                break;
                            case DATA_CHOICE_OUTPUT_BUFFER_SIZE:
                                metric = (double) snapShot.getOutputBufferSize();
                                break;
                            default:
                                break;
                        }
                        if (metric != null) {
                            dataset.addValue(metric, dataType, xLabel);
                            if (calcMoving) {
                                movingTotal += metric;
                                movingList.add(metric);
                                if (movingList.size() > movingSize) {
                                    movingTotal -= movingList.get(0);
                                    movingList.remove(0);
                                }
                                double movingAverage = movingTotal / movingList.size();
                                dataset.addValue(movingAverage, dataType + "(Avg)", xLabel);
                            // System.out.println("moving average = "+movingAverage+", movingTotal="+movingTotal+", m");
                            }
                        }
                    }
                }
            }
        }
    }
    String chartTitle = title;
    if (multiStep) {
        chartTitle += " (" + dataChoices[dataIndices[0]] + ")";
    } else {
        chartTitle += " (" + selectedSteps[0] + ")";
    }
    final JFreeChart chart = // chart title
    ChartFactory.createLineChart(// chart title
    chartTitle, BaseMessages.getString(PKG, "StepPerformanceSnapShotDialog.TimeInSeconds.Label", Integer.toString(totalTimeInSeconds), // domain axis label
    Long.toString(timeDifference)), // range axis label
    BaseMessages.getString(PKG, "StepPerformanceSnapShotDialog.RowsPerSecond.Label"), // data
    dataset, // orientation
    PlotOrientation.VERTICAL, // include legend
    true, // tooltips
    true, // urls
    false);
    chart.setBackgroundPaint(Color.white);
    TextTitle title = new TextTitle(chartTitle);
    // title.setExpandToFitSpace(true);
    // org.eclipse.swt.graphics.Color pentahoColor = GUIResource.getInstance().getColorPentaho();
    // java.awt.Color color = new java.awt.Color(pentahoColor.getRed(), pentahoColor.getGreen(),pentahoColor.getBlue());
    // title.setBackgroundPaint(color);
    title.setFont(new java.awt.Font("SansSerif", java.awt.Font.BOLD, 12));
    chart.setTitle(title);
    CategoryPlot plot = (CategoryPlot) chart.getPlot();
    plot.setBackgroundPaint(Color.white);
    plot.setForegroundAlpha(0.5f);
    plot.setRangeGridlinesVisible(true);
    NumberAxis rangeAxis = (NumberAxis) plot.getRangeAxis();
    rangeAxis.setStandardTickUnits(NumberAxis.createIntegerTickUnits());
    CategoryAxis domainAxis = plot.getDomainAxis();
    domainAxis.setTickLabelsVisible(false);
    // Customize the renderer...
    // 
    LineAndShapeRenderer renderer = (LineAndShapeRenderer) plot.getRenderer();
    renderer.setBaseShapesVisible(true);
    renderer.setDrawOutlines(true);
    renderer.setUseFillPaint(true);
    renderer.setBaseFillPaint(Color.white);
    renderer.setSeriesStroke(0, new BasicStroke(1.5f));
    renderer.setSeriesOutlineStroke(0, new BasicStroke(1.5f));
    renderer.setSeriesStroke(1, new BasicStroke(2.5f));
    renderer.setSeriesOutlineStroke(1, new BasicStroke(2.5f));
    renderer.setSeriesShape(0, new Ellipse2D.Double(-3.0, -3.0, 6.0, 6.0));
    BufferedImage bufferedImage = chart.createBufferedImage(bounds.width, bounds.height);
    ImageData imageData = ImageUtil.convertToSWT(bufferedImage);
    // 
    if (image != null) {
        image.dispose();
    }
    image = new Image(transGraph.getDisplay(), imageData);
    // Draw the image on the canvas...
    // 
    canvas.redraw();
}

Example 5 with StepPerformanceSnapShot

use of org.pentaho.di.trans.performance.StepPerformanceSnapShot in project pentaho-kettle by pentaho.

the class PerformanceLogTable method getLogRecord.

/**
 * This method calculates all the values that are required
 *
 * @param id
 *          the id to use or -1 if no id is needed
 * @param status
 *          the log status to use
 */
public RowMetaAndData getLogRecord(LogStatus status, Object subject, Object parent) {
    if (subject == null || subject instanceof StepPerformanceSnapShot) {
        StepPerformanceSnapShot snapShot = (StepPerformanceSnapShot) subject;
        RowMetaAndData row = new RowMetaAndData();
        for (LogTableField field : fields) {
            if (field.isEnabled()) {
                Object value = null;
                if (subject != null) {
                    switch(ID.valueOf(field.getId())) {
                        case ID_BATCH:
                            value = new Long(snapShot.getBatchId());
                            break;
                        case SEQ_NR:
                            value = new Long(snapShot.getSeqNr());
                            break;
                        case LOGDATE:
                            value = snapShot.getDate();
                            break;
                        case TRANSNAME:
                            value = snapShot.getTransName();
                            break;
                        case STEPNAME:
                            value = snapShot.getStepName();
                            break;
                        case STEP_COPY:
                            value = new Long(snapShot.getStepCopy());
                            break;
                        case LINES_READ:
                            value = new Long(snapShot.getLinesRead());
                            break;
                        case LINES_WRITTEN:
                            value = new Long(snapShot.getLinesWritten());
                            break;
                        case LINES_INPUT:
                            value = new Long(snapShot.getLinesInput());
                            break;
                        case LINES_OUTPUT:
                            value = new Long(snapShot.getLinesOutput());
                            break;
                        case LINES_UPDATED:
                            value = new Long(snapShot.getLinesUpdated());
                            break;
                        case LINES_REJECTED:
                            value = new Long(snapShot.getLinesRejected());
                            break;
                        case ERRORS:
                            value = new Long(snapShot.getErrors());
                            break;
                        case INPUT_BUFFER_ROWS:
                            value = new Long(snapShot.getInputBufferSize());
                            break;
                        case OUTPUT_BUFFER_ROWS:
                            value = new Long(snapShot.getOutputBufferSize());
                            break;
                        default:
                            break;
                    }
                }
                row.addValue(field.getFieldName(), field.getDataType(), value);
                row.getRowMeta().getValueMeta(row.size() - 1).setLength(field.getLength());
            }
        }
        return row;
    } else {
        return null;
    }
}