Examples with StepMetaDataCombi org.pentaho.di.trans.step.StepMetaDataCombi used on opensource projects

Example 1 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-kettle by pentaho.

the class Trans method getRunThread.

/**
 * Gets the run thread for the step with the specified name and copy number.
 *
 * @param name
 *          the step name
 * @param copy
 *          the copy number
 * @return a StepInterface object corresponding to the run thread for the specified step
 */
public StepInterface getRunThread(String name, int copy) {
    if (steps == null) {
        return null;
    }
    for (int i = 0; i < steps.size(); i++) {
        StepMetaDataCombi sid = steps.get(i);
        StepInterface step = sid.step;
        if (step.getStepname().equalsIgnoreCase(name) && step.getCopy() == copy) {
            return step;
        }
    }
    return null;
}

Example 2 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-kettle by pentaho.

the class Trans method doTopologySortOfSteps.

public synchronized void doTopologySortOfSteps() {
    // The bubble sort algorithm in contrast to the QuickSort or MergeSort
    // algorithms
    // does indeed cover all possibilities.
    // Sorting larger transformations with hundreds of steps might be too slow
    // though.
    // We should consider caching TransMeta.findPrevious() results in that case.
    // 
    transMeta.clearCaches();
    // 
    // Cocktail sort (bi-directional bubble sort)
    // 
    // Original sort was taking 3ms for 30 steps
    // cocktail sort takes about 8ms for the same 30, but it works :)
    // 
    int stepsMinSize = 0;
    int stepsSize = steps.size();
    // Noticed a problem with an immediate shrinking iteration window
    // trapping rows that need to be sorted.
    // This threshold buys us some time to get the sorting close before
    // starting to decrease the window size.
    // 
    // TODO: this could become much smarter by tracking row movement
    // and reacting to that each outer iteration verses
    // using a threshold.
    // 
    // After this many iterations enable trimming inner iteration
    // window on no change being detected.
    // 
    int windowShrinkThreshold = (int) Math.round(stepsSize * 0.75);
    // give ourselves some room to sort big lists. the window threshold should
    // stop us before reaching this anyway.
    // 
    int totalIterations = stepsSize * 2;
    boolean isBefore = false;
    boolean forwardChange = false;
    boolean backwardChange = false;
    boolean lastForwardChange = true;
    boolean keepSortingForward = true;
    StepMetaDataCombi one = null;
    StepMetaDataCombi two = null;
    for (int x = 0; x < totalIterations; x++) {
        // 
        if (keepSortingForward) {
            for (int y = stepsMinSize; y < stepsSize - 1; y++) {
                one = steps.get(y);
                two = steps.get(y + 1);
                if (one.stepMeta.equals(two.stepMeta)) {
                    isBefore = one.copy > two.copy;
                } else {
                    isBefore = transMeta.findPrevious(one.stepMeta, two.stepMeta);
                }
                if (isBefore) {
                    // two was found to be positioned BEFORE one so we need to
                    // switch them...
                    // 
                    steps.set(y, two);
                    steps.set(y + 1, one);
                    forwardChange = true;
                }
            }
        }
        // 
        for (int z = stepsSize - 1; z > stepsMinSize; z--) {
            one = steps.get(z);
            two = steps.get(z - 1);
            if (one.stepMeta.equals(two.stepMeta)) {
                isBefore = one.copy > two.copy;
            } else {
                isBefore = transMeta.findPrevious(one.stepMeta, two.stepMeta);
            }
            if (!isBefore) {
                // two was found NOT to be positioned BEFORE one so we need to
                // switch them...
                // 
                steps.set(z, two);
                steps.set(z - 1, one);
                backwardChange = true;
            }
        }
        // 
        if (x > windowShrinkThreshold && !forwardChange) {
            // should we keep going? check the window size
            // 
            stepsSize--;
            if (stepsSize <= stepsMinSize) {
                break;
            }
        }
        // 
        if (x > windowShrinkThreshold && !backwardChange) {
            // should we keep going? check the window size
            // 
            stepsMinSize++;
            if (stepsMinSize >= stepsSize) {
                break;
            }
        }
        // 
        if (!forwardChange && !backwardChange) {
            break;
        }
        // 
        if (keepSortingForward && x > 0 && !lastForwardChange && !forwardChange) {
            keepSortingForward = false;
        }
        lastForwardChange = forwardChange;
        forwardChange = false;
        backwardChange = false;
    }
// finished sorting
}

Example 3 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-kettle by pentaho.

the class Trans method printStats.

/**
 * Logs the execution statistics for the transformation for the specified time interval. If the total length of
 * execution is supplied as the interval, then the statistics represent the average throughput (lines
 * read/written/updated/rejected/etc. per second) for the entire execution.
 *
 * @param seconds
 *          the time interval (in seconds)
 */
public void printStats(int seconds) {
    log.logBasic(" ");
    if (steps == null) {
        return;
    }
    for (int i = 0; i < steps.size(); i++) {
        StepMetaDataCombi sid = steps.get(i);
        StepInterface step = sid.step;
        long proc = step.getProcessed();
        if (seconds != 0) {
            if (step.getErrors() == 0) {
                log.logBasic(BaseMessages.getString(PKG, "Trans.Log.ProcessSuccessfullyInfo", step.getStepname(), "." + step.getCopy(), String.valueOf(proc), String.valueOf((proc / seconds))));
            } else {
                log.logError(BaseMessages.getString(PKG, "Trans.Log.ProcessErrorInfo", step.getStepname(), "." + step.getCopy(), String.valueOf(step.getErrors()), String.valueOf(proc), String.valueOf(proc / seconds)));
            }
        } else {
            if (step.getErrors() == 0) {
                log.logBasic(BaseMessages.getString(PKG, "Trans.Log.ProcessSuccessfullyInfo", step.getStepname(), "." + step.getCopy(), String.valueOf(proc), seconds != 0 ? String.valueOf((proc / seconds)) : "-"));
            } else {
                log.logError(BaseMessages.getString(PKG, "Trans.Log.ProcessErrorInfo2", step.getStepname(), "." + step.getCopy(), String.valueOf(step.getErrors()), String.valueOf(proc), String.valueOf(seconds)));
            }
        }
    }
}

Example 4 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-kettle by pentaho.

the class Trans method findBaseSteps.

/**
 * Find the base steps for the step with the specified name.
 *
 * @param stepname
 *          the step name
 * @return the list of base steps for the specified step
 */
public List<StepInterface> findBaseSteps(String stepname) {
    List<StepInterface> baseSteps = new ArrayList<>();
    if (steps == null) {
        return baseSteps;
    }
    for (int i = 0; i < steps.size(); i++) {
        StepMetaDataCombi sid = steps.get(i);
        StepInterface stepInterface = sid.step;
        if (stepInterface.getStepname().equalsIgnoreCase(stepname)) {
            baseSteps.add(stepInterface);
        }
    }
    return baseSteps;
}

Example 5 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-kettle by pentaho.

the class Trans method hasStepStarted.

/**
 * Checks whether the specified step (or step copy) has started.
 *
 * @param sname
 *          the step name
 * @param copy
 *          the copy number
 * @return true the specified step (or step copy) has started, false otherwise
 */
public boolean hasStepStarted(String sname, int copy) {
    // log.logDetailed("DIS: hasStepStarted() looking in "+threads.size()+" threads");
    for (int i = 0; i < steps.size(); i++) {
        StepMetaDataCombi sid = steps.get(i);
        boolean started = (sid.stepname != null && sid.stepname.equalsIgnoreCase(sname)) && sid.copy == copy;
        if (started) {
            return true;
        }
    }
    return false;
}