Examples with StepInterface org.pentaho.di.trans.step.StepInterface used on opensource projects

Example 41 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class TimedTransRunner method runEngine.

public boolean runEngine(boolean printDescription) throws KettleException {
    System.gc();
    KettleEnvironment.init();
    transMeta = new TransMeta(filename);
    transMeta.setVariable("NR_OF_ROWS", Long.toString(records));
    if (printDescription) {
        printTransDescription();
    }
    // Replace the TARGET database connection settings with the one provided
    if (targetDatabaseMeta != null) {
        transMeta.addOrReplaceDatabase(targetDatabaseMeta);
    }
    // OK, now run this transFormation.
    Trans trans = new Trans(transMeta);
    trans.setLogLevel(logLevel);
    try {
        trans.prepareExecution(null);
    } catch (Exception e) {
        System.err.println(KettleLogStore.getAppender().getBuffer(trans.getLogChannelId(), true));
        trans.getLogChannel().logError("Error preparing / initializing transformation", e);
        return false;
    }
    if (!Utils.isEmpty(rowListenerStep)) {
        StepInterface step = trans.findRunThread(rowListenerStep);
        if (step != null) {
            step.addRowListener(rowListener);
        }
    }
    long startTime = System.currentTimeMillis();
    trans.startThreads();
    trans.waitUntilFinished();
    long stopTime = System.currentTimeMillis();
    result = trans.getResult();
    runTime = (double) (stopTime - startTime) / 1000;
    speed = records / (runTime);
    printStats("V3 results", records, runTime, speed);
    return true;
}

Example 42 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class HopIT method testCopyHops.

/**
 * Test case for hop using copy.
 *
 * The transformation is as follows: an injector step links to a dummy step, which in turn links to 2 target dummy
 * steps.
 *
 * Both dummy1 and dummy2 should get all rows.
 */
public void testCopyHops() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("hop test default");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep);
    transMeta.addTransHop(hi);
    // THIS DETERMINES THE COPY OR DISTRIBUTE BEHAVIOUR
    dummyStep.setDistributes(false);
    // 
    // Create a dummy target step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hop1 = new TransHopMeta(dummyStep, dummyStep1);
    transMeta.addTransHop(hop1);
    // 
    // Create a dummy target step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hop2 = new TransHopMeta(dummyStep, dummyStep2);
    transMeta.addTransHop(hop2);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si1 = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector rc1 = new RowStepCollector();
    si1.addRowListener(rc1);
    StepInterface si2 = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector rc2 = new RowStepCollector();
    si2.addRowListener(rc2);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = rc1.getRowsWritten();
    checkRows(resultRows, inputList);
}

Example 43 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class NormalExecutionIT method testWaitUntilFinished.

/**
 * Tests that all rows are written out to {@link org.pentaho.di.trans.step.RowListener}s
 * before {@link Trans#waitUntilFinished} returns.
 */
public void testWaitUntilFinished() throws Exception {
    // The number of rows we'll pump through our RowProducer.
    final Long ROWS = 10L;
    final int ITERATIONS = 100;
    // Load transformation
    TransMeta transMeta = new TransMeta("src/it/resources/NormalExecutionTest - WaitUntilFinished.ktr");
    transMeta.setSizeRowset(5);
    for (int t = 0; t < ITERATIONS; t++) {
        Trans trans = new Trans(transMeta);
        trans.setLogLevel(LogLevel.NOTHING);
        trans.prepareExecution(null);
        StepInterface injector = trans.findRunThread("Injector");
        StepInterface output = trans.findRunThread("Output");
        // Get the RowMeta for the injector step (it will be an Integer named 'a' of length 1)
        RowMeta injectorRowMeta = new RowMeta();
        ((BaseStepMeta) injector.getStepMeta().getStepMetaInterface()).getFields(injectorRowMeta, null, null, null, null, null, null);
        RowProducer producer = trans.addRowProducer(injector.getStepname(), 0);
        // Our counting row listener will record how many rows we receive
        CountingRowListener countingListener = new CountingRowListener();
        output.addRowListener(countingListener);
        // Feed input to transformation
        Object[] row = new Object[1];
        trans.startThreads();
        for (Integer i = 0; i < ROWS; i++) {
            row[0] = i;
            producer.putRow(injectorRowMeta, row);
        }
        producer.finished();
        // This should block until everything is complete
        trans.waitUntilFinished();
        countingListener.setListening(false);
        assertTrue(trans.isFinished());
        /*
       *
       * // Make sure we collect all output so we can report how long it actually took long start =
       * System.currentTimeMillis(); while (countingListener.getWritten() + countingListener.getIgnoredWritten() !=
       * ROWS) { Thread.sleep(0, 10); } long end = System.currentTimeMillis();
       *
       * System.out.println("Run report for RowListener on last step in transformation, iteration #"+(t+1)+" :\n");
       * System.out.println("Rows read             : " + countingListener.getRead());
       * System.out.println("Rows written          : " + countingListener.getWritten());
       * System.out.println("Rows error            : " + countingListener.getError());
       * System.out.println("Rows ignored (read)   : " + countingListener.getIgnoredRead());
       * System.out.println("Rows ignored (written): " + countingListener.getIgnoredWritten());
       * System.out.println("Rows ignored (error)  : " + countingListener.getIgnoredError());
       * System.out.println("Had to wait " + (end - start) + "ms for all data to be received by the row listener.");
       */
        assertEquals("Incorrect number of read rows received", ROWS, countingListener.getRead());
        assertEquals("Incorrect number of written rows received", ROWS, countingListener.getWritten());
        assertEquals("Incorrect number of error rows received", new Long(0), countingListener.getError());
    }
}

Example 44 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class WordCountSampleIT method testWordCountMapper.

@Test
public void testWordCountMapper() throws Exception {
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta("src/it/resources/wordcount-mapper.ktr");
    transMeta.setTransformationType(TransformationType.SingleThreaded);
    long transStart = System.currentTimeMillis();
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.setLogLevel(LogLevel.MINIMAL);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface("Output", 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer("Injector", 0);
    trans.startThreads();
    String metricsStep = "Remove garbage";
    // The single threaded transformation type expects us to run the steps
    // ourselves.
    // 
    SingleThreadedTransExecutor executor = new SingleThreadedTransExecutor(trans);
    // Initialize all steps
    // 
    executor.init();
    int iterations = 1000000;
    long totalWait = 0;
    List<RowMetaAndData> inputList = createMapperData();
    for (int i = 0; i < iterations; i++) {
        // add rows
        for (RowMetaAndData rm : inputList) {
            Object[] copy = rm.getRowMeta().cloneRow(rm.getData());
            rp.putRow(rm.getRowMeta(), copy);
        }
        long start = System.currentTimeMillis();
        boolean cont = executor.oneIteration();
        if (!cont) {
            fail("We don't expect any step or the transformation to be done before the end of all iterations.");
        }
        long end = System.currentTimeMillis();
        long delay = end - start;
        totalWait += delay;
        if (i > 0 && (i % 100000) == 0) {
            long rowsProcessed = trans.findRunThread(metricsStep).getLinesRead();
            double speed = Const.round((rowsProcessed) / ((double) (end - transStart) / 1000), 1);
            int totalRows = 0;
            for (StepMetaDataCombi combi : trans.getSteps()) {
                for (RowSet rowSet : combi.step.getInputRowSets()) {
                    totalRows += rowSet.size();
                }
                for (RowSet rowSet : combi.step.getOutputRowSets()) {
                    totalRows += rowSet.size();
                }
            }
            System.out.println("#" + i + " : Finished processing one iteration in " + delay + "ms, average is: " + Const.round(((double) totalWait / (i + 1)), 1) + ", speed=" + speed + " row/s, total rows buffered: " + totalRows);
        }
        List<RowMetaAndData> resultRows = rc.getRowsWritten();
        // Result has one row less because we filter out one.
        // We also join with 3 identical rows in a data grid, giving 9 rows of which 3 are filtered out
        // 
        assertEquals("Error found in iteration " + i + " : not the expected amount of output rows.", 9, resultRows.size());
        rc.clear();
    }
    rp.finished();
    // Dispose all steps.
    // 
    executor.dispose();
    long rowsProcessed = trans.findRunThread(metricsStep).getLinesRead();
    long transEnd = System.currentTimeMillis();
    long transTime = transEnd - transStart;
    System.out.println("Average delay before idle : " + Const.round(((double) totalWait / iterations), 1));
    double transTimeSeconds = Const.round(((double) transTime / 1000), 1);
    System.out.println("Total transformation runtime for " + iterations + " iterations :" + transTimeSeconds + " seconds");
    double transTimePerIteration = Const.round(((double) transTime / iterations), 2);
    System.out.println("Runtime per iteration: " + transTimePerIteration + " miliseconds");
    double rowsPerSecond = Const.round((rowsProcessed) / ((double) transTime / 1000), 1);
    System.out.println("Average speed: " + rowsPerSecond + " rows/second");
}

Example 45 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class AddSequenceIT method testAddSequence.

/**
 * Test case for add sequence step. Row generator attached to several add sequence steps and checking whether the end
 * result is as expected.
 */
public void testAddSequence() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("addsequencetest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create a row generator step...
    // 
    String rowGeneratorStepname = "row generator step";
    RowGeneratorMeta rm = new RowGeneratorMeta();
    // Set the information of the row generator.
    String rowGeneratorPid = registry.getPluginId(rm);
    StepMeta rowGeneratorStep = new StepMeta(rowGeneratorPid, rowGeneratorStepname, rm);
    transMeta.addStep(rowGeneratorStep);
    // 
    // Generate 10 empty rows
    // 
    String[] fieldName = {};
    String[] type = {};
    String[] value = {};
    String[] fieldFormat = {};
    String[] group = {};
    String[] decimal = {};
    int[] intDummies = {};
    rm.setDefault();
    rm.setFieldName(fieldName);
    rm.setFieldType(type);
    rm.setValue(value);
    rm.setFieldLength(intDummies);
    rm.setFieldPrecision(intDummies);
    rm.setRowLimit("10");
    rm.setFieldFormat(fieldFormat);
    rm.setGroup(group);
    rm.setDecimal(decimal);
    // 
    // Create first add sequence
    // 
    String seqStepname1 = "add sequence 1";
    AddSequenceMeta asm1 = new AddSequenceMeta();
    asm1.setUseCounter(true);
    asm1.setValuename("counter1");
    asm1.setStartAt(10);
    asm1.setIncrementBy(1);
    asm1.setMaxValue(100);
    String addSeqPid1 = registry.getPluginId(asm1);
    StepMeta addSeqStep1 = new StepMeta(addSeqPid1, seqStepname1, asm1);
    transMeta.addStep(addSeqStep1);
    TransHopMeta hi1 = new TransHopMeta(rowGeneratorStep, addSeqStep1);
    transMeta.addTransHop(hi1);
    // 
    // Create second add sequence
    // 
    String seqStepname2 = "add sequence 2";
    AddSequenceMeta asm2 = new AddSequenceMeta();
    asm2.setUseCounter(true);
    asm2.setValuename("valuename2");
    asm2.setStartAt(1);
    asm2.setIncrementBy(1);
    asm2.setMaxValue(5);
    String addSeqPid2 = registry.getPluginId(asm2);
    StepMeta addSeqStep2 = new StepMeta(addSeqPid2, seqStepname2, asm2);
    transMeta.addStep(addSeqStep2);
    TransHopMeta hi2 = new TransHopMeta(addSeqStep1, addSeqStep2);
    transMeta.addTransHop(hi2);
    // 
    // Create third add sequence
    // 
    String seqStepname3 = "add sequence 3";
    AddSequenceMeta asm3 = new AddSequenceMeta();
    asm3.setUseCounter(true);
    asm3.setValuename("valuename3");
    asm3.setStartAt(1);
    asm3.setIncrementBy(3);
    asm3.setMaxValue(17);
    String addSeqPid3 = registry.getPluginId(asm3);
    StepMeta addSeqStep3 = new StepMeta(addSeqPid3, seqStepname3, asm3);
    transMeta.addStep(addSeqStep3);
    TransHopMeta hi3 = new TransHopMeta(addSeqStep2, addSeqStep3);
    transMeta.addTransHop(hi3);
    // 
    // Create fourth add sequence
    // 
    String seqStepname4 = "add sequence 4";
    AddSequenceMeta asm4 = new AddSequenceMeta();
    asm4.setUseCounter(true);
    asm4.setValuename("valuename4");
    asm4.setStartAt(10);
    asm4.setIncrementBy(-2);
    asm4.setMaxValue(3);
    String addSeqPid4 = registry.getPluginId(asm4);
    StepMeta addSeqStep4 = new StepMeta(addSeqPid4, seqStepname4, asm4);
    transMeta.addStep(addSeqStep4);
    TransHopMeta hi4 = new TransHopMeta(addSeqStep3, addSeqStep4);
    transMeta.addTransHop(hi4);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(seqStepname4, 0);
    RowStepCollector endRc = new RowStepCollector();
    si.addRowListener(endRc);
    trans.startThreads();
    trans.waitUntilFinished();
    // Now check whether the output is still as we expect.
    List<RowMetaAndData> goldenImageRows = createResultData1();
    List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
}