Examples with RowSet org.pentaho.di.core.RowSet used on opensource projects

Example 96 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class SwitchCaseTest method processRow_NullsArePutIntoDefaultWhenNotSpecified.

@Test
public void processRow_NullsArePutIntoDefaultWhenNotSpecified() throws Exception {
    SwitchCaseCustom step = new SwitchCaseCustom(mockHelper);
    step.meta.loadXML(loadStepXmlMetadata("SwitchCaseTest_PDI-12671.xml"), Collections.<DatabaseMeta>emptyList(), mock(IMetaStore.class));
    List<RowSet> outputRowSets = new LinkedList<RowSet>();
    for (SwitchCaseTarget item : step.meta.getCaseTargets()) {
        StepMetaInterface smInt = new DummyTransMeta();
        item.caseTargetStep = new StepMeta(item.caseTargetStepname, smInt);
        RowSet rw = new QueueRowSet();
        step.map.put(item.caseTargetStepname, rw);
        outputRowSets.add(rw);
    }
    // create a default step
    StepMetaInterface smInt = new DummyTransMeta();
    StepMeta stepMeta = new StepMeta(step.meta.getDefaultTargetStepname(), smInt);
    step.meta.setDefaultTargetStep(stepMeta);
    RowSet defaultRowSet = new QueueRowSet();
    step.map.put(step.meta.getDefaultTargetStepname(), defaultRowSet);
    step.input.add(new Object[] { null });
    step.processRow();
    assertEquals(1, defaultRowSet.size());
    for (RowSet rowSet : outputRowSets) {
        assertEquals(0, rowSet.size());
    }
    assertNull(defaultRowSet.getRow()[0]);
}

Example 97 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class CloneRowTest method nullNrCloneField.

@Test(expected = KettleException.class)
public void nullNrCloneField() throws Exception {
    CloneRow step = new CloneRow(stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta, stepMockHelper.trans);
    step.init(stepMockHelper.initStepMetaInterface, stepMockHelper.initStepDataInterface);
    RowMetaInterface inputRowMeta = mock(RowMetaInterface.class);
    when(inputRowMeta.getInteger(any(Object[].class), anyInt())).thenReturn(null);
    RowSet inputRowSet = stepMockHelper.getMockInputRowSet(new Integer[] { null });
    when(inputRowSet.getRowMeta()).thenReturn(inputRowMeta);
    step.setInputRowSets(Collections.singletonList(inputRowSet));
    when(stepMockHelper.processRowsStepMetaInterface.isNrCloneInField()).thenReturn(true);
    when(stepMockHelper.processRowsStepMetaInterface.getNrCloneField()).thenReturn("field");
    step.processRow(stepMockHelper.processRowsStepMetaInterface, stepMockHelper.processRowsStepDataInterface);
}

Example 98 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class TransSingleThreadIT method testSingleThreadedTrans.

public void testSingleThreadedTrans() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta("src/it/resources/SingleThreadedTest - Stream Lookup.ktr");
    transMeta.setTransformationType(TransformationType.SingleThreaded);
    long transStart = System.currentTimeMillis();
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.setLogLevel(LogLevel.MINIMAL);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface("OUTPUT", 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer("INPUT", 0);
    trans.startThreads();
    // The single threaded transformation type expects us to run the steps
    // ourselves.
    // 
    SingleThreadedTransExecutor executor = new SingleThreadedTransExecutor(trans);
    // Initialize all steps
    // 
    executor.init();
    int iterations = 1000000;
    long totalWait = 0;
    List<RowMetaAndData> inputList = createData();
    for (int i = 0; i < iterations; i++) {
        // add rows
        for (RowMetaAndData rm : inputList) {
            Object[] copy = rm.getRowMeta().cloneRow(rm.getData());
            rp.putRow(rm.getRowMeta(), copy);
        }
        long start = System.currentTimeMillis();
        boolean cont = executor.oneIteration();
        if (!cont) {
            fail("We don't expect any step or the transformation to be done before the end of all iterations.");
        }
        long end = System.currentTimeMillis();
        long delay = end - start;
        totalWait += delay;
        if (i > 0 && (i % 100000) == 0) {
            long rowsProcessed = trans.findRunThread("bottles").getLinesRead();
            double speed = Const.round((rowsProcessed) / ((double) (end - transStart) / 1000), 1);
            int totalRows = 0;
            for (StepMetaDataCombi combi : trans.getSteps()) {
                for (RowSet rowSet : combi.step.getInputRowSets()) {
                    totalRows += rowSet.size();
                }
                for (RowSet rowSet : combi.step.getOutputRowSets()) {
                    totalRows += rowSet.size();
                }
            }
            System.out.println("#" + i + " : Finished processing one iteration in " + delay + "ms, average is: " + Const.round(((double) totalWait / (i + 1)), 1) + ", speed=" + speed + " row/s, total rows buffered: " + totalRows);
        }
        List<RowMetaAndData> resultRows = rc.getRowsWritten();
        // Result has one row less because we filter out one.
        // We also join with 3 identical rows in a data grid, giving 9 rows of which 3 are filtered out
        // 
        assertEquals("Error found in iteration " + i, 6, resultRows.size());
        rc.clear();
    }
    rp.finished();
    // Dispose all steps.
    // 
    executor.dispose();
    long rowsProcessed = trans.findRunThread("bottles").getLinesRead();
    long transEnd = System.currentTimeMillis();
    long transTime = transEnd - transStart;
    System.out.println("Average delay before idle : " + Const.round(((double) totalWait / iterations), 1));
    double transTimeSeconds = Const.round(((double) transTime / 1000), 1);
    System.out.println("Total transformation runtime for " + iterations + " iterations :" + transTimeSeconds + " seconds");
    double transTimePerIteration = Const.round(((double) transTime / iterations), 2);
    System.out.println("Runtime per iteration: " + transTimePerIteration + " miliseconds");
    double rowsPerSecond = Const.round((rowsProcessed) / ((double) transTime / 1000), 1);
    System.out.println("Average speed: " + rowsPerSecond + " rows/second");
}

Example 99 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class CalculatorBackwardCompatibilityUnitTest method assertRoundStd.

public void assertRoundStd(final double expectedResult, final double value) throws KettleException {
    RowMeta inputRowMeta = new RowMeta();
    ValueMetaNumber valueMeta = new ValueMetaNumber("Value");
    inputRowMeta.addValueMeta(valueMeta);
    RowSet inputRowSet = smh.getMockInputRowSet(new Object[] { value });
    inputRowSet.setRowMeta(inputRowMeta);
    Calculator calculator = new Calculator(smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans);
    calculator.addRowSetToInputRowSets(inputRowSet);
    calculator.setInputRowMeta(inputRowMeta);
    calculator.init(smh.initStepMetaInterface, smh.initStepDataInterface);
    CalculatorMeta meta = new CalculatorMeta();
    meta.setCalculation(new CalculatorMetaFunction[] { new CalculatorMetaFunction("test", CalculatorMetaFunction.CALC_ROUND_STD_1, "Value", null, null, ValueMetaInterface.TYPE_NUMBER, 2, 0, false, "", "", "", "") });
    // Verify output
    try {
        calculator.addRowListener(new RowAdapter() {

            @Override
            public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                assertEquals(expectedResult, row[1]);
            }
        });
        calculator.processRow(meta, new CalculatorData());
    } catch (KettleException ke) {
        ke.printStackTrace();
        fail();
    }
}

Example 100 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class CalculatorBackwardCompatibilityUnitTest method assertRoundStd2.

public void assertRoundStd2(final double expectedResult, final double value, final long precision) throws KettleException {
    RowMeta inputRowMeta = new RowMeta();
    ValueMetaNumber valueMeta = new ValueMetaNumber("Value");
    ValueMetaInteger precisionMeta = new ValueMetaInteger("Precision");
    inputRowMeta.addValueMeta(valueMeta);
    inputRowMeta.addValueMeta(precisionMeta);
    RowSet inputRowSet = smh.getMockInputRowSet(new Object[] { value, precision });
    inputRowSet.setRowMeta(inputRowMeta);
    Calculator calculator = new Calculator(smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans);
    calculator.addRowSetToInputRowSets(inputRowSet);
    calculator.setInputRowMeta(inputRowMeta);
    calculator.init(smh.initStepMetaInterface, smh.initStepDataInterface);
    CalculatorMeta meta = new CalculatorMeta();
    meta.setCalculation(new CalculatorMetaFunction[] { new CalculatorMetaFunction("test", CalculatorMetaFunction.CALC_ROUND_STD_2, "Value", "Precision", null, ValueMetaInterface.TYPE_NUMBER, 2, 0, false, "", "", "", "") });
    // Verify output
    try {
        calculator.addRowListener(new RowAdapter() {

            @Override
            public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                assertEquals(expectedResult, row[2]);
            }
        });
        calculator.processRow(meta, new CalculatorData());
    } catch (KettleException ke) {
        ke.printStackTrace();
        fail();
    }
}