Examples with RowSet org.pentaho.di.core.RowSet used on opensource projects

Example 41 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class CalculatorUnitTest method testReturnDigitsOnly.

@Test
public void testReturnDigitsOnly() throws KettleException {
    RowMeta inputRowMeta = new RowMeta();
    ValueMetaString nameMeta = new ValueMetaString("Name");
    inputRowMeta.addValueMeta(nameMeta);
    ValueMetaString valueMeta = new ValueMetaString("Value");
    inputRowMeta.addValueMeta(valueMeta);
    RowSet inputRowSet = smh.getMockInputRowSet(new Object[][] { { "name1", "qwe123asd456zxc" }, { "name2", null } });
    inputRowSet.setRowMeta(inputRowMeta);
    Calculator calculator = new Calculator(smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans);
    calculator.addRowSetToInputRowSets(inputRowSet);
    calculator.setInputRowMeta(inputRowMeta);
    calculator.init(smh.initStepMetaInterface, smh.initStepDataInterface);
    CalculatorMeta meta = new CalculatorMeta();
    meta.setCalculation(new CalculatorMetaFunction[] { new CalculatorMetaFunction("digits", CalculatorMetaFunction.CALC_GET_ONLY_DIGITS, "Value", null, null, ValueMetaInterface.TYPE_STRING, 0, 0, false, "", "", "", "") });
    // Verify output
    try {
        calculator.addRowListener(new RowAdapter() {

            @Override
            public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                assertEquals("123456", row[2]);
            }
        });
        calculator.processRow(meta, new CalculatorData());
    } catch (KettleException ke) {
        ke.printStackTrace();
        fail();
    }
}

Example 42 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class CsvInputEnclosureTest method doTest.

public void doTest(String content, String enclosure) throws Exception {
    RowSet output = new QueueRowSet();
    File tmp = createTestFile("utf-8", content);
    try {
        CsvInputMeta meta = createMeta(tmp, createInputFileFields("f1", "f2"), enclosure);
        CsvInputData data = new CsvInputData();
        csvInput.init(meta, data);
        csvInput.addRowSetToOutputRowSets(output);
        try {
            csvInput.processRow(meta, data);
        } finally {
            csvInput.dispose(meta, data);
        }
    } finally {
        tmp.delete();
    }
    Object[] row = output.getRowImmediate();
    assertNotNull(row);
    assertEquals("value1", row[0]);
    assertEquals("value2", row[1]);
    assertNull(output.getRowImmediate());
}

Example 43 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class CsvInputMultiCharDelimiterTest method doTest.

private void doTest(String content) throws Exception {
    RowSet output = new QueueRowSet();
    File tmp = createTestFile("utf-8", content);
    try {
        CsvInputMeta meta = createMeta(tmp, createInputFileFields("f1", "f2", "f3"));
        CsvInputData data = new CsvInputData();
        csvInput.init(meta, data);
        csvInput.addRowSetToOutputRowSets(output);
        try {
            csvInput.processRow(meta, data);
        } finally {
            csvInput.dispose(meta, data);
        }
    } finally {
        tmp.delete();
    }
    Object[] row = output.getRowImmediate();
    assertNotNull(row);
    assertEquals("value1", row[0]);
    assertEquals("value2", row[1]);
    assertEquals("value3", row[2]);
    assertNull(output.getRowImmediate());
}

Example 44 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class PDI5436Test method mockInputRowSet.

private RowSet mockInputRowSet() {
    RowSet inputRowSet = smh.getMockInputRowSet(new Object[][] { { "name".getBytes(), "1".getBytes() } });
    inputRowSet.setRowMeta(mockInputRowMeta());
    return inputRowSet;
}

Example 45 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class WordCountSampleIT method testWordCountMapper.

@Test
public void testWordCountMapper() throws Exception {
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta("src/it/resources/wordcount-mapper.ktr");
    transMeta.setTransformationType(TransformationType.SingleThreaded);
    long transStart = System.currentTimeMillis();
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.setLogLevel(LogLevel.MINIMAL);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface("Output", 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer("Injector", 0);
    trans.startThreads();
    String metricsStep = "Remove garbage";
    // The single threaded transformation type expects us to run the steps
    // ourselves.
    // 
    SingleThreadedTransExecutor executor = new SingleThreadedTransExecutor(trans);
    // Initialize all steps
    // 
    executor.init();
    int iterations = 1000000;
    long totalWait = 0;
    List<RowMetaAndData> inputList = createMapperData();
    for (int i = 0; i < iterations; i++) {
        // add rows
        for (RowMetaAndData rm : inputList) {
            Object[] copy = rm.getRowMeta().cloneRow(rm.getData());
            rp.putRow(rm.getRowMeta(), copy);
        }
        long start = System.currentTimeMillis();
        boolean cont = executor.oneIteration();
        if (!cont) {
            fail("We don't expect any step or the transformation to be done before the end of all iterations.");
        }
        long end = System.currentTimeMillis();
        long delay = end - start;
        totalWait += delay;
        if (i > 0 && (i % 100000) == 0) {
            long rowsProcessed = trans.findRunThread(metricsStep).getLinesRead();
            double speed = Const.round((rowsProcessed) / ((double) (end - transStart) / 1000), 1);
            int totalRows = 0;
            for (StepMetaDataCombi combi : trans.getSteps()) {
                for (RowSet rowSet : combi.step.getInputRowSets()) {
                    totalRows += rowSet.size();
                }
                for (RowSet rowSet : combi.step.getOutputRowSets()) {
                    totalRows += rowSet.size();
                }
            }
            System.out.println("#" + i + " : Finished processing one iteration in " + delay + "ms, average is: " + Const.round(((double) totalWait / (i + 1)), 1) + ", speed=" + speed + " row/s, total rows buffered: " + totalRows);
        }
        List<RowMetaAndData> resultRows = rc.getRowsWritten();
        // Result has one row less because we filter out one.
        // We also join with 3 identical rows in a data grid, giving 9 rows of which 3 are filtered out
        // 
        assertEquals("Error found in iteration " + i + " : not the expected amount of output rows.", 9, resultRows.size());
        rc.clear();
    }
    rp.finished();
    // Dispose all steps.
    // 
    executor.dispose();
    long rowsProcessed = trans.findRunThread(metricsStep).getLinesRead();
    long transEnd = System.currentTimeMillis();
    long transTime = transEnd - transStart;
    System.out.println("Average delay before idle : " + Const.round(((double) totalWait / iterations), 1));
    double transTimeSeconds = Const.round(((double) transTime / 1000), 1);
    System.out.println("Total transformation runtime for " + iterations + " iterations :" + transTimeSeconds + " seconds");
    double transTimePerIteration = Const.round(((double) transTime / iterations), 2);
    System.out.println("Runtime per iteration: " + transTimePerIteration + " miliseconds");
    double rowsPerSecond = Const.round((rowsProcessed) / ((double) transTime / 1000), 1);
    System.out.println("Average speed: " + rowsPerSecond + " rows/second");
}