Examples with RowSet org.pentaho.di.core.RowSet used on opensource projects

Example 66 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class SingleThreadedTransExecutor method oneIteration.

/**
 * Give all steps in the transformation the chance to process all rows on input...
 *
 * @return true if more iterations can be performed. False if this is not the case.
 */
public boolean oneIteration() throws KettleException {
    for (int s = 0; s < steps.size() && !trans.isStopped(); s++) {
        if (!done[s]) {
            StepMetaDataCombi combi = steps.get(s);
            // If this step is waiting for data (text, db, and so on), we simply read all the data
            // This means that it is impractical to use this transformation type to load large files.
            // 
            boolean stepDone = false;
            // For every input row we call the processRow() method of the step.
            // 
            List<RowSet> infoRowSets = stepInfoRowSets.get(s);
            // 
            for (RowSet rowSet : infoRowSets) {
                boolean once = true;
                while (once || (rowSet.size() > 0 && !stepDone)) {
                    once = false;
                    stepDone = !combi.step.processRow(combi.meta, combi.data);
                    if (combi.step.getErrors() > 0) {
                        return false;
                    }
                }
            }
            // Do normal processing of input rows...
            // 
            List<RowSet> rowSets = combi.step.getInputRowSets();
            // 
            if (rowSets.size() == 0) {
                while (!stepDone && !trans.isStopped()) {
                    stepDone = !combi.step.processRow(combi.meta, combi.data);
                    if (combi.step.getErrors() > 0) {
                        return false;
                    }
                }
            } else {
                // Since we can't be sure that the step actually reads from the row sets where we measure rows,
                // we simply count the total nr of rows on input. The steps will find the rows in either row set.
                // 
                int nrRows = 0;
                for (RowSet rowSet : rowSets) {
                    nrRows += rowSet.size();
                }
                // 
                for (int i = 0; i < nrRows; i++) {
                    stepDone = !combi.step.processRow(combi.meta, combi.data);
                    if (combi.step.getErrors() > 0) {
                        return false;
                    }
                }
            }
            // Signal the step that a batch of rows has passed for this iteration (sort rows and all)
            // 
            combi.step.batchComplete();
            if (stepDone) {
                nrDone++;
            }
            done[s] = stepDone;
        }
    }
    return nrDone < steps.size() && !trans.isStopped();
}

Example 67 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class Trans method clearError.

/**
 * Clear the error in the transformation, clear all the rows from all the row sets, to make sure the transformation
 * can continue with other data. This is intended for use when running single threaded.
 */
public void clearError() {
    setStopped(false);
    errors.set(0);
    setFinished(false);
    for (StepMetaDataCombi combi : steps) {
        StepInterface step = combi.step;
        for (RowSet rowSet : step.getInputRowSets()) {
            rowSet.clear();
        }
        step.setStopped(false);
    }
}

Example 68 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class RowProducerTest method testGetSetRowSet.

@Test
public void testGetSetRowSet() throws Exception {
    assertEquals(rowSet, rowProducer.getRowSet());
    rowProducer.setRowSet(null);
    assertNull(rowProducer.getRowSet());
    RowSet newRowSet = mock(RowSet.class);
    rowProducer.setRowSet(newRowSet);
    assertEquals(newRowSet, rowProducer.getRowSet());
}

Example 69 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class TransPartitioningTest method testOneToPartitioningSchema.

/**
 * Test one to one partitioning step transformation organization.
 *
 * @throws KettleException
 */
@Test
public void testOneToPartitioningSchema() throws KettleException {
    prepareStepMetas_1_cl1();
    trans.prepareExecution(new String[] {});
    List<RowSet> rowsets = trans.getRowsets();
    assertTrue(!rowsets.isEmpty());
    assertEquals("We have 2 rowsets finally", 2, rowsets.size());
    assertEquals("We have 3 steps: 1 producer and 2 copies of consumer since it is partitioned", 3, trans.getSteps().size());
    // Ok, examine initialized steps now.
    StepInterface stepOne0 = getStepByName(S10);
    assertTrue("1 step have no input row sets", stepOne0.getInputRowSets().isEmpty());
    assertEquals("1 step have 2 output rowsets", 2, stepOne0.getOutputRowSets().size());
    StepInterface stepTwo0 = getStepByName(SP20);
    assertEquals("2.0 step have one input row sets", 1, stepTwo0.getInputRowSets().size());
    assertTrue("2.0 step have no output rowsets", stepTwo0.getOutputRowSets().isEmpty());
    StepInterface stepTwo1 = getStepByName(SP21);
    Assert.assertEquals("2.1 step have 1 input row sets", 1, stepTwo1.getInputRowSets().size());
    Assert.assertTrue("2.1 step have no output row sets", stepTwo1.getOutputRowSets().isEmpty());
}

Example 70 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class TransPartitioningTest method testManyCopiesToPartitioningFlow.

/**
 * This is a case when step running in many copies meets partitioning one.
 *
 * @throws KettleException
 */
@Test
public void testManyCopiesToPartitioningFlow() throws KettleException {
    prepareStepMetas_x2_cl1();
    trans.prepareExecution(new String[] {});
    List<RowSet> rowsets = trans.getRowsets();
    assertTrue(!rowsets.isEmpty());
    assertEquals("We have 4 rowsets finally since repartitioning happens", 4, rowsets.size());
    assertEquals("We have 4 steps: 2 producer copies and 2 copies of consumer since consumer is partitioned", 4, trans.getSteps().size());
    // Ok, examine initialized steps now.
    StepInterface stepOne0 = getStepByName(S10);
    assertTrue("1.0 step have no input row sets", stepOne0.getInputRowSets().isEmpty());
    assertEquals("1.0 step have 2 output rowsets", 2, stepOne0.getOutputRowSets().size());
    StepInterface stepOne1 = getStepByName(S11);
    assertTrue("1.1 step have no input row sets", stepOne1.getInputRowSets().isEmpty());
    assertEquals("1.1 step have 2 output rowsets", 2, stepOne1.getOutputRowSets().size());
    StepInterface stepTwo0 = getStepByName(SP20);
    assertTrue("2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty());
    assertEquals("2.0 step have 2 input rowsets", 2, stepTwo0.getInputRowSets().size());
    StepInterface stepTwo2 = getStepByName(SP21);
    assertTrue("2.1 step have no output row sets", stepTwo2.getOutputRowSets().isEmpty());
    assertEquals("2.2 step have 2 input rowsets", 2, stepTwo2.getInputRowSets().size());
}