Examples with RowSet org.pentaho.di.core.RowSet used on opensource projects

Example 71 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class TransPartitioningTest method testDifferentPartitioningFlow.

/**
 * This is PDI-12140 case. 2 steps with same partitions ID's count but different partitioner. This is not a swim lines
 * cases and we need repartitioning here.
 *
 * @throws KettleException
 */
@Test
public void testDifferentPartitioningFlow() throws KettleException {
    prepareStepMetas_cl1_cl2();
    trans.prepareExecution(new String[] {});
    List<RowSet> rowsets = trans.getRowsets();
    assertTrue(!rowsets.isEmpty());
    assertEquals("We have 4 rowsets finally since repartitioning happens", 4, rowsets.size());
    assertEquals("We have 4 steps: 2 producer copies and 2 copies of consumer since they both partitioned", 4, trans.getSteps().size());
    // Ok, examine initialized steps now.
    StepInterface stepOne0 = getStepByName(SP10);
    assertTrue("1.0 step have no input row sets", stepOne0.getInputRowSets().isEmpty());
    assertEquals("1.0 step have 2 output rowsets", 2, stepOne0.getOutputRowSets().size());
    StepInterface stepOne1 = getStepByName(SP11);
    assertTrue("1.1 step have no input row sets", stepOne1.getInputRowSets().isEmpty());
    assertEquals("1.1 step have 2 output rowsets", 2, stepOne1.getOutputRowSets().size());
    StepInterface stepTwo0 = getStepByName(SP20);
    assertTrue("2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty());
    assertEquals("2.0 step have 1 input rowsets", 2, stepTwo0.getInputRowSets().size());
    StepInterface stepTwo2 = getStepByName(SP21);
    assertTrue("2.1 step have no output row sets", stepTwo2.getOutputRowSets().isEmpty());
    assertEquals("2.2 step have 2 input rowsets", 2, stepTwo2.getInputRowSets().size());
}

Example 72 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class TransPartitioningTest method testSwimLanesPartitioning.

/**
 * Test 'Swim lines partitioning'
 *
 * @throws KettleException
 */
@Test
public void testSwimLanesPartitioning() throws KettleException {
    prepareStepMetas_cl1_cl1();
    trans.prepareExecution(new String[] {});
    List<RowSet> rowsets = trans.getRowsets();
    assertTrue(!rowsets.isEmpty());
    assertEquals("We have 2 rowsets finally", 2, rowsets.size());
    assertEquals("We have 3 steps: 1 producer and 2 copies of consumer since it is partitioned", 4, trans.getSteps().size());
    // Ok, examine initialized steps now.
    StepInterface stepOne0 = getStepByName(SP10);
    assertTrue("1.0 step have no input row sets", stepOne0.getInputRowSets().isEmpty());
    assertEquals("1.0 step have 1 output rowsets", 1, stepOne0.getOutputRowSets().size());
    StepInterface stepOne1 = getStepByName(SP11);
    assertTrue("1.1 step have no input row sets", stepOne1.getInputRowSets().isEmpty());
    assertEquals("1.1 step have 1 output rowsets", 1, stepOne1.getOutputRowSets().size());
    StepInterface stepTwo0 = getStepByName(SP20);
    assertEquals("2.0 step have 2 input row sets", 1, stepTwo0.getInputRowSets().size());
    assertTrue("2.0 step have no output rowsets", stepTwo0.getOutputRowSets().isEmpty());
    StepInterface stepTwo2 = getStepByName(SP21);
    assertTrue("2.2 step have no output row sets", stepTwo2.getOutputRowSets().isEmpty());
    assertEquals("2.2 step have 2 output rowsets", 1, stepTwo2.getInputRowSets().size());
}

Example 73 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class TransPartitioningTest method testManyToOneCopies.

/**
 * This checks transformation initialization when using many copies to one next step
 *
 * @throws KettleException
 */
@Test
public void testManyToOneCopies() throws KettleException {
    prepareStepMetas_x2_1();
    trans.prepareExecution(new String[] {});
    List<RowSet> rowsets = trans.getRowsets();
    assertTrue(!rowsets.isEmpty());
    assertEquals("We have 2 rowsets finally", 2, rowsets.size());
    assertEquals("We have 4 steps: 2 copies of producer and 2 copies of consumer", 3, trans.getSteps().size());
    // Ok, examine initialized steps now.
    StepInterface stepOne0 = getStepByName(S10);
    assertTrue("1 step have no input row sets", stepOne0.getInputRowSets().isEmpty());
    assertEquals("1 step have 1 output rowsets", 1, stepOne0.getOutputRowSets().size());
    StepInterface stepOne1 = getStepByName(S11);
    assertTrue("1 step have no input row sets", stepOne1.getInputRowSets().isEmpty());
    assertEquals("1 step have 1 output rowsets", 1, stepOne1.getOutputRowSets().size());
    StepInterface stepTwo0 = getStepByName(S20);
    Assert.assertEquals("2.0 step have 2 input row sets", 2, stepTwo0.getInputRowSets().size());
    Assert.assertTrue("2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty());
}

Example 74 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class BaseStepTest method testBaseStepPutRowLocalSpecialPartitioning.

/**
 * This test checks that data from one non-partitioned step copies to 2 partitioned steps right.
 *
 * @throws KettleException
 * @see {@link <a href="http://jira.pentaho.com/browse/PDI-12211">http://jira.pentaho.com/browse/PDI-12211<a>}
 */
@Test
public void testBaseStepPutRowLocalSpecialPartitioning() throws KettleException {
    List<StepMeta> stepMetas = new ArrayList<StepMeta>();
    stepMetas.add(mockHelper.stepMeta);
    stepMetas.add(mockHelper.stepMeta);
    StepPartitioningMeta stepPartitioningMeta = spy(new StepPartitioningMeta());
    BasePartitioner partitioner = mock(BasePartitioner.class);
    when(mockHelper.logChannelInterfaceFactory.create(any(), any(LoggingObjectInterface.class))).thenAnswer(new Answer<LogChannelInterface>() {

        @Override
        public LogChannelInterface answer(InvocationOnMock invocation) throws Throwable {
            ((BaseStep) invocation.getArguments()[0]).getLogLevel();
            return mockHelper.logChannelInterface;
        }
    });
    when(mockHelper.trans.isRunning()).thenReturn(true);
    when(mockHelper.transMeta.findNextSteps(any(StepMeta.class))).thenReturn(stepMetas);
    when(mockHelper.stepMeta.getStepPartitioningMeta()).thenReturn(stepPartitioningMeta);
    when(stepPartitioningMeta.getPartitioner()).thenReturn(partitioner);
    when(partitioner.getNrPartitions()).thenReturn(2);
    Object object0 = "name0";
    ValueMetaInterface meta0 = new ValueMetaString(object0.toString());
    Object object1 = "name1";
    ValueMetaInterface meta2 = new ValueMetaString(object1.toString());
    RowMetaInterface rowMeta0 = new RowMeta();
    rowMeta0.addValueMeta(meta0);
    Object[] objects0 = { object0 };
    RowMetaInterface rowMeta1 = new RowMeta();
    rowMeta1.addValueMeta(meta2);
    Object[] objects1 = { object1 };
    when(stepPartitioningMeta.getPartition(rowMeta0, objects0)).thenReturn(0);
    when(stepPartitioningMeta.getPartition(rowMeta1, objects1)).thenReturn(1);
    BlockingRowSet[] rowSet = { new BlockingRowSet(2), new BlockingRowSet(2), new BlockingRowSet(2), new BlockingRowSet(2) };
    List<RowSet> outputRowSets = new ArrayList<RowSet>();
    outputRowSets.addAll(Arrays.asList(rowSet));
    BaseStep baseStep = new BaseStep(mockHelper.stepMeta, mockHelper.stepDataInterface, 0, mockHelper.transMeta, mockHelper.trans);
    baseStep.setStopped(false);
    baseStep.setRepartitioning(StepPartitioningMeta.PARTITIONING_METHOD_SPECIAL);
    baseStep.setOutputRowSets(outputRowSets);
    baseStep.putRow(rowMeta0, objects0);
    baseStep.putRow(rowMeta1, objects1);
    assertEquals(object0, baseStep.getOutputRowSets().get(0).getRow()[0]);
    assertEquals(object1, baseStep.getOutputRowSets().get(1).getRow()[0]);
    assertEquals(object0, baseStep.getOutputRowSets().get(2).getRow()[0]);
    assertEquals(object1, baseStep.getOutputRowSets().get(3).getRow()[0]);
}

Example 75 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class RegexEvalUnitTest method testOutputIsMuchBiggerThanInputDoesntThrowArrayIndexOutOfBounds.

@Test
public void testOutputIsMuchBiggerThanInputDoesntThrowArrayIndexOutOfBounds() throws KettleException {
    RegexEval regexEval = new RegexEval(stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta, stepMockHelper.trans);
    when(stepMockHelper.processRowsStepMetaInterface.isAllowCaptureGroupsFlagSet()).thenReturn(true);
    String[] outFields = new String[] { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k" };
    when(stepMockHelper.processRowsStepMetaInterface.getFieldName()).thenReturn(outFields);
    when(stepMockHelper.processRowsStepMetaInterface.getMatcher()).thenReturn("\\.+");
    stepMockHelper.processRowsStepDataInterface.pattern = Pattern.compile("(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)");
    Object[] inputRow = new Object[] {};
    RowSet inputRowSet = stepMockHelper.getMockInputRowSet(inputRow);
    RowMetaInterface mockInputRowMeta = mock(RowMetaInterface.class);
    RowMetaInterface mockOutputRoMeta = mock(RowMetaInterface.class);
    when(mockOutputRoMeta.size()).thenReturn(outFields.length);
    when(mockInputRowMeta.size()).thenReturn(0);
    when(inputRowSet.getRowMeta()).thenReturn(mockInputRowMeta);
    when(mockInputRowMeta.clone()).thenReturn(mockOutputRoMeta);
    when(mockInputRowMeta.isNull(any(Object[].class), anyInt())).thenReturn(true);
    regexEval.addRowSetToInputRowSets(inputRowSet);
    regexEval.init(stepMockHelper.initStepMetaInterface, stepMockHelper.initStepDataInterface);
    regexEval.processRow(stepMockHelper.processRowsStepMetaInterface, stepMockHelper.processRowsStepDataInterface);
}