Examples with BlockingRowSet org.pentaho.di.core.BlockingRowSet used on opensource projects

Example 6 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class MappingInputFieldsTest method testSelectingAndSortingUnspecifiedFields.

/**
 * verifies: If SelectingAndSortingUnspecifiedFields checkbox is checked, then
 * <ol>
 * <li>all fields throw to the next step;
 * <li>fields are resorted: mapped fields, then alphabetical sorted not mapped fields.
 * </ol>
 */
@Test
public void testSelectingAndSortingUnspecifiedFields() throws Exception {
    meta.setSelectingAndSortingUnspecifiedFields(true);
    MappingInputData sdi = new MappingInputData();
    sdi.linked = true;
    sdi.valueRenames = new ArrayList<MappingValueRename>();
    sdi.valueRenames.add(new MappingValueRename("number2", "n2"));
    sdi.valueRenames.add(new MappingValueRename("number4", "n4"));
    BlockingRowSet in = new BlockingRowSet(10);
    BlockingRowSet out = new BlockingRowSet(10);
    RowMeta rm = new RowMeta();
    rm.addValueMeta(new ValueMetaString("string"));
    rm.addValueMeta(new ValueMetaInteger("number1"));
    rm.addValueMeta(new ValueMetaInteger("number2"));
    rm.addValueMeta(new ValueMetaInteger("number3"));
    rm.addValueMeta(new ValueMetaInteger("number"));
    rm.addValueMeta(new ValueMetaInteger("number4"));
    rm.addValueMeta(new ValueMetaInteger("number5"));
    in.putRow(rm, new Object[] { "str", new Integer(100501), new Integer(100502), new Integer(100503), new Integer(100500), new Integer(100504), new Integer(100505) });
    in.putRow(rm, new Object[] { "str_1", new Integer(200501), new Integer(200502), new Integer(200503), new Integer(200500), new Integer(200504), new Integer(200505) });
    step.addRowSetToInputRowSets(in);
    step.addRowSetToOutputRowSets(out);
    assertTrue(step.init(meta, sdi));
    assertTrue(step.processRow(meta, sdi));
    Object[] outRowData = out.getRow();
    RowMetaInterface outMeta = out.getRowMeta();
    assertEquals("All fields are expected.", 7, outMeta.size());
    int i = 0;
    // Check if row-meta is formed according to the step specification
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_INTEGER, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "n2", outMeta.getValueMeta(i++).getName());
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_INTEGER, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "n4", outMeta.getValueMeta(i++).getName());
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_INTEGER, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "number", outMeta.getValueMeta(i++).getName());
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_INTEGER, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "number1", outMeta.getValueMeta(i++).getName());
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_INTEGER, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "number3", outMeta.getValueMeta(i++).getName());
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_INTEGER, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "number5", outMeta.getValueMeta(i++).getName());
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_STRING, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "string", outMeta.getValueMeta(i++).getName());
    // Check if row-data corresponds to the row-meta
    assertEquals("the field value mismatch.", new Integer(100502), outRowData[0]);
    assertEquals("the field value mismatch.", new Integer(100504), outRowData[1]);
    assertEquals("the field value mismatch.", new Integer(100500), outRowData[2]);
    assertEquals("the field value mismatch.", new Integer(100501), outRowData[3]);
    assertEquals("the field value mismatch.", new Integer(100503), outRowData[4]);
    assertEquals("the field value mismatch.", new Integer(100505), outRowData[5]);
    assertEquals("the field value mismatch.", "str", outRowData[6]);
    assertTrue(step.processRow(meta, sdi));
    outRowData = out.getRow();
    outMeta = out.getRowMeta();
    assertEquals("All fields are expected.", 7, outMeta.size());
    i = 0;
    // Check if row-meta is formed according to the step specification
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_INTEGER, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "n2", outMeta.getValueMeta(i++).getName());
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_INTEGER, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "n4", outMeta.getValueMeta(i++).getName());
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_INTEGER, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "number", outMeta.getValueMeta(i++).getName());
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_INTEGER, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "number1", outMeta.getValueMeta(i++).getName());
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_INTEGER, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "number3", outMeta.getValueMeta(i++).getName());
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_INTEGER, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "number5", outMeta.getValueMeta(i++).getName());
    assertEquals("the field type-meta mismatch.", ValueMetaInterface.TYPE_STRING, outMeta.getValueMeta(i).getType());
    assertEquals("the field name-meta mismatch.", "string", outMeta.getValueMeta(i++).getName());
    // Check if row-data corresponds to the row-meta
    assertEquals("the field value mismatch.", new Integer(200502), outRowData[0]);
    assertEquals("the field value mismatch.", new Integer(200504), outRowData[1]);
    assertEquals("the field value mismatch.", new Integer(200500), outRowData[2]);
    assertEquals("the field value mismatch.", new Integer(200501), outRowData[3]);
    assertEquals("the field value mismatch.", new Integer(200503), outRowData[4]);
    assertEquals("the field value mismatch.", new Integer(200505), outRowData[5]);
    assertEquals("the field value mismatch.", "str_1", outRowData[6]);
}

Example 7 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class GroupByTest method testProcessRow.

@Test
public void testProcessRow() throws KettleException {
    GroupByMeta groupByMeta = mock(GroupByMeta.class);
    GroupByData groupByData = mock(GroupByData.class);
    GroupBy groupBySpy = Mockito.spy(new GroupBy(mockHelper.stepMeta, mockHelper.stepDataInterface, 0, mockHelper.transMeta, mockHelper.trans));
    doReturn(null).when(groupBySpy).getRow();
    doReturn(null).when(groupBySpy).getInputRowMeta();
    RowMetaInterface rowMeta = new RowMeta();
    rowMeta.addValueMeta(new ValueMetaInteger("ROWNR"));
    List<RowSet> outputRowSets = new ArrayList<RowSet>();
    BlockingRowSet rowSet = new BlockingRowSet(1);
    rowSet.putRow(rowMeta, new Object[] { new Long(0) });
    outputRowSets.add(rowSet);
    groupBySpy.setOutputRowSets(outputRowSets);
    final String[] sub = { "b" };
    doReturn(sub).when(groupByMeta).getSubjectField();
    final String[] groupField = { "a" };
    doReturn(groupField).when(groupByMeta).getGroupField();
    final String[] aggFields = { "b_g" };
    doReturn(aggFields).when(groupByMeta).getAggregateField();
    final int[] aggType = { GroupByMeta.TYPE_GROUP_CONCAT_COMMA };
    doReturn(aggType).when(groupByMeta).getAggregateType();
    when(mockHelper.transMeta.getPrevStepFields(mockHelper.stepMeta)).thenReturn(new RowMeta());
    groupBySpy.processRow(groupByMeta, groupByData);
    assertTrue(groupBySpy.getOutputRowSets().get(0).isDone());
}

Example 8 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class TransTestingUtil method execute.

public static List<Object[]> execute(BaseStep step, StepMetaInterface meta, StepDataInterface data, int expectedRowsAmount, boolean checkIsDone) throws Exception {
    RowSet output = new BlockingRowSet(Math.max(1, expectedRowsAmount));
    step.setOutputRowSets(Collections.singletonList(output));
    int i = 0;
    List<Object[]> result = new ArrayList<>(expectedRowsAmount);
    while (step.processRow(meta, data) && i < expectedRowsAmount) {
        Object[] row = output.getRowImmediate();
        assertNotNull(Integer.toString(i), row);
        result.add(row);
        i++;
    }
    assertEquals("The amount of executions should be equal to expected", expectedRowsAmount, i);
    if (checkIsDone) {
        assertTrue(output.isDone());
    }
    return result;
}

Example 9 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class BaseRowSetConcurrentTest method test.

@Test
public void test() throws Exception {
    BlockingRowSet sharedBlockingRowSet = new BlockingRowSet(100);
    // fill data with initial values
    sharedBlockingRowSet.setThreadNameFromToCopy("1", 1, "1", 1);
    AtomicBoolean condition = new AtomicBoolean(true);
    List<Mutator> mutators = generateMutators(sharedBlockingRowSet, condition);
    List<Getter> getters = generateGetters(sharedBlockingRowSet, condition);
    ConcurrencyTestRunner.runAndCheckNoExceptionRaised(mutators, getters, condition);
}

Example 10 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class BaseStep method swapFirstInputRowSetIfExists.

protected void swapFirstInputRowSetIfExists(String stepName) {
    synchronized (inputRowSetsLock) {
        for (int i = 0; i < inputRowSets.size(); i++) {
            BlockingRowSet rs = (BlockingRowSet) inputRowSets.get(i);
            if (rs.getOriginStepName().equalsIgnoreCase(stepName)) {
                // swap this one and position 0...that means, the main stream is always stream 0 --> easy!
                // 
                BlockingRowSet zero = (BlockingRowSet) inputRowSets.get(0);
                inputRowSets.set(0, rs);
                inputRowSets.set(i, zero);
            }
        }
    }
}