Examples with BlockingRowSet org.pentaho.di.core.BlockingRowSet used on opensource projects

Example 16 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class BaseStepTest method testBaseStepPutRowLocalSpecialPartitioning.

/**
 * This test checks that data from one non-partitioned step copies to 2 partitioned steps right.
 *
 * @throws KettleException
 * @see {@link <a href="http://jira.pentaho.com/browse/PDI-12211">http://jira.pentaho.com/browse/PDI-12211<a>}
 */
@Test
public void testBaseStepPutRowLocalSpecialPartitioning() throws KettleException {
    List<StepMeta> stepMetas = new ArrayList<StepMeta>();
    stepMetas.add(mockHelper.stepMeta);
    stepMetas.add(mockHelper.stepMeta);
    StepPartitioningMeta stepPartitioningMeta = spy(new StepPartitioningMeta());
    BasePartitioner partitioner = mock(BasePartitioner.class);
    when(mockHelper.logChannelInterfaceFactory.create(any(), any(LoggingObjectInterface.class))).thenAnswer(new Answer<LogChannelInterface>() {

        @Override
        public LogChannelInterface answer(InvocationOnMock invocation) throws Throwable {
            ((BaseStep) invocation.getArguments()[0]).getLogLevel();
            return mockHelper.logChannelInterface;
        }
    });
    when(mockHelper.trans.isRunning()).thenReturn(true);
    when(mockHelper.transMeta.findNextSteps(any(StepMeta.class))).thenReturn(stepMetas);
    when(mockHelper.stepMeta.getStepPartitioningMeta()).thenReturn(stepPartitioningMeta);
    when(stepPartitioningMeta.getPartitioner()).thenReturn(partitioner);
    when(partitioner.getNrPartitions()).thenReturn(2);
    Object object0 = "name0";
    ValueMetaInterface meta0 = new ValueMetaString(object0.toString());
    Object object1 = "name1";
    ValueMetaInterface meta2 = new ValueMetaString(object1.toString());
    RowMetaInterface rowMeta0 = new RowMeta();
    rowMeta0.addValueMeta(meta0);
    Object[] objects0 = { object0 };
    RowMetaInterface rowMeta1 = new RowMeta();
    rowMeta1.addValueMeta(meta2);
    Object[] objects1 = { object1 };
    when(stepPartitioningMeta.getPartition(rowMeta0, objects0)).thenReturn(0);
    when(stepPartitioningMeta.getPartition(rowMeta1, objects1)).thenReturn(1);
    BlockingRowSet[] rowSet = { new BlockingRowSet(2), new BlockingRowSet(2), new BlockingRowSet(2), new BlockingRowSet(2) };
    List<RowSet> outputRowSets = new ArrayList<RowSet>();
    outputRowSets.addAll(Arrays.asList(rowSet));
    BaseStep baseStep = new BaseStep(mockHelper.stepMeta, mockHelper.stepDataInterface, 0, mockHelper.transMeta, mockHelper.trans);
    baseStep.setStopped(false);
    baseStep.setRepartitioning(StepPartitioningMeta.PARTITIONING_METHOD_SPECIAL);
    baseStep.setOutputRowSets(outputRowSets);
    baseStep.putRow(rowMeta0, objects0);
    baseStep.putRow(rowMeta1, objects1);
    assertEquals(object0, baseStep.getOutputRowSets().get(0).getRow()[0]);
    assertEquals(object1, baseStep.getOutputRowSets().get(1).getRow()[0]);
    assertEquals(object0, baseStep.getOutputRowSets().get(2).getRow()[0]);
    assertEquals(object1, baseStep.getOutputRowSets().get(3).getRow()[0]);
}

Example 17 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class BaseStepTest method testGetRowSafeModeEnabled.

@Test
public void testGetRowSafeModeEnabled() throws KettleException {
    Trans transMock = mock(Trans.class);
    when(transMock.isSafeModeEnabled()).thenReturn(true);
    BaseStep baseStepSpy = spy(new BaseStep(mockHelper.stepMeta, mockHelper.stepDataInterface, 0, mockHelper.transMeta, transMock));
    doNothing().when(baseStepSpy).waitUntilTransformationIsStarted();
    doNothing().when(baseStepSpy).openRemoteInputStepSocketsOnce();
    BlockingRowSet rowSet = new BlockingRowSet(1);
    List<ValueMetaInterface> valueMetaList = Arrays.asList(new ValueMetaInteger("x"), new ValueMetaString("a"));
    RowMeta rowMeta = new RowMeta();
    rowMeta.setValueMetaList(valueMetaList);
    final Object[] row = new Object[] {};
    rowSet.putRow(rowMeta, row);
    baseStepSpy.setInputRowSets(Arrays.asList(rowSet));
    doReturn(rowSet).when(baseStepSpy).currentInputStream();
    baseStepSpy.getRow();
    verify(mockHelper.transMeta, times(1)).checkRowMixingStatically(any(StepMeta.class), anyObject());
}

Example 18 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class JoinRowsTest method getRowSetWithData.

BlockingRowSet getRowSetWithData(int size, String dataPrefix, boolean isMainStep) {
    BlockingRowSet blockingRowSet = new BlockingRowSet(size);
    RowMeta rowMeta = new RowMeta();
    ValueMetaInterface valueMetaString = new ValueMetaString(dataPrefix + " first value name");
    ValueMetaInterface valueMetaInteger = new ValueMetaString(dataPrefix + " second value name");
    ValueMetaInterface valueMetaBoolean = new ValueMetaString(dataPrefix + " third value name");
    rowMeta.addValueMeta(valueMetaString);
    rowMeta.addValueMeta(valueMetaInteger);
    rowMeta.addValueMeta(valueMetaBoolean);
    blockingRowSet.setRowMeta(rowMeta);
    for (int i = 0; i < size; i++) {
        Object[] rowData = new Object[3];
        rowData[0] = dataPrefix + " row[" + i + "]-first value";
        rowData[1] = dataPrefix + " row[" + i + "]-second value";
        rowData[2] = dataPrefix + " row[" + i + "]-third value";
        blockingRowSet.putRow(rowMeta, rowData);
    }
    if (isMainStep) {
        blockingRowSet.setThreadNameFromToCopy("main step name", 0, null, 0);
    } else {
        blockingRowSet.setThreadNameFromToCopy("secondary step name", 0, null, 0);
    }
    blockingRowSet.setDone();
    return blockingRowSet;
}

Example 19 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class TextFileInputDialogTest method testMinimalWidth_PDI_14253.

@Test
public void testMinimalWidth_PDI_14253() throws Exception {
    final String virtualFile = "ram://pdi-14253.txt";
    KettleVFS.getFileObject(virtualFile).createFile();
    final String content = "r1c1,  r1c2\nr2c1  ,  r2c2  ";
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    bos.write(content.getBytes());
    OutputStream os = KettleVFS.getFileObject(virtualFile).getContent().getOutputStream();
    IOUtils.copy(new ByteArrayInputStream(bos.toByteArray()), os);
    os.close();
    TextFileInputMeta meta = new TextFileInputMeta();
    meta.content.lineWrapped = false;
    meta.inputFields = new BaseFileField[] { new BaseFileField("col1", -1, -1), new BaseFileField("col2", -1, -1) };
    meta.content.fileCompression = "None";
    meta.content.fileType = "CSV";
    meta.content.header = false;
    meta.content.nrHeaderLines = -1;
    meta.content.footer = false;
    meta.content.nrFooterLines = -1;
    TextFileInputData data = new TextFileInputData();
    data.files = new FileInputList();
    data.files.addFile(KettleVFS.getFileObject(virtualFile));
    data.outputRowMeta = new RowMeta();
    data.outputRowMeta.addValueMeta(new ValueMetaString("col1"));
    data.outputRowMeta.addValueMeta(new ValueMetaString("col2"));
    data.dataErrorLineHandler = mock(FileErrorHandler.class);
    data.fileFormatType = TextFileInputMeta.FILE_FORMAT_UNIX;
    data.separator = ",";
    data.filterProcessor = new TextFileFilterProcessor(new TextFileFilter[0], new Variables() {
    });
    data.filePlayList = new FilePlayListAll();
    TextFileInputDialog dialog = new TextFileInputDialog(mock(Shell.class), meta, mock(TransMeta.class), "TFIMinimalWidthTest");
    TableView tv = mock(TableView.class);
    when(tv.nrNonEmpty()).thenReturn(0);
    // click the Minimal width button
    dialog.setMinimalWidth(tv);
    RowSet output = new BlockingRowSet(5);
    TextFileInput input = StepMockUtil.getStep(TextFileInput.class, TextFileInputMeta.class, "test");
    input.setOutputRowSets(Collections.singletonList(output));
    while (input.processRow(meta, data)) {
    // wait until the step completes executing
    }
    Object[] row1 = output.getRowImmediate();
    assertRow(row1, "r1c1", "r1c2");
    Object[] row2 = output.getRowImmediate();
    assertRow(row2, "r2c1", "r2c2");
    KettleVFS.getFileObject(virtualFile).delete();
}