Examples with RowProducer org.pentaho.di.trans.RowProducer used on opensource projects

Example 31 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class FilterRowsIT method testFilterConditionRefersToNonExistingFields.

@Test
public void testFilterConditionRefersToNonExistingFields() throws Exception {
    KettleEnvironment.init();
    // Create a new transformation...
    TransMeta transMeta = new TransMeta();
    transMeta.setName("filterrowstest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // create an injector step...
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // Create a filter rows step
    String filterStepName = "filter rows step";
    FilterRowsMeta frm = new FilterRowsMeta();
    Condition condition = new Condition();
    String nonExistingFieldName = "non-existing-field";
    condition.setLeftValuename(nonExistingFieldName);
    // IS NOT
    condition.setFunction(8);
    condition.setRightValuename(null);
    condition.setOperator(0);
    frm.setCondition(condition);
    String filterRowsStepPid = registry.getPluginId(StepPluginType.class, frm);
    StepMeta filterRowsStep = new StepMeta(filterRowsStepPid, filterStepName, frm);
    transMeta.addStep(filterRowsStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, filterRowsStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    // add rows
    List<RowMetaAndData> inputList = createIntegerData();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.startThreads();
    trans.waitUntilFinished();
    // expect errors
    assertEquals(1, trans.getErrors());
}

Example 32 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pdi-dataservice-server-plugin by pentaho.

the class DefaultTransWiring method run.

@Override
public void run() {
    // This is where we will inject the rows from the service transformation step
    // 
    final RowProducer rowProducer;
    final Trans serviceTrans = dataServiceExecutor.getServiceTrans();
    final Trans genTrans = dataServiceExecutor.getGenTrans();
    try {
        rowProducer = dataServiceExecutor.addRowProducer();
    } catch (KettleException e) {
        throw Throwables.propagate(e);
    }
    // Now connect the 2 transformations with listeners and injector
    // 
    StepInterface serviceStep = serviceTrans.findRunThread(dataServiceExecutor.getService().getStepname());
    if (serviceStep == null) {
        throw Throwables.propagate(new KettleException("Service step is not accessible"));
    }
    serviceStep.addRowListener(new DefaultTransWiringRowAdapter(serviceTrans, genTrans, rowProducer));
    // Let the other transformation know when there are no more rows
    // 
    serviceStep.addStepListener(new StepAdapter() {

        @Override
        public void stepFinished(Trans trans, StepMeta stepMeta, StepInterface step) {
            rowProducer.finished();
        }
    });
    dataServiceExecutor.getGenTrans().findRunThread(dataServiceExecutor.getResultStepName()).addStepListener(new StepAdapter() {

        @Override
        public void stepFinished(Trans trans, StepMeta stepMeta, StepInterface step) {
            if (serviceTrans.isRunning()) {
                trans.getLogChannel().logBasic("Query finished, stopping service transformation");
                serviceTrans.stopAll();
            }
        }
    });
}

Example 33 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pdi-dataservice-server-plugin by pentaho.

the class DataServiceExecutorTest method testExecuteQueryNoResults.

@Test
public void testExecuteQueryNoResults() throws Exception {
    SQL sql = new SQL("SELECT * FROM " + DATA_SERVICE_NAME);
    StepInterface serviceStep = serviceTrans.findRunThread(DATA_SERVICE_STEP);
    StepInterface resultStep = genTrans.findRunThread(RESULT_STEP_NAME);
    when(serviceTrans.getTransMeta().listParameters()).thenReturn(new String[0]);
    PushDownOptimizationMeta optimization = mock(PushDownOptimizationMeta.class);
    when(optimization.isEnabled()).thenReturn(true);
    dataService.getPushDownOptimizationMeta().add(optimization);
    IMetaStore metastore = mock(IMetaStore.class);
    DataServiceExecutor executor = new DataServiceExecutor.Builder(sql, dataService, context).serviceTrans(serviceTrans).sqlTransGenerator(sqlTransGenerator).genTrans(genTrans).metastore(metastore).build();
    ArgumentCaptor<String> objectIds = ArgumentCaptor.forClass(String.class);
    verify(serviceTrans).setContainerObjectId(objectIds.capture());
    when(serviceTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
    verify(genTrans).setContainerObjectId(objectIds.capture());
    when(genTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
    verify(serviceTrans).setMetaStore(metastore);
    verify(genTrans).setMetaStore(metastore);
    RowProducer sqlTransRowProducer = mock(RowProducer.class);
    when(genTrans.addRowProducer(INJECTOR_STEP_NAME, 0)).thenReturn(sqlTransRowProducer);
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    // Start Execution
    executor.executeQuery(new DataOutputStream(outputStream));
    // Check header was written
    assertThat(outputStream.size(), greaterThan(0));
    outputStream.reset();
    InOrder genTransStartup = inOrder(genTrans, resultStep);
    InOrder serviceTransStartup = inOrder(optimization, serviceTrans, serviceStep);
    ArgumentCaptor<RowListener> listenerArgumentCaptor = ArgumentCaptor.forClass(RowListener.class);
    ArgumentCaptor<StepListener> resultStepListener = ArgumentCaptor.forClass(StepListener.class);
    ArgumentCaptor<TransListener> transListenerCaptor = ArgumentCaptor.forClass(TransListener.class);
    genTransStartup.verify(genTrans).addTransListener(transListenerCaptor.capture());
    genTransStartup.verify(genTrans).addRowProducer(INJECTOR_STEP_NAME, 0);
    genTransStartup.verify(resultStep).addStepListener(resultStepListener.capture());
    genTransStartup.verify(resultStep).addRowListener(listenerArgumentCaptor.capture());
    RowListener clientRowListener = listenerArgumentCaptor.getValue();
    genTransStartup.verify(genTrans).startThreads();
    serviceTransStartup.verify(optimization).activate(executor);
    serviceTransStartup.verify(serviceStep).addRowListener(listenerArgumentCaptor.capture());
    serviceTransStartup.verify(serviceTrans).startThreads();
    // Verify linkage
    RowListener serviceRowListener = listenerArgumentCaptor.getValue();
    assertNotNull(serviceRowListener);
    // Push row from service to sql Trans
    RowMetaInterface rowMeta = genTrans.getTransMeta().getStepFields(RESULT_STEP_NAME);
    doReturn(true).when(serviceTrans).isRunning();
    resultStepListener.getValue().stepFinished(genTrans, resultStep.getStepMeta(), resultStep);
    verify(serviceTrans).stopAll();
    // Verify Service Trans finished
    ArgumentCaptor<StepListener> serviceStepListener = ArgumentCaptor.forClass(StepListener.class);
    verify(serviceStep).addStepListener(serviceStepListener.capture());
    serviceStepListener.getValue().stepFinished(serviceTrans, serviceStep.getStepMeta(), serviceStep);
    verify(sqlTransRowProducer).finished();
    // finish transformation, so that the listener runs
    transListenerCaptor.getValue().transFinished(genTrans);
    InOrder writeRows = inOrder(rowMeta);
    ArgumentCaptor<DataOutputStream> streamCaptor = ArgumentCaptor.forClass(DataOutputStream.class);
    writeRows.verify(rowMeta).writeMeta(streamCaptor.capture());
    DataOutputStream dataOutputStream = streamCaptor.getValue();
    writeRows.verify(rowMeta, times(0)).writeData(same(dataOutputStream), argThat(arrayWithSize(1)));
    writeRows.verifyNoMoreInteractions();
    executor.waitUntilFinished();
    verify(serviceTrans).waitUntilFinished();
    verify(genTrans).waitUntilFinished();
}

Example 34 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pdi-dataservice-server-plugin by pentaho.

the class DataServiceExecutorTest method testExecuteQuery.

@Test
public void testExecuteQuery() throws Exception {
    SQL sql = new SQL("SELECT * FROM " + DATA_SERVICE_NAME);
    StepInterface serviceStep = serviceTrans.findRunThread(DATA_SERVICE_STEP);
    StepInterface resultStep = genTrans.findRunThread(RESULT_STEP_NAME);
    when(serviceTrans.getTransMeta().listParameters()).thenReturn(new String[0]);
    PushDownOptimizationMeta optimization = mock(PushDownOptimizationMeta.class);
    when(optimization.isEnabled()).thenReturn(true);
    dataService.getPushDownOptimizationMeta().add(optimization);
    IMetaStore metastore = mock(IMetaStore.class);
    DataServiceExecutor executor = new DataServiceExecutor.Builder(sql, dataService, context).serviceTrans(serviceTrans).sqlTransGenerator(sqlTransGenerator).genTrans(genTrans).metastore(metastore).build();
    ArgumentCaptor<String> objectIds = ArgumentCaptor.forClass(String.class);
    verify(serviceTrans).setContainerObjectId(objectIds.capture());
    when(serviceTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
    verify(genTrans).setContainerObjectId(objectIds.capture());
    when(genTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
    verify(serviceTrans).setMetaStore(metastore);
    verify(genTrans).setMetaStore(metastore);
    RowProducer sqlTransRowProducer = mock(RowProducer.class);
    when(genTrans.addRowProducer(INJECTOR_STEP_NAME, 0)).thenReturn(sqlTransRowProducer);
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    // Start Execution
    executor.executeQuery(new DataOutputStream(outputStream));
    // Check header was written
    assertThat(outputStream.size(), greaterThan(0));
    outputStream.reset();
    InOrder genTransStartup = inOrder(genTrans, resultStep);
    InOrder serviceTransStartup = inOrder(optimization, serviceTrans, serviceStep);
    ArgumentCaptor<RowListener> listenerArgumentCaptor = ArgumentCaptor.forClass(RowListener.class);
    ArgumentCaptor<StepListener> resultStepListener = ArgumentCaptor.forClass(StepListener.class);
    ArgumentCaptor<TransListener> transListenerCaptor = ArgumentCaptor.forClass(TransListener.class);
    genTransStartup.verify(genTrans).addTransListener(transListenerCaptor.capture());
    genTransStartup.verify(genTrans).addRowProducer(INJECTOR_STEP_NAME, 0);
    genTransStartup.verify(resultStep).addStepListener(resultStepListener.capture());
    genTransStartup.verify(resultStep).addRowListener(listenerArgumentCaptor.capture());
    RowListener clientRowListener = listenerArgumentCaptor.getValue();
    genTransStartup.verify(genTrans).startThreads();
    serviceTransStartup.verify(optimization).activate(executor);
    serviceTransStartup.verify(serviceStep).addRowListener(listenerArgumentCaptor.capture());
    serviceTransStartup.verify(serviceTrans).startThreads();
    // Verify linkage
    RowListener serviceRowListener = listenerArgumentCaptor.getValue();
    assertNotNull(serviceRowListener);
    // Push row from service to sql Trans
    RowMetaInterface rowMeta = genTrans.getTransMeta().getStepFields(RESULT_STEP_NAME);
    Object[] data;
    for (int i = 0; i < 50; i++) {
        data = new Object[] { i };
        Object[] dataClone = { i };
        when(rowMeta.cloneRow(data)).thenReturn(dataClone);
        serviceRowListener.rowWrittenEvent(rowMeta, data);
        verify(sqlTransRowProducer).putRowWait(same(rowMeta), and(eq(dataClone), not(same(data))), any(Long.class), any(TimeUnit.class));
        verify(rowMeta).cloneRow(data);
    }
    doReturn(true).when(serviceTrans).isRunning();
    resultStepListener.getValue().stepFinished(genTrans, resultStep.getStepMeta(), resultStep);
    verify(serviceTrans).stopAll();
    // Verify Service Trans finished
    ArgumentCaptor<StepListener> serviceStepListener = ArgumentCaptor.forClass(StepListener.class);
    verify(serviceStep).addStepListener(serviceStepListener.capture());
    serviceStepListener.getValue().stepFinished(serviceTrans, serviceStep.getStepMeta(), serviceStep);
    verify(sqlTransRowProducer).finished();
    // Push row from service to sql Trans
    for (int i = 0; i < 50; i++) {
        Object[] row = { i };
        clientRowListener.rowWrittenEvent(rowMeta, row);
    }
    transListenerCaptor.getValue().transFinished(genTrans);
    InOrder writeRows = inOrder(rowMeta);
    ArgumentCaptor<DataOutputStream> streamCaptor = ArgumentCaptor.forClass(DataOutputStream.class);
    writeRows.verify(rowMeta).writeMeta(streamCaptor.capture());
    DataOutputStream dataOutputStream = streamCaptor.getValue();
    writeRows.verify(rowMeta, times(50)).writeData(same(dataOutputStream), argThat(arrayWithSize(1)));
    writeRows.verifyNoMoreInteractions();
    executor.waitUntilFinished();
    verify(serviceTrans).waitUntilFinished();
    verify(genTrans).waitUntilFinished();
}

Example 35 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pdi-dataservice-server-plugin by pentaho.

the class CachedServiceTest method testReplayFullCache.

@Test
public void testReplayFullCache() throws Exception {
    DataServiceExecutor executor = dataServiceExecutor(BASE_QUERY);
    CachedService cachedService = CachedService.complete(testData);
    RowProducer rowProducer = genTrans.addRowProducer(INJECTOR_STEP, 0);
    // Activate cachedServiceLoader
    Executor mockExecutor = mock(Executor.class);
    final CachedServiceLoader cachedServiceLoader = new CachedServiceLoader(cachedService, mockExecutor);
    ListenableFuture<Integer> replay = cachedServiceLoader.replay(executor);
    ArgumentCaptor<Runnable> replayRunnable = ArgumentCaptor.forClass(Runnable.class);
    verify(mockExecutor).execute(replayRunnable.capture());
    stepMetaDataCombi.step = inputStep;
    stepMetaDataCombi.meta = inputStepMetaInterface;
    stepMetaDataCombi.data = inputStepDataInterface;
    List<StepMetaDataCombi> stepMetaDataCombis = new ArrayList<>();
    stepMetaDataCombis.add(stepMetaDataCombi);
    when(serviceTrans.getSteps()).thenReturn(stepMetaDataCombis);
    // Simulate executing data service
    executor.executeListeners(DataServiceExecutor.ExecutionPoint.READY);
    executor.executeListeners(DataServiceExecutor.ExecutionPoint.START);
    // Verify that serviceTrans never started, genTrans is accepting rows
    verify(serviceTrans).stopAll();
    verify(inputStep).setOutputDone();
    verify(inputStep).dispose(inputStepMetaInterface, inputStepDataInterface);
    verify(inputStep).markStop();
    verify(serviceTrans, never()).startThreads();
    verify(genTrans).startThreads();
    when(rowProducer.putRowWait(any(RowMetaInterface.class), any(Object[].class), anyInt(), any(TimeUnit.class))).then(new Answer<Boolean>() {

        int calls = 0;

        @Override
        public Boolean answer(InvocationOnMock invocation) throws Throwable {
            // Simulate full row set on tenth call
            return ++calls != 10;
        }
    });
    when(genTrans.isRunning()).thenReturn(true);
    // Run cache loader (would be asynchronous)
    replayRunnable.getValue().run();
    assertThat(replay.get(1, TimeUnit.SECONDS), equalTo(testData.size()));
    InOrder rowsProduced = inOrder(rowProducer);
    for (int i = 0; i < testData.size(); i++) {
        RowMetaAndData metaAndData = testData.get(i);
        // Tenth row was called twice, since row set was full
        Object[] data = metaAndData.getData();
        rowsProduced.verify(rowProducer, times(i == 9 ? 2 : 1)).putRowWait(eq(metaAndData.getRowMeta()), and(eq(data), AdditionalMatchers.not(same(data))), anyInt(), any(TimeUnit.class));
    }
    rowsProduced.verify(rowProducer).finished();
    rowsProduced.verifyNoMoreInteractions();
}