Examples with StepInterface org.pentaho.di.trans.step.StepInterface used on opensource projects

Example 76 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class TransGridDelegate method refreshView.

private void refreshView() {
    boolean insert = true;
    int nrSteps = -1;
    int totalSteps = -1;
    if (transGridView == null || transGridView.isDisposed()) {
        return;
    }
    if (refresh_busy) {
        return;
    }
    List<StepMeta> selectedSteps = new ArrayList<StepMeta>();
    if (showSelectedSteps) {
        selectedSteps = transGraph.trans.getTransMeta().getSelectedSteps();
    }
    int topIdx = transGridView.getTable().getTopIndex();
    refresh_busy = true;
    Table table = transGridView.table;
    long time = new Date().getTime();
    long msSinceLastUpdate = time - lastUpdateView;
    if (transGraph.trans != null && !transGraph.trans.isPreparing() && msSinceLastUpdate > UPDATE_TIME_VIEW) {
        lastUpdateView = time;
        nrSteps = transGraph.trans.nrSteps();
        totalSteps = nrSteps;
        if (hideInactiveSteps) {
            nrSteps = transGraph.trans.nrActiveSteps();
        }
        StepExecutionStatus[] stepStatusLookup = transGraph.trans.getTransStepExecutionStatusLookup();
        boolean[] isRunningLookup = transGraph.trans.getTransStepIsRunningLookup();
        int sortColumn = transGridView.getSortField();
        boolean sortDescending = transGridView.isSortingDescending();
        int[] selectedItems = transGridView.getSelectionIndices();
        if (table.getItemCount() != nrSteps) {
            table.removeAll();
        } else {
            insert = false;
        }
        if (nrSteps == 0 && table.getItemCount() == 0) {
            new TableItem(table, SWT.NONE);
            refresh_busy = false;
            return;
        }
        int nr = 0;
        for (int i = 0; i < totalSteps; i++) {
            StepInterface baseStep = transGraph.trans.getRunThread(i);
            // See if the step is selected & in need of display
            // 
            boolean showSelected;
            if (showSelectedSteps) {
                if (selectedSteps.size() == 0) {
                    showSelected = true;
                } else {
                    showSelected = false;
                    for (StepMeta stepMeta : selectedSteps) {
                        if (baseStep.getStepMeta().equals(stepMeta)) {
                            showSelected = true;
                            break;
                        }
                    }
                }
            } else {
                showSelected = true;
            }
            // 
            if (showSelected && (hideInactiveSteps && (isRunningLookup[i] || stepStatusLookup[i] != StepExecutionStatus.STATUS_FINISHED)) || (!hideInactiveSteps && stepStatusLookup[i] != StepExecutionStatus.STATUS_EMPTY)) {
                TableItem ti = null;
                if (insert) {
                    ti = new TableItem(table, SWT.NONE);
                } else {
                    ti = table.getItem(nr);
                }
                if (ti == null) {
                    continue;
                }
                String num = "" + (i + 1);
                if (ti.getText(0).length() < 1) {
                    ti.setText(0, num);
                }
                if (ti.getText(0).length() > 0) {
                    Integer tIndex = Integer.parseInt(ti.getText(0));
                    tIndex--;
                    baseStep = transGraph.trans.getRunThread(tIndex);
                }
                StepStatus stepStatus = new StepStatus(baseStep);
                String[] fields = stepStatus.getTransLogFields();
                // screen!
                for (int f = 1; f < fields.length; f++) {
                    if (!fields[f].equalsIgnoreCase(ti.getText(f))) {
                        ti.setText(f, fields[f]);
                    }
                }
                // Error lines should appear in red:
                if (baseStep.getErrors() > 0) {
                    ti.setBackground(GUIResource.getInstance().getColorRed());
                }
                nr++;
                Collection<StepStatus> stepStatuses = baseStep.subStatuses();
                int subIndex = 1;
                for (StepStatus status : stepStatuses) {
                    String[] subFields = status.getTransLogFields(baseStep.getStatus().getDescription());
                    subFields[1] = "     " + subFields[1];
                    TableItem subItem = new TableItem(table, SWT.NONE);
                    subItem.setText(0, num + "." + subIndex++);
                    for (int f = 1; f < subFields.length; f++) {
                        subItem.setText(f, subFields[f]);
                    }
                }
            }
        }
        for (int i = 0; i < table.getItems().length; i++) {
            TableItem item = table.getItem(i);
            item.setForeground(GUIResource.getInstance().getColorBlack());
            if (!item.getBackground().equals(GUIResource.getInstance().getColorRed())) {
                item.setBackground(i % 2 == 0 ? GUIResource.getInstance().getColorWhite() : GUIResource.getInstance().getColorBlueCustomGrid());
            }
        }
        // default
        if (table.getItemCount() > 0 && (sortColumn != 0 || !sortDescending)) {
            transGridView.sortTable(sortColumn, sortDescending, false);
        }
        // if (updateRowNumbers) { transGridView.setRowNums(); }
        transGridView.optWidth(true);
        if (selectedItems != null && selectedItems.length > 0) {
            transGridView.setSelection(selectedItems);
        }
        // transGridView.getTable().setTopIndex(topIdx);
        if (transGridView.getTable().getTopIndex() != topIdx) {
            transGridView.getTable().setTopIndex(topIdx);
        }
    } else {
        // We need at least one table-item in a table!
        if (table.getItemCount() == 0) {
            new TableItem(table, SWT.NONE);
        }
    }
    refresh_busy = false;
}

Example 77 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pdi-dataservice-server-plugin by pentaho.

the class DefaultTransWiring method run.

@Override
public void run() {
    // This is where we will inject the rows from the service transformation step
    // 
    final RowProducer rowProducer;
    final Trans serviceTrans = dataServiceExecutor.getServiceTrans();
    final Trans genTrans = dataServiceExecutor.getGenTrans();
    try {
        rowProducer = dataServiceExecutor.addRowProducer();
    } catch (KettleException e) {
        throw Throwables.propagate(e);
    }
    // Now connect the 2 transformations with listeners and injector
    // 
    StepInterface serviceStep = serviceTrans.findRunThread(dataServiceExecutor.getService().getStepname());
    if (serviceStep == null) {
        throw Throwables.propagate(new KettleException("Service step is not accessible"));
    }
    serviceStep.addRowListener(new DefaultTransWiringRowAdapter(serviceTrans, genTrans, rowProducer));
    // Let the other transformation know when there are no more rows
    // 
    serviceStep.addStepListener(new StepAdapter() {

        @Override
        public void stepFinished(Trans trans, StepMeta stepMeta, StepInterface step) {
            rowProducer.finished();
        }
    });
    dataServiceExecutor.getGenTrans().findRunThread(dataServiceExecutor.getResultStepName()).addStepListener(new StepAdapter() {

        @Override
        public void stepFinished(Trans trans, StepMeta stepMeta, StepInterface step) {
            if (serviceTrans.isRunning()) {
                trans.getLogChannel().logBasic("Query finished, stopping service transformation");
                serviceTrans.stopAll();
            }
        }
    });
}

Example 78 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pdi-dataservice-server-plugin by pentaho.

the class DataServiceExecutorTest method testExecuteQueryNoResults.

@Test
public void testExecuteQueryNoResults() throws Exception {
    SQL sql = new SQL("SELECT * FROM " + DATA_SERVICE_NAME);
    StepInterface serviceStep = serviceTrans.findRunThread(DATA_SERVICE_STEP);
    StepInterface resultStep = genTrans.findRunThread(RESULT_STEP_NAME);
    when(serviceTrans.getTransMeta().listParameters()).thenReturn(new String[0]);
    PushDownOptimizationMeta optimization = mock(PushDownOptimizationMeta.class);
    when(optimization.isEnabled()).thenReturn(true);
    dataService.getPushDownOptimizationMeta().add(optimization);
    IMetaStore metastore = mock(IMetaStore.class);
    DataServiceExecutor executor = new DataServiceExecutor.Builder(sql, dataService, context).serviceTrans(serviceTrans).sqlTransGenerator(sqlTransGenerator).genTrans(genTrans).metastore(metastore).build();
    ArgumentCaptor<String> objectIds = ArgumentCaptor.forClass(String.class);
    verify(serviceTrans).setContainerObjectId(objectIds.capture());
    when(serviceTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
    verify(genTrans).setContainerObjectId(objectIds.capture());
    when(genTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
    verify(serviceTrans).setMetaStore(metastore);
    verify(genTrans).setMetaStore(metastore);
    RowProducer sqlTransRowProducer = mock(RowProducer.class);
    when(genTrans.addRowProducer(INJECTOR_STEP_NAME, 0)).thenReturn(sqlTransRowProducer);
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    // Start Execution
    executor.executeQuery(new DataOutputStream(outputStream));
    // Check header was written
    assertThat(outputStream.size(), greaterThan(0));
    outputStream.reset();
    InOrder genTransStartup = inOrder(genTrans, resultStep);
    InOrder serviceTransStartup = inOrder(optimization, serviceTrans, serviceStep);
    ArgumentCaptor<RowListener> listenerArgumentCaptor = ArgumentCaptor.forClass(RowListener.class);
    ArgumentCaptor<StepListener> resultStepListener = ArgumentCaptor.forClass(StepListener.class);
    ArgumentCaptor<TransListener> transListenerCaptor = ArgumentCaptor.forClass(TransListener.class);
    genTransStartup.verify(genTrans).addTransListener(transListenerCaptor.capture());
    genTransStartup.verify(genTrans).addRowProducer(INJECTOR_STEP_NAME, 0);
    genTransStartup.verify(resultStep).addStepListener(resultStepListener.capture());
    genTransStartup.verify(resultStep).addRowListener(listenerArgumentCaptor.capture());
    RowListener clientRowListener = listenerArgumentCaptor.getValue();
    genTransStartup.verify(genTrans).startThreads();
    serviceTransStartup.verify(optimization).activate(executor);
    serviceTransStartup.verify(serviceStep).addRowListener(listenerArgumentCaptor.capture());
    serviceTransStartup.verify(serviceTrans).startThreads();
    // Verify linkage
    RowListener serviceRowListener = listenerArgumentCaptor.getValue();
    assertNotNull(serviceRowListener);
    // Push row from service to sql Trans
    RowMetaInterface rowMeta = genTrans.getTransMeta().getStepFields(RESULT_STEP_NAME);
    doReturn(true).when(serviceTrans).isRunning();
    resultStepListener.getValue().stepFinished(genTrans, resultStep.getStepMeta(), resultStep);
    verify(serviceTrans).stopAll();
    // Verify Service Trans finished
    ArgumentCaptor<StepListener> serviceStepListener = ArgumentCaptor.forClass(StepListener.class);
    verify(serviceStep).addStepListener(serviceStepListener.capture());
    serviceStepListener.getValue().stepFinished(serviceTrans, serviceStep.getStepMeta(), serviceStep);
    verify(sqlTransRowProducer).finished();
    // finish transformation, so that the listener runs
    transListenerCaptor.getValue().transFinished(genTrans);
    InOrder writeRows = inOrder(rowMeta);
    ArgumentCaptor<DataOutputStream> streamCaptor = ArgumentCaptor.forClass(DataOutputStream.class);
    writeRows.verify(rowMeta).writeMeta(streamCaptor.capture());
    DataOutputStream dataOutputStream = streamCaptor.getValue();
    writeRows.verify(rowMeta, times(0)).writeData(same(dataOutputStream), argThat(arrayWithSize(1)));
    writeRows.verifyNoMoreInteractions();
    executor.waitUntilFinished();
    verify(serviceTrans).waitUntilFinished();
    verify(genTrans).waitUntilFinished();
}

Example 79 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pdi-dataservice-server-plugin by pentaho.

the class DataServiceExecutorTest method testExecuteQuery.

@Test
public void testExecuteQuery() throws Exception {
    SQL sql = new SQL("SELECT * FROM " + DATA_SERVICE_NAME);
    StepInterface serviceStep = serviceTrans.findRunThread(DATA_SERVICE_STEP);
    StepInterface resultStep = genTrans.findRunThread(RESULT_STEP_NAME);
    when(serviceTrans.getTransMeta().listParameters()).thenReturn(new String[0]);
    PushDownOptimizationMeta optimization = mock(PushDownOptimizationMeta.class);
    when(optimization.isEnabled()).thenReturn(true);
    dataService.getPushDownOptimizationMeta().add(optimization);
    IMetaStore metastore = mock(IMetaStore.class);
    DataServiceExecutor executor = new DataServiceExecutor.Builder(sql, dataService, context).serviceTrans(serviceTrans).sqlTransGenerator(sqlTransGenerator).genTrans(genTrans).metastore(metastore).build();
    ArgumentCaptor<String> objectIds = ArgumentCaptor.forClass(String.class);
    verify(serviceTrans).setContainerObjectId(objectIds.capture());
    when(serviceTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
    verify(genTrans).setContainerObjectId(objectIds.capture());
    when(genTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
    verify(serviceTrans).setMetaStore(metastore);
    verify(genTrans).setMetaStore(metastore);
    RowProducer sqlTransRowProducer = mock(RowProducer.class);
    when(genTrans.addRowProducer(INJECTOR_STEP_NAME, 0)).thenReturn(sqlTransRowProducer);
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    // Start Execution
    executor.executeQuery(new DataOutputStream(outputStream));
    // Check header was written
    assertThat(outputStream.size(), greaterThan(0));
    outputStream.reset();
    InOrder genTransStartup = inOrder(genTrans, resultStep);
    InOrder serviceTransStartup = inOrder(optimization, serviceTrans, serviceStep);
    ArgumentCaptor<RowListener> listenerArgumentCaptor = ArgumentCaptor.forClass(RowListener.class);
    ArgumentCaptor<StepListener> resultStepListener = ArgumentCaptor.forClass(StepListener.class);
    ArgumentCaptor<TransListener> transListenerCaptor = ArgumentCaptor.forClass(TransListener.class);
    genTransStartup.verify(genTrans).addTransListener(transListenerCaptor.capture());
    genTransStartup.verify(genTrans).addRowProducer(INJECTOR_STEP_NAME, 0);
    genTransStartup.verify(resultStep).addStepListener(resultStepListener.capture());
    genTransStartup.verify(resultStep).addRowListener(listenerArgumentCaptor.capture());
    RowListener clientRowListener = listenerArgumentCaptor.getValue();
    genTransStartup.verify(genTrans).startThreads();
    serviceTransStartup.verify(optimization).activate(executor);
    serviceTransStartup.verify(serviceStep).addRowListener(listenerArgumentCaptor.capture());
    serviceTransStartup.verify(serviceTrans).startThreads();
    // Verify linkage
    RowListener serviceRowListener = listenerArgumentCaptor.getValue();
    assertNotNull(serviceRowListener);
    // Push row from service to sql Trans
    RowMetaInterface rowMeta = genTrans.getTransMeta().getStepFields(RESULT_STEP_NAME);
    Object[] data;
    for (int i = 0; i < 50; i++) {
        data = new Object[] { i };
        Object[] dataClone = { i };
        when(rowMeta.cloneRow(data)).thenReturn(dataClone);
        serviceRowListener.rowWrittenEvent(rowMeta, data);
        verify(sqlTransRowProducer).putRowWait(same(rowMeta), and(eq(dataClone), not(same(data))), any(Long.class), any(TimeUnit.class));
        verify(rowMeta).cloneRow(data);
    }
    doReturn(true).when(serviceTrans).isRunning();
    resultStepListener.getValue().stepFinished(genTrans, resultStep.getStepMeta(), resultStep);
    verify(serviceTrans).stopAll();
    // Verify Service Trans finished
    ArgumentCaptor<StepListener> serviceStepListener = ArgumentCaptor.forClass(StepListener.class);
    verify(serviceStep).addStepListener(serviceStepListener.capture());
    serviceStepListener.getValue().stepFinished(serviceTrans, serviceStep.getStepMeta(), serviceStep);
    verify(sqlTransRowProducer).finished();
    // Push row from service to sql Trans
    for (int i = 0; i < 50; i++) {
        Object[] row = { i };
        clientRowListener.rowWrittenEvent(rowMeta, row);
    }
    transListenerCaptor.getValue().transFinished(genTrans);
    InOrder writeRows = inOrder(rowMeta);
    ArgumentCaptor<DataOutputStream> streamCaptor = ArgumentCaptor.forClass(DataOutputStream.class);
    writeRows.verify(rowMeta).writeMeta(streamCaptor.capture());
    DataOutputStream dataOutputStream = streamCaptor.getValue();
    writeRows.verify(rowMeta, times(50)).writeData(same(dataOutputStream), argThat(arrayWithSize(1)));
    writeRows.verifyNoMoreInteractions();
    executor.waitUntilFinished();
    verify(serviceTrans).waitUntilFinished();
    verify(genTrans).waitUntilFinished();
}

Example 80 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pdi-dataservice-server-plugin by pentaho.

the class TableInputParameterGenerationTest method testFailures.

@Test
public void testFailures() throws Exception {
    // Throw exception if type is not TableInput
    try {
        service.pushDown(mock(Condition.class), mock(ParameterGeneration.class), mock(StepInterface.class));
        fail();
    } catch (PushDownOptimizationException thrown) {
        assertThat(thrown.getMessage(), notNullValue());
    }
    // Throw exception if connection fails
    KettleDatabaseException expected = new KettleDatabaseException();
    data.db = mock(DatabaseWrapper.class);
    doThrow(expected).when(data.db).connect();
    try {
        service.pushDown(mock(Condition.class), mock(ParameterGeneration.class), stepInterface);
        fail();
    } catch (PushDownOptimizationException thrown) {
        assertThat(thrown.getCause(), equalTo((Throwable) expected));
    }
}