Examples with StepMetaDataCombi org.pentaho.di.trans.step.StepMetaDataCombi used on opensource projects

Example 16 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-kettle by pentaho.

the class MQTTProducerTest method setup.

@Before
public void setup() throws Exception {
    KettleLogStore.setLogChannelInterfaceFactory(logChannelFactory);
    when(logChannelFactory.create(any(), any())).thenReturn(logChannel);
    when(logChannelFactory.create(any())).thenReturn(logChannel);
    TransMeta transMeta = new TransMeta(getClass().getResource("/ProduceFourRows.ktr").getPath());
    trans = new Trans(transMeta);
    trans.setVariable("mqttServer", "127.0.0.1:1883");
    trans.setVariable("clientId", "client1");
    trans.setVariable("topic", "TestWinning");
    trans.setVariable("messageField", "message");
    trans.setVariable("qos", "0");
    trans.prepareExecution(new String[] {});
    StepMetaDataCombi combi = trans.getSteps().get(1);
    MQTTProducer step = (MQTTProducer) combi.step;
    MQTTProducerData data = (MQTTProducerData) combi.data;
    data.mqttClient = mqttClient;
    step.first = false;
}

Example 17 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-kettle by pentaho.

the class Mapping method lookupStatusStepNumbers.

private void lookupStatusStepNumbers() {
    MappingData mappingData = getData();
    if (mappingData.getMappingTrans() != null) {
        List<StepMetaDataCombi> steps = mappingData.getMappingTrans().getSteps();
        for (int i = 0; i < steps.size(); i++) {
            StepMetaDataCombi sid = steps.get(i);
            BaseStep rt = (BaseStep) sid.step;
            if (rt.getStepname().equals(getData().mappingTransMeta.getTransLogTable().getStepnameRead())) {
                mappingData.linesReadStepNr = i;
            }
            if (rt.getStepname().equals(getData().mappingTransMeta.getTransLogTable().getStepnameInput())) {
                mappingData.linesInputStepNr = i;
            }
            if (rt.getStepname().equals(getData().mappingTransMeta.getTransLogTable().getStepnameWritten())) {
                mappingData.linesWrittenStepNr = i;
            }
            if (rt.getStepname().equals(getData().mappingTransMeta.getTransLogTable().getStepnameOutput())) {
                mappingData.linesOutputStepNr = i;
            }
            if (rt.getStepname().equals(getData().mappingTransMeta.getTransLogTable().getStepnameUpdated())) {
                mappingData.linesUpdatedStepNr = i;
            }
            if (rt.getStepname().equals(getData().mappingTransMeta.getTransLogTable().getStepnameRejected())) {
                mappingData.linesRejectedStepNr = i;
            }
        }
    }
}

Example 18 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-kettle by pentaho.

the class TransWebSocketEngineAdapterTest method testOpsIncludeSubTrans.

@Test
public void testOpsIncludeSubTrans() throws Exception {
    TransMeta transMeta = new TransMeta(getClass().getResource("grid-to-subtrans.ktr").getPath());
    TransWebSocketEngineAdapter adapter = new TransWebSocketEngineAdapter(transMeta, "", "", false);
    adapter.prepareExecution(new String[] {});
    List<StepMetaDataCombi> steps = adapter.getSteps();
    steps.sort(Comparator.comparing(s -> s.stepname));
    assertEquals(2, steps.size());
    assertEquals(0, steps.get(0).step.subStatuses().size());
    assertEquals(2, steps.get(1).step.subStatuses().size());
}

Example 19 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-kettle by pentaho.

the class CsvProcessRowInParallelTest method createBaseCombi.

private StepMetaDataCombi createBaseCombi(File sharedFile, boolean headerPresent, String delimiter) {
    StepMetaDataCombi combi = new StepMetaDataCombi();
    CsvInputData data = new CsvInputData();
    CsvInputMeta meta = createMeta(sharedFile, createInputFileFields("Field_000", "Field_001"), headerPresent, delimiter);
    CsvInput csvInput = createCsvInput();
    csvInput.init(meta, data);
    combi.step = csvInput;
    combi.data = data;
    combi.meta = meta;
    return combi;
}

Example 20 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-kettle by pentaho.

the class WordCountSampleIT method testWordCountMapper.

@Test
public void testWordCountMapper() throws Exception {
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta("src/it/resources/wordcount-mapper.ktr");
    transMeta.setTransformationType(TransformationType.SingleThreaded);
    long transStart = System.currentTimeMillis();
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.setLogLevel(LogLevel.MINIMAL);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface("Output", 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer("Injector", 0);
    trans.startThreads();
    String metricsStep = "Remove garbage";
    // The single threaded transformation type expects us to run the steps
    // ourselves.
    // 
    SingleThreadedTransExecutor executor = new SingleThreadedTransExecutor(trans);
    // Initialize all steps
    // 
    executor.init();
    int iterations = 1000000;
    long totalWait = 0;
    List<RowMetaAndData> inputList = createMapperData();
    for (int i = 0; i < iterations; i++) {
        // add rows
        for (RowMetaAndData rm : inputList) {
            Object[] copy = rm.getRowMeta().cloneRow(rm.getData());
            rp.putRow(rm.getRowMeta(), copy);
        }
        long start = System.currentTimeMillis();
        boolean cont = executor.oneIteration();
        if (!cont) {
            fail("We don't expect any step or the transformation to be done before the end of all iterations.");
        }
        long end = System.currentTimeMillis();
        long delay = end - start;
        totalWait += delay;
        if (i > 0 && (i % 100000) == 0) {
            long rowsProcessed = trans.findRunThread(metricsStep).getLinesRead();
            double speed = Const.round((rowsProcessed) / ((double) (end - transStart) / 1000), 1);
            int totalRows = 0;
            for (StepMetaDataCombi combi : trans.getSteps()) {
                for (RowSet rowSet : combi.step.getInputRowSets()) {
                    totalRows += rowSet.size();
                }
                for (RowSet rowSet : combi.step.getOutputRowSets()) {
                    totalRows += rowSet.size();
                }
            }
            System.out.println("#" + i + " : Finished processing one iteration in " + delay + "ms, average is: " + Const.round(((double) totalWait / (i + 1)), 1) + ", speed=" + speed + " row/s, total rows buffered: " + totalRows);
        }
        List<RowMetaAndData> resultRows = rc.getRowsWritten();
        // Result has one row less because we filter out one.
        // We also join with 3 identical rows in a data grid, giving 9 rows of which 3 are filtered out
        // 
        assertEquals("Error found in iteration " + i + " : not the expected amount of output rows.", 9, resultRows.size());
        rc.clear();
    }
    rp.finished();
    // Dispose all steps.
    // 
    executor.dispose();
    long rowsProcessed = trans.findRunThread(metricsStep).getLinesRead();
    long transEnd = System.currentTimeMillis();
    long transTime = transEnd - transStart;
    System.out.println("Average delay before idle : " + Const.round(((double) totalWait / iterations), 1));
    double transTimeSeconds = Const.round(((double) transTime / 1000), 1);
    System.out.println("Total transformation runtime for " + iterations + " iterations :" + transTimeSeconds + " seconds");
    double transTimePerIteration = Const.round(((double) transTime / iterations), 2);
    System.out.println("Runtime per iteration: " + transTimePerIteration + " miliseconds");
    double rowsPerSecond = Const.round((rowsProcessed) / ((double) transTime / 1000), 1);
    System.out.println("Average speed: " + rowsPerSecond + " rows/second");
}