Examples with RowStepCollector org.pentaho.di.trans.RowStepCollector used on opensource projects

Example 11 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class TextFileOutputIT method testTextFileOutput1.

/**
 * Tests output rows
 */
@Test
public void testTextFileOutput1() throws Exception {
    KettleEnvironment.init();
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("testTextFileOutput1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // create an injector step
    String injectorStepName = "injector step";
    StepMeta injectorStep = TestUtilities.createInjectorStep(injectorStepName, registry);
    transMeta.addStep(injectorStep);
    // create a row generator step
    StepMeta rowGeneratorStep = createRowGeneratorStep("Create rows for testTextFileOutput1", registry);
    transMeta.addStep(rowGeneratorStep);
    // create a TransHopMeta for injector and add it to the transMeta
    TransHopMeta hop_injectory_rowGenerator = new TransHopMeta(injectorStep, rowGeneratorStep);
    transMeta.addTransHop(hop_injectory_rowGenerator);
    // create the text file output step
    // but first lets get a filename
    String textFileName = TestUtilities.createEmptyTempFile("testTextFileOutput1");
    StepMeta textFileOutputStep = createTextFileOutputStep("text file output step", textFileName, "None", registry);
    transMeta.addStep(textFileOutputStep);
    // create a TransHopMeta for textFileOutputStep and add it to the transMeta
    TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta(rowGeneratorStep, textFileOutputStep);
    transMeta.addTransHop(hop_RowGenerator_outputTextFile);
    // Create a dummy step and add it to the tranMeta
    String dummyStepName = "dummy step";
    StepMeta dummyStep = createDummyStep(dummyStepName, registry);
    transMeta.addStep(dummyStep);
    // create a TransHopMeta for the
    TransHopMeta hop_outputTextFile_dummyStep = new TransHopMeta(textFileOutputStep, dummyStep);
    transMeta.addTransHop(hop_outputTextFile_dummyStep);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    // Create a row collector and add it to the dummy step interface
    StepInterface dummyStepInterface = trans.getStepInterface(dummyStepName, 0);
    RowStepCollector dummyRowCollector = new RowStepCollector();
    dummyStepInterface.addRowListener(dummyRowCollector);
    // RowProducer rowProducer = trans.addRowProducer(injectorStepName, 0);
    trans.startThreads();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRowCollector.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    try {
        TestUtilities.checkRows(goldenImageRows, resultRows);
    } catch (TestFailedException tfe) {
        fail(tfe.getMessage());
    }
}

Example 12 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class TextFileOutputIT method testTextFileOutput4.

/**
 * Tests the ZIP output capability of the TextFileOutput step
 *
 * @throws Exception
 */
@Test
public void testTextFileOutput4() throws Exception {
    KettleEnvironment.init();
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("testTextFileOutput4");
    PluginRegistry registry = PluginRegistry.getInstance();
    // create an injector step
    String injectorStepName = "injector step";
    StepMeta injectorStep = TestUtilities.createInjectorStep(injectorStepName, registry);
    transMeta.addStep(injectorStep);
    // create a row generator step
    StepMeta rowGeneratorStep = createRowGeneratorStep("Create rows for testTextFileOutput4", registry);
    transMeta.addStep(rowGeneratorStep);
    // create a TransHopMeta for injector and add it to the transMeta
    TransHopMeta hop_injectory_rowGenerator = new TransHopMeta(injectorStep, rowGeneratorStep);
    transMeta.addTransHop(hop_injectory_rowGenerator);
    // create the text file output step with ZIP compression
    // but first lets get a filename
    String textFileName = "testTextFileOutput4";
    String textFileOutputStepName = "text file output step";
    StepMeta textFileOutputStep = createTextFileOutputStep(textFileOutputStepName, textFileName, "Zip", registry);
    transMeta.addStep(textFileOutputStep);
    // create a TransHopMeta for textFileOutputStep and add it to the transMeta
    TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta(rowGeneratorStep, textFileOutputStep);
    transMeta.addTransHop(hop_RowGenerator_outputTextFile);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    // Create a row collector and add it to the dummy step interface
    StepInterface dummyStepInterface = trans.getStepInterface(textFileOutputStepName, 0);
    RowStepCollector dummyRowCollector = new RowStepCollector();
    dummyStepInterface.addRowListener(dummyRowCollector);
    trans.startThreads();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRowCollector.getRowsWritten();
    Object[][] rows = new Object[10][3];
    File f = new File(textFileName + ".zip");
    f.deleteOnExit();
    try {
        ZipFile zf = new ZipFile(f);
        int zipEntryCount = 0;
        Enumeration<? extends ZipEntry> entries = zf.entries();
        while (entries.hasMoreElements()) {
            ZipEntry ze = entries.nextElement();
            zipEntryCount++;
            BufferedReader input = new BufferedReader(new InputStreamReader(zf.getInputStream(ze)));
            readData1Rows(rows, input);
        }
        zf.close();
        assertEquals(zipEntryCount, 1);
    } catch (IOException e) {
        fail(e.getLocalizedMessage());
    }
    List<RowMetaAndData> outFileRows = createResultDataFromObjects(rows);
    try {
        TestUtilities.checkRows(resultRows, outFileRows);
    } catch (TestFailedException tfe) {
        fail(tfe.getMessage());
    }
}

Example 13 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class TextFileOutputIT method testTextFileOutput5.

/**
 * Tests the GZIP output capability of the TextFileOutput step
 *
 * @throws Exception
 */
@Test
public void testTextFileOutput5() throws Exception {
    KettleEnvironment.init();
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("testTextFileOutput5");
    PluginRegistry registry = PluginRegistry.getInstance();
    // create an injector step
    String injectorStepName = "injector step";
    StepMeta injectorStep = TestUtilities.createInjectorStep(injectorStepName, registry);
    transMeta.addStep(injectorStep);
    // create a row generator step
    StepMeta rowGeneratorStep = createRowGeneratorStep("Create rows for testTextFileOutput5", registry);
    transMeta.addStep(rowGeneratorStep);
    // create a TransHopMeta for injector and add it to the transMeta
    TransHopMeta hop_injectory_rowGenerator = new TransHopMeta(injectorStep, rowGeneratorStep);
    transMeta.addTransHop(hop_injectory_rowGenerator);
    // create the text file output step with GZIP compression
    // but first lets get a filename
    String textFileName = "testTextFileOutput5";
    String textFileOutputStepName = "text file output step";
    StepMeta textFileOutputStep = createTextFileOutputStep(textFileOutputStepName, textFileName, "GZip", registry);
    transMeta.addStep(textFileOutputStep);
    // create a TransHopMeta for textFileOutputStep and add it to the transMeta
    TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta(rowGeneratorStep, textFileOutputStep);
    transMeta.addTransHop(hop_RowGenerator_outputTextFile);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    // Create a row collector and add it to the dummy step interface
    StepInterface dummyStepInterface = trans.getStepInterface(textFileOutputStepName, 0);
    RowStepCollector dummyRowCollector = new RowStepCollector();
    dummyStepInterface.addRowListener(dummyRowCollector);
    trans.startThreads();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRowCollector.getRowsWritten();
    Object[][] rows = new Object[10][3];
    File f = new File(textFileName + "." + EXTENSION + ".gz");
    f.deleteOnExit();
    try {
        FileInputStream fin = new FileInputStream(f);
        GZIPInputStream gzis = new GZIPInputStream(fin);
        InputStreamReader xover = new InputStreamReader(gzis);
        BufferedReader input = new BufferedReader(xover);
        readData1Rows(rows, input);
        fin.close();
    } catch (IOException e) {
        fail(e.getLocalizedMessage());
    }
    List<RowMetaAndData> outFileRows = createResultDataFromObjects(rows);
    try {
        TestUtilities.checkRows(resultRows, outFileRows);
    } catch (TestFailedException tfe) {
        fail(tfe.getMessage());
    }
}

Example 14 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class TransExecutorIT method subTransExecutionStatisticsIsCollected.

@Test
public void subTransExecutionStatisticsIsCollected() throws Exception {
    TransExecutorMeta executorMeta = getExecutorMeta(transExecutor);
    executorMeta.setExecutionTimeField("time");
    executorMeta.setExecutionResultTargetStepMeta(dummy);
    Trans trans = createTrans(transMeta);
    RowStepCollector endRc = listenExecutor(trans);
    RowProducer rp = trans.addRowProducer(injector.getName(), 0);
    trans.startThreads();
    RowMetaAndData testInput = new RowMetaAndData(createRowMetaForOneField(), SAMPLE_INPUT);
    rp.putRow(testInput.getRowMeta(), testInput.getData());
    rp.finished();
    trans.waitUntilFinished();
    assertFalse(endRc.getRowsWritten().isEmpty());
    // execution time field
    assertNotNull(endRc.getRowsWritten().get(0).getData()[0]);
}

Example 15 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class TransExecutorIT method listenExecutor.

private RowStepCollector listenExecutor(Trans trans) {
    StepInterface transExecutorStep = trans.getStepInterface(transExecutor.getName(), 0);
    RowStepCollector rc = new RowStepCollector();
    transExecutorStep.addRowListener(rc);
    return rc;
}