Examples with Variables org.pentaho.di.core.variables.Variables used on opensource projects

Example 51 with Variables

use of org.pentaho.di.core.variables.Variables in project pentaho-kettle by pentaho.

the class StepErrorMetaTest method testGetErrorRowMeta.

@Test
public void testGetErrorRowMeta() {
    VariableSpace vars = new Variables();
    vars.setVariable("VarNumberErrors", "nbrErrors");
    vars.setVariable("VarErrorDescription", "errorDescription");
    vars.setVariable("VarErrorFields", "errorFields");
    vars.setVariable("VarErrorCodes", "errorCodes");
    StepErrorMeta testObject = new StepErrorMeta(vars, new StepMeta(), new StepMeta(), "${VarNumberErrors}", "${VarErrorDescription}", "${VarErrorFields}", "${VarErrorCodes}");
    RowMetaInterface result = testObject.getErrorRowMeta(10, "some data was bad", "factId", "BAD131");
    assertNotNull(result);
    assertEquals(4, result.size());
    assertEquals(ValueMetaInterface.TYPE_INTEGER, result.getValueMeta(0).getType());
    assertEquals("nbrErrors", result.getValueMeta(0).getName());
    assertEquals(ValueMetaInterface.TYPE_STRING, result.getValueMeta(1).getType());
    assertEquals("errorDescription", result.getValueMeta(1).getName());
    assertEquals(ValueMetaInterface.TYPE_STRING, result.getValueMeta(2).getType());
    assertEquals("errorFields", result.getValueMeta(2).getName());
    assertEquals(ValueMetaInterface.TYPE_STRING, result.getValueMeta(3).getType());
    assertEquals("errorCodes", result.getValueMeta(3).getName());
}

Example 52 with Variables

use of org.pentaho.di.core.variables.Variables in project pentaho-kettle by pentaho.

the class UniqueRowsIT method testAllUnique.

@Test
public void testAllUnique() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("uniquerowstest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a unique rows step
    // 
    String uniqueRowsStepname = "unique rows step";
    UniqueRowsMeta urm = new UniqueRowsMeta();
    urm.setCompareFields(new String[] { "KEY" });
    urm.setCaseInsensitive(new boolean[] { true });
    urm.setRejectDuplicateRow(true);
    String uniqueRowsStepPid = registry.getPluginId(StepPluginType.class, urm);
    StepMeta uniqueRowsStep = new StepMeta(uniqueRowsStepPid, uniqueRowsStepname, urm);
    uniqueRowsStep.setDistributes(false);
    transMeta.addStep(uniqueRowsStep);
    transMeta.addTransHop(new TransHopMeta(injectorStep, uniqueRowsStep));
    // 
    // Create a dummy step to receive the unique rows
    // 
    String dummyStepname1 = "dummy step";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    transMeta.addTransHop(new TransHopMeta(uniqueRowsStep, dummyStep1));
    // 
    // Create a dummy step to receive the duplicate rows (errors)
    // 
    String dummyStepname2 = "dummy step2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    // Set up error (aka duplicates) handling info
    StepErrorMeta stepErrorMeta = new StepErrorMeta(new Variables(), uniqueRowsStep);
    stepErrorMeta.setTargetStep(dummyStep2);
    stepErrorMeta.setEnabled(true);
    stepErrorMeta.setNrErrorsValuename("numErrors");
    stepErrorMeta.setErrorDescriptionsValuename("duplicates");
    stepErrorMeta.setErrorFieldsValuename("KEY");
    stepErrorMeta.setErrorCodesValuename("errorCodes");
    stepErrorMeta.setMaxErrors("9999");
    stepErrorMeta.setMaxPercentErrors("");
    stepErrorMeta.setMinPercentRows("");
    uniqueRowsStep.setStepErrorMeta(stepErrorMeta);
    transMeta.addTransHop(new TransHopMeta(uniqueRowsStep, dummyStep2));
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    StepInterface si2 = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector dummyRc2 = new RowStepCollector();
    si2.addRowListener(dummyRc2);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataAllUnique();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
    checkRows(createResultDataAllUnique(), resultRows);
    List<RowMetaAndData> errorRows = dummyRc2.getRowsWritten();
    // There should be no duplicates for this test
    assertEquals(errorRows.size(), 0);
}

Example 53 with Variables

use of org.pentaho.di.core.variables.Variables in project pentaho-kettle by pentaho.

the class BlackBoxIT method runTrans.

public Result runTrans(String fileName, LogChannelInterface log) throws KettleException {
    // Bootstrap the Kettle API...
    // 
    KettleEnvironment.init();
    TransMeta transMeta = new TransMeta(fileName);
    Trans trans = new Trans(transMeta);
    Result result;
    try {
        trans.setLogLevel(LogLevel.ERROR);
        result = trans.getResult();
    } catch (Exception e) {
        result = trans.getResult();
        String message = "Processing has stopped because of an error: " + getPath(fileName);
        addFailure(message);
        log.logError(message, e);
        fail(message);
        return result;
    }
    try {
        trans.initializeVariablesFrom(null);
        trans.getTransMeta().setInternalKettleVariables(trans);
        trans.setSafeModeEnabled(true);
        // see if the transformation checks ok
        List<CheckResultInterface> remarks = new ArrayList<CheckResultInterface>();
        trans.getTransMeta().checkSteps(remarks, false, null, new Variables(), null, null);
        for (CheckResultInterface remark : remarks) {
            if (remark.getType() == CheckResultInterface.TYPE_RESULT_ERROR) {
                // add this to the log
                addFailure("Check error: " + getPath(fileName) + ", " + remark.getErrorCode());
                log.logError("BlackBoxTest", "Check error: " + getPath(fileName) + ", " + remark.getErrorCode());
            }
        }
        // allocate & run the required sub-threads
        try {
            trans.execute(null);
        } catch (Exception e) {
            addFailure("Unable to prepare and initialize this transformation: " + getPath(fileName));
            log.logError("BlackBoxTest", "Unable to prepare and initialize this transformation: " + getPath(fileName));
            fail("Unable to prepare and initialize this transformation: " + getPath(fileName));
            return null;
        }
        trans.waitUntilFinished();
        result = trans.getResult();
        // The result flag is not set to true by a transformation - set it to true if got no errors
        // FIXME: Find out if there is a better way to check if a transformation has thrown an error
        result.setResult(result.getNrErrors() == 0);
        return result;
    } catch (Exception e) {
        addFailure("Unexpected error occurred: " + getPath(fileName));
        log.logError("BlackBoxTest", "Unexpected error occurred: " + getPath(fileName), e);
        result.setResult(false);
        result.setNrErrors(1);
        fail("Unexpected error occurred: " + getPath(fileName));
        return result;
    }
}

Example 54 with Variables

use of org.pentaho.di.core.variables.Variables in project pentaho-kettle by pentaho.

the class JaninoStepIT method testIntegerReturnType.

@Test
public void testIntegerReturnType() throws KettleException {
    String STEP_NAME = "JaninoStep";
    JaninoMeta meta = new JaninoMeta();
    meta.setDefault();
    JaninoMetaFunction[] formulas = { new JaninoMetaFunction("intretval", "2", ValueMetaInterface.TYPE_INTEGER, -1, -1, null) };
    meta.setFormula(formulas);
    TransMeta transMeta = TransTestFactory.generateTestTransformation(new Variables(), meta, STEP_NAME);
    List<RowMetaAndData> inputData = new ArrayList<RowMetaAndData>();
    inputData.add(new RowMetaAndData(new RowMeta(), new Object[0]));
    List<RowMetaAndData> resultData = TransTestFactory.executeTestTransformation(transMeta, TransTestFactory.INJECTOR_STEPNAME, STEP_NAME, TransTestFactory.DUMMY_STEPNAME, inputData);
    assertNotNull(resultData);
    assertEquals(1, resultData.size());
    assertEquals(1, resultData.get(0).size());
    assertEquals("intretval", resultData.get(0).getValueMeta(0).getName());
    assertEquals(ValueMetaInterface.TYPE_INTEGER, resultData.get(0).getValueMeta(0).getType());
    assertEquals(Long.valueOf(2), resultData.get(0).getInteger(0));
}

Example 55 with Variables

use of org.pentaho.di.core.variables.Variables in project pentaho-kettle by pentaho.

the class JavaFilterIT method testVariableCondition.

@Test
public void testVariableCondition() throws KettleException {
    Variables varSpace = new Variables();
    varSpace.setVariable("myVarCondition", realCondition);
    JavaFilterMeta meta = new JavaFilterMeta();
    meta.setCondition("${myVarCondition}");
    TransMeta trans = TransTestFactory.generateTestTransformation(varSpace, meta, stepName);
    List<RowMetaAndData> input = getInputData();
    List<RowMetaAndData> result = TransTestFactory.executeTestTransformation(trans, TransTestFactory.INJECTOR_STEPNAME, stepName, TransTestFactory.DUMMY_STEPNAME, input);
    // Only the "Car" row should be returned, based upon the Java Filter condition
    assertTrue(result.size() == 1);
    assertTrue(result.get(0).getString(fieldName, null).equals("Car"));
}