Examples with StepMeta org.pentaho.di.trans.step.StepMeta used on opensource projects

Example 81 with StepMeta

use of org.pentaho.di.trans.step.StepMeta in project pentaho-kettle by pentaho.

the class KettleDatabaseRepositoryTransDelegate method insertTransformation.

// CHECKSTYLE:LineLength:OFF
private synchronized void insertTransformation(TransMeta transMeta) throws KettleException {
    RowMetaAndData table = new RowMetaAndData();
    table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_TRANSFORMATION_ID_TRANSFORMATION), new LongObjectId(transMeta.getObjectId()));
    table.addValue(new ValueMetaString(KettleDatabaseRepository.FIELD_TRANSFORMATION_NAME), transMeta.getName());
    table.addValue(new ValueMetaString(KettleDatabaseRepository.FIELD_TRANSFORMATION_DESCRIPTION), transMeta.getDescription());
    table.addValue(new ValueMetaString(KettleDatabaseRepository.FIELD_TRANSFORMATION_EXTENDED_DESCRIPTION), transMeta.getExtendedDescription());
    table.addValue(new ValueMetaString(KettleDatabaseRepository.FIELD_TRANSFORMATION_TRANS_VERSION), transMeta.getTransversion());
    table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_TRANSFORMATION_TRANS_STATUS), new Long(transMeta.getTransstatus() < 0 ? -1L : transMeta.getTransstatus()));
    TransLogTable logTable = transMeta.getTransLogTable();
    StepMeta step = (StepMeta) logTable.getSubject(TransLogTable.ID.LINES_READ);
    table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_TRANSFORMATION_ID_STEP_READ), step == null ? null : step.getObjectId());
    step = (StepMeta) logTable.getSubject(TransLogTable.ID.LINES_WRITTEN);
    table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_TRANSFORMATION_ID_STEP_WRITE), step == null ? null : step.getObjectId());
    step = (StepMeta) logTable.getSubject(TransLogTable.ID.LINES_INPUT);
    table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_TRANSFORMATION_ID_STEP_INPUT), step == null ? null : step.getObjectId());
    step = (StepMeta) logTable.getSubject(TransLogTable.ID.LINES_OUTPUT);
    table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_TRANSFORMATION_ID_STEP_OUTPUT), step == null ? null : step.getObjectId());
    step = (StepMeta) logTable.getSubject(TransLogTable.ID.LINES_UPDATED);
    table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_TRANSFORMATION_ID_STEP_UPDATE), step == null ? null : step.getObjectId());
    table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_TRANSFORMATION_ID_DATABASE_LOG), logTable.getDatabaseMeta() == null ? new LongObjectId(-1L).longValue() : new LongObjectId(logTable.getDatabaseMeta().getObjectId()).longValue());
    table.addValue(new ValueMetaString(KettleDatabaseRepository.FIELD_TRANSFORMATION_TABLE_NAME_LOG), logTable.getDatabaseMeta());
    table.addValue(new ValueMetaBoolean(KettleDatabaseRepository.FIELD_TRANSFORMATION_USE_BATCHID), Boolean.valueOf(logTable.isBatchIdUsed()));
    table.addValue(new ValueMetaBoolean(KettleDatabaseRepository.FIELD_TRANSFORMATION_USE_LOGFIELD), Boolean.valueOf(logTable.isLogFieldUsed()));
    table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_TRANSFORMATION_ID_DATABASE_MAXDATE), transMeta.getMaxDateConnection() == null ? new LongObjectId(-1L).longValue() : new LongObjectId(transMeta.getMaxDateConnection().getObjectId()).longValue());
    table.addValue(new ValueMetaString(KettleDatabaseRepository.FIELD_TRANSFORMATION_TABLE_NAME_MAXDATE), transMeta.getMaxDateTable());
    table.addValue(new ValueMetaString(KettleDatabaseRepository.FIELD_TRANSFORMATION_FIELD_NAME_MAXDATE), transMeta.getMaxDateField());
    table.addValue(new ValueMetaNumber(KettleDatabaseRepository.FIELD_TRANSFORMATION_OFFSET_MAXDATE), new Double(transMeta.getMaxDateOffset()));
    table.addValue(new ValueMetaNumber(KettleDatabaseRepository.FIELD_TRANSFORMATION_DIFF_MAXDATE), new Double(transMeta.getMaxDateDifference()));
    table.addValue(new ValueMetaString(KettleDatabaseRepository.FIELD_TRANSFORMATION_CREATED_USER), transMeta.getCreatedUser());
    table.addValue(new ValueMetaDate(KettleDatabaseRepository.FIELD_TRANSFORMATION_CREATED_DATE), transMeta.getCreatedDate());
    table.addValue(new ValueMetaString(KettleDatabaseRepository.FIELD_TRANSFORMATION_MODIFIED_USER), transMeta.getModifiedUser());
    table.addValue(new ValueMetaDate(KettleDatabaseRepository.FIELD_TRANSFORMATION_MODIFIED_DATE), transMeta.getModifiedDate());
    table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_TRANSFORMATION_SIZE_ROWSET), new Long(transMeta.getSizeRowset()));
    table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_TRANSFORMATION_ID_DIRECTORY), transMeta.getRepositoryDirectory().getObjectId());
    repository.connectionDelegate.getDatabase().prepareInsert(table.getRowMeta(), KettleDatabaseRepository.TABLE_R_TRANSFORMATION);
    repository.connectionDelegate.getDatabase().setValuesInsert(table);
    repository.connectionDelegate.getDatabase().insertRow();
    repository.connectionDelegate.getDatabase().closeInsert();
    step = (StepMeta) logTable.getSubject(TransLogTable.ID.LINES_REJECTED);
    if (step != null) {
        ObjectId rejectedId = step.getObjectId();
        repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_ID_STEP_REJECTED, rejectedId == null ? null : Long.valueOf(rejectedId.toString()), null);
    }
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_UNIQUE_CONNECTIONS, 0, transMeta.isUsingUniqueConnections() ? "Y" : "N");
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_FEEDBACK_SHOWN, 0, transMeta.isFeedbackShown() ? "Y" : "N");
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_FEEDBACK_SIZE, transMeta.getFeedbackSize(), "");
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_USING_THREAD_PRIORITIES, 0, transMeta.isUsingThreadPriorityManagment() ? "Y" : "N");
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_SHARED_FILE, 0, transMeta.getSharedObjectsFile());
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_CAPTURE_STEP_PERFORMANCE, 0, transMeta.isCapturingStepPerformanceSnapShots() ? "Y" : "N");
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_STEP_PERFORMANCE_CAPTURING_DELAY, transMeta.getStepPerformanceCapturingDelay(), "");
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_STEP_PERFORMANCE_CAPTURING_SIZE_LIMIT, 0, transMeta.getStepPerformanceCapturingSizeLimit());
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_STEP_PERFORMANCE_LOG_TABLE, 0, transMeta.getPerformanceLogTable().getTableName());
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_LOG_SIZE_LIMIT, 0, transMeta.getTransLogTable().getLogSizeLimit());
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_LOG_INTERVAL, 0, transMeta.getTransLogTable().getLogInterval());
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_TRANSFORMATION_TYPE, 0, transMeta.getTransformationType().getCode());
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_SLEEP_TIME_EMPTY, transMeta.getSleepTimeEmpty(), null);
    repository.connectionDelegate.insertTransAttribute(transMeta.getObjectId(), 0, KettleDatabaseRepository.TRANS_ATTRIBUTE_SLEEP_TIME_FULL, transMeta.getSleepTimeFull(), null);
    // Save the logging connection link...
    if (logTable.getDatabaseMeta() != null) {
        repository.insertStepDatabase(transMeta.getObjectId(), null, logTable.getDatabaseMeta().getObjectId());
    }
    // Save the maxdate connection link...
    if (transMeta.getMaxDateConnection() != null) {
        repository.insertStepDatabase(transMeta.getObjectId(), null, transMeta.getMaxDateConnection().getObjectId());
    }
    // Save the logging tables too..
    // 
    RepositoryAttributeInterface attributeInterface = new KettleDatabaseRepositoryTransAttribute(repository.connectionDelegate, transMeta.getObjectId());
    transMeta.getTransLogTable().saveToRepository(attributeInterface);
    transMeta.getStepLogTable().saveToRepository(attributeInterface);
    transMeta.getPerformanceLogTable().saveToRepository(attributeInterface);
    transMeta.getChannelLogTable().saveToRepository(attributeInterface);
}

Example 82 with StepMeta

use of org.pentaho.di.trans.step.StepMeta in project pentaho-kettle by pentaho.

the class KettleDatabaseRepositoryTransDelegate method loadTransHopMeta.

public TransHopMeta loadTransHopMeta(ObjectId id_trans_hop, List<StepMeta> steps) throws KettleException {
    TransHopMeta hopTransMeta = new TransHopMeta();
    try {
        hopTransMeta.setObjectId(id_trans_hop);
        RowMetaAndData r = getTransHop(id_trans_hop);
        hopTransMeta.setEnabled(r.getBoolean("ENABLED", false));
        long id_step_from = r.getInteger("ID_STEP_FROM", 0);
        long id_step_to = r.getInteger("ID_STEP_TO", 0);
        StepMeta fromStep = StepMeta.findStep(steps, new LongObjectId(id_step_from));
        // 
        if (fromStep == null && id_step_from > 0) {
            // Simply load this, we only want the name, we don't care about the
            // rest...
            // 
            StepMeta stepMeta = repository.stepDelegate.loadStepMeta(new LongObjectId(id_step_from), new ArrayList<DatabaseMeta>(), new ArrayList<PartitionSchema>());
            fromStep = StepMeta.findStep(steps, stepMeta.getName());
        }
        if (fromStep == null) {
            log.logError("Unable to determine source step of transformation hop with ID: " + id_trans_hop);
            // Invalid hop, simply ignore. See: PDI-2446
            return null;
        }
        hopTransMeta.setFromStep(fromStep);
        hopTransMeta.getFromStep().setDraw(true);
        hopTransMeta.setToStep(StepMeta.findStep(steps, new LongObjectId(id_step_to)));
        // 
        if (hopTransMeta.getToStep() == null && id_step_to > 0) {
            // Simply load this, we only want the name, we don't care about
            // the rest...
            StepMeta stepMeta = repository.stepDelegate.loadStepMeta(new LongObjectId(id_step_to), new ArrayList<DatabaseMeta>(), new ArrayList<PartitionSchema>());
            hopTransMeta.setToStep(StepMeta.findStep(steps, stepMeta.getName()));
        }
        if (hopTransMeta.getFromStep() == null || hopTransMeta.getFromStep() == null) {
            // 
            return null;
        }
        hopTransMeta.getToStep().setDraw(true);
        return hopTransMeta;
    } catch (KettleDatabaseException dbe) {
        throw new KettleException(BaseMessages.getString(PKG, "TransHopMeta.Exception.LoadTransformationHopInfo") + id_trans_hop, dbe);
    }
}

Example 83 with StepMeta

use of org.pentaho.di.trans.step.StepMeta in project pentaho-kettle by pentaho.

the class GetXMLDataTest method testInit.

public void testInit() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("getxmldata1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a Get XML Data step
    // 
    String getXMLDataName = "get xml data step";
    GetXMLDataMeta gxdm = new GetXMLDataMeta();
    String getXMLDataPid = registry.getPluginId(StepPluginType.class, gxdm);
    StepMeta getXMLDataStep = new StepMeta(getXMLDataPid, getXMLDataName, gxdm);
    transMeta.addStep(getXMLDataStep);
    GetXMLDataField[] fields = new GetXMLDataField[5];
    for (int idx = 0; idx < fields.length; idx++) {
        fields[idx] = new GetXMLDataField();
    }
    fields[0].setName("objectid");
    fields[0].setXPath("${xml_path}");
    fields[0].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
    fields[0].setType(ValueMetaInterface.TYPE_STRING);
    fields[0].setFormat("");
    fields[0].setLength(-1);
    fields[0].setPrecision(-1);
    fields[0].setCurrencySymbol("");
    fields[0].setDecimalSymbol("");
    fields[0].setGroupSymbol("");
    fields[0].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
    gxdm.setEncoding("UTF-8");
    gxdm.setIsAFile(false);
    gxdm.setInFields(true);
    gxdm.setLoopXPath("Level1/Level2/Props");
    gxdm.setXMLField("field1");
    gxdm.setInputFields(fields);
    TransHopMeta hi = new TransHopMeta(injectorStep, getXMLDataStep);
    transMeta.addTransHop(hi);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi1 = new TransHopMeta(getXMLDataStep, dummyStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    GetXMLDataData getXMLDataData = new GetXMLDataData();
    GetXMLData getXmlData = new GetXMLData(dummyStep1, getXMLDataData, 0, transMeta, trans);
    getXmlData.setVariable("xml_path", "data/owner");
    getXmlData.init(gxdm, getXMLDataData);
    assertEquals(gxdm.getInputFields()[0].getXPath(), "${xml_path}");
}

Example 84 with StepMeta

use of org.pentaho.di.trans.step.StepMeta in project pentaho-kettle by pentaho.

the class XmlJoinMetaGetFieldsTest method testGetFieldsReturnTargetStepFieldsPlusResultXmlField.

@Test
public void testGetFieldsReturnTargetStepFieldsPlusResultXmlField() throws Exception {
    String sourceXmlStep = "source xml step name";
    String sourceStepField = "source field test name";
    String targetStepField = "target field test name";
    String resultXmlFieldName = "result xml field name";
    RowMeta rowMetaPreviousSteps = new RowMeta();
    rowMetaPreviousSteps.addValueMeta(new ValueMeta(sourceStepField, ValueMetaInterface.TYPE_STRING));
    xmlJoinMeta.setSourceXMLstep(sourceXmlStep);
    xmlJoinMeta.setValueXMLfield("result xml field name");
    StepMeta sourceStepMeta = new StepMeta();
    sourceStepMeta.setName(sourceXmlStep);
    when(transMeta.findStep(sourceXmlStep)).thenReturn(sourceStepMeta);
    when(transMeta.getStepFields(sourceStepMeta, null, null)).thenReturn(rowMetaPreviousSteps);
    RowMeta rowMeta = new RowMeta();
    ValueMetaString keepValueMeta = new ValueMetaString(targetStepField);
    ValueMetaString removeValueMeta = new ValueMetaString(sourceStepField);
    rowMeta.addValueMeta(keepValueMeta);
    rowMeta.addValueMeta(removeValueMeta);
    xmlJoinMeta.getFields(rowMeta, "testStepName", null, null, transMeta, null, null);
    Assert.assertEquals(2, rowMeta.size());
    String[] strings = rowMeta.getFieldNames();
    Assert.assertEquals(targetStepField, strings[0]);
    Assert.assertEquals(resultXmlFieldName, strings[1]);
}

Example 85 with StepMeta

use of org.pentaho.di.trans.step.StepMeta in project pentaho-kettle by pentaho.

the class XMLOutputMetaTest method testCheck.

@Test
public void testCheck() throws Exception {
    XMLOutputMeta xmlOutputMeta = new XMLOutputMeta();
    xmlOutputMeta.setDefault();
    TransMeta transMeta = mock(TransMeta.class);
    StepMeta stepInfo = mock(StepMeta.class);
    RowMetaInterface prev = mock(RowMetaInterface.class);
    Repository repos = mock(Repository.class);
    IMetaStore metastore = mock(IMetaStore.class);
    RowMetaInterface info = mock(RowMetaInterface.class);
    ArrayList<CheckResultInterface> remarks = new ArrayList<>();
    xmlOutputMeta.check(remarks, transMeta, stepInfo, prev, new String[] { "input" }, new String[] { "output" }, info, new Variables(), repos, metastore);
    assertEquals(2, remarks.size());
    assertEquals("Step is receiving info from other steps.", remarks.get(0).getText());
    assertEquals("File specifications are not checked.", remarks.get(1).getText());
    XMLField xmlField = new XMLField();
    xmlField.setFieldName("aField");
    xmlField.setType(1);
    xmlField.setLength(10);
    xmlField.setPrecision(3);
    xmlOutputMeta.setOutputFields(new XMLField[] { xmlField });
    when(prev.size()).thenReturn(1);
    remarks.clear();
    xmlOutputMeta.check(remarks, transMeta, stepInfo, prev, new String[] { "input" }, new String[] { "output" }, info, new Variables(), repos, metastore);
    assertEquals(4, remarks.size());
    assertEquals("Step is connected to previous one, receiving 1 fields", remarks.get(0).getText());
    assertEquals("All output fields are found in the input stream.", remarks.get(1).getText());
    assertEquals("Step is receiving info from other steps.", remarks.get(2).getText());
    assertEquals("File specifications are not checked.", remarks.get(3).getText());
}