Examples with TransHopMeta org.pentaho.di.trans.TransHopMeta used on opensource projects

Example 16 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class JobGenerator method generateDimensionTransformation.

/**
 * Generates a template
 * @param databaseMeta
 * @param logicalModel
 * @return
 */
public TransMeta generateDimensionTransformation(DatabaseMeta databaseMeta, LogicalTable logicalTable) {
    TransMeta transMeta = new TransMeta();
    String tableName = ConceptUtil.getName(logicalTable, locale);
    String tableDescription = ConceptUtil.getDescription(logicalTable, locale);
    DimensionType dimensionType = ConceptUtil.getDimensionType(logicalTable);
    transMeta.setName("Update dimension '" + tableName + "'");
    transMeta.setDescription(tableDescription);
    // Let's not forget to add the target database
    // 
    transMeta.addDatabase(databaseMeta);
    Point location = new Point(GRAPH_LEFT, GRAPH_TOP);
    // Find all the source columns and source tables and put them into a table input step...
    // 
    StepMeta inputStep = generateTableInputStepFromLogicalTable(logicalTable);
    DatabaseMeta sourceDatabaseMeta = ((TableInputMeta) inputStep.getStepMetaInterface()).getDatabaseMeta();
    if (sourceDatabaseMeta != null)
        transMeta.addOrReplaceDatabase(sourceDatabaseMeta);
    inputStep.setLocation(location.x, location.y);
    nextLocation(location);
    transMeta.addStep(inputStep);
    StepMeta lastStep = inputStep;
    // Generate an dimension lookup/update step for each table
    // 
    StepMeta dimensionStep;
    if (dimensionType == DimensionType.SLOWLY_CHANGING_DIMENSION) {
        dimensionStep = generateDimensionLookupStepFromLogicalTable(databaseMeta, logicalTable);
    } else {
        dimensionStep = generateCombinationLookupStepFromLogicalTable(databaseMeta, logicalTable);
    }
    dimensionStep.setLocation(location.x, location.y);
    nextLocation(location);
    transMeta.addStep(dimensionStep);
    TransHopMeta transHop = new TransHopMeta(lastStep, dimensionStep);
    transMeta.addTransHop(transHop);
    return transMeta;
}

Example 17 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class TransMetaConverter method removeInactivePaths.

/**
 * Removes steps which cannot be reached using enabled hops. Steps removed along with every input and
 * output hops they have. Downstream steps processed recursively in the same way. Should be invoked with null second arg.
 *
 * @param trans trans object to process
 * @param steps
 */
private static void removeInactivePaths(TransMeta trans, List<StepMeta> steps) {
    if (steps == null) {
        List<TransHopMeta> disabledHops = findHops(trans, hop -> !hop.isEnabled());
        List<StepMeta> disabledSteps = disabledHops.stream().map(hop -> hop.getToStep()).collect(Collectors.toList());
        removeInactivePaths(trans, disabledSteps);
    } else {
        for (StepMeta step : steps) {
            List<TransHopMeta> enabledInHops = findHops(trans, hop -> hop.getToStep().equals(step) && hop.isEnabled());
            List<TransHopMeta> disabledInHops = findHops(trans, hop -> hop.getToStep().equals(step) && !hop.isEnabled());
            if (enabledInHops.size() == 0) {
                List<StepMeta> nextSteps = findHops(trans, hop -> hop.getFromStep().equals(step)).stream().map(TransHopMeta::getToStep).collect(Collectors.toList());
                findHops(trans, hop -> hop.getToStep().equals(step) || hop.getFromStep().equals(step)).forEach(trans::removeTransHop);
                trans.getSteps().remove(step);
                removeInactivePaths(trans, nextSteps);
            } else {
                disabledInHops.forEach(trans::removeTransHop);
            }
        }
    }
}

Example 18 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class MultiMergeJoin method processFirstRow.

private boolean processFirstRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (MultiMergeJoinMeta) smi;
    data = (MultiMergeJoinData) sdi;
    TransMeta transMeta = getTransMeta();
    TransHopMeta transHopMeta;
    StepIOMetaInterface stepIOMeta = meta.getStepIOMeta();
    List<StreamInterface> infoStreams = stepIOMeta.getInfoStreams();
    StreamInterface stream;
    StepMeta toStepMeta = meta.getParentStepMeta();
    StepMeta fromStepMeta;
    ArrayList<String> inputStepNameList = new ArrayList<String>();
    String[] inputStepNames = meta.getInputSteps();
    String inputStepName;
    for (int i = 0; i < infoStreams.size(); i++) {
        inputStepName = inputStepNames[i];
        stream = infoStreams.get(i);
        fromStepMeta = stream.getStepMeta();
        if (fromStepMeta == null) {
            // should not arrive here, shoud typically have been caught by init.
            throw new KettleException(BaseMessages.getString(PKG, "MultiMergeJoin.Log.UnableToFindReferenceStream", inputStepName));
        }
        // check the hop
        transHopMeta = transMeta.findTransHop(fromStepMeta, toStepMeta, true);
        // there is no hop: this is unexpected.
        if (transHopMeta == null) {
            // should not arrive here, shoud typically have been caught by init.
            throw new KettleException(BaseMessages.getString(PKG, "MultiMergeJoin.Log.UnableToFindReferenceStream", inputStepName));
        } else if (transHopMeta.isEnabled()) {
            inputStepNameList.add(inputStepName);
        } else {
            logDetailed(BaseMessages.getString(PKG, "MultiMergeJoin.Log.IgnoringStep", inputStepName));
        }
    }
    int streamSize = inputStepNameList.size();
    if (streamSize == 0) {
        return false;
    }
    String keyField;
    String[] keyFields;
    data.rowSets = new RowSet[streamSize];
    RowSet rowSet;
    Object[] row;
    data.rows = new Object[streamSize][];
    data.metas = new RowMetaInterface[streamSize];
    data.rowLengths = new int[streamSize];
    MultiMergeJoinData.QueueComparator comparator = new MultiMergeJoinData.QueueComparator(data);
    data.queue = new PriorityQueue<MultiMergeJoinData.QueueEntry>(streamSize, comparator);
    data.results = new ArrayList<List<Object[]>>(streamSize);
    MultiMergeJoinData.QueueEntry queueEntry;
    data.queueEntries = new MultiMergeJoinData.QueueEntry[streamSize];
    data.drainIndices = new int[streamSize];
    data.keyNrs = new int[streamSize][];
    data.dummy = new Object[streamSize][];
    RowMetaInterface rowMeta;
    data.outputRowMeta = new RowMeta();
    for (int i = 0, j = 0; i < inputStepNames.length; i++) {
        inputStepName = inputStepNames[i];
        if (!inputStepNameList.contains(inputStepName)) {
            // ignore step with disabled hop.
            continue;
        }
        queueEntry = new MultiMergeJoinData.QueueEntry();
        queueEntry.index = j;
        data.queueEntries[j] = queueEntry;
        data.results.add(new ArrayList<Object[]>());
        rowSet = findInputRowSet(inputStepName);
        if (rowSet == null) {
            throw new KettleException(BaseMessages.getString(PKG, "MultiMergeJoin.Exception.UnableToFindSpecifiedStep", inputStepName));
        }
        data.rowSets[j] = rowSet;
        row = getRowFrom(rowSet);
        data.rows[j] = row;
        if (row == null) {
            rowMeta = getTransMeta().getStepFields(inputStepName);
            data.metas[j] = rowMeta;
        } else {
            queueEntry.row = row;
            rowMeta = rowSet.getRowMeta();
            keyField = meta.getKeyFields()[i];
            String[] keyFieldParts = keyField.split(",");
            String keyFieldPart;
            data.keyNrs[j] = new int[keyFieldParts.length];
            for (int k = 0; k < keyFieldParts.length; k++) {
                keyFieldPart = keyFieldParts[k];
                data.keyNrs[j][k] = rowMeta.indexOfValue(keyFieldPart);
                if (data.keyNrs[j][k] < 0) {
                    String message = BaseMessages.getString(PKG, "MultiMergeJoin.Exception.UnableToFindFieldInReferenceStream", keyFieldPart, inputStepName);
                    logError(message);
                    throw new KettleStepException(message);
                }
            }
            data.metas[j] = rowMeta;
            data.queue.add(data.queueEntries[j]);
        }
        data.outputRowMeta.mergeRowMeta(rowMeta.clone());
        data.rowLengths[j] = rowMeta.size();
        data.dummy[j] = RowDataUtil.allocateRowData(rowMeta.size());
        j++;
    }
    return true;
}

Example 19 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class JsonOutputTest method test.

public String test(boolean compatibilityMode) throws Exception {
    KettleEnvironment.init();
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("testJsonOutput");
    PluginRegistry registry = PluginRegistry.getInstance();
    // create an injector step
    String injectorStepName = "injector step";
    StepMeta injectorStep = TestUtilities.createInjectorStep(injectorStepName, registry);
    transMeta.addStep(injectorStep);
    // create a row generator step
    StepMeta rowGeneratorStep = createRowGeneratorStep("Create rows for testJsonOutput1", registry);
    transMeta.addStep(rowGeneratorStep);
    // create a TransHopMeta for injector and add it to the transMeta
    TransHopMeta hop_injectory_rowGenerator = new TransHopMeta(injectorStep, rowGeneratorStep);
    transMeta.addTransHop(hop_injectory_rowGenerator);
    // create the json output step
    // but first lets get a filename
    String jsonFileName = TestUtilities.createEmptyTempFile("testJsonOutput1_");
    StepMeta jsonOutputStep = createJsonOutputStep("json output step", jsonFileName, registry);
    ((JsonOutputMeta) jsonOutputStep.getStepMetaInterface()).setCompatibilityMode(compatibilityMode);
    transMeta.addStep(jsonOutputStep);
    // create a TransHopMeta for jsonOutputStep and add it to the transMeta
    TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta(rowGeneratorStep, jsonOutputStep);
    transMeta.addTransHop(hop_RowGenerator_outputTextFile);
    // Create a dummy step and add it to the tranMeta
    String dummyStepName = "dummy step";
    StepMeta dummyStep = createDummyStep(dummyStepName, registry);
    transMeta.addStep(dummyStep);
    // create a TransHopMeta for the
    TransHopMeta hop_outputJson_dummyStep = new TransHopMeta(jsonOutputStep, dummyStep);
    transMeta.addTransHop(hop_outputJson_dummyStep);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    // Create a row collector and add it to the dummy step interface
    StepInterface dummyStepInterface = trans.getStepInterface(dummyStepName, 0);
    RowStepCollector dummyRowCollector = new RowStepCollector();
    dummyStepInterface.addRowListener(dummyRowCollector);
    // RowProducer rowProducer = trans.addRowProducer(injectorStepName, 0);
    trans.startThreads();
    trans.waitUntilFinished();
    // get the results and return it
    File outputFile = new File(jsonFileName + ".js");
    String jsonStructure = FileUtils.readFileToString(outputFile);
    return jsonStructure;
}

Example 20 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class TransDelegate method elementToDataNode.

public DataNode elementToDataNode(final RepositoryElementInterface element) throws KettleException {
    TransMeta transMeta = (TransMeta) element;
    DataNode rootNode = new DataNode(NODE_TRANS);
    if (transMeta.getPrivateDatabases() != null) {
        // save all private transformations database name http://jira.pentaho.com/browse/PPP-3405
        String privateDatabaseNames = StringUtils.join(transMeta.getPrivateDatabases(), TRANS_PRIVATE_DATABASE_DELIMITER);
        DataNode privateDatabaseNode = rootNode.addNode(NODE_TRANS_PRIVATE_DATABASES);
        privateDatabaseNode.setProperty(PROP_TRANS_PRIVATE_DATABASE_NAMES, privateDatabaseNames);
    }
    DataNode stepsNode = rootNode.addNode(NODE_STEPS);
    // Also save all the steps in the transformation!
    // 
    int stepNr = 0;
    for (StepMeta step : transMeta.getSteps()) {
        stepNr++;
        // $NON-NLS-1$
        DataNode stepNode = stepsNode.addNode(sanitizeNodeName(step.getName()) + "_" + stepNr + EXT_STEP);
        // Store the main data
        // 
        stepNode.setProperty(PROP_NAME, step.getName());
        stepNode.setProperty(PROP_DESCRIPTION, step.getDescription());
        stepNode.setProperty(PROP_STEP_TYPE, step.getStepID());
        stepNode.setProperty(PROP_STEP_DISTRIBUTE, step.isDistributes());
        stepNode.setProperty(PROP_STEP_ROW_DISTRIBUTION, step.getRowDistribution() == null ? null : step.getRowDistribution().getCode());
        stepNode.setProperty(PROP_STEP_COPIES, step.getCopies());
        stepNode.setProperty(PROP_STEP_COPIES_STRING, step.getCopiesString());
        stepNode.setProperty(PROP_STEP_GUI_LOCATION_X, step.getLocation().x);
        stepNode.setProperty(PROP_STEP_GUI_LOCATION_Y, step.getLocation().y);
        stepNode.setProperty(PROP_STEP_GUI_DRAW, step.isDrawn());
        // Also save the step group attributes map
        // 
        AttributesMapUtil.saveAttributesMap(stepNode, step);
        // Save the step metadata using the repository save method, NOT XML
        // That is because we want to keep the links to databases, conditions, etc by ID, not name.
        // 
        StepMetaInterface stepMetaInterface = step.getStepMetaInterface();
        DataNode stepCustomNode = new DataNode(NODE_STEP_CUSTOM);
        Repository proxy = new RepositoryProxy(stepCustomNode);
        compatibleSaveRep(stepMetaInterface, proxy, null, null);
        stepMetaInterface.saveRep(proxy, proxy.getMetaStore(), null, null);
        stepNode.addNode(stepCustomNode);
        // Save the partitioning information by reference as well...
        // 
        StepPartitioningMeta partitioningMeta = step.getStepPartitioningMeta();
        if (partitioningMeta != null && partitioningMeta.getPartitionSchema() != null && partitioningMeta.isPartitioned()) {
            DataNodeRef ref = new DataNodeRef(partitioningMeta.getPartitionSchema().getObjectId().getId());
            stepNode.setProperty(PROP_PARTITIONING_SCHEMA, ref);
            // method of partitioning
            stepNode.setProperty(PROP_PARTITIONING_METHOD, partitioningMeta.getMethodCode());
            if (partitioningMeta.getPartitioner() != null) {
                DataNode partitionerCustomNode = new DataNode(NODE_PARTITIONER_CUSTOM);
                proxy = new RepositoryProxy(partitionerCustomNode);
                partitioningMeta.getPartitioner().saveRep(proxy, null, null);
                stepNode.addNode(partitionerCustomNode);
            }
        }
        // Save the clustering information as well...
        // 
        stepNode.setProperty(PROP_CLUSTER_SCHEMA, step.getClusterSchema() == null ? "" : // $NON-NLS-1$
        step.getClusterSchema().getName());
        // Save the error hop metadata
        // 
        StepErrorMeta stepErrorMeta = step.getStepErrorMeta();
        if (stepErrorMeta != null) {
            stepNode.setProperty(PROP_STEP_ERROR_HANDLING_SOURCE_STEP, stepErrorMeta.getSourceStep() != null ? stepErrorMeta.getSourceStep().getName() : // $NON-NLS-1$
            "");
            stepNode.setProperty(PROP_STEP_ERROR_HANDLING_TARGET_STEP, stepErrorMeta.getTargetStep() != null ? stepErrorMeta.getTargetStep().getName() : // $NON-NLS-1$
            "");
            stepNode.setProperty(PROP_STEP_ERROR_HANDLING_IS_ENABLED, stepErrorMeta.isEnabled());
            stepNode.setProperty(PROP_STEP_ERROR_HANDLING_NR_VALUENAME, stepErrorMeta.getNrErrorsValuename());
            stepNode.setProperty(PROP_STEP_ERROR_HANDLING_DESCRIPTIONS_VALUENAME, stepErrorMeta.getErrorDescriptionsValuename());
            stepNode.setProperty(PROP_STEP_ERROR_HANDLING_FIELDS_VALUENAME, stepErrorMeta.getErrorFieldsValuename());
            stepNode.setProperty(PROP_STEP_ERROR_HANDLING_CODES_VALUENAME, stepErrorMeta.getErrorCodesValuename());
            stepNode.setProperty(PROP_STEP_ERROR_HANDLING_MAX_ERRORS, stepErrorMeta.getMaxErrors());
            stepNode.setProperty(PROP_STEP_ERROR_HANDLING_MAX_PCT_ERRORS, stepErrorMeta.getMaxPercentErrors());
            stepNode.setProperty(PROP_STEP_ERROR_HANDLING_MIN_PCT_ROWS, stepErrorMeta.getMinPercentRows());
        }
    }
    // Save the notes
    // 
    DataNode notesNode = rootNode.addNode(NODE_NOTES);
    notesNode.setProperty(PROP_NR_NOTES, transMeta.nrNotes());
    for (int i = 0; i < transMeta.nrNotes(); i++) {
        NotePadMeta note = transMeta.getNote(i);
        DataNode noteNode = notesNode.addNode(NOTE_PREFIX + i);
        noteNode.setProperty(PROP_XML, note.getXML());
    }
    // Finally, save the hops
    // 
    DataNode hopsNode = rootNode.addNode(NODE_HOPS);
    hopsNode.setProperty(PROP_NR_HOPS, transMeta.nrTransHops());
    for (int i = 0; i < transMeta.nrTransHops(); i++) {
        TransHopMeta hop = transMeta.getTransHop(i);
        DataNode hopNode = hopsNode.addNode(TRANS_HOP_PREFIX + i);
        hopNode.setProperty(TRANS_HOP_FROM, hop.getFromStep().getName());
        hopNode.setProperty(TRANS_HOP_TO, hop.getToStep().getName());
        hopNode.setProperty(TRANS_HOP_ENABLED, hop.isEnabled());
    }
    // Parameters
    // 
    String[] paramKeys = transMeta.listParameters();
    DataNode paramsNode = rootNode.addNode(NODE_PARAMETERS);
    paramsNode.setProperty(PROP_NR_PARAMETERS, paramKeys == null ? 0 : paramKeys.length);
    for (int idx = 0; idx < paramKeys.length; idx++) {
        DataNode paramNode = paramsNode.addNode(TRANS_PARAM_PREFIX + idx);
        String key = paramKeys[idx];
        String description = transMeta.getParameterDescription(paramKeys[idx]);
        String defaultValue = transMeta.getParameterDefault(paramKeys[idx]);
        // $NON-NLS-1$
        paramNode.setProperty(PARAM_KEY, key != null ? key : "");
        // $NON-NLS-1$
        paramNode.setProperty(PARAM_DEFAULT, defaultValue != null ? defaultValue : "");
        // $NON-NLS-1$
        paramNode.setProperty(PARAM_DESC, description != null ? description : "");
    }
    // Let's not forget to save the details of the transformation itself.
    // This includes logging information, parameters, etc.
    // 
    saveTransformationDetails(rootNode, transMeta);
    return rootNode;
}