Examples with IRowSet org.apache.hop.core.IRowSet used on opensource projects

Example 1 with IRowSet

use of org.apache.hop.core.IRowSet in project hop by apache.

the class TransformClassBase method findTargetRowSet.

public IRowSet findTargetRowSet(String tag) throws HopException {
    if (tag == null) {
        return null;
    }
    String transformName = data.targetMap.get(tag);
    if (Utils.isEmpty(transformName)) {
        throw new HopException(BaseMessages.getString(PKG, "TransformClassBase.Exception.UnableToFindTargetTransformNameForTag", tag));
    }
    IRowSet rowSet = findOutputRowSet(transformName);
    if (rowSet == null) {
        throw new HopException(BaseMessages.getString(PKG, "TransformClassBase.Exception.UnableToFindTargetRowSetForTransform", transformName));
    }
    return rowSet;
}

Example 2 with IRowSet

use of org.apache.hop.core.IRowSet in project hop by apache.

the class TransformClassBase method findInfoRowSet.

public IRowSet findInfoRowSet(String tag) throws HopException {
    if (tag == null) {
        return null;
    }
    String transformName = data.infoMap.get(tag);
    if (Utils.isEmpty(transformName)) {
        throw new HopException(BaseMessages.getString(PKG, "TransformClassBase.Exception.UnableToFindInfoTransformNameForTag", tag));
    }
    IRowSet rowSet = findInputRowSet(transformName);
    if (rowSet == null) {
        throw new HopException(BaseMessages.getString(PKG, "TransformClassBase.Exception.UnableToFindInfoRowSetForTransform", transformName));
    }
    return rowSet;
}

Example 3 with IRowSet

use of org.apache.hop.core.IRowSet in project hop by apache.

the class SingleThreadedPipelineExecutor method initializeObject.

public void initializeObject(final Pipeline pipeline, boolean handleExceptionsExternally) {
    this.pipeline = pipeline;
    this.handleExceptionsExternally = handleExceptionsExternally;
    this.log = pipeline.getLogChannel();
    transforms = pipeline.getTransforms();
    sortTransforms();
    done = new boolean[transforms.size()];
    nrDone = 0;
    transformInfoStreams = new ArrayList<>();
    transformInfoRowSets = new ArrayList<>();
    for (TransformMetaDataCombi combi : transforms) {
        List<IStream> infoStreams = combi.transformMeta.getTransform().getTransformIOMeta().getInfoStreams();
        transformInfoStreams.add(infoStreams);
        List<IRowSet> infoRowSets = new ArrayList<>();
        for (IStream infoStream : infoStreams) {
            IRowSet infoRowSet = pipeline.findRowSet(infoStream.getTransformName(), 0, combi.transformName, 0);
            if (infoRowSet != null) {
                infoRowSets.add(infoRowSet);
            }
        }
        transformInfoRowSets.add(infoRowSets);
    }
}

Example 4 with IRowSet

use of org.apache.hop.core.IRowSet in project hop by apache.

the class SingleThreadedPipelineExecutor method oneIteration.

/**
 * Give all transforms in the pipeline the chance to process all rows on input...
 *
 * @return true if more iterations can be performed. False if this is not the case.
 */
public boolean oneIteration() throws HopException {
    this.exceptionsRaisedCounter = 0;
    try {
        for (int s = 0; s < transforms.size() && !pipeline.isStopped(); s++) {
            if (!done[s]) {
                TransformMetaDataCombi combi = transforms.get(s);
                this.inProcessCombi = combi;
                // If this transform is waiting for data (text, db, and so on), we simply read all the
                // data
                // This means that it is impractical to use this pipeline type to load large files.
                // 
                boolean transformDone = false;
                // For every input row we call the processRow() method of the transform.
                // 
                List<IRowSet> infoRowSets = transformInfoRowSets.get(s);
                // 
                for (IRowSet rowSet : infoRowSets) {
                    boolean once = true;
                    while (once || (rowSet.size() > 0 && !transformDone)) {
                        once = false;
                        transformDone = !combi.transform.processRow();
                        if (combi.transform.getErrors() > 0) {
                            return false;
                        }
                    }
                }
                // Do normal processing of input rows...
                // 
                List<IRowSet> rowSets = combi.transform.getInputRowSets();
                // 
                if (rowSets.size() == 0) {
                    while (!transformDone && !pipeline.isStopped()) {
                        transformDone = !combi.transform.processRow();
                        if (combi.transform.getErrors() > 0) {
                            return false;
                        }
                    }
                } else {
                    // Since we can't be sure that the transform actually reads from the row sets where we
                    // measure rows,
                    // we simply count the total nr of rows on input. The transforms will find the rows in
                    // either row set.
                    // 
                    int nrRows = 0;
                    for (IRowSet rowSet : rowSets) {
                        nrRows += rowSet.size();
                    }
                    // 
                    for (int i = 0; i < nrRows; i++) {
                        transformDone = !combi.transform.processRow();
                        if (combi.transform.getErrors() > 0) {
                            return false;
                        }
                    }
                }
                // Signal the transform that a batch of rows has passed for this iteration (sort rows and
                // all)
                // 
                combi.transform.batchComplete();
                if (transformDone) {
                    nrDone++;
                }
                done[s] = transformDone;
            }
        }
    } catch (Exception e) {
        if (handleExceptionsExternally) {
            log.logDebug("An exception was raised during a transform's execution: " + inProcessCombi.transformName);
            this.exceptionsRaisedCounter += 1;
        } else
            throw new HopException("Error performing an iteration in a single threaded pipeline", e);
    }
    return nrDone < transforms.size() && !pipeline.isStopped();
}

Example 5 with IRowSet

use of org.apache.hop.core.IRowSet in project hop by apache.

the class JoinRows method batchComplete.

@Override
public void batchComplete() throws HopException {
    IRowSet rowSet = getFirstInputRowSet();
    int repeats = 0;
    for (int i = 0; i < data.cache.length; i++) {
        if (repeats == 0) {
            repeats = 1;
        }
        if (data.cache[i] != null) {
            repeats *= data.cache[i].size();
        }
    }
    while (rowSet.size() > 0 && !isStopped()) {
        init();
    }
    // 
    for (int i = 0; i < repeats; i++) {
        init();
    }
}