Examples with HopTransformException org.apache.hop.core.exception.HopTransformException used on opensource projects

Example 1 with HopTransformException

use of org.apache.hop.core.exception.HopTransformException in project hop by apache.

the class RowMetaUtils method getRowMetaForUpdate.

public static IRowMeta getRowMetaForUpdate(IRowMeta prev, String[] keyLookup, String[] keyStream, String[] updateLookup, String[] updateStream) throws HopTransformException {
    IRowMeta tableFields = new RowMeta();
    // the key fields
    if (keyLookup != null) {
        for (int i = 0; i < keyLookup.length; i++) {
            IValueMeta v = prev.searchValueMeta(keyStream[i]);
            if (v != null) {
                IValueMeta tableField = v.clone();
                tableField.setName(keyLookup[i]);
                tableFields.addValueMeta(tableField);
            } else {
                throw new HopTransformException("Unable to find field [" + keyStream[i] + "] in the input rows");
            }
        }
    }
    // the lookup fields
    for (int i = 0; i < updateLookup.length; i++) {
        IValueMeta v = prev.searchValueMeta(updateStream[i]);
        if (v != null) {
            IValueMeta vk = tableFields.searchValueMeta(updateLookup[i]);
            if (vk == null) {
                // do not add again when already added as key fields
                IValueMeta tableField = v.clone();
                tableField.setName(updateLookup[i]);
                tableFields.addValueMeta(tableField);
            }
        } else {
            throw new HopTransformException("Unable to find field [" + updateStream[i] + "] in the input rows");
        }
    }
    return tableFields;
}

Example 2 with HopTransformException

use of org.apache.hop.core.exception.HopTransformException in project hop by apache.

the class BaseTransform method handlePutRow.

private void handlePutRow(IRowMeta rowMeta, Object[] row) throws HopTransformException {
    // 
    while (paused.get() && !stopped.get()) {
        try {
            Thread.sleep(1);
        } catch (InterruptedException e) {
            throw new HopTransformException(e);
        }
    }
    // 
    if (stopped.get() && !safeStopped.get()) {
        if (log.isDebug()) {
            logDebug(BaseMessages.getString(PKG, "BaseTransform.Log.StopPuttingARow"));
        }
        stopAll();
        return;
    }
    // 
    if (this.checkPipelineRunning == false) {
        int counter = 0;
        while (!pipeline.isRunning() && !stopped.get()) {
            try {
                Thread.sleep(1000);
                counter++;
            } catch (InterruptedException e) {
            // Ignore
            }
            // wait 3s max
            if (counter >= 3) {
                break;
            }
        }
        this.checkPipelineRunning = true;
    }
    // 
    for (IRowListener listener : rowListeners) {
        listener.rowWrittenEvent(rowMeta, row);
    }
    // 
    if (terminator && terminatorRows != null) {
        try {
            terminatorRows.add(rowMeta.cloneRow(row));
        } catch (HopValueException e) {
            throw new HopTransformException("Unable to clone row while adding rows to the terminator rows.", e);
        }
    }
    outputRowSetsLock.readLock().lock();
    try {
        if (outputRowSets.isEmpty()) {
            // No more output rowsets!
            // Still update the nr of lines written.
            // 
            incrementLinesWritten();
            // we're done here!
            return;
        }
        // 
        switch(repartitioning) {
            case TransformPartitioningMeta.PARTITIONING_METHOD_NONE:
                noPartitioning(rowMeta, row);
                break;
            case TransformPartitioningMeta.PARTITIONING_METHOD_SPECIAL:
                specialPartitioning(rowMeta, row);
                break;
            case TransformPartitioningMeta.PARTITIONING_METHOD_MIRROR:
                mirrorPartitioning(rowMeta, row);
                break;
            default:
                throw new HopTransformException("Internal error: invalid repartitioning type: " + repartitioning);
        }
    } finally {
        outputRowSetsLock.readLock().unlock();
    }
}

Example 3 with HopTransformException

use of org.apache.hop.core.exception.HopTransformException in project hop by apache.

the class InjectDataSetIntoTransformExtensionPoint method callExtensionPoint.

@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, final IPipelineEngine<PipelineMeta> pipeline) throws HopException {
    if (!(pipeline instanceof LocalPipelineEngine)) {
        throw new HopPluginException("Unit tests can only run using a local pipeline engine type");
    }
    final PipelineMeta pipelineMeta = pipeline.getPipelineMeta();
    boolean dataSetEnabled = "Y".equalsIgnoreCase(pipeline.getVariable(DataSetConst.VAR_RUN_UNIT_TEST));
    if (log.isDetailed()) {
        log.logDetailed("Data Set enabled? " + dataSetEnabled);
    }
    if (!dataSetEnabled) {
        return;
    }
    String unitTestName = pipeline.getVariable(DataSetConst.VAR_UNIT_TEST_NAME);
    if (log.isDetailed()) {
        log.logDetailed("Unit test name: " + unitTestName);
    }
    try {
        IHopMetadataProvider metadataProvider = pipelineMeta.getMetadataProvider();
        // 
        if (StringUtil.isEmpty(unitTestName)) {
            return;
        }
        PipelineUnitTest unitTest = metadataProvider.getSerializer(PipelineUnitTest.class).load(unitTestName);
        if (unitTest == null) {
            if (log.isDetailed()) {
                log.logDetailed("Unit test '" + unitTestName + "' could not be found");
            }
            return;
        }
        // 
        for (final TransformMeta transformMeta : pipeline.getPipelineMeta().getTransforms()) {
            String transformName = transformMeta.getName();
            PipelineUnitTestSetLocation inputLocation = unitTest.findInputLocation(transformName);
            if (inputLocation != null && StringUtils.isNotEmpty(inputLocation.getDataSetName())) {
                String inputDataSetName = inputLocation.getDataSetName();
                log.logDetailed("Data Set location found for transform '" + transformName + "' and data set  " + inputDataSetName);
                // We need to inject data from the data set with the specified name into the transform
                // 
                injectDataSetIntoTransform((LocalPipelineEngine) pipeline, inputDataSetName, metadataProvider, transformMeta, inputLocation);
            }
            // How about capturing rows for golden data review?
            // 
            PipelineUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation(transformName);
            if (goldenLocation != null) {
                String goldenDataSetName = goldenLocation.getDataSetName();
                if (!StringUtil.isEmpty(goldenDataSetName)) {
                    log.logDetailed("Capturing rows for validation at pipeline end, transform='" + transformMeta.getName() + "', golden set '" + goldenDataSetName);
                    final RowCollection rowCollection = new RowCollection();
                    // Create a row collection map if it's missing...
                    // 
                    @SuppressWarnings("unchecked") Map<String, RowCollection> collectionMap = (Map<String, RowCollection>) pipeline.getExtensionDataMap().get(DataSetConst.ROW_COLLECTION_MAP);
                    if (collectionMap == null) {
                        collectionMap = new HashMap<>();
                        pipeline.getExtensionDataMap().put(DataSetConst.ROW_COLLECTION_MAP, collectionMap);
                    }
                    // Keep the map for safe keeping...
                    // 
                    collectionMap.put(transformMeta.getName(), rowCollection);
                    // We'll capture the rows from this one and then evaluate them after execution...
                    // 
                    IEngineComponent component = pipeline.findComponent(transformMeta.getName(), 0);
                    component.addRowListener(new RowAdapter() {

                        @Override
                        public void rowReadEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                            if (rowCollection.getRowMeta() == null) {
                                rowCollection.setRowMeta(rowMeta);
                            }
                            rowCollection.getRows().add(row);
                        }
                    });
                }
            }
        }
    } catch (Throwable e) {
        throw new HopException("Unable to inject data set rows", e);
    }
}

Example 4 with HopTransformException

use of org.apache.hop.core.exception.HopTransformException in project hop by apache.

the class WriteToDataSetExtensionPoint method passTransformRowsToDataSet.

private void passTransformRowsToDataSet(final IPipelineEngine<PipelineMeta> pipeline, final PipelineMeta pipelineMeta, final TransformMeta transformMeta, final List<SourceToTargetMapping> mappings, final DataSet dataSet) throws HopException {
    // This is the transform to inject into the specified data set
    // 
    final IRowMeta setRowMeta = dataSet.getSetRowMeta();
    IEngineComponent component = pipeline.findComponent(transformMeta.getName(), 0);
    final List<Object[]> transformsForDbRows = new ArrayList<>();
    component.addRowListener(new RowAdapter() {

        @Override
        public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
            Object[] transformForDbRow = RowDataUtil.allocateRowData(setRowMeta.size());
            for (SourceToTargetMapping mapping : mappings) {
                transformForDbRow[mapping.getTargetPosition()] = row[mapping.getSourcePosition()];
            }
            transformsForDbRows.add(transformForDbRow);
        }
    });
    // At the end of the pipeline, write it...
    // 
    pipeline.addExecutionFinishedListener(engine -> DataSetCsvUtil.writeDataSetData(pipeline, dataSet, setRowMeta, transformsForDbRows));
}

Example 5 with HopTransformException

use of org.apache.hop.core.exception.HopTransformException in project hop by apache.

the class CypherMeta method getFields.

@Override
public void getFields(IRowMeta rowMeta, String name, IRowMeta[] info, TransformMeta nextStep, IVariables space, IHopMetadataProvider metadataProvider) throws HopTransformException {
    if (usingUnwind) {
        // Unwind only outputs results, not input
        // 
        rowMeta.clear();
    }
    if (returningGraph) {
        // We send out a single Graph value per input row
        // 
        IValueMeta valueMetaGraph = new ValueMetaGraph(Const.NVL(returnGraphField, "graph"));
        valueMetaGraph.setOrigin(name);
        rowMeta.addValueMeta(valueMetaGraph);
    } else {
        // 
        for (ReturnValue returnValue : returnValues) {
            try {
                int type = ValueMetaFactory.getIdForValueMeta(returnValue.getType());
                IValueMeta valueMeta = ValueMetaFactory.createValueMeta(returnValue.getName(), type);
                valueMeta.setOrigin(name);
                rowMeta.addValueMeta(valueMeta);
            } catch (HopPluginException e) {
                throw new HopTransformException("Unknown data type '" + returnValue.getType() + "' for value named '" + returnValue.getName() + "'");
            }
        }
    }
}