Examples with HopTransformException org.apache.hop.core.exception.HopTransformException used on opensource projects

Example 16 with HopTransformException

use of org.apache.hop.core.exception.HopTransformException in project hop by apache.

the class PipelineDebugMeta method addRowListenersToPipeline.

public synchronized void addRowListenersToPipeline(final IPipelineEngine<PipelineMeta> pipeline) {
    // for every transform in the map, add a row listener...
    // 
    dataShown = false;
    for (final TransformMeta transformMeta : transformDebugMetaMap.keySet()) {
        final TransformDebugMeta transformDebugMeta = transformDebugMetaMap.get(transformMeta);
        // 
        for (IEngineComponent component : pipeline.getComponentCopies(transformMeta.getName())) {
            // 
            if (component instanceof ITransform) {
                ITransform baseTransform = (ITransform) component;
                baseTransform.addRowListener(new RowAdapter() {

                    @Override
                    public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                        try {
                            synchronized (transformDebugMeta) {
                                // This block of code is called whenever there is a row written by the
                                // transform
                                // So we want to execute the debugging actions that are specified by the
                                // transform...
                                // 
                                int rowCount = transformDebugMeta.getRowCount();
                                if (transformDebugMeta.isReadingFirstRows() && rowCount > 0) {
                                    int bufferSize = transformDebugMeta.getRowBuffer().size();
                                    if (bufferSize < rowCount) {
                                        // This is the classic preview mode.
                                        // We add simply add the row to the buffer.
                                        // 
                                        transformDebugMeta.setRowBufferMeta(rowMeta);
                                        transformDebugMeta.getRowBuffer().add(rowMeta.cloneRow(row));
                                    } else {
                                        // pause the pipeline...
                                        // 
                                        pipeline.pauseExecution();
                                        // Also call the pause / break-point listeners on the transform
                                        // debugger...
                                        // 
                                        dataShown = true;
                                        transformDebugMeta.fireBreakPointListeners(PipelineDebugMeta.this);
                                    }
                                } else if (transformDebugMeta.isPausingOnBreakPoint() && transformDebugMeta.getCondition() != null) {
                                    // 
                                    if (rowCount > 0) {
                                        // Keep a number of rows in memory
                                        // Store them in a reverse order to keep it intuitive for the user.
                                        // 
                                        transformDebugMeta.setRowBufferMeta(rowMeta);
                                        transformDebugMeta.getRowBuffer().add(0, rowMeta.cloneRow(row));
                                        // Only keep a number of rows in memory
                                        // If we have too many, remove the last (oldest)
                                        // 
                                        int bufferSize = transformDebugMeta.getRowBuffer().size();
                                        if (bufferSize > rowCount) {
                                            transformDebugMeta.getRowBuffer().remove(bufferSize - 1);
                                        }
                                    } else {
                                        // 
                                        if (transformDebugMeta.getRowBuffer().isEmpty()) {
                                            transformDebugMeta.getRowBuffer().add(rowMeta.cloneRow(row));
                                        } else {
                                            transformDebugMeta.getRowBuffer().set(0, rowMeta.cloneRow(row));
                                        }
                                    }
                                    // 
                                    if (transformDebugMeta.getCondition().evaluate(rowMeta, row)) {
                                        // We hit the break-point: pause the pipeline
                                        // 
                                        pipeline.pauseExecution();
                                        // Also fire off the break point listeners...
                                        // 
                                        transformDebugMeta.fireBreakPointListeners(PipelineDebugMeta.this);
                                    }
                                }
                            }
                        } catch (HopException e) {
                            throw new HopTransformException(e);
                        }
                    }
                });
            }
        }
    }
    // 
    try {
        pipeline.addExecutionFinishedListener(p -> {
            if (dataShown) {
                return;
            }
            for (TransformMeta transformMeta : transformDebugMetaMap.keySet()) {
                TransformDebugMeta transformDebugMeta = transformDebugMetaMap.get(transformMeta);
                if (transformDebugMeta != null) {
                    List<Object[]> rowBuffer = transformDebugMeta.getRowBuffer();
                    if (rowBuffer != null && !rowBuffer.isEmpty()) {
                        transformDebugMeta.fireBreakPointListeners(this);
                    }
                }
            }
        });
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 17 with HopTransformException

use of org.apache.hop.core.exception.HopTransformException in project hop by apache.

the class PipelineDataProbeXp method executeProbingPipeline.

/**
 * Execute a probing pipeline for the current pipeline. Add a listener to the transform copies.
 * Send the data to the PipelineDataProbe transform(s) in the probing pipeline
 *
 * @param pipelineProbe
 * @param dataProbeLocation
 * @param loggingPipelineFilename The pipeline to start for the location
 * @param pipeline The parent pipeline to listen to
 * @param variables
 * @throws HopException
 */
private synchronized void executeProbingPipeline(PipelineProbe pipelineProbe, DataProbeLocation dataProbeLocation, String loggingPipelineFilename, IPipelineEngine<PipelineMeta> pipeline, IVariables variables) throws HopException {
    PipelineMeta probingPipelineMeta = new PipelineMeta(loggingPipelineFilename, pipeline.getMetadataProvider(), true, variables);
    // Create a local pipeline engine...
    // 
    LocalPipelineEngine probingPipeline = new LocalPipelineEngine(probingPipelineMeta, variables, pipeline);
    // Flag it as a probing and logging pipeline so we don't try to probe or log ourselves...
    // 
    probingPipeline.getExtensionDataMap().put(PIPELINE_DATA_PROBE_FLAG, "Y");
    probingPipeline.getExtensionDataMap().put(PipelineStartLoggingXp.PIPELINE_LOGGING_FLAG, "Y");
    // Only log errors
    // 
    probingPipeline.setLogLevel(LogLevel.ERROR);
    probingPipeline.prepareExecution();
    List<IEngineComponent> componentCopies = pipeline.getComponentCopies(dataProbeLocation.getSourceTransformName());
    for (IEngineComponent componentCopy : componentCopies) {
        // 
        for (TransformMetaDataCombi combi : probingPipeline.getTransforms()) {
            if (combi.transform instanceof PipelineDataProbe) {
                // Give the transform a bit more information to work with...
                // 
                PipelineDataProbe pipelineDataProbe = (PipelineDataProbe) combi.transform;
                pipelineDataProbe.setSourcePipelineName(pipeline.getPipelineMeta().getName());
                pipelineDataProbe.setSourceTransformLogChannelId(pipeline.getLogChannelId());
                pipelineDataProbe.setSourceTransformName(componentCopy.getName());
                pipelineDataProbe.setSourceTransformCopy(componentCopy.getCopyNr());
                try {
                    final RowProducer rowProducer = probingPipeline.addRowProducer(combi.transformName, combi.copy);
                    // For every copy of the component, add an input row set to the parent pipeline...
                    // 
                    componentCopy.addRowListener(new RowAdapter() {

                        @Override
                        public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                            // Pass this row to the row producer...
                            // 
                            rowProducer.putRow(rowMeta, row);
                        }
                    });
                    // If the pipeline we're the transform is and we can safely stop streaming...
                    // 
                    pipeline.addExecutionFinishedListener(pe -> rowProducer.finished());
                } catch (HopException e) {
                    throw new HopTransformException("Error adding row producer to transform '" + combi.transformName + "'", e);
                }
            }
        }
    }
    // Execute the logging pipeline to save the logging information
    // 
    probingPipeline.startThreads();
    // We'll not wait around until this is finished...
    // The pipeline should stop automatically when the parent does
    // 
    pipeline.addExecutionStoppedListener(e -> probingPipeline.stopAll());
}

Example 18 with HopTransformException

use of org.apache.hop.core.exception.HopTransformException in project hop by apache.

the class DatabaseJoin method lookupValues.

private void lookupValues(IRowMeta rowMeta, Object[] rowData) throws HopException {
    dbLock.lock();
    if (first) {
        first = false;
        data.outputRowMeta = rowMeta.clone();
        meta.getFields(data.outputRowMeta, getTransformName(), new IRowMeta[] { meta.getTableFields(this) }, null, this, metadataProvider);
        data.lookupRowMeta = new RowMeta();
        if (log.isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "DatabaseJoin.Log.CheckingRow") + rowMeta.getString(rowData));
        }
        data.keynrs = new int[meta.getParameters().size()];
        for (int i = 0; i < data.keynrs.length; i++) {
            ParameterField field = meta.getParameters().get(i);
            data.keynrs[i] = rowMeta.indexOfValue(field.getName());
            if (data.keynrs[i] < 0) {
                throw new HopTransformException(BaseMessages.getString(PKG, "DatabaseJoin.Exception.FieldNotFound", field.getName()));
            }
            data.lookupRowMeta.addValueMeta(rowMeta.getValueMeta(data.keynrs[i]).clone());
        }
    }
    final ResultSet rs;
    try {
        // Construct the parameters row...
        Object[] lookupRowData = new Object[data.lookupRowMeta.size()];
        for (int i = 0; i < data.keynrs.length; i++) {
            lookupRowData[i] = rowData[data.keynrs[i]];
        }
        // Set the values on the prepared statement (for faster exec.)
        rs = data.db.openQuery(data.pstmt, data.lookupRowMeta, lookupRowData);
        // Get a row from the database...
        // 
        Object[] add = data.db.getRow(rs);
        IRowMeta addMeta = data.db.getReturnRowMeta();
        incrementLinesInput();
        int counter = 0;
        while (add != null && (meta.getRowLimit() == 0 || counter < meta.getRowLimit())) {
            counter++;
            Object[] newRow = RowDataUtil.resizeArray(rowData, data.outputRowMeta.size());
            int newIndex = rowMeta.size();
            for (int i = 0; i < addMeta.size(); i++) {
                newRow[newIndex++] = add[i];
            }
            // we have to clone, otherwise we only get the last new value
            putRow(data.outputRowMeta, data.outputRowMeta.cloneRow(newRow));
            if (log.isRowLevel()) {
                logRowlevel(BaseMessages.getString(PKG, "DatabaseJoin.Log.PutoutRow") + data.outputRowMeta.getString(newRow));
            }
            // Get a new row
            if (meta.getRowLimit() == 0 || counter < meta.getRowLimit()) {
                add = data.db.getRow(rs);
                incrementLinesInput();
            }
        }
        // Nothing found? Perhaps we have to put something out after all?
        if (counter == 0 && meta.isOuterJoin()) {
            if (data.notfound == null) {
                // Just return null values for all values...
                // 
                data.notfound = new Object[data.db.getReturnRowMeta().size()];
            }
            Object[] newRow = RowDataUtil.resizeArray(rowData, data.outputRowMeta.size());
            int newIndex = rowMeta.size();
            for (int i = 0; i < data.notfound.length; i++) {
                newRow[newIndex++] = data.notfound[i];
            }
            putRow(data.outputRowMeta, newRow);
        }
        data.db.closeQuery(rs);
    } finally {
        dbLock.unlock();
    }
}

Example 19 with HopTransformException

use of org.apache.hop.core.exception.HopTransformException in project hop by apache.

the class ColumnExists method processRow.

@Override
public boolean processRow() throws HopException {
    boolean sendToErrorRow = false;
    String errorMessage = null;
    // Get row from input rowset & set row busy!
    Object[] r = getRow();
    // if no more input to be expected set done
    if (r == null) {
        setOutputDone();
        return false;
    }
    boolean columnexists = false;
    if (first) {
        first = false;
        data.outputRowMeta = getInputRowMeta().clone();
        meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider);
        // Check is columnname field is provided
        if (Utils.isEmpty(meta.getColumnnamefield())) {
            logError(BaseMessages.getString(PKG, "ColumnExists.Error.ColumnnameFieldMissing"));
            throw new HopException(BaseMessages.getString(PKG, "ColumnExists.Error.ColumnnameFieldMissing"));
        }
        if (meta.isTablenameInfield()) {
            // Check is tablename field is provided
            if (Utils.isEmpty(meta.getTablenamefield())) {
                logError(BaseMessages.getString(PKG, "ColumnExists.Error.TablenameFieldMissing"));
                throw new HopException(BaseMessages.getString(PKG, "ColumnExists.Error.TablenameFieldMissing"));
            }
            // cache the position of the field
            if (data.indexOfTablename < 0) {
                data.indexOfTablename = getInputRowMeta().indexOfValue(meta.getTablenamefield());
                if (data.indexOfTablename < 0) {
                    // The field is unreachable !
                    logError(BaseMessages.getString(PKG, PKG_COULD_NOT_FIND_FIELD) + "[" + meta.getTablenamefield() + "]");
                    throw new HopException(BaseMessages.getString(PKG, PKG_COULD_NOT_FIND_FIELD, meta.getTablenamefield()));
                }
            }
        } else {
            if (!Utils.isEmpty(data.schemaname)) {
                data.tableName = data.db.getDatabaseMeta().getQuotedSchemaTableCombination(this, data.schemaname, data.tableName);
            } else {
                data.tableName = data.db.getDatabaseMeta().quoteField(data.tableName);
            }
        }
        // cache the position of the column field
        if (data.indexOfColumnname < 0) {
            data.indexOfColumnname = getInputRowMeta().indexOfValue(meta.getColumnnamefield());
            if (data.indexOfColumnname < 0) {
                // The field is unreachable !
                logError(BaseMessages.getString(PKG, PKG_COULD_NOT_FIND_FIELD) + "[" + meta.getColumnnamefield() + "]");
                throw new HopException(BaseMessages.getString(PKG, PKG_COULD_NOT_FIND_FIELD, meta.getColumnnamefield()));
            }
        }
    // End If first
    }
    try {
        // get tablename
        if (meta.isTablenameInfield()) {
            data.tableName = getInputRowMeta().getString(r, data.indexOfTablename);
            data.tableName = data.db.getDatabaseMeta().quoteField(data.tableName);
        }
        // get columnname
        String columnname = getInputRowMeta().getString(r, data.indexOfColumnname);
        columnname = data.db.getDatabaseMeta().quoteField(columnname);
        // Check if table exists on the specified connection
        columnexists = data.db.checkColumnExists(data.schemaname, data.tableName, columnname);
        Object[] outputRowData = RowDataUtil.addValueData(r, getInputRowMeta().size(), columnexists);
        // add new values to the row.
        // copy row to output rowset(s)
        putRow(data.outputRowMeta, outputRowData);
        if (log.isRowLevel()) {
            logRowlevel(BaseMessages.getString(PKG, "ColumnExists.LineNumber", getLinesRead() + " : " + getInputRowMeta().getString(r)));
        }
    } catch (HopException e) {
        if (getTransformMeta().isDoingErrorHandling()) {
            sendToErrorRow = true;
            errorMessage = e.toString();
        } else {
            logError(BaseMessages.getString(PKG, "ColumnExists.ErrorInTransformRunning" + " : " + e.getMessage()));
            throw new HopTransformException(BaseMessages.getString(PKG, "ColumnExists.Log.ErrorInTransform"), e);
        }
        if (sendToErrorRow) {
            // Simply add this row to the error row
            putError(getInputRowMeta(), r, 1, errorMessage, meta.getResultfieldname(), "ColumnExists001");
        }
    }
    return true;
}

Example 20 with HopTransformException

use of org.apache.hop.core.exception.HopTransformException in project hop by apache.

the class JsonInputMeta method getFields.

@Override
public void getFields(IRowMeta rowMeta, String name, IRowMeta[] info, TransformMeta nextTransform, IVariables variables, IHopMetadataProvider metadataProvider) throws HopTransformException {
    if (inFields && removeSourceField && !Utils.isEmpty(valueField)) {
        int index = rowMeta.indexOfValue(valueField);
        if (index != -1) {
            rowMeta.removeValueMeta(index);
        }
    }
    for (JsonInputField field : getInputFields()) {
        try {
            rowMeta.addValueMeta(field.toValueMeta(name, variables));
        } catch (Exception e) {
            throw new HopTransformException(e);
        }
    }
    if (includeFilename) {
        IValueMeta v = new ValueMetaString(variables.resolve(filenameField));
        v.setLength(250);
        v.setPrecision(-1);
        v.setOrigin(name);
        rowMeta.addValueMeta(v);
    }
    if (includeRowNumber) {
        IValueMeta v = new ValueMetaInteger(variables.resolve(rowNumberField));
        v.setLength(IValueMeta.DEFAULT_INTEGER_LENGTH, 0);
        v.setOrigin(name);
        rowMeta.addValueMeta(v);
    }
    // Add additional fields
    additionalOutputFields.normalize();
    additionalOutputFields.getFields(rowMeta, name, info, variables, metadataProvider);
}