Examples with SourceToTargetMapping org.apache.hop.core.SourceToTargetMapping used on opensource projects

Example 1 with SourceToTargetMapping

use of org.apache.hop.core.SourceToTargetMapping in project hop by apache.

the class WriteToDataSetExtensionPoint method passTransformRowsToDataSet.

private void passTransformRowsToDataSet(final IPipelineEngine<PipelineMeta> pipeline, final PipelineMeta pipelineMeta, final TransformMeta transformMeta, final List<SourceToTargetMapping> mappings, final DataSet dataSet) throws HopException {
    // This is the transform to inject into the specified data set
    // 
    final IRowMeta setRowMeta = dataSet.getSetRowMeta();
    IEngineComponent component = pipeline.findComponent(transformMeta.getName(), 0);
    final List<Object[]> transformsForDbRows = new ArrayList<>();
    component.addRowListener(new RowAdapter() {

        @Override
        public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
            Object[] transformForDbRow = RowDataUtil.allocateRowData(setRowMeta.size());
            for (SourceToTargetMapping mapping : mappings) {
                transformForDbRow[mapping.getTargetPosition()] = row[mapping.getSourcePosition()];
            }
            transformsForDbRows.add(transformForDbRow);
        }
    });
    // At the end of the pipeline, write it...
    // 
    pipeline.addExecutionFinishedListener(engine -> DataSetCsvUtil.writeDataSetData(pipeline, dataSet, setRowMeta, transformsForDbRows));
}

Example 2 with SourceToTargetMapping

use of org.apache.hop.core.SourceToTargetMapping in project hop by apache.

the class WriteToDataSetExtensionPoint method callExtensionPoint.

@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, IPipelineEngine<PipelineMeta> pipeline) throws HopException {
    final PipelineMeta pipelineMeta = pipeline.getPipelineMeta();
    boolean writeToDataSet = "Y".equalsIgnoreCase(pipeline.getVariable(DataSetConst.VAR_WRITE_TO_DATASET));
    if (!writeToDataSet) {
        return;
    }
    pipeline.addExecutionFinishedListener(engine -> {
        // Remove the flag when done.
        // We don't want to write to the data set every time we run
        // 
        pipeline.setVariable(DataSetConst.VAR_WRITE_TO_DATASET, null);
        // Prevent memory leaking as well
        // 
        WriteToDataSetExtensionPoint.transformsMap.remove(pipelineMeta.getName());
        WriteToDataSetExtensionPoint.mappingsMap.remove(pipelineMeta.getName());
        WriteToDataSetExtensionPoint.setsMap.remove(pipelineMeta.getName());
    });
    try {
        IHopMetadataProvider metadataProvider = pipelineMeta.getMetadataProvider();
        if (metadataProvider == null) {
            // Nothing to do here, we can't reference data sets.
            return;
        }
        // 
        for (final TransformMeta transformMeta : pipeline.getPipelineMeta().getTransforms()) {
            // We might want to pass the data from this transform into a data set all by itself...
            // For this we want to attach a row listener which writes the data.
            // 
            TransformMeta injectMeta = transformsMap.get(pipelineMeta.getName());
            if (injectMeta != null && injectMeta.equals(transformMeta)) {
                final List<SourceToTargetMapping> mappings = mappingsMap.get(pipelineMeta.getName());
                final DataSet dataSet = setsMap.get(pipelineMeta.getName());
                if (mappings != null && dataSet != null) {
                    passTransformRowsToDataSet(pipeline, pipelineMeta, transformMeta, mappings, dataSet);
                }
            }
        }
    } catch (Throwable e) {
        throw new HopException("Unable to pass rows to data set", e);
    }
}

Example 3 with SourceToTargetMapping

use of org.apache.hop.core.SourceToTargetMapping in project hop by apache.

the class PGBulkLoaderDialog method generateMappings.

/**
 * Reads in the fields from the previous transforms and from the ONE next transform and opens an
 * EnterMappingDialog with this information. After the user did the mapping, those information is
 * put into the Select/Rename table.
 */
private void generateMappings() {
    // Determine the source and target fields...
    // 
    IRowMeta sourceFields;
    IRowMeta targetFields;
    try {
        sourceFields = pipelineMeta.getPrevTransformFields(variables, transformMeta);
    } catch (HopException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "PGBulkLoaderDialog.DoMapping.UnableToFindSourceFields.Title"), BaseMessages.getString(PKG, "PGBulkLoaderDialog.DoMapping.UnableToFindSourceFields.Message"), e);
        return;
    }
    // refresh data
    input.setDatabaseMeta(pipelineMeta.findDatabase(wConnection.getText()));
    input.setTableName(variables.resolve(wTable.getText()));
    ITransformMeta transformMetaInterface = transformMeta.getTransform();
    try {
        targetFields = transformMetaInterface.getRequiredFields(variables);
    } catch (HopException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "PGBulkLoaderDialog.DoMapping.UnableToFindTargetFields.Title"), BaseMessages.getString(PKG, "PGBulkLoaderDialog.DoMapping.UnableToFindTargetFields.Message"), e);
        return;
    }
    String[] inputNames = new String[sourceFields.size()];
    for (int i = 0; i < sourceFields.size(); i++) {
        IValueMeta value = sourceFields.getValueMeta(i);
        inputNames[i] = value.getName();
    }
    // Create the existing mapping list...
    // 
    List<SourceToTargetMapping> mappings = new ArrayList<>();
    StringBuilder missingSourceFields = new StringBuilder();
    StringBuilder missingTargetFields = new StringBuilder();
    int nrFields = wReturn.nrNonEmpty();
    for (int i = 0; i < nrFields; i++) {
        TableItem item = wReturn.getNonEmpty(i);
        String source = item.getText(2);
        String target = item.getText(1);
        int sourceIndex = sourceFields.indexOfValue(source);
        if (sourceIndex < 0) {
            missingSourceFields.append(Const.CR).append("   ").append(source).append(" --> ").append(target);
        }
        int targetIndex = targetFields.indexOfValue(target);
        if (targetIndex < 0) {
            missingTargetFields.append(Const.CR).append("   ").append(source).append(" --> ").append(target);
        }
        if (sourceIndex < 0 || targetIndex < 0) {
            continue;
        }
        SourceToTargetMapping mapping = new SourceToTargetMapping(sourceIndex, targetIndex);
        mappings.add(mapping);
    }
    // 
    if (missingSourceFields.length() > 0 || missingTargetFields.length() > 0) {
        String message = "";
        if (missingSourceFields.length() > 0) {
            message += BaseMessages.getString(PKG, "PGBulkLoaderDialog.DoMapping.SomeSourceFieldsNotFound", missingSourceFields.toString()) + Const.CR;
        }
        if (missingTargetFields.length() > 0) {
            message += BaseMessages.getString(PKG, "PGBulkLoaderDialog.DoMapping.SomeTargetFieldsNotFound", missingSourceFields.toString()) + Const.CR;
        }
        message += Const.CR;
        message += BaseMessages.getString(PKG, "PGBulkLoaderDialog.DoMapping.SomeFieldsNotFoundContinue") + Const.CR;
        int answer = BaseDialog.openMessageBox(shell, BaseMessages.getString(PKG, "PGBulkLoaderDialog.DoMapping.SomeFieldsNotFoundTitle"), message, SWT.ICON_QUESTION | SWT.OK | SWT.CANCEL);
        boolean goOn = (answer & SWT.OK) != 0;
        if (!goOn) {
            return;
        }
    }
    EnterMappingDialog d = new EnterMappingDialog(PGBulkLoaderDialog.this.shell, sourceFields.getFieldNames(), targetFields.getFieldNames(), mappings);
    mappings = d.open();
    // 
    if (mappings != null) {
        // Clear and re-populate!
        // 
        wReturn.table.removeAll();
        wReturn.table.setItemCount(mappings.size());
        for (int i = 0; i < mappings.size(); i++) {
            SourceToTargetMapping mapping = mappings.get(i);
            TableItem item = wReturn.table.getItem(i);
            item.setText(2, sourceFields.getValueMeta(mapping.getSourcePosition()).getName());
            item.setText(1, targetFields.getValueMeta(mapping.getTargetPosition()).getName());
        }
        wReturn.setRowNums();
        wReturn.optWidth(true);
    }
}

Example 4 with SourceToTargetMapping

use of org.apache.hop.core.SourceToTargetMapping in project hop by apache.

the class InsertUpdateDialog method generateMappings.

/**
 * Reads in the fields from the previous transforms and from the ONE next transform and opens an
 * EnterMappingDialog with this information. After the user did the mapping, those information is
 * put into the Select/Rename table.
 */
private void generateMappings() {
    // Determine the source and target fields...
    // 
    IRowMeta sourceFields;
    IRowMeta targetFields;
    try {
        sourceFields = pipelineMeta.getPrevTransformFields(variables, transformMeta);
    } catch (HopException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "InsertUpdateDialog.DoMapping.UnableToFindSourceFields.Title"), BaseMessages.getString(PKG, "InsertUpdateDialog.DoMapping.UnableToFindSourceFields.Message"), e);
        return;
    }
    // refresh data
    input.setConnection(wConnection.getText());
    input.getInsertUpdateLookupField().setTableName(variables.resolve(wTable.getText()));
    ITransformMeta transformMetaInterface = transformMeta.getTransform();
    try {
        targetFields = transformMetaInterface.getRequiredFields(variables);
    } catch (HopException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "InsertUpdateDialog.DoMapping.UnableToFindTargetFields.Title"), BaseMessages.getString(PKG, "InsertUpdateDialog.DoMapping.UnableToFindTargetFields.Message"), e);
        return;
    }
    String[] inputNames = new String[sourceFields.size()];
    for (int i = 0; i < sourceFields.size(); i++) {
        IValueMeta value = sourceFields.getValueMeta(i);
        inputNames[i] = value.getName();
    }
    // Create the existing mapping list...
    // Also copy the update status of targets in to a hashmap
    // 
    List<SourceToTargetMapping> mappings = new ArrayList<>();
    Map<String, String> targetUpdateStatus = new HashMap<>();
    StringBuilder missingSourceFields = new StringBuilder();
    StringBuilder missingTargetFields = new StringBuilder();
    int nrFields = wReturn.nrNonEmpty();
    for (int i = 0; i < nrFields; i++) {
        TableItem item = wReturn.getNonEmpty(i);
        String source = item.getText(2);
        String target = item.getText(1);
        targetUpdateStatus.put(item.getText(1), item.getText(3));
        int sourceIndex = sourceFields.indexOfValue(source);
        if (sourceIndex < 0) {
            missingSourceFields.append(Const.CR).append("   ").append(source).append(" --> ").append(target);
        }
        int targetIndex = targetFields.indexOfValue(target);
        if (targetIndex < 0) {
            missingTargetFields.append(Const.CR).append("   ").append(source).append(" --> ").append(target);
        }
        if (sourceIndex < 0 || targetIndex < 0) {
            continue;
        }
        SourceToTargetMapping mapping = new SourceToTargetMapping(sourceIndex, targetIndex);
        mappings.add(mapping);
    }
    // 
    if (missingSourceFields.length() > 0 || missingTargetFields.length() > 0) {
        String message = "";
        if (missingSourceFields.length() > 0) {
            message += BaseMessages.getString(PKG, "InsertUpdateDialog.DoMapping.SomeSourceFieldsNotFound", missingSourceFields.toString()) + Const.CR;
        }
        if (missingTargetFields.length() > 0) {
            message += BaseMessages.getString(PKG, "InsertUpdateDialog.DoMapping.SomeTargetFieldsNotFound", missingSourceFields.toString()) + Const.CR;
        }
        message += Const.CR;
        message += BaseMessages.getString(PKG, "InsertUpdateDialog.DoMapping.SomeFieldsNotFoundContinue") + Const.CR;
        int answer = BaseDialog.openMessageBox(shell, BaseMessages.getString(PKG, "InsertUpdateDialog.DoMapping.SomeFieldsNotFoundTitle"), message, SWT.ICON_QUESTION | SWT.OK | SWT.CANCEL);
        boolean goOn = (answer & SWT.OK) != 0;
        if (!goOn) {
            return;
        }
    }
    EnterMappingDialog d = new EnterMappingDialog(InsertUpdateDialog.this.shell, sourceFields.getFieldNames(), targetFields.getFieldNames(), mappings);
    mappings = d.open();
    // 
    if (mappings != null) {
        // Clear and re-populate!
        // 
        wReturn.table.removeAll();
        wReturn.table.setItemCount(mappings.size());
        for (int i = 0; i < mappings.size(); i++) {
            SourceToTargetMapping mapping = mappings.get(i);
            TableItem item = wReturn.table.getItem(i);
            item.setText(2, sourceFields.getValueMeta(mapping.getSourcePosition()).getName());
            item.setText(1, targetFields.getValueMeta(mapping.getTargetPosition()).getName());
            if (targetUpdateStatus.get(item.getText(1)) == null) {
                item.setText(3, "Y");
            } else {
                item.setText(3, targetUpdateStatus.get(item.getText(1)));
            }
        }
        wReturn.setRowNums();
        wReturn.optWidth(true);
    }
}

Example 5 with SourceToTargetMapping

use of org.apache.hop.core.SourceToTargetMapping in project hop by apache.

the class SalesforceInsertDialog method generateMappings.

/**
 * Reads in the fields from the previous transforms and from the ONE next transform and opens an
 * EnterMappingDialog with this information. After the user did the mapping, those information is
 * put into the Select/Rename table.
 */
private void generateMappings() {
    if (!checkInput()) {
        return;
    }
    // Determine the source and target fields...
    // 
    IRowMeta sourceFields;
    IRowMeta targetFields = new RowMeta();
    try {
        sourceFields = pipelineMeta.getPrevTransformFields(variables, transformMeta);
    } catch (HopException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "SalesforceInsertDialog.DoMapping.UnableToFindSourceFields.Title"), BaseMessages.getString(PKG, "SalesforceInsertDialog.DoMapping.UnableToFindSourceFields.Message"), e);
        return;
    }
    try {
        SalesforceConnection connection = getConnection();
        Field[] fields = connection.getObjectFields(variables.resolve(wModule.getText()));
        String[] fieldNames = connection.getFields(fields);
        FieldType dateType = FieldType.date;
        for (int i = 0; i < fields.length; i++) {
            if (dateType.equals(fields[i].getType())) {
                // Mark date columns as TYPE_DATE to strip time part later
                targetFields.addValueMeta(ValueMetaFactory.createValueMeta(fieldNames[i], IValueMeta.TYPE_DATE));
            } else {
                targetFields.addValueMeta(new ValueMetaNone(fieldNames[i]));
            }
        }
    } catch (Exception e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "SalesforceInsertDialog.DoMapping.UnableToFindTargetFields.Title"), BaseMessages.getString(PKG, "SalesforceInsertDialog.DoMapping.UnableToFindTargetFields.Message"), e);
        return;
    }
    String[] inputNames = new String[sourceFields.size()];
    for (int i = 0; i < sourceFields.size(); i++) {
        IValueMeta value = sourceFields.getValueMeta(i);
        inputNames[i] = value.getName();
    }
    // Create the existing mapping list...
    // 
    List<SourceToTargetMapping> mappings = new ArrayList<>();
    StringBuffer missingSourceFields = new StringBuffer();
    StringBuffer missingTargetFields = new StringBuffer();
    int nrFields = wReturn.nrNonEmpty();
    for (int i = 0; i < nrFields; i++) {
        TableItem item = wReturn.getNonEmpty(i);
        String source = item.getText(2);
        String target = item.getText(1);
        int sourceIndex = sourceFields.indexOfValue(source);
        if (sourceIndex < 0) {
            missingSourceFields.append(Const.CR + "   " + source + " --> " + target);
        }
        int targetIndex = targetFields.indexOfValue(target);
        if (targetIndex < 0) {
            missingTargetFields.append(Const.CR + "   " + source + " --> " + target);
        }
        if (sourceIndex < 0 || targetIndex < 0) {
            continue;
        }
        SourceToTargetMapping mapping = new SourceToTargetMapping(sourceIndex, targetIndex);
        mappings.add(mapping);
    }
    // 
    if (missingSourceFields.length() > 0 || missingTargetFields.length() > 0) {
        String message = "";
        if (missingSourceFields.length() > 0) {
            message += BaseMessages.getString(PKG, "SalesforceInsertDialog.DoMapping.SomeSourceFieldsNotFound", missingSourceFields.toString()) + Const.CR;
        }
        if (missingTargetFields.length() > 0) {
            message += BaseMessages.getString(PKG, "SalesforceInsertDialog.DoMapping.SomeTargetFieldsNotFound", missingSourceFields.toString()) + Const.CR;
        }
        message += Const.CR;
        message += BaseMessages.getString(PKG, "SalesforceInsertDialog.DoMapping.SomeFieldsNotFoundContinue") + Const.CR;
        int answer = BaseDialog.openMessageBox(shell, BaseMessages.getString(PKG, "SalesforceInsertDialog.DoMapping.SomeFieldsNotFoundTitle"), message, SWT.ICON_QUESTION | SWT.OK | SWT.CANCEL);
        boolean goOn = (answer & SWT.OK) != 0;
        if (!goOn) {
            return;
        }
    }
    EnterMappingDialog d = new EnterMappingDialog(SalesforceInsertDialog.this.shell, sourceFields.getFieldNames(), targetFields.getFieldNames(), mappings);
    mappings = d.open();
    // 
    if (mappings != null) {
        // Clear and re-populate!
        // 
        wReturn.table.removeAll();
        wReturn.table.setItemCount(mappings.size());
        for (int i = 0; i < mappings.size(); i++) {
            SourceToTargetMapping mapping = mappings.get(i);
            TableItem item = wReturn.table.getItem(i);
            item.setText(2, sourceFields.getValueMeta(mapping.getSourcePosition()).getName());
            item.setText(1, targetFields.getValueMeta(mapping.getTargetPosition()).getName());
        }
        wReturn.setRowNums();
        wReturn.optWidth(true);
    }
}