Examples with IgnoreRowException org.jumpmind.symmetric.io.data.transform.IgnoreRowException used on opensource projects

Example 1 with IgnoreRowException

use of org.jumpmind.symmetric.io.data.transform.IgnoreRowException in project symmetric-ds by JumpMind.

the class TransformWriter method transform.

protected List<TransformedData> transform(DataEventType eventType, DataContext context, TransformTable transformation, Map<String, String> sourceKeyValues, Map<String, String> oldSourceValues, Map<String, String> sourceValues) {
    try {
        List<TransformedData> dataToTransform = create(context, eventType, transformation, sourceKeyValues, oldSourceValues, sourceValues);
        List<TransformedData> dataThatHasBeenTransformed = new ArrayList<TransformedData>(dataToTransform.size());
        if (log.isDebugEnabled()) {
            log.debug("{} target data was created for the {} transformation.  The target table is {}", new Object[] { dataToTransform.size(), transformation.getTransformId(), transformation.getFullyQualifiedTargetTableName() });
        }
        int transformNumber = 0;
        for (TransformedData targetData : dataToTransform) {
            transformNumber++;
            if (perform(context, targetData, transformation, sourceValues, oldSourceValues)) {
                if (log.isDebugEnabled()) {
                    log.debug("Data has been transformed to a {} for the #{} transform.  The mapped target columns are: {}. The mapped target values are: {}", new Object[] { targetData.getTargetDmlType().toString(), transformNumber, ArrayUtils.toString(targetData.getColumnNames()), ArrayUtils.toString(targetData.getColumnValues()) });
                }
                dataThatHasBeenTransformed.add(targetData);
            } else {
                log.debug("Data has not been transformed for the #{} transform", transformNumber);
            }
        }
        return dataThatHasBeenTransformed;
    } catch (IgnoreRowException ex) {
        // ignore this row
        if (log.isDebugEnabled()) {
            log.debug("Transform indicated that the target row should be ignored with a target key of: {}", "unknown.  Transformation aborted during tranformation of key");
        }
        return new ArrayList<TransformedData>(0);
    }
}

Example 2 with IgnoreRowException

use of org.jumpmind.symmetric.io.data.transform.IgnoreRowException in project symmetric-ds by JumpMind.

the class TransformWriter method perform.

protected boolean perform(DataContext context, TransformedData data, TransformTable transformation, Map<String, String> sourceValues, Map<String, String> oldSourceValues) throws IgnoreRowException {
    boolean persistData = false;
    try {
        DataEventType eventType = data.getSourceDmlType();
        for (TransformColumn transformColumn : transformation.getTransformColumns()) {
            if (!transformColumn.isPk()) {
                IncludeOnType includeOn = transformColumn.getIncludeOn();
                if (includeOn == IncludeOnType.ALL || (includeOn == IncludeOnType.INSERT && eventType == DataEventType.INSERT) || (includeOn == IncludeOnType.UPDATE && eventType == DataEventType.UPDATE) || (includeOn == IncludeOnType.DELETE && eventType == DataEventType.DELETE)) {
                    if (StringUtils.isBlank(transformColumn.getSourceColumnName()) || sourceValues.containsKey(transformColumn.getSourceColumnName())) {
                        try {
                            Object value = transformColumn(context, data, transformColumn, sourceValues, oldSourceValues);
                            if (value instanceof NewAndOldValue) {
                                data.put(transformColumn, ((NewAndOldValue) value).getNewValue(), oldSourceValues != null ? ((NewAndOldValue) value).getOldValue() : null, false);
                            } else if (value == null || value instanceof String) {
                                data.put(transformColumn, (String) value, null, false);
                            } else if (value instanceof List) {
                                throw new IllegalStateException(String.format("Column transform failed %s.%s. Transforms that multiply rows must be marked as part of the primary key", transformColumn.getTransformId(), transformColumn.getTargetColumnName()));
                            } else {
                                throw new IllegalStateException(String.format("Column transform failed %s.%s. It returned an unexpected type of %s", transformColumn.getTransformId(), transformColumn.getTargetColumnName(), value.getClass().getSimpleName()));
                            }
                        } catch (IgnoreColumnException e) {
                            // Do nothing. We are ignoring the column
                            if (log.isDebugEnabled()) {
                                log.debug("A transform indicated we should ignore the target column {}", transformColumn.getTargetColumnName());
                            }
                        }
                    } else {
                        if (eventType != DataEventType.DELETE) {
                            log.warn("Could not find a source column of {} for the transformation: {}", transformColumn.getSourceColumnName(), transformation.getTransformId());
                        } else {
                            log.debug("Could not find a source column of {} for the transformation: {}.  This is probably because this was a DELETE event and no old data was captured.", transformColumn.getSourceColumnName(), transformation.getTransformId());
                        }
                    }
                }
            }
        }
        // transformation
        if (data.getColumnNames().length > 0) {
            TargetDmlAction targetAction = null;
            switch(data.getTargetDmlType()) {
                case INSERT:
                    targetAction = TargetDmlAction.INS_ROW;
                    break;
                case UPDATE:
                    targetAction = transformation.evaluateTargetDmlAction(context, data);
                    break;
                case DELETE:
                    targetAction = transformation.getDeleteAction();
                    break;
                default:
                    persistData = true;
            }
            if (targetAction != null) {
                // how to handle the update/delete action on target..
                switch(targetAction) {
                    case DEL_ROW:
                        data.setTargetDmlType(DataEventType.DELETE);
                        persistData = true;
                        break;
                    case UPDATE_COL:
                    case UPD_ROW:
                        data.setTargetDmlType(DataEventType.UPDATE);
                        persistData = true;
                        break;
                    case INS_ROW:
                        data.setTargetDmlType(DataEventType.INSERT);
                        persistData = true;
                        break;
                    case NONE:
                    default:
                        if (log.isDebugEnabled()) {
                            log.debug("The {} transformation is not configured to delete row.  Not sending the delete through.", transformation.getTransformId());
                        }
                        break;
                }
            }
        }
    } catch (IgnoreRowException ex) {
        // ignore this row
        if (log.isDebugEnabled()) {
            log.debug("Transform indicated that the target row should be ignored with a target key of: {}", ArrayUtils.toString(data.getKeyValues()));
        }
    }
    return persistData;
}

Example 3 with IgnoreRowException

use of org.jumpmind.symmetric.io.data.transform.IgnoreRowException in project symmetric-ds by JumpMind.

the class TransformWriter method create.

protected List<TransformedData> create(DataContext context, DataEventType dataEventType, TransformTable transformation, Map<String, String> sourceKeyValues, Map<String, String> oldSourceValues, Map<String, String> sourceValues) throws IgnoreRowException {
    List<TransformColumn> columns = transformation.getPrimaryKeyColumns();
    if (columns == null || columns.size() == 0) {
        log.error("No primary key defined for the transformation: {}", transformation.getTransformId());
        return new ArrayList<TransformedData>(0);
    } else {
        List<TransformedData> datas = new ArrayList<TransformedData>();
        TransformedData data = new TransformedData(transformation, dataEventType, sourceKeyValues, oldSourceValues, sourceValues);
        datas.add(data);
        DataEventType eventType = data.getSourceDmlType();
        for (TransformColumn transformColumn : columns) {
            IncludeOnType includeOn = transformColumn.getIncludeOn();
            if (includeOn == IncludeOnType.ALL || (includeOn == IncludeOnType.INSERT && eventType == DataEventType.INSERT) || (includeOn == IncludeOnType.UPDATE && eventType == DataEventType.UPDATE) || (includeOn == IncludeOnType.DELETE && eventType == DataEventType.DELETE)) {
                List<TransformedData> newDatas = null;
                try {
                    Object columnValue = transformColumn(context, data, transformColumn, sourceValues, oldSourceValues);
                    if (columnValue instanceof List) {
                        @SuppressWarnings("unchecked") List<String> values = (List<String>) columnValue;
                        if (values.size() > 0) {
                            data.put(transformColumn, values.get(0), oldSourceValues != null ? values.get(0) : null, true);
                            if (values.size() > 1) {
                                if (newDatas == null) {
                                    newDatas = new ArrayList<TransformedData>(values.size() - 1);
                                }
                                for (int i = 1; i < values.size(); i++) {
                                    TransformedData newData = data.copy();
                                    newData.put(transformColumn, values.get(i), oldSourceValues != null ? values.get(i) : null, true);
                                    newDatas.add(newData);
                                }
                            }
                        } else {
                            throw new IgnoreRowException();
                        }
                    } else if (columnValue instanceof NewAndOldValue) {
                        data.put(transformColumn, ((NewAndOldValue) columnValue).getNewValue(), oldSourceValues != null ? ((NewAndOldValue) columnValue).getOldValue() : null, true);
                    } else {
                        data.put(transformColumn, (String) columnValue, oldSourceValues != null ? (String) columnValue : null, true);
                    }
                } catch (IgnoreColumnException e) {
                // Do nothing. We are suppose to ignore the column.
                }
                if (newDatas != null) {
                    datas.addAll(newDatas);
                    newDatas = null;
                }
            }
        }
        return datas;
    }
}