Examples with TransformedData org.jumpmind.symmetric.io.data.transform.TransformedData used on opensource projects

Example 1 with TransformedData

use of org.jumpmind.symmetric.io.data.transform.TransformedData in project symmetric-ds by JumpMind.

the class JavaColumnTransformTest method testSimple.

@Test
public void testSimple() throws Exception {
    String javaCode = "return \"transValue\";";
    TransformColumn column = new TransformColumn("sColumn", "tColumn", false, "java", javaCode);
    TransformTable table = new TransformTable("sTable", "tTable", TransformPoint.LOAD, column);
    Map<String, String> sourceKeyValues = new HashMap<String, String>();
    Map<String, String> sourceValues = new HashMap<String, String>();
    sourceValues.put("sColumn", "aNewValue");
    Map<String, String> oldSourceValues = new HashMap<String, String>();
    oldSourceValues.put("sColumn", "anOldValue");
    TransformedData data = new TransformedData(table, DataEventType.INSERT, sourceKeyValues, oldSourceValues, sourceValues);
    JavaColumnTransform transform = new JavaColumnTransform(extensionService);
    String out = transform.transform(platform, context, column, data, sourceValues, "aNewValue", "anOldValue");
    assertEquals("transValue", out);
}

Example 2 with TransformedData

use of org.jumpmind.symmetric.io.data.transform.TransformedData in project symmetric-ds by JumpMind.

the class TransformWriter method transform.

protected List<TransformedData> transform(DataEventType eventType, DataContext context, TransformTable transformation, Map<String, String> sourceKeyValues, Map<String, String> oldSourceValues, Map<String, String> sourceValues) {
    try {
        List<TransformedData> dataToTransform = create(context, eventType, transformation, sourceKeyValues, oldSourceValues, sourceValues);
        List<TransformedData> dataThatHasBeenTransformed = new ArrayList<TransformedData>(dataToTransform.size());
        if (log.isDebugEnabled()) {
            log.debug("{} target data was created for the {} transformation.  The target table is {}", new Object[] { dataToTransform.size(), transformation.getTransformId(), transformation.getFullyQualifiedTargetTableName() });
        }
        int transformNumber = 0;
        for (TransformedData targetData : dataToTransform) {
            transformNumber++;
            if (perform(context, targetData, transformation, sourceValues, oldSourceValues)) {
                if (log.isDebugEnabled()) {
                    log.debug("Data has been transformed to a {} for the #{} transform.  The mapped target columns are: {}. The mapped target values are: {}", new Object[] { targetData.getTargetDmlType().toString(), transformNumber, ArrayUtils.toString(targetData.getColumnNames()), ArrayUtils.toString(targetData.getColumnValues()) });
                }
                dataThatHasBeenTransformed.add(targetData);
            } else {
                log.debug("Data has not been transformed for the #{} transform", transformNumber);
            }
        }
        return dataThatHasBeenTransformed;
    } catch (IgnoreRowException ex) {
        // ignore this row
        if (log.isDebugEnabled()) {
            log.debug("Transform indicated that the target row should be ignored with a target key of: {}", "unknown.  Transformation aborted during tranformation of key");
        }
        return new ArrayList<TransformedData>(0);
    }
}

Example 3 with TransformedData

use of org.jumpmind.symmetric.io.data.transform.TransformedData in project symmetric-ds by JumpMind.

the class TransformWriter method write.

public void write(CsvData data) {
    DataEventType eventType = data.getDataEventType();
    if (activeTransforms != null && activeTransforms.size() > 0 && isTransformable(eventType)) {
        if (data.requiresTable() && sourceTable == null && context.getLastParsedTable() != null) {
            // if we cross batches and the table isn't specified, then
            // use the last table we used
            start(context.getLastParsedTable());
        }
        long ts = System.currentTimeMillis();
        Map<String, String> sourceValues = data.toColumnNameValuePairs(this.sourceTable.getColumnNames(), CsvData.ROW_DATA);
        Map<String, String> oldSourceValues = null;
        if (data.contains(CsvData.OLD_DATA)) {
            oldSourceValues = data.toColumnNameValuePairs(this.sourceTable.getColumnNames(), CsvData.OLD_DATA);
        }
        Map<String, String> sourceKeyValues = null;
        if (data.contains(CsvData.PK_DATA)) {
            sourceKeyValues = data.toKeyColumnValuePairs(this.sourceTable);
        }
        if (eventType == DataEventType.DELETE) {
            sourceValues = oldSourceValues;
            if (sourceValues == null || sourceValues.size() == 0) {
                sourceValues = sourceKeyValues;
            }
        }
        if (log.isDebugEnabled()) {
            log.debug("{} transformation(s) started because of {} on {}.  The original row data was: {}", new Object[] { activeTransforms.size(), eventType.toString(), this.sourceTable.getFullyQualifiedTableName(), sourceValues });
        }
        List<TransformedData> dataThatHasBeenTransformed = new ArrayList<TransformedData>();
        TransformTable[] transformTables = activeTransforms.toArray(new TransformTable[activeTransforms.size()]);
        if (eventType == DataEventType.DELETE) {
            CollectionUtils.reverseArray(transformTables);
        }
        for (TransformTable transformation : transformTables) {
            if (eventType == DataEventType.INSERT && transformation.isUpdateFirst()) {
                eventType = DataEventType.UPDATE;
            }
            dataThatHasBeenTransformed.addAll(transform(eventType, context, transformation, sourceKeyValues, oldSourceValues, sourceValues));
        }
        for (TransformedData transformedData : dataThatHasBeenTransformed) {
            Table transformedTable = transformedData.buildTargetTable();
            CsvData csvData = transformedData.buildTargetCsvData();
            long transformTimeInMs = System.currentTimeMillis() - ts;
            boolean processData = true;
            if (lastTransformedTable == null || !lastTransformedTable.equals(transformedTable)) {
                if (lastTransformedTable != null) {
                    this.nestedWriter.end(lastTransformedTable);
                }
                processData = this.nestedWriter.start(transformedTable);
                if (!processData) {
                    lastTransformedTable = null;
                } else {
                    lastTransformedTable = transformedTable;
                }
            }
            if (processData || !csvData.requiresTable()) {
                this.nestedWriter.write(csvData);
            }
            Statistics stats = this.nestedWriter.getStatistics().get(batch);
            if (stats != null) {
                stats.increment(DataWriterStatisticConstants.TRANSFORMMILLIS, transformTimeInMs);
            }
            ts = System.currentTimeMillis();
        }
    } else {
        if (sourceTable != null) {
            super.start(sourceTable);
        }
        super.write(data);
        if (sourceTable != null) {
            super.end(sourceTable);
        }
    }
}

Example 4 with TransformedData

use of org.jumpmind.symmetric.io.data.transform.TransformedData in project symmetric-ds by JumpMind.

the class JavaColumnTransformTest method testInnerClass.

@Test
public void testInnerClass() throws Exception {
    String javaCode = "final DataContext ctx = context;" + "HashMap namedParams = new HashMap();" + "context.findTransaction().query(\"sql\", new ISqlRowMapper<Object>() {" + "        public Object mapRow(Row row) {" + "            ctx.put(\"a\", row.getString(\"b\"));" + "            return null;" + "        }" + "}, namedParams);" + "return \"transValue\";";
    TransformColumn column = new TransformColumn("sColumn", "tColumn", false, "java", javaCode);
    TransformTable table = new TransformTable("sTable", "tTable", TransformPoint.LOAD, column);
    Map<String, String> sourceKeyValues = new HashMap<String, String>();
    Map<String, String> sourceValues = new HashMap<String, String>();
    sourceValues.put("sColumn", "aNewValue");
    Map<String, String> oldSourceValues = new HashMap<String, String>();
    oldSourceValues.put("sColumn", "anOldValue");
    TransformedData data = new TransformedData(table, DataEventType.INSERT, sourceKeyValues, oldSourceValues, sourceValues);
    JavaColumnTransform transform = new JavaColumnTransform(extensionService);
    String out = transform.transform(platform, context, column, data, sourceValues, "aNewValue", "anOldValue");
    assertEquals("transValue", out);
}

Example 5 with TransformedData

use of org.jumpmind.symmetric.io.data.transform.TransformedData in project symmetric-ds by JumpMind.

the class TransformWriter method create.

protected List<TransformedData> create(DataContext context, DataEventType dataEventType, TransformTable transformation, Map<String, String> sourceKeyValues, Map<String, String> oldSourceValues, Map<String, String> sourceValues) throws IgnoreRowException {
    List<TransformColumn> columns = transformation.getPrimaryKeyColumns();
    if (columns == null || columns.size() == 0) {
        log.error("No primary key defined for the transformation: {}", transformation.getTransformId());
        return new ArrayList<TransformedData>(0);
    } else {
        List<TransformedData> datas = new ArrayList<TransformedData>();
        TransformedData data = new TransformedData(transformation, dataEventType, sourceKeyValues, oldSourceValues, sourceValues);
        datas.add(data);
        DataEventType eventType = data.getSourceDmlType();
        for (TransformColumn transformColumn : columns) {
            IncludeOnType includeOn = transformColumn.getIncludeOn();
            if (includeOn == IncludeOnType.ALL || (includeOn == IncludeOnType.INSERT && eventType == DataEventType.INSERT) || (includeOn == IncludeOnType.UPDATE && eventType == DataEventType.UPDATE) || (includeOn == IncludeOnType.DELETE && eventType == DataEventType.DELETE)) {
                List<TransformedData> newDatas = null;
                try {
                    Object columnValue = transformColumn(context, data, transformColumn, sourceValues, oldSourceValues);
                    if (columnValue instanceof List) {
                        @SuppressWarnings("unchecked") List<String> values = (List<String>) columnValue;
                        if (values.size() > 0) {
                            data.put(transformColumn, values.get(0), oldSourceValues != null ? values.get(0) : null, true);
                            if (values.size() > 1) {
                                if (newDatas == null) {
                                    newDatas = new ArrayList<TransformedData>(values.size() - 1);
                                }
                                for (int i = 1; i < values.size(); i++) {
                                    TransformedData newData = data.copy();
                                    newData.put(transformColumn, values.get(i), oldSourceValues != null ? values.get(i) : null, true);
                                    newDatas.add(newData);
                                }
                            }
                        } else {
                            throw new IgnoreRowException();
                        }
                    } else if (columnValue instanceof NewAndOldValue) {
                        data.put(transformColumn, ((NewAndOldValue) columnValue).getNewValue(), oldSourceValues != null ? ((NewAndOldValue) columnValue).getOldValue() : null, true);
                    } else {
                        data.put(transformColumn, (String) columnValue, oldSourceValues != null ? (String) columnValue : null, true);
                    }
                } catch (IgnoreColumnException e) {
                // Do nothing. We are suppose to ignore the column.
                }
                if (newDatas != null) {
                    datas.addAll(newDatas);
                    newDatas = null;
                }
            }
        }
        return datas;
    }
}