Examples with TransformTable org.jumpmind.symmetric.io.data.transform.TransformTable used on opensource projects

Example 1 with TransformTable

use of org.jumpmind.symmetric.io.data.transform.TransformTable in project symmetric-ds by JumpMind.

the class JavaColumnTransformTest method testSimple.

@Test
public void testSimple() throws Exception {
    String javaCode = "return \"transValue\";";
    TransformColumn column = new TransformColumn("sColumn", "tColumn", false, "java", javaCode);
    TransformTable table = new TransformTable("sTable", "tTable", TransformPoint.LOAD, column);
    Map<String, String> sourceKeyValues = new HashMap<String, String>();
    Map<String, String> sourceValues = new HashMap<String, String>();
    sourceValues.put("sColumn", "aNewValue");
    Map<String, String> oldSourceValues = new HashMap<String, String>();
    oldSourceValues.put("sColumn", "anOldValue");
    TransformedData data = new TransformedData(table, DataEventType.INSERT, sourceKeyValues, oldSourceValues, sourceValues);
    JavaColumnTransform transform = new JavaColumnTransform(extensionService);
    String out = transform.transform(platform, context, column, data, sourceValues, "aNewValue", "anOldValue");
    assertEquals("transValue", out);
}

Example 2 with TransformTable

use of org.jumpmind.symmetric.io.data.transform.TransformTable in project symmetric-ds by JumpMind.

the class TransformWriter method start.

@Override
public boolean start(Table table) {
    List<TransformTable> activeTransformsTemp = transformsBySourceTable.get(table.getFullyQualifiedTableNameLowerCase());
    if (activeTransformsTemp != null && activeTransformsTemp.size() > 0) {
        this.sourceTable = table;
        activeTransforms = new ArrayList<TransformTable>(activeTransformsTemp.size());
        for (TransformTable transformation : activeTransformsTemp) {
            activeTransforms.add(transformation.enhanceWithImpliedColumns(this.sourceTable.getPrimaryKeyColumnNames(), this.sourceTable.getColumnNames()));
        }
        return true;
    } else {
        this.sourceTable = null;
        return super.start(table);
    }
}

Example 3 with TransformTable

use of org.jumpmind.symmetric.io.data.transform.TransformTable in project symmetric-ds by JumpMind.

the class TransformWriter method write.

public void write(CsvData data) {
    DataEventType eventType = data.getDataEventType();
    if (activeTransforms != null && activeTransforms.size() > 0 && isTransformable(eventType)) {
        if (data.requiresTable() && sourceTable == null && context.getLastParsedTable() != null) {
            // if we cross batches and the table isn't specified, then
            // use the last table we used
            start(context.getLastParsedTable());
        }
        long ts = System.currentTimeMillis();
        Map<String, String> sourceValues = data.toColumnNameValuePairs(this.sourceTable.getColumnNames(), CsvData.ROW_DATA);
        Map<String, String> oldSourceValues = null;
        if (data.contains(CsvData.OLD_DATA)) {
            oldSourceValues = data.toColumnNameValuePairs(this.sourceTable.getColumnNames(), CsvData.OLD_DATA);
        }
        Map<String, String> sourceKeyValues = null;
        if (data.contains(CsvData.PK_DATA)) {
            sourceKeyValues = data.toKeyColumnValuePairs(this.sourceTable);
        }
        if (eventType == DataEventType.DELETE) {
            sourceValues = oldSourceValues;
            if (sourceValues == null || sourceValues.size() == 0) {
                sourceValues = sourceKeyValues;
            }
        }
        if (log.isDebugEnabled()) {
            log.debug("{} transformation(s) started because of {} on {}.  The original row data was: {}", new Object[] { activeTransforms.size(), eventType.toString(), this.sourceTable.getFullyQualifiedTableName(), sourceValues });
        }
        List<TransformedData> dataThatHasBeenTransformed = new ArrayList<TransformedData>();
        TransformTable[] transformTables = activeTransforms.toArray(new TransformTable[activeTransforms.size()]);
        if (eventType == DataEventType.DELETE) {
            CollectionUtils.reverseArray(transformTables);
        }
        for (TransformTable transformation : transformTables) {
            if (eventType == DataEventType.INSERT && transformation.isUpdateFirst()) {
                eventType = DataEventType.UPDATE;
            }
            dataThatHasBeenTransformed.addAll(transform(eventType, context, transformation, sourceKeyValues, oldSourceValues, sourceValues));
        }
        for (TransformedData transformedData : dataThatHasBeenTransformed) {
            Table transformedTable = transformedData.buildTargetTable();
            CsvData csvData = transformedData.buildTargetCsvData();
            long transformTimeInMs = System.currentTimeMillis() - ts;
            boolean processData = true;
            if (lastTransformedTable == null || !lastTransformedTable.equals(transformedTable)) {
                if (lastTransformedTable != null) {
                    this.nestedWriter.end(lastTransformedTable);
                }
                processData = this.nestedWriter.start(transformedTable);
                if (!processData) {
                    lastTransformedTable = null;
                } else {
                    lastTransformedTable = transformedTable;
                }
            }
            if (processData || !csvData.requiresTable()) {
                this.nestedWriter.write(csvData);
            }
            Statistics stats = this.nestedWriter.getStatistics().get(batch);
            if (stats != null) {
                stats.increment(DataWriterStatisticConstants.TRANSFORMMILLIS, transformTimeInMs);
            }
            ts = System.currentTimeMillis();
        }
    } else {
        if (sourceTable != null) {
            super.start(sourceTable);
        }
        super.write(data);
        if (sourceTable != null) {
            super.end(sourceTable);
        }
    }
}

Example 4 with TransformTable

use of org.jumpmind.symmetric.io.data.transform.TransformTable in project symmetric-ds by JumpMind.

the class TransformWriter method toMap.

protected Map<String, List<TransformTable>> toMap(TransformTable[] transforms) {
    Map<String, List<TransformTable>> transformsByTable = new HashMap<String, List<TransformTable>>();
    if (transforms != null) {
        for (TransformTable transformTable : transforms) {
            if (transformPoint == transformTable.getTransformPoint()) {
                String sourceTableName = transformTable.getFullyQualifiedSourceTableName().toLowerCase();
                List<TransformTable> tables = transformsByTable.get(sourceTableName);
                if (tables == null) {
                    tables = new ArrayList<TransformTable>();
                    transformsByTable.put(sourceTableName, tables);
                }
                tables.add(transformTable);
            }
        }
    }
    return transformsByTable;
}

Example 5 with TransformTable

use of org.jumpmind.symmetric.io.data.transform.TransformTable in project symmetric-ds by JumpMind.

the class DataLoaderService method buildDataWriter.

protected IDataWriter buildDataWriter(ProcessInfo processInfo, String sourceNodeId, String channelId, long batchId, boolean isRetry) {
    TransformTable[] transforms = null;
    NodeGroupLink link = null;
    List<ResolvedData> resolvedDatas = new ArrayList<ResolvedData>();
    List<IDatabaseWriterFilter> filters = extensionService.getExtensionPointList(IDatabaseWriterFilter.class);
    List<IDatabaseWriterFilter> dynamicFilters = filters;
    List<IDatabaseWriterErrorHandler> errorHandlers = extensionService.getExtensionPointList(IDatabaseWriterErrorHandler.class);
    List<IDatabaseWriterErrorHandler> dynamicErrorHandlers = errorHandlers;
    if (sourceNodeId != null) {
        Node sourceNode = nodeService.findNode(sourceNodeId, true);
        if (sourceNode != null) {
            link = new NodeGroupLink(sourceNode.getNodeGroupId(), parameterService.getNodeGroupId());
        }
        Map<LoadFilterType, Map<String, List<LoadFilter>>> loadFilters = loadFilterService.findLoadFiltersFor(link, true);
        List<DynamicDatabaseWriterFilter> databaseWriterFilters = DynamicDatabaseWriterFilter.getDatabaseWriterFilters(engine, loadFilters);
        if (loadFilters != null && loadFilters.size() > 0) {
            dynamicFilters = new ArrayList<IDatabaseWriterFilter>(filters.size() + 1);
            dynamicFilters.addAll(filters);
            dynamicFilters.addAll(databaseWriterFilters);
            dynamicErrorHandlers = new ArrayList<IDatabaseWriterErrorHandler>(errorHandlers.size() + 1);
            dynamicErrorHandlers.addAll(errorHandlers);
            dynamicErrorHandlers.addAll(databaseWriterFilters);
        }
        List<TransformTableNodeGroupLink> transformsList = transformService.findTransformsFor(link, TransformPoint.LOAD);
        transforms = transformsList != null ? transformsList.toArray(new TransformTable[transformsList.size()]) : null;
        if (isRetry) {
            List<IncomingError> incomingErrors = getIncomingErrors(batchId, sourceNodeId);
            for (IncomingError incomingError : incomingErrors) {
                if (incomingError.isResolveIgnore() || StringUtils.isNotBlank(incomingError.getResolveData())) {
                    resolvedDatas.add(new ResolvedData(incomingError.getFailedRowNumber(), incomingError.getResolveData(), incomingError.isResolveIgnore()));
                }
            }
        }
    }
    TransformWriter transformWriter = new TransformWriter(platform, TransformPoint.LOAD, null, transformService.getColumnTransforms(), transforms);
    IDataWriter targetWriter = getFactory(channelId).getDataWriter(sourceNodeId, symmetricDialect, transformWriter, dynamicFilters, dynamicErrorHandlers, getConflictSettingsNodeGroupLinks(link, false), resolvedDatas);
    transformWriter.setNestedWriter(new ProcessInfoDataWriter(targetWriter, processInfo));
    return transformWriter;
}