Examples with Statistics org.jumpmind.util.Statistics used on opensource projects

Example 11 with Statistics

use of org.jumpmind.util.Statistics in project symmetric-ds by JumpMind.

the class TransformWriter method write.

public void write(CsvData data) {
    DataEventType eventType = data.getDataEventType();
    if (activeTransforms != null && activeTransforms.size() > 0 && isTransformable(eventType)) {
        if (data.requiresTable() && sourceTable == null && context.getLastParsedTable() != null) {
            // if we cross batches and the table isn't specified, then
            // use the last table we used
            start(context.getLastParsedTable());
        }
        long ts = System.currentTimeMillis();
        Map<String, String> sourceValues = data.toColumnNameValuePairs(this.sourceTable.getColumnNames(), CsvData.ROW_DATA);
        Map<String, String> oldSourceValues = null;
        if (data.contains(CsvData.OLD_DATA)) {
            oldSourceValues = data.toColumnNameValuePairs(this.sourceTable.getColumnNames(), CsvData.OLD_DATA);
        }
        Map<String, String> sourceKeyValues = null;
        if (data.contains(CsvData.PK_DATA)) {
            sourceKeyValues = data.toKeyColumnValuePairs(this.sourceTable);
        }
        if (eventType == DataEventType.DELETE) {
            sourceValues = oldSourceValues;
            if (sourceValues == null || sourceValues.size() == 0) {
                sourceValues = sourceKeyValues;
            }
        }
        if (log.isDebugEnabled()) {
            log.debug("{} transformation(s) started because of {} on {}.  The original row data was: {}", new Object[] { activeTransforms.size(), eventType.toString(), this.sourceTable.getFullyQualifiedTableName(), sourceValues });
        }
        List<TransformedData> dataThatHasBeenTransformed = new ArrayList<TransformedData>();
        TransformTable[] transformTables = activeTransforms.toArray(new TransformTable[activeTransforms.size()]);
        if (eventType == DataEventType.DELETE) {
            CollectionUtils.reverseArray(transformTables);
        }
        for (TransformTable transformation : transformTables) {
            if (eventType == DataEventType.INSERT && transformation.isUpdateFirst()) {
                eventType = DataEventType.UPDATE;
            }
            dataThatHasBeenTransformed.addAll(transform(eventType, context, transformation, sourceKeyValues, oldSourceValues, sourceValues));
        }
        for (TransformedData transformedData : dataThatHasBeenTransformed) {
            Table transformedTable = transformedData.buildTargetTable();
            CsvData csvData = transformedData.buildTargetCsvData();
            long transformTimeInMs = System.currentTimeMillis() - ts;
            boolean processData = true;
            if (lastTransformedTable == null || !lastTransformedTable.equals(transformedTable)) {
                if (lastTransformedTable != null) {
                    this.nestedWriter.end(lastTransformedTable);
                }
                processData = this.nestedWriter.start(transformedTable);
                if (!processData) {
                    lastTransformedTable = null;
                } else {
                    lastTransformedTable = transformedTable;
                }
            }
            if (processData || !csvData.requiresTable()) {
                this.nestedWriter.write(csvData);
            }
            Statistics stats = this.nestedWriter.getStatistics().get(batch);
            if (stats != null) {
                stats.increment(DataWriterStatisticConstants.TRANSFORMMILLIS, transformTimeInMs);
            }
            ts = System.currentTimeMillis();
        }
    } else {
        if (sourceTable != null) {
            super.start(sourceTable);
        }
        super.write(data);
        if (sourceTable != null) {
            super.end(sourceTable);
        }
    }
}

Example 12 with Statistics

use of org.jumpmind.util.Statistics in project symmetric-ds by JumpMind.

the class AbstractProtocolDataWriter method start.

public void start(Batch batch) {
    this.statistics.put(batch, new Statistics());
    this.batch = batch;
    if (listeners != null) {
        for (IProtocolDataWriterListener listener : listeners) {
            listener.start(context, batch);
        }
    }
    if (StringUtils.isBlank(sourceNodeId)) {
        sourceNodeId = batch.getSourceNodeId();
    }
    if (flushNodeId) {
        if (StringUtils.isNotBlank(sourceNodeId)) {
            println(CsvConstants.NODEID, sourceNodeId);
        }
        if (!backwardsCompatible) {
            printBinary();
        }
        flushNodeId = false;
    }
    if (!backwardsCompatible && StringUtils.isNotBlank(batch.getChannelId())) {
        println(CsvConstants.CHANNEL, batch.getChannelId());
    }
    println(CsvConstants.BATCH, Long.toString(batch.getBatchId()));
    if (backwardsCompatible) {
        printBinary();
    }
}

Example 13 with Statistics

use of org.jumpmind.util.Statistics in project symmetric-ds by JumpMind.

the class AbstractWriterTest method writeData.

protected long writeData(IDataWriter writer, TableCsvData... datas) {
    this.lastDataWriterUsed = writer;
    DataContext context = new DataContext();
    writer.open(context);
    try {
        for (TableCsvData tableCsvData : datas) {
            Batch batch = new Batch(BatchType.LOAD, getNextBatchId(), "default", BinaryEncoding.BASE64, "00000", "00001", false);
            try {
                writer.start(batch);
                if (writer.start(tableCsvData.table)) {
                    for (CsvData d : tableCsvData.data) {
                        writer.write(d);
                    }
                    writer.end(tableCsvData.table);
                }
                writer.end(batch, false);
            } catch (IgnoreBatchException ex) {
                writer.end(batch, false);
            } catch (Exception ex) {
                writer.end(batch, true);
                if (!isErrorExpected()) {
                    if (ex instanceof RuntimeException) {
                        throw (RuntimeException) ex;
                    } else {
                        throw new RuntimeException(ex);
                    }
                }
            }
        }
    } finally {
        writer.close();
    }
    long statementCount = 0;
    Collection<Statistics> stats = writer.getStatistics().values();
    for (Statistics statistics : stats) {
        statementCount += statistics.get(DataWriterStatisticConstants.STATEMENTCOUNT);
    }
    return statementCount;
}

Example 14 with Statistics

use of org.jumpmind.util.Statistics in project symmetric-ds by JumpMind.

the class StructureDataWriter method start.

public void start(Batch batch) {
    List<String> payloadData = new ArrayList<String>();
    this.currentBatch = batch.getBatchId();
    this.statistics.put(batch, new Statistics());
    this.payloadMap.put(currentBatch, payloadData);
}

Example 15 with Statistics

use of org.jumpmind.util.Statistics in project symmetric-ds by JumpMind.

the class AbstractDatabaseWriter method start.

public void start(Batch batch) {
    this.batch = batch;
    this.statistics.put(batch, new Statistics());
}