Examples with IgnoreBatchException org.jumpmind.symmetric.io.data.writer.IgnoreBatchException used on opensource projects

Example 1 with IgnoreBatchException

use of org.jumpmind.symmetric.io.data.writer.IgnoreBatchException in project symmetric-ds by JumpMind.

the class DataProcessor method forEachDataInTable.

protected int forEachDataInTable(DataContext context, boolean processTable, Batch batch) {
    int dataRow = 0;
    IgnoreBatchException ignore = null;
    long startTime = System.currentTimeMillis();
    long ts = System.currentTimeMillis();
    do {
        batch.startTimer(STAT_READ_DATA);
        currentData = dataReader.nextData();
        context.setData(currentData);
        batch.incrementDataReadMillis(batch.endTimer(STAT_READ_DATA));
        if (currentData != null) {
            dataRow++;
            if (processTable || !currentData.requiresTable()) {
                try {
                    batch.startTimer(STAT_WRITE_DATA);
                    batch.incrementLineCount();
                    context.getWriter().write(currentData);
                    batch.incrementDataWriteMillis(batch.endTimer(STAT_WRITE_DATA));
                } catch (IgnoreBatchException ex) {
                    ignore = ex;
                    processTable = false;
                }
            }
        }
        if (System.currentTimeMillis() - ts > 60000 && context.getWriter() != null) {
            Statistics stats = context.getWriter().getStatistics().get(batch);
            if (stats != null) {
                log.info("Batch '{}', for node '{}', for process '{}' has been processing for {} seconds.  The following stats have been gathered: {}", new Object[] { batch.getBatchId(), batch.getTargetNodeId(), name, (System.currentTimeMillis() - startTime) / 1000, stats.toString() });
            }
            ts = System.currentTimeMillis();
        }
        if (Thread.currentThread().isInterrupted()) {
            throw new IoException("This thread was interrupted");
        }
    } while (currentData != null);
    if (ignore != null) {
        throw ignore;
    }
    return dataRow;
}

Example 2 with IgnoreBatchException

use of org.jumpmind.symmetric.io.data.writer.IgnoreBatchException in project symmetric-ds by JumpMind.

the class AbstractWriterTest method writeData.

protected long writeData(IDataWriter writer, TableCsvData... datas) {
    this.lastDataWriterUsed = writer;
    DataContext context = new DataContext();
    writer.open(context);
    try {
        for (TableCsvData tableCsvData : datas) {
            Batch batch = new Batch(BatchType.LOAD, getNextBatchId(), "default", BinaryEncoding.BASE64, "00000", "00001", false);
            try {
                writer.start(batch);
                if (writer.start(tableCsvData.table)) {
                    for (CsvData d : tableCsvData.data) {
                        writer.write(d);
                    }
                    writer.end(tableCsvData.table);
                }
                writer.end(batch, false);
            } catch (IgnoreBatchException ex) {
                writer.end(batch, false);
            } catch (Exception ex) {
                writer.end(batch, true);
                if (!isErrorExpected()) {
                    if (ex instanceof RuntimeException) {
                        throw (RuntimeException) ex;
                    } else {
                        throw new RuntimeException(ex);
                    }
                }
            }
        }
    } finally {
        writer.close();
    }
    long statementCount = 0;
    Collection<Statistics> stats = writer.getStatistics().values();
    for (Statistics statistics : stats) {
        statementCount += statistics.get(DataWriterStatisticConstants.STATEMENTCOUNT);
    }
    return statementCount;
}