Examples with BatchBufferedWriter org.jumpmind.symmetric.transport.BatchBufferedWriter used on opensource projects

Example 1 with BatchBufferedWriter

use of org.jumpmind.symmetric.transport.BatchBufferedWriter in project symmetric-ds by JumpMind.

the class DataExtractorService method transferFromStaging.

protected void transferFromStaging(ExtractMode mode, BatchType batchType, OutgoingBatch batch, boolean isRetry, IStagedResource stagedResource, BufferedWriter writer, DataContext context, BigDecimal maxKBytesPerSec) {
    final int MAX_WRITE_LENGTH = 32768;
    BufferedReader reader = stagedResource.getReader();
    try {
        // retry the batch from the target's staging
        if (isRetry) {
            String line = null;
            while ((line = reader.readLine()) != null) {
                if (line.startsWith(CsvConstants.BATCH)) {
                    writer.write(CsvConstants.RETRY + "," + batch.getBatchId());
                    writer.newLine();
                    writer.write(CsvConstants.COMMIT + "," + batch.getBatchId());
                    writer.newLine();
                    break;
                } else {
                    writer.write(line);
                    writer.newLine();
                }
            }
        } else {
            long totalCharsRead = 0, totalBytesRead = 0;
            int numCharsRead = 0, numBytesRead = 0;
            long startTime = System.currentTimeMillis(), ts = startTime, bts = startTime;
            boolean isThrottled = maxKBytesPerSec != null && maxKBytesPerSec.compareTo(BigDecimal.ZERO) > 0;
            long totalThrottleTime = 0;
            int bufferSize = MAX_WRITE_LENGTH;
            if (isThrottled) {
                bufferSize = maxKBytesPerSec.multiply(new BigDecimal(1024)).intValue();
            }
            char[] buffer = new char[bufferSize];
            while ((numCharsRead = reader.read(buffer)) != -1) {
                writer.write(buffer, 0, numCharsRead);
                totalCharsRead += numCharsRead;
                if (Thread.currentThread().isInterrupted()) {
                    throw new IoException("This thread was interrupted");
                }
                long batchStatusUpdateMillis = parameterService.getLong(ParameterConstants.OUTGOING_BATCH_UPDATE_STATUS_MILLIS);
                if (System.currentTimeMillis() - ts > batchStatusUpdateMillis && batch.getStatus() != Status.SE && batch.getStatus() != Status.RS) {
                    changeBatchStatus(Status.SE, batch, mode);
                }
                if (System.currentTimeMillis() - ts > 60000) {
                    log.info("Batch '{}', for node '{}', for process 'send from stage' has been processing for {} seconds.  " + "The following stats have been gathered: {}", new Object[] { batch.getBatchId(), batch.getNodeId(), (System.currentTimeMillis() - startTime) / 1000, "CHARS=" + totalCharsRead });
                    ts = System.currentTimeMillis();
                }
                if (isThrottled) {
                    numBytesRead += new String(buffer, 0, numCharsRead).getBytes().length;
                    totalBytesRead += numBytesRead;
                    if (numBytesRead >= bufferSize) {
                        long expectedMillis = (long) (((numBytesRead / 1024f) / maxKBytesPerSec.floatValue()) * 1000);
                        long actualMillis = System.currentTimeMillis() - bts;
                        if (actualMillis < expectedMillis) {
                            totalThrottleTime += expectedMillis - actualMillis;
                            Thread.sleep(expectedMillis - actualMillis);
                        }
                        numBytesRead = 0;
                        bts = System.currentTimeMillis();
                    }
                } else {
                    totalBytesRead += new String(buffer, 0, numCharsRead).getBytes().length;
                }
            }
            statisticManager.incrementDataSent(batch.getChannelId(), batch.getDataEventCount());
            statisticManager.incrementDataBytesSent(batch.getChannelId(), totalBytesRead);
            if (log.isDebugEnabled() && totalThrottleTime > 0) {
                log.debug("Batch '{}' for node '{}' took {}ms for {} bytes and was throttled for {}ms because limit is set to {} KB/s", batch.getBatchId(), batch.getNodeId(), (System.currentTimeMillis() - startTime), totalBytesRead, totalThrottleTime, maxKBytesPerSec);
            }
        }
        if (writer instanceof BatchBufferedWriter) {
            ((BatchBufferedWriter) writer).getBatchIds().add(batch.getBatchId());
        }
    } catch (Throwable t) {
        throw new RuntimeException(t);
    } finally {
        stagedResource.close();
        stagedResource.dereference();
        if (!stagedResource.isFileResource() && !stagedResource.isInUse()) {
            stagedResource.delete();
        }
    }
}