Search in sources :

Example 6 with IDataReader

use of org.jumpmind.symmetric.io.data.IDataReader in project symmetric-ds by JumpMind.

the class DataLoaderService method loadDataFromTransport.

/**
 * Load database from input stream and return a list of batch statuses. This
 * is used for a pull request that responds with data, and the
 * acknowledgment is sent later.
 */
protected List<IncomingBatch> loadDataFromTransport(final ProcessInfo processInfo, final Node sourceNode, IIncomingTransport transport) throws IOException {
    final ManageIncomingBatchListener listener = new ManageIncomingBatchListener();
    final DataContext ctx = new DataContext();
    Throwable error = null;
    try {
        Node targetNode = nodeService.findIdentity();
        ctx.put(Constants.DATA_CONTEXT_ENGINE, engine);
        if (targetNode != null) {
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode);
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_ID, targetNode.getNodeId());
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_GROUP_ID, targetNode.getNodeGroupId());
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_EXTERNAL_ID, targetNode.getExternalId());
        }
        if (sourceNode != null) {
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE, sourceNode);
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_ID, sourceNode.getNodeId());
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_GROUP_ID, sourceNode.getNodeGroupId());
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_EXTERNAL_ID, sourceNode.getExternalId());
        }
        for (ILoadSyncLifecycleListener l : extensionService.getExtensionPointList(ILoadSyncLifecycleListener.class)) {
            l.syncStarted(ctx);
        }
        long memoryThresholdInBytes = parameterService.getLong(ParameterConstants.STREAM_TO_FILE_THRESHOLD);
        long totalNetworkMillis = System.currentTimeMillis();
        String targetNodeId = nodeService.findIdentityNodeId();
        if (parameterService.is(ParameterConstants.STREAM_TO_FILE_ENABLED)) {
            processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
            IDataReader dataReader = new ProtocolDataReader(BatchType.LOAD, targetNodeId, transport.openReader());
            IDataWriter dataWriter = new StagingDataWriter(memoryThresholdInBytes, sourceNode.getNodeId(), Constants.STAGING_CATEGORY_INCOMING, stagingManager, new LoadIntoDatabaseOnArrivalListener(processInfo, sourceNode.getNodeId(), listener));
            new DataProcessor(dataReader, dataWriter, "transfer to stage").process(ctx);
            totalNetworkMillis = System.currentTimeMillis() - totalNetworkMillis;
        } else {
            DataProcessor processor = new DataProcessor(new ProtocolDataReader(BatchType.LOAD, targetNodeId, transport.openReader()), null, listener, "data load") {

                @Override
                protected IDataWriter chooseDataWriter(Batch batch) {
                    return buildDataWriter(processInfo, sourceNode.getNodeId(), batch.getChannelId(), batch.getBatchId());
                }
            };
            processor.process(ctx);
        }
    } catch (Throwable ex) {
        error = ex;
        logAndRethrow(sourceNode, ex);
    } finally {
        transport.close();
        for (ILoadSyncLifecycleListener l : extensionService.getExtensionPointList(ILoadSyncLifecycleListener.class)) {
            l.syncEnded(ctx, listener.getBatchesProcessed(), error);
        }
    }
    return listener.getBatchesProcessed();
}
Also used : IDataReader(org.jumpmind.symmetric.io.data.IDataReader) ILoadSyncLifecycleListener(org.jumpmind.symmetric.load.ILoadSyncLifecycleListener) Node(org.jumpmind.symmetric.model.Node) DataProcessor(org.jumpmind.symmetric.io.data.DataProcessor) DataContext(org.jumpmind.symmetric.io.data.DataContext) IncomingBatch(org.jumpmind.symmetric.model.IncomingBatch) Batch(org.jumpmind.symmetric.io.data.Batch) ProtocolDataReader(org.jumpmind.symmetric.io.data.reader.ProtocolDataReader) StagingDataWriter(org.jumpmind.symmetric.io.data.writer.StagingDataWriter) IDataWriter(org.jumpmind.symmetric.io.data.IDataWriter)

Aggregations

DataContext (org.jumpmind.symmetric.io.data.DataContext)6 DataProcessor (org.jumpmind.symmetric.io.data.DataProcessor)6 IDataReader (org.jumpmind.symmetric.io.data.IDataReader)6 Node (org.jumpmind.symmetric.model.Node)5 ExtractDataReader (org.jumpmind.symmetric.io.data.reader.ExtractDataReader)3 ProtocolDataReader (org.jumpmind.symmetric.io.data.reader.ProtocolDataReader)3 IStagedResource (org.jumpmind.symmetric.io.stage.IStagedResource)3 OutgoingBatch (org.jumpmind.symmetric.model.OutgoingBatch)3 ProcessInfoDataWriter (org.jumpmind.symmetric.model.ProcessInfoDataWriter)3 Statistics (org.jumpmind.util.Statistics)3 Batch (org.jumpmind.symmetric.io.data.Batch)2 ProtocolDataWriter (org.jumpmind.symmetric.io.data.writer.ProtocolDataWriter)2 StagingDataWriter (org.jumpmind.symmetric.io.data.writer.StagingDataWriter)2 ProcessInfo (org.jumpmind.symmetric.model.ProcessInfo)2 IOException (java.io.IOException)1 SQLException (java.sql.SQLException)1 CancellationException (java.util.concurrent.CancellationException)1 ExecutionException (java.util.concurrent.ExecutionException)1 Semaphore (java.util.concurrent.Semaphore)1 TimeoutException (java.util.concurrent.TimeoutException)1