Examples with SimpleStagingDataWriter org.jumpmind.symmetric.io.data.writer.SimpleStagingDataWriter used on opensource projects

Example 1 with SimpleStagingDataWriter

use of org.jumpmind.symmetric.io.data.writer.SimpleStagingDataWriter in project symmetric-ds by JumpMind.

the class DataLoaderService method loadDataFromTransport.

/**
     * Load database from input stream and return a list of batch statuses. This
     * is used for a pull request that responds with data, and the
     * acknowledgment is sent later.
     */
protected List<IncomingBatch> loadDataFromTransport(final ProcessInfo processInfo, final Node sourceNode, IIncomingTransport transport, OutputStream out) throws IOException {
    final ManageIncomingBatchListener listener = new ManageIncomingBatchListener();
    final DataContext ctx = new DataContext();
    Throwable error = null;
    try {
        Node targetNode = nodeService.findIdentity();
        ctx.put(Constants.DATA_CONTEXT_ENGINE, engine);
        if (targetNode != null) {
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode);
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_ID, targetNode.getNodeId());
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_GROUP_ID, targetNode.getNodeGroupId());
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_EXTERNAL_ID, targetNode.getExternalId());
        }
        if (sourceNode != null) {
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE, sourceNode);
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_ID, sourceNode.getNodeId());
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_GROUP_ID, sourceNode.getNodeGroupId());
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_EXTERNAL_ID, sourceNode.getExternalId());
        }
        for (ILoadSyncLifecycleListener l : extensionService.getExtensionPointList(ILoadSyncLifecycleListener.class)) {
            l.syncStarted(ctx);
        }
        long memoryThresholdInBytes = parameterService.getLong(ParameterConstants.STREAM_TO_FILE_THRESHOLD);
        String targetNodeId = nodeService.findIdentityNodeId();
        if (parameterService.is(ParameterConstants.STREAM_TO_FILE_ENABLED)) {
            processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
            ExecutorService executor = Executors.newFixedThreadPool(1, new CustomizableThreadFactory(String.format("dataloader-%s-%s", sourceNode.getNodeGroupId(), sourceNode.getNodeId())));
            LoadIntoDatabaseOnArrivalListener loadListener = new LoadIntoDatabaseOnArrivalListener(processInfo, sourceNode.getNodeId(), listener, executor);
            new SimpleStagingDataWriter(transport.openReader(), stagingManager, Constants.STAGING_CATEGORY_INCOMING, memoryThresholdInBytes, BatchType.LOAD, targetNodeId, ctx, loadListener).process();
            /* Previously submitted tasks will still be executed */
            executor.shutdown();
            OutputStreamWriter outWriter = null;
            if (out != null) {
                outWriter = new OutputStreamWriter(out, IoConstants.ENCODING);
                long keepAliveMillis = parameterService.getLong(ParameterConstants.DATA_LOADER_SEND_ACK_KEEPALIVE);
                while (!executor.awaitTermination(keepAliveMillis, TimeUnit.MILLISECONDS)) {
                    outWriter.write("1=1&");
                    outWriter.flush();
                }
            } else {
                executor.awaitTermination(12, TimeUnit.HOURS);
            }
            loadListener.isDone();
        } else {
            DataProcessor processor = new DataProcessor(new ProtocolDataReader(BatchType.LOAD, targetNodeId, transport.openReader()), null, listener, "data load") {

                @Override
                protected IDataWriter chooseDataWriter(Batch batch) {
                    return buildDataWriter(processInfo, sourceNode.getNodeId(), batch.getChannelId(), batch.getBatchId(), ((ManageIncomingBatchListener) listener).getCurrentBatch().isRetry());
                }
            };
            processor.process(ctx);
        }
    } catch (Throwable ex) {
        error = ex;
        logAndRethrow(ex);
    } finally {
        transport.close();
        for (ILoadSyncLifecycleListener l : extensionService.getExtensionPointList(ILoadSyncLifecycleListener.class)) {
            l.syncEnded(ctx, listener.getBatchesProcessed(), error);
        }
    }
    return listener.getBatchesProcessed();
}