Search in sources :

Example 1 with ProtocolDataReader

use of org.jumpmind.symmetric.io.data.reader.ProtocolDataReader in project symmetric-ds by JumpMind.

the class DataExtractorService method sendOutgoingBatch.

protected OutgoingBatch sendOutgoingBatch(ProcessInfo processInfo, Node targetNode, OutgoingBatch currentBatch, boolean isRetry, IDataWriter dataWriter, BufferedWriter writer, ExtractMode mode) {
    if (currentBatch.getStatus() != Status.OK || ExtractMode.EXTRACT_ONLY == mode) {
        currentBatch.setSentCount(currentBatch.getSentCount() + 1);
        long ts = System.currentTimeMillis();
        IStagedResource extractedBatch = getStagedResource(currentBatch);
        if (extractedBatch != null) {
            if (mode == ExtractMode.FOR_SYM_CLIENT && writer != null) {
                if (!isRetry && parameterService.is(ParameterConstants.OUTGOING_BATCH_COPY_TO_INCOMING_STAGING) && !parameterService.is(ParameterConstants.NODE_OFFLINE, false)) {
                    ISymmetricEngine targetEngine = AbstractSymmetricEngine.findEngineByUrl(targetNode.getSyncUrl());
                    if (targetEngine != null && extractedBatch.isFileResource()) {
                        try {
                            Node sourceNode = nodeService.findIdentity();
                            IStagedResource targetResource = targetEngine.getStagingManager().create(Constants.STAGING_CATEGORY_INCOMING, Batch.getStagedLocation(false, sourceNode.getNodeId()), currentBatch.getBatchId());
                            SymmetricUtils.copyFile(extractedBatch.getFile(), targetResource.getFile());
                            targetResource.setState(State.DONE);
                            isRetry = true;
                        } catch (Exception e) {
                            throw new RuntimeException(e);
                        }
                    }
                }
                Channel channel = configurationService.getChannel(currentBatch.getChannelId());
                DataContext ctx = new DataContext();
                transferFromStaging(mode, BatchType.EXTRACT, currentBatch, isRetry, extractedBatch, writer, ctx, channel.getMaxKBytesPerSecond());
            } else {
                IDataReader dataReader = new ProtocolDataReader(BatchType.EXTRACT, currentBatch.getNodeId(), extractedBatch);
                DataContext ctx = new DataContext();
                ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode);
                ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE, nodeService.findIdentity());
                new DataProcessor(dataReader, new ProcessInfoDataWriter(dataWriter, processInfo), "send from stage").process(ctx);
                if (dataReader.getStatistics().size() > 0) {
                    Statistics stats = dataReader.getStatistics().values().iterator().next();
                    statisticManager.incrementDataSent(currentBatch.getChannelId(), stats.get(DataReaderStatistics.READ_RECORD_COUNT));
                    long byteCount = stats.get(DataReaderStatistics.READ_BYTE_COUNT);
                    statisticManager.incrementDataBytesSent(currentBatch.getChannelId(), byteCount);
                } else {
                    log.warn("Could not find recorded statistics for batch {}", currentBatch.getNodeBatchId());
                }
            }
        } else {
            throw new IllegalStateException(String.format("Could not find the staged resource for batch %s", currentBatch.getNodeBatchId()));
        }
        currentBatch = requeryIfEnoughTimeHasPassed(ts, currentBatch);
    }
    return currentBatch;
}
Also used : IDataReader(org.jumpmind.symmetric.io.data.IDataReader) Node(org.jumpmind.symmetric.model.Node) NodeChannel(org.jumpmind.symmetric.model.NodeChannel) Channel(org.jumpmind.symmetric.model.Channel) ISymmetricEngine(org.jumpmind.symmetric.ISymmetricEngine) DataProcessor(org.jumpmind.symmetric.io.data.DataProcessor) Statistics(org.jumpmind.util.Statistics) DataReaderStatistics(org.jumpmind.symmetric.io.data.reader.DataReaderStatistics) CancellationException(java.util.concurrent.CancellationException) SymmetricException(org.jumpmind.symmetric.SymmetricException) SQLException(java.sql.SQLException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) ProtocolException(org.jumpmind.symmetric.io.data.ProtocolException) TimeoutException(java.util.concurrent.TimeoutException) IoException(org.jumpmind.exception.IoException) DataContext(org.jumpmind.symmetric.io.data.DataContext) IStagedResource(org.jumpmind.symmetric.io.stage.IStagedResource) ProtocolDataReader(org.jumpmind.symmetric.io.data.reader.ProtocolDataReader) ProcessInfoDataWriter(org.jumpmind.symmetric.model.ProcessInfoDataWriter)

Example 2 with ProtocolDataReader

use of org.jumpmind.symmetric.io.data.reader.ProtocolDataReader in project symmetric-ds by JumpMind.

the class DataLoaderService method loadDataFromTransport.

/**
     * Load database from input stream and return a list of batch statuses. This
     * is used for a pull request that responds with data, and the
     * acknowledgment is sent later.
     */
protected List<IncomingBatch> loadDataFromTransport(final ProcessInfo processInfo, final Node sourceNode, IIncomingTransport transport, OutputStream out) throws IOException {
    final ManageIncomingBatchListener listener = new ManageIncomingBatchListener();
    final DataContext ctx = new DataContext();
    Throwable error = null;
    try {
        Node targetNode = nodeService.findIdentity();
        ctx.put(Constants.DATA_CONTEXT_ENGINE, engine);
        if (targetNode != null) {
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode);
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_ID, targetNode.getNodeId());
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_GROUP_ID, targetNode.getNodeGroupId());
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_EXTERNAL_ID, targetNode.getExternalId());
        }
        if (sourceNode != null) {
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE, sourceNode);
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_ID, sourceNode.getNodeId());
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_GROUP_ID, sourceNode.getNodeGroupId());
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_EXTERNAL_ID, sourceNode.getExternalId());
        }
        for (ILoadSyncLifecycleListener l : extensionService.getExtensionPointList(ILoadSyncLifecycleListener.class)) {
            l.syncStarted(ctx);
        }
        long memoryThresholdInBytes = parameterService.getLong(ParameterConstants.STREAM_TO_FILE_THRESHOLD);
        String targetNodeId = nodeService.findIdentityNodeId();
        if (parameterService.is(ParameterConstants.STREAM_TO_FILE_ENABLED)) {
            processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
            ExecutorService executor = Executors.newFixedThreadPool(1, new CustomizableThreadFactory(String.format("dataloader-%s-%s", sourceNode.getNodeGroupId(), sourceNode.getNodeId())));
            LoadIntoDatabaseOnArrivalListener loadListener = new LoadIntoDatabaseOnArrivalListener(processInfo, sourceNode.getNodeId(), listener, executor);
            new SimpleStagingDataWriter(transport.openReader(), stagingManager, Constants.STAGING_CATEGORY_INCOMING, memoryThresholdInBytes, BatchType.LOAD, targetNodeId, ctx, loadListener).process();
            /* Previously submitted tasks will still be executed */
            executor.shutdown();
            OutputStreamWriter outWriter = null;
            if (out != null) {
                outWriter = new OutputStreamWriter(out, IoConstants.ENCODING);
                long keepAliveMillis = parameterService.getLong(ParameterConstants.DATA_LOADER_SEND_ACK_KEEPALIVE);
                while (!executor.awaitTermination(keepAliveMillis, TimeUnit.MILLISECONDS)) {
                    outWriter.write("1=1&");
                    outWriter.flush();
                }
            } else {
                executor.awaitTermination(12, TimeUnit.HOURS);
            }
            loadListener.isDone();
        } else {
            DataProcessor processor = new DataProcessor(new ProtocolDataReader(BatchType.LOAD, targetNodeId, transport.openReader()), null, listener, "data load") {

                @Override
                protected IDataWriter chooseDataWriter(Batch batch) {
                    return buildDataWriter(processInfo, sourceNode.getNodeId(), batch.getChannelId(), batch.getBatchId(), ((ManageIncomingBatchListener) listener).getCurrentBatch().isRetry());
                }
            };
            processor.process(ctx);
        }
    } catch (Throwable ex) {
        error = ex;
        logAndRethrow(ex);
    } finally {
        transport.close();
        for (ILoadSyncLifecycleListener l : extensionService.getExtensionPointList(ILoadSyncLifecycleListener.class)) {
            l.syncEnded(ctx, listener.getBatchesProcessed(), error);
        }
    }
    return listener.getBatchesProcessed();
}
Also used : CustomizableThreadFactory(org.jumpmind.util.CustomizableThreadFactory) ILoadSyncLifecycleListener(org.jumpmind.symmetric.load.ILoadSyncLifecycleListener) Node(org.jumpmind.symmetric.model.Node) SimpleStagingDataWriter(org.jumpmind.symmetric.io.data.writer.SimpleStagingDataWriter) DataProcessor(org.jumpmind.symmetric.io.data.DataProcessor) DataContext(org.jumpmind.symmetric.io.data.DataContext) IncomingBatch(org.jumpmind.symmetric.model.IncomingBatch) Batch(org.jumpmind.symmetric.io.data.Batch) ExecutorService(java.util.concurrent.ExecutorService) OutputStreamWriter(java.io.OutputStreamWriter) ProtocolDataReader(org.jumpmind.symmetric.io.data.reader.ProtocolDataReader)

Example 3 with ProtocolDataReader

use of org.jumpmind.symmetric.io.data.reader.ProtocolDataReader in project symmetric-ds by JumpMind.

the class StagingDataWriterTest method readThenWrite.

public void readThenWrite(long threshold) throws Exception {
    InputStreamReader is = new InputStreamReader(getClass().getResourceAsStream("FileCsvDataWriterTest.1.csv"));
    String origCsv = IOUtils.toString(is);
    is.close();
    StagingManager stagingManager = new StagingManager(DIR.getAbsolutePath());
    ProtocolDataReader reader = new ProtocolDataReader(BatchType.LOAD, "test", origCsv);
    StagingDataWriter writer = new StagingDataWriter(threshold, false, "aaa", "test", stagingManager, new BatchListener());
    DataProcessor processor = new DataProcessor(reader, writer, "test");
    processor.process(new DataContext());
    assertEquals(1, batchesWritten.size());
    assertEquals(convertEol(origCsv), convertEol(batchesWritten.get(0)));
    IStagedResource resource = (IStagedResource) stagingManager.find("test", "aaa", 1);
    assertNotNull(resource);
    if (threshold > origCsv.length()) {
        assertFalse(resource.getFile().exists());
    } else {
        assertTrue(resource.getFile().exists());
    }
    resource.delete();
    assertFalse(resource.getFile().exists());
}
Also used : DataContext(org.jumpmind.symmetric.io.data.DataContext) InputStreamReader(java.io.InputStreamReader) IStagedResource(org.jumpmind.symmetric.io.stage.IStagedResource) ProtocolDataReader(org.jumpmind.symmetric.io.data.reader.ProtocolDataReader) DataProcessor(org.jumpmind.symmetric.io.data.DataProcessor) StagingManager(org.jumpmind.symmetric.io.stage.StagingManager)

Aggregations

DataContext (org.jumpmind.symmetric.io.data.DataContext)3 DataProcessor (org.jumpmind.symmetric.io.data.DataProcessor)3 ProtocolDataReader (org.jumpmind.symmetric.io.data.reader.ProtocolDataReader)3 IStagedResource (org.jumpmind.symmetric.io.stage.IStagedResource)2 Node (org.jumpmind.symmetric.model.Node)2 IOException (java.io.IOException)1 InputStreamReader (java.io.InputStreamReader)1 OutputStreamWriter (java.io.OutputStreamWriter)1 SQLException (java.sql.SQLException)1 CancellationException (java.util.concurrent.CancellationException)1 ExecutionException (java.util.concurrent.ExecutionException)1 ExecutorService (java.util.concurrent.ExecutorService)1 TimeoutException (java.util.concurrent.TimeoutException)1 IoException (org.jumpmind.exception.IoException)1 ISymmetricEngine (org.jumpmind.symmetric.ISymmetricEngine)1 SymmetricException (org.jumpmind.symmetric.SymmetricException)1 Batch (org.jumpmind.symmetric.io.data.Batch)1 IDataReader (org.jumpmind.symmetric.io.data.IDataReader)1 ProtocolException (org.jumpmind.symmetric.io.data.ProtocolException)1 DataReaderStatistics (org.jumpmind.symmetric.io.data.reader.DataReaderStatistics)1