Examples with OutgoingBatch org.jumpmind.symmetric.model.OutgoingBatch used on opensource projects

Example 26 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class DataExtractorService method extractOnlyOutgoingBatch.

/**
     * This method will extract an outgoing batch, but will not update the outgoing batch status
     */
public boolean extractOnlyOutgoingBatch(String nodeId, long batchId, Writer writer) {
    boolean extracted = false;
    Node targetNode = null;
    if (Constants.UNROUTED_NODE_ID.equals(nodeId)) {
        targetNode = new Node(nodeId, parameterService.getNodeGroupId());
    } else {
        targetNode = nodeService.findNode(nodeId);
    }
    if (targetNode != null) {
        OutgoingBatch batch = outgoingBatchService.findOutgoingBatch(batchId, nodeId);
        if (batch != null) {
            IDataWriter dataWriter = new ProtocolDataWriter(nodeService.findIdentityNodeId(), writer, targetNode.requires13Compatiblity());
            List<OutgoingBatch> batches = new ArrayList<OutgoingBatch>(1);
            batches.add(batch);
            batches = extract(new ProcessInfo(), targetNode, batches, dataWriter, null, ExtractMode.EXTRACT_ONLY);
            extracted = batches.size() > 0;
        }
    }
    return extracted;
}

Example 27 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class FileSyncService method sendFiles.

public List<OutgoingBatch> sendFiles(ProcessInfo processInfo, Node targetNode, IOutgoingTransport outgoingTransport) {
    List<OutgoingBatch> batchesToProcess = getBatchesToProcess(targetNode);
    if (batchesToProcess.isEmpty()) {
        return batchesToProcess;
    }
    IStagingManager stagingManager = engine.getStagingManager();
    long maxBytesToSync = parameterService.getLong(ParameterConstants.TRANSPORT_MAX_BYTES_TO_SYNC);
    List<OutgoingBatch> processedBatches = new ArrayList<OutgoingBatch>();
    OutgoingBatch currentBatch = null;
    IStagedResource stagedResource = null;
    IStagedResource previouslyStagedResource = null;
    FileSyncZipDataWriter dataWriter = null;
    try {
        long syncedBytes = 0;
        try {
            for (int i = 0; i < batchesToProcess.size(); i++) {
                currentBatch = batchesToProcess.get(i);
                previouslyStagedResource = getStagedResource(currentBatch);
                if (isWaitForExtractionRequired(currentBatch, previouslyStagedResource) || isFlushBatchesRequired(currentBatch, processedBatches, previouslyStagedResource)) {
                    // previously staged batch will have to wait for the next push/pull.
                    break;
                }
                if (previouslyStagedResource != null) {
                    log.debug("Using existing extraction for file sync batch {}", currentBatch.getNodeBatchId());
                    stagedResource = previouslyStagedResource;
                } else {
                    if (dataWriter == null) {
                        stagedResource = stagingManager.create(Constants.STAGING_CATEGORY_OUTGOING, processInfo.getSourceNodeId(), targetNode.getNodeId(), "filesync.zip");
                        dataWriter = new FileSyncZipDataWriter(maxBytesToSync, this, engine.getNodeService(), stagedResource);
                    }
                    log.debug("Extracting batch {} for filesync.", currentBatch.getNodeBatchId());
                    ((DataExtractorService) engine.getDataExtractorService()).extractOutgoingBatch(processInfo, targetNode, dataWriter, currentBatch, false, true, DataExtractorService.ExtractMode.FOR_SYM_CLIENT);
                }
                processedBatches.add(currentBatch);
                syncedBytes += stagedResource.getSize();
                processInfo.incrementBatchCount();
                processInfo.setCurrentBatchId(currentBatch.getBatchId());
                log.debug("Processed file sync batch {}. syncedBytes={}, maxBytesToSync={}", currentBatch, syncedBytes, maxBytesToSync);
                /*
                     * check to see if max bytes to sync has been reached and
                     * stop processing batches
                     */
                if (previouslyStagedResource != null || dataWriter.readyToSend()) {
                    break;
                }
            }
        } finally {
            if (dataWriter != null) {
                dataWriter.finish();
            }
        }
        processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
        for (OutgoingBatch outgoingBatch : processedBatches) {
            outgoingBatch.setStatus(Status.SE);
        }
        engine.getOutgoingBatchService().updateOutgoingBatches(processedBatches);
        try {
            if (stagedResource != null && stagedResource.exists()) {
                InputStream is = stagedResource.getInputStream();
                try {
                    OutputStream os = outgoingTransport.openStream();
                    IOUtils.copy(is, os);
                    os.flush();
                } catch (IOException e) {
                    throw new IoException(e);
                }
            }
            for (int i = 0; i < batchesToProcess.size(); i++) {
                batchesToProcess.get(i).setStatus(Status.LD);
            }
            engine.getOutgoingBatchService().updateOutgoingBatches(batchesToProcess);
        } finally {
            if (stagedResource != null) {
                stagedResource.close();
            }
        }
    } catch (RuntimeException e) {
        if (stagedResource == previouslyStagedResource) {
            // on error, don't let the load extract be deleted.
            stagedResource = null;
        }
        if (currentBatch != null) {
            engine.getStatisticManager().incrementDataExtractedErrors(currentBatch.getChannelId(), 1);
            currentBatch.setSqlMessage(getRootMessage(e));
            currentBatch.revertStatsOnError();
            if (currentBatch.getStatus() != Status.IG) {
                currentBatch.setStatus(Status.ER);
            }
            currentBatch.setErrorFlag(true);
            engine.getOutgoingBatchService().updateOutgoingBatch(currentBatch);
            if (isStreamClosedByClient(e)) {
                log.warn("Failed to extract file sync batch {}.  The stream was closed by the client.  The error was: {}", currentBatch, getRootMessage(e));
            } else {
                log.error("Failed to extract file sync batch " + currentBatch, e);
            }
        } else {
            log.error("Could not log the outgoing batch status because the batch was null", e);
        }
        throw e;
    } finally {
        if (stagedResource != null) {
            stagedResource.delete();
        }
    }
    return processedBatches;
}

Example 28 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class FileSyncService method getBatchesToProcess.

protected List<OutgoingBatch> getBatchesToProcess(Node targetNode) {
    List<OutgoingBatch> batchesToProcess = new ArrayList<OutgoingBatch>();
    List<Channel> fileSyncChannels = engine.getConfigurationService().getFileSyncChannels();
    OutgoingBatches batches = engine.getOutgoingBatchService().getOutgoingBatches(targetNode.getNodeId(), false);
    for (Channel channel : fileSyncChannels) {
        batchesToProcess.addAll(batches.filterBatchesForChannel(channel));
    }
    return batchesToProcess;
}

Example 29 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class MultiBatchStagingWriter method nextBatch.

protected void nextBatch() {
    if (this.outgoingBatch != null) {
        this.finishedBatches.add(outgoingBatch);
        rowCount += this.outgoingBatch.getDataEventCount();
        byteCount += this.outgoingBatch.getByteCount();
    }
    this.outgoingBatch = this.batches.remove(0);
    this.outgoingBatch.setDataEventCount(0);
    this.outgoingBatch.setInsertEventCount(0);
    if (this.finishedBatches.size() > 0) {
        this.outgoingBatch.setExtractCount(this.outgoingBatch.getExtractCount() + 1);
    }
    /*
         * Update the last update time so the batch 
         * isn't purged prematurely
         */
    for (OutgoingBatch batch : finishedBatches) {
        IStagedResource resource = this.dataExtractorService.getStagedResource(batch);
        if (resource != null) {
            resource.refreshLastUpdateTime();
        }
    }
}

Example 30 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class PushService method pushToNode.

private void pushToNode(Node remote, RemoteNodeStatus status) {
    Node identity = nodeService.findIdentity();
    NodeSecurity identitySecurity = nodeService.findNodeSecurity(identity.getNodeId(), true);
    IOutgoingWithResponseTransport transport = null;
    ProcessInfo processInfo = statisticManager.newProcessInfo(new ProcessInfoKey(identity.getNodeId(), status.getChannelId(), remote.getNodeId(), ProcessType.PUSH_JOB));
    Map<String, String> requestProperties = new HashMap<String, String>();
    requestProperties.put(WebConstants.THREAD_CHANNEL, status.getChannelId());
    try {
        transport = transportManager.getPushTransport(remote, identity, identitySecurity.getNodePassword(), requestProperties, parameterService.getRegistrationUrl());
        List<OutgoingBatch> extractedBatches = dataExtractorService.extract(processInfo, remote, status.getChannelId(), transport);
        if (extractedBatches.size() > 0) {
            log.info("Push data sent to {}", remote);
            List<BatchAck> batchAcks = readAcks(extractedBatches, transport, transportManager, acknowledgeService);
            status.updateOutgoingStatus(extractedBatches, batchAcks);
        }
        if (processInfo.getStatus() != Status.ERROR) {
            processInfo.setStatus(Status.OK);
        }
        fireOnline(remote, status);
    } catch (Exception ex) {
        processInfo.setStatus(Status.ERROR);
        fireOffline(ex, remote, status);
    } finally {
        try {
            transport.close();
        } catch (Exception e) {
        }
    }
}