Examples with OutgoingBatch org.jumpmind.symmetric.model.OutgoingBatch used on opensource projects

Example 16 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class OutgoingBatchService method markAllAsSentForNode.

public void markAllAsSentForNode(String nodeId, boolean includeConfigChannel) {
    OutgoingBatches batches = null;
    int configCount;
    do {
        configCount = 0;
        batches = getOutgoingBatches(nodeId, true);
        List<OutgoingBatch> list = batches.getBatches();
        /*
             * Sort in reverse order so we don't get fk errors for batches that
             * are currently processing. We don't make the update transactional
             * to prevent contention in highly loaded systems
             */
        Collections.sort(list, new Comparator<OutgoingBatch>() {

            public int compare(OutgoingBatch o1, OutgoingBatch o2) {
                return -new Long(o1.getBatchId()).compareTo(o2.getBatchId());
            }
        });
        for (OutgoingBatch outgoingBatch : list) {
            if (includeConfigChannel || !outgoingBatch.getChannelId().equals(Constants.CHANNEL_CONFIG)) {
                outgoingBatch.setStatus(Status.OK);
                outgoingBatch.setErrorFlag(false);
                updateOutgoingBatch(outgoingBatch);
            } else {
                configCount++;
            }
        }
    } while (batches.getBatches().size() > configCount);
}

Example 17 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class OutgoingBatchService method getBatchesForChannelWindows.

public List<OutgoingBatch> getBatchesForChannelWindows(OutgoingBatches batches, String targetNodeId, NodeChannel channel, List<NodeGroupChannelWindow> windows) {
    List<OutgoingBatch> keeping = new ArrayList<OutgoingBatch>();
    List<OutgoingBatch> current = batches.getBatches();
    if (current != null && current.size() > 0) {
        if (inTimeWindow(windows, targetNodeId)) {
            int maxBatchesToSend = channel.getMaxBatchToSend();
            for (OutgoingBatch outgoingBatch : current) {
                if (channel.getChannelId().equals(outgoingBatch.getChannelId()) && maxBatchesToSend > 0) {
                    keeping.add(outgoingBatch);
                    maxBatchesToSend--;
                }
            }
        }
    }
    return keeping;
}

Example 18 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class DataService method insertDataEventAndOutgoingBatch.

protected long insertDataEventAndOutgoingBatch(ISqlTransaction transaction, long dataId, String channelId, String nodeId, DataEventType eventType, String routerId, boolean isLoad, long loadId, String createBy, Status status, String tableName) {
    OutgoingBatch outgoingBatch = new OutgoingBatch(nodeId, channelId, status);
    outgoingBatch.setLoadId(loadId);
    outgoingBatch.setCreateBy(createBy);
    outgoingBatch.setLoadFlag(isLoad);
    outgoingBatch.incrementEventCount(eventType);
    if (tableName != null) {
        outgoingBatch.incrementTableCount(tableName.toLowerCase());
    }
    if (status == Status.RQ) {
        outgoingBatch.setExtractJobFlag(true);
    }
    engine.getOutgoingBatchService().insertOutgoingBatch(transaction, outgoingBatch);
    insertDataEvent(transaction, new DataEvent(dataId, outgoingBatch.getBatchId(), routerId));
    return outgoingBatch.getBatchId();
}

Example 19 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class FileSyncService method pushFilesToNode.

protected void pushFilesToNode(NodeCommunication nodeCommunication, RemoteNodeStatus status, Node identity, NodeSecurity security) {
    ProcessInfo processInfo = engine.getStatisticManager().newProcessInfo(new ProcessInfoKey(nodeCommunication.getNodeId(), identity.getNodeId(), ProcessType.FILE_SYNC_PUSH_JOB));
    IOutgoingWithResponseTransport transport = null;
    ITransportManager transportManager = null;
    try {
        if (!engine.getParameterService().is(ParameterConstants.NODE_OFFLINE)) {
            transportManager = engine.getTransportManager();
            transport = transportManager.getFilePushTransport(nodeCommunication.getNode(), identity, security.getNodePassword(), parameterService.getRegistrationUrl());
        } else {
            transportManager = ((AbstractSymmetricEngine) engine).getOfflineTransportManager();
            transport = transportManager.getFilePushTransport(nodeCommunication.getNode(), identity, security.getNodePassword(), parameterService.getRegistrationUrl());
        }
        List<OutgoingBatch> batches = sendFiles(processInfo, nodeCommunication.getNode(), transport);
        if (batches.size() > 0) {
            if (transport instanceof FileOutgoingTransport) {
                ((FileOutgoingTransport) transport).setProcessedBatches(batches);
            }
            List<BatchAck> batchAcks = readAcks(batches, transport, transportManager, engine.getAcknowledgeService());
            status.updateOutgoingStatus(batches, batchAcks);
        }
        if (!status.failed() && batches.size() > 0) {
            log.info("Pushed files to {}. {} files and {} batches were processed", new Object[] { nodeCommunication.getNodeId(), status.getDataProcessed(), status.getBatchesProcessed() });
        } else if (status.failed()) {
            log.info("There was a failure while pushing files to {}. {} files and {} batches were processed", new Object[] { nodeCommunication.getNodeId(), status.getDataProcessed(), status.getBatchesProcessed() });
        }
    } catch (Exception e) {
        fireOffline(e, nodeCommunication.getNode(), status);
    } finally {
        if (processInfo.getStatus() != ProcessInfo.Status.ERROR) {
            processInfo.setStatus(ProcessInfo.Status.OK);
        }
        if (transport != null) {
            transport.close();
            if (transport instanceof FileOutgoingTransport) {
                ((FileOutgoingTransport) transport).complete(processInfo.getStatus() == ProcessInfo.Status.OK);
            }
        }
    }
}

Example 20 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class RouterService method insertDataEvents.

protected int insertDataEvents(ProcessInfo processInfo, ChannelRouterContext context, DataMetaData dataMetaData, Collection<String> nodeIds) {
    int numberOfDataEventsInserted = 0;
    if (nodeIds == null || nodeIds.size() == 0) {
        nodeIds = new HashSet<String>(1);
        nodeIds.add(Constants.UNROUTED_NODE_ID);
    }
    long ts = System.currentTimeMillis();
    long batchIdToReuse = -1;
    boolean dataEventAdded = false;
    for (String nodeId : nodeIds) {
        if (nodeId != null) {
            Map<String, OutgoingBatch> batches = context.getBatchesByNodes();
            OutgoingBatch batch = batches.get(nodeId);
            if (batch == null) {
                batch = new OutgoingBatch(nodeId, dataMetaData.getNodeChannel().getChannelId(), Status.RT);
                batch.setBatchId(batchIdToReuse);
                batch.setCommonFlag(context.isProduceCommonBatches());
                log.debug("About to insert a new batch for node {} on the '{}' channel.  Batches in progress are: {}.", new Object[] { nodeId, batch.getChannelId(), context.getBatchesByNodes().values() });
                engine.getOutgoingBatchService().insertOutgoingBatch(batch);
                processInfo.incrementBatchCount();
                context.getBatchesByNodes().put(nodeId, batch);
                // if in reuse mode, then share the batch id
                if (context.isProduceCommonBatches()) {
                    batchIdToReuse = batch.getBatchId();
                }
            }
            if (dataMetaData.getData().getDataEventType() == DataEventType.RELOAD) {
                long loadId = context.getLastLoadId();
                if (loadId < 0) {
                    loadId = engine.getSequenceService().nextVal(context.getSqlTransaction(), Constants.SEQUENCE_OUTGOING_BATCH_LOAD_ID);
                    context.setLastLoadId(loadId);
                }
                batch.setLoadId(loadId);
            } else {
                context.setLastLoadId(-1);
            }
            batch.incrementEventCount(dataMetaData.getData().getDataEventType());
            batch.incrementDataEventCount();
            batch.incrementTableCount(dataMetaData.getTable().getNameLowerCase());
            if (!context.isProduceCommonBatches() || (context.isProduceCommonBatches() && !dataEventAdded)) {
                Router router = dataMetaData.getRouter();
                context.addDataEvent(dataMetaData.getData().getDataId(), batch.getBatchId(), router != null ? router.getRouterId() : Constants.UNKNOWN_ROUTER_ID);
                numberOfDataEventsInserted++;
                dataEventAdded = true;
            }
            Map<String, IBatchAlgorithm> batchAlgorithms = extensionService.getExtensionPointMap(IBatchAlgorithm.class);
            if (batchAlgorithms.get(context.getChannel().getBatchAlgorithm()).isBatchComplete(batch, dataMetaData, context)) {
                context.setNeedsCommitted(true);
            }
        }
    }
    context.incrementStat(System.currentTimeMillis() - ts, ChannelRouterContext.STAT_INSERT_DATA_EVENTS_MS);
    return numberOfDataEventsInserted;
}