Examples with OutgoingBatch org.jumpmind.symmetric.model.OutgoingBatch used on opensource projects

Example 21 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class AbstractService method readAcks.

protected List<BatchAck> readAcks(List<OutgoingBatch> batches, IOutgoingWithResponseTransport transport, ITransportManager transportManager, IAcknowledgeService acknowledgeService) throws IOException {
    Set<Long> batchIds = new HashSet<Long>(batches.size());
    for (OutgoingBatch outgoingBatch : batches) {
        if (outgoingBatch.getStatus() == OutgoingBatch.Status.LD) {
            batchIds.add(outgoingBatch.getBatchId());
        }
    }
    BufferedReader reader = transport.readResponse();
    String ackString = reader.readLine();
    String ackExtendedString = reader.readLine();
    log.debug("Reading ack: {}", ackString);
    log.debug("Reading extend ack: {}", ackExtendedString);
    String line = null;
    do {
        line = reader.readLine();
        if (line != null) {
            log.info("Read another unexpected line {}", line);
        }
    } while (line != null);
    if (StringUtils.isBlank(ackString)) {
        throw new SymmetricException("Did not receive an acknowledgement for the batches sent.  " + "The 'ack string' was: '%s' and the 'extended ack string' was: '%s'", ackString, ackExtendedString);
    }
    List<BatchAck> batchAcks = transportManager.readAcknowledgement(ackString, ackExtendedString);
    long batchIdInError = Long.MAX_VALUE;
    for (BatchAck batchInfo : batchAcks) {
        batchIds.remove(batchInfo.getBatchId());
        if (!batchInfo.isOk()) {
            batchIdInError = batchInfo.getBatchId();
        }
        log.debug("Saving ack: {}, {}", batchInfo.getBatchId(), (batchInfo.isOk() ? "OK" : "ER"));
        acknowledgeService.ack(batchInfo);
    }
    for (Long batchId : batchIds) {
        if (batchId < batchIdInError) {
            log.error("We expected but did not receive an ack for batch {}", batchId);
        }
    }
    return batchAcks;
}

Example 22 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class AcknowledgeService method ack.

public BatchAckResult ack(final BatchAck batch) {
    IRegistrationService registrationService = engine.getRegistrationService();
    IOutgoingBatchService outgoingBatchService = engine.getOutgoingBatchService();
    BatchAckResult result = new BatchAckResult(batch);
    for (IAcknowledgeEventListener listener : engine.getExtensionService().getExtensionPointList(IAcknowledgeEventListener.class)) {
        listener.onAcknowledgeEvent(batch);
    }
    if (batch.getBatchId() == Constants.VIRTUAL_BATCH_FOR_REGISTRATION) {
        if (batch.isOk()) {
            registrationService.markNodeAsRegistered(batch.getNodeId());
        }
    } else {
        OutgoingBatch outgoingBatch = outgoingBatchService.findOutgoingBatch(batch.getBatchId(), batch.getNodeId());
        Status status = batch.isOk() ? Status.OK : batch.isResend() ? Status.RS : Status.ER;
        if (outgoingBatch != null) {
            // is OK.
            if (outgoingBatch.getStatus() != Status.OK && outgoingBatch.getStatus() != Status.IG) {
                outgoingBatch.setStatus(status);
                outgoingBatch.setErrorFlag(!batch.isOk());
            } else {
                // clearing the error flag in case the user set the batch
                // status to OK
                Status oldStatus = outgoingBatch.getStatus();
                outgoingBatch.setStatus(Status.OK);
                outgoingBatch.setErrorFlag(false);
                log.info("Batch {} for {} was set to {}.  Updating the status to OK", new Object[] { batch.getBatchId(), batch.getNodeId(), oldStatus.name() });
            }
            if (batch.isIgnored()) {
                outgoingBatch.incrementIgnoreCount();
            }
            outgoingBatch.setNetworkMillis(batch.getNetworkMillis());
            outgoingBatch.setFilterMillis(batch.getFilterMillis());
            outgoingBatch.setLoadMillis(batch.getDatabaseMillis());
            outgoingBatch.setSqlCode(batch.getSqlCode());
            outgoingBatch.setSqlState(batch.getSqlState());
            outgoingBatch.setSqlMessage(batch.getSqlMessage());
            boolean isNewError = false;
            if (!batch.isOk() && batch.getErrorLine() != 0) {
                List<Number> ids = sqlTemplateDirty.query(getSql("selectDataIdSql"), new NumberMapper(), outgoingBatch.getBatchId());
                if (ids.size() >= batch.getErrorLine()) {
                    long failedDataId = ids.get((int) batch.getErrorLine() - 1).longValue();
                    if (outgoingBatch.getFailedDataId() == 0 || outgoingBatch.getFailedDataId() != failedDataId) {
                        isNewError = true;
                    }
                    outgoingBatch.setFailedDataId(failedDataId);
                }
            }
            if (status == Status.ER) {
                log.error("The outgoing batch {} failed: {}{}", outgoingBatch.getNodeBatchId(), (batch.getSqlCode() != 0 ? "[" + batch.getSqlState() + "," + batch.getSqlCode() + "] " : ""), batch.getSqlMessage());
                RouterStats routerStats = engine.getStatisticManager().getRouterStatsByBatch(batch.getBatchId());
                if (routerStats != null) {
                    log.info("Router stats for batch " + outgoingBatch.getBatchId() + ": " + routerStats.toString());
                }
                if (isNewError && outgoingBatch.getSqlCode() == ErrorConstants.FK_VIOLATION_CODE && parameterService.is(ParameterConstants.AUTO_RESOLVE_FOREIGN_KEY_VIOLATION)) {
                    Channel channel = engine.getConfigurationService().getChannel(outgoingBatch.getChannelId());
                    if (channel != null && !channel.isReloadFlag()) {
                        engine.getDataService().reloadMissingForeignKeyRows(outgoingBatch.getNodeId(), outgoingBatch.getFailedDataId());
                    }
                }
            } else if (status == Status.RS) {
                log.info("The outgoing batch {} received resend request", outgoingBatch.getNodeBatchId());
            }
            outgoingBatchService.updateOutgoingBatch(outgoingBatch);
            if (status == Status.OK) {
                Channel channel = engine.getConfigurationService().getChannel(outgoingBatch.getChannelId());
                if (channel != null && channel.isFileSyncFlag()) {
                    /* Acknowledge the file_sync in case the file needs deleted. */
                    engine.getFileSyncService().acknowledgeFiles(outgoingBatch);
                }
                engine.getStatisticManager().removeRouterStatsByBatch(batch.getBatchId());
            }
        } else {
            log.error("Could not find batch {}-{} to acknowledge as {}", new Object[] { batch.getNodeId(), batch.getBatchId(), status.name() });
            result.setOk(false);
        }
    }
    return result;
}

Example 23 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class DataExtractorService method execute.

/**
     * This is a callback method used by the NodeCommunicationService that extracts an initial load
     * in the background.
     */
public void execute(NodeCommunication nodeCommunication, RemoteNodeStatus status) {
    if (!isApplicable(nodeCommunication, status)) {
        log.debug("{} failed isApplicable check and will not run.", this);
        return;
    }
    List<ExtractRequest> requests = getExtractRequestsForNode(nodeCommunication);
    long ts = System.currentTimeMillis();
    /*
         * Process extract requests until it has taken longer than 30 seconds, and then
         * allow the process to return so progress status can be seen.
         */
    for (int i = 0; i < requests.size() && (System.currentTimeMillis() - ts) <= Constants.LONG_OPERATION_THRESHOLD; i++) {
        ExtractRequest request = requests.get(i);
        Node identity = nodeService.findIdentity();
        Node targetNode = nodeService.findNode(nodeCommunication.getNodeId());
        log.info("Extracting batches for request {}. Starting at batch {}.  Ending at batch {}", new Object[] { request.getRequestId(), request.getStartBatchId(), request.getEndBatchId() });
        List<OutgoingBatch> batches = outgoingBatchService.getOutgoingBatchRange(request.getStartBatchId(), request.getEndBatchId()).getBatches();
        ProcessInfo processInfo = statisticManager.newProcessInfo(new ProcessInfoKey(identity.getNodeId(), nodeCommunication.getQueue(), nodeCommunication.getNodeId(), getProcessType()));
        processInfo.setBatchCount(batches.size());
        try {
            boolean areBatchesOk = true;
            /*
                 * check to see if batches have been OK'd by another reload
                 * request 
                 */
            for (OutgoingBatch outgoingBatch : batches) {
                if (outgoingBatch.getStatus() != Status.OK) {
                    areBatchesOk = false;
                    break;
                }
            }
            if (!areBatchesOk) {
                Channel channel = configurationService.getChannel(batches.get(0).getChannelId());
                /*
                     * "Trick" the extractor to extract one reload batch, but we
                     * will split it across the N batches when writing it
                     */
                OutgoingBatch firstBatch = batches.get(0);
                processInfo.setCurrentLoadId(firstBatch.getLoadId());
                IStagedResource resource = getStagedResource(firstBatch);
                if (resource != null && resource.exists() && resource.getState() != State.CREATE) {
                    resource.delete();
                }
                MultiBatchStagingWriter multiBatchStatingWriter = buildMultiBatchStagingWriter(request, identity, targetNode, batches, processInfo, channel);
                extractOutgoingBatch(processInfo, targetNode, multiBatchStatingWriter, firstBatch, false, false, ExtractMode.FOR_SYM_CLIENT);
                for (OutgoingBatch outgoingBatch : batches) {
                    resource = getStagedResource(outgoingBatch);
                    if (resource != null) {
                        resource.setState(State.DONE);
                    }
                }
            } else {
                log.info("Batches already had an OK status for request {}, batches {} to {}.  Not extracting", new Object[] { request.getRequestId(), request.getStartBatchId(), request.getEndBatchId() });
            }
            /*
                 * re-query the batches to see if they have been OK'd while
                 * extracting
                 */
            List<OutgoingBatch> checkBatches = outgoingBatchService.getOutgoingBatchRange(request.getStartBatchId(), request.getEndBatchId()).getBatches();
            areBatchesOk = true;
            /*
                 * check to see if batches have been OK'd by another reload
                 * request while extracting
                 */
            for (OutgoingBatch outgoingBatch : checkBatches) {
                if (outgoingBatch.getStatus() != Status.OK) {
                    areBatchesOk = false;
                    break;
                }
            }
            ISqlTransaction transaction = null;
            try {
                transaction = sqlTemplate.startSqlTransaction();
                updateExtractRequestStatus(transaction, request.getRequestId(), ExtractStatus.OK);
                if (!areBatchesOk) {
                    for (OutgoingBatch outgoingBatch : batches) {
                        if (!parameterService.is(ParameterConstants.INITIAL_LOAD_EXTRACT_AND_SEND_WHEN_STAGED, false)) {
                            outgoingBatch.setStatus(Status.NE);
                            outgoingBatchService.updateOutgoingBatch(transaction, outgoingBatch);
                        }
                    }
                } else {
                    log.info("Batches already had an OK status for request {}, batches {} to {}.  Not updating the status to NE", new Object[] { request.getRequestId(), request.getStartBatchId(), request.getEndBatchId() });
                }
                transaction.commit();
                log.info("Done extracting {} batches for request {}", (request.getEndBatchId() - request.getStartBatchId()) + 1, request.getRequestId());
            } catch (Error ex) {
                if (transaction != null) {
                    transaction.rollback();
                }
                throw ex;
            } catch (RuntimeException ex) {
                if (transaction != null) {
                    transaction.rollback();
                }
                throw ex;
            } finally {
                close(transaction);
            }
            processInfo.setStatus(org.jumpmind.symmetric.model.ProcessInfo.Status.OK);
        } catch (CancellationException ex) {
            log.info("Cancelled extract request {}. Starting at batch {}.  Ending at batch {}", new Object[] { request.getRequestId(), request.getStartBatchId(), request.getEndBatchId() });
            processInfo.setStatus(org.jumpmind.symmetric.model.ProcessInfo.Status.OK);
        } catch (RuntimeException ex) {
            log.debug("Failed to extract batches for request {}. Starting at batch {}.  Ending at batch {}", new Object[] { request.getRequestId(), request.getStartBatchId(), request.getEndBatchId() });
            processInfo.setStatus(org.jumpmind.symmetric.model.ProcessInfo.Status.ERROR);
            throw ex;
        }
    }
}

Example 24 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class DataExtractorService method extractBatchRange.

public boolean extractBatchRange(Writer writer, String nodeId, long startBatchId, long endBatchId) {
    boolean foundBatch = false;
    Node sourceNode = nodeService.findIdentity();
    for (long batchId = startBatchId; batchId <= endBatchId; batchId++) {
        OutgoingBatch batch = outgoingBatchService.findOutgoingBatch(batchId, nodeId);
        if (batch != null) {
            Node targetNode = nodeService.findNode(nodeId);
            if (targetNode == null && Constants.UNROUTED_NODE_ID.equals(nodeId)) {
                targetNode = new Node();
                targetNode.setNodeId("-1");
            }
            if (targetNode != null) {
                IDataReader dataReader = new ExtractDataReader(symmetricDialect.getPlatform(), new SelectFromSymDataSource(batch, sourceNode, targetNode, new ProcessInfo()));
                DataContext ctx = new DataContext();
                ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode);
                ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE, nodeService.findIdentity());
                new DataProcessor(dataReader, createTransformDataWriter(nodeService.findIdentity(), targetNode, new ProtocolDataWriter(nodeService.findIdentityNodeId(), writer, targetNode.requires13Compatiblity())), "extract range").process(ctx);
                foundBatch = true;
            }
        }
    }
    return foundBatch;
}

Example 25 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class DataExtractorService method cleanupIgnoredBatch.

protected void cleanupIgnoredBatch(Node sourceNode, Node targetNode, OutgoingBatch currentBatch, IDataWriter writer) {
    Batch batch = new Batch(BatchType.EXTRACT, currentBatch.getBatchId(), currentBatch.getChannelId(), symmetricDialect.getBinaryEncoding(), sourceNode.getNodeId(), currentBatch.getNodeId(), currentBatch.isCommonFlag());
    batch.setIgnored(true);
    try {
        IStagedResource resource = getStagedResource(currentBatch);
        if (resource != null) {
            resource.delete();
        }
        DataContext ctx = new DataContext(batch);
        ctx.put("targetNode", targetNode);
        ctx.put("sourceNode", sourceNode);
        writer.open(ctx);
        writer.start(batch);
        writer.end(batch, false);
    } finally {
        writer.close();
    }
}