Search in sources :

Example 6 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class DataExtractorService method extract.

protected List<OutgoingBatch> extract(final ProcessInfo processInfo, final Node targetNode, final List<OutgoingBatch> activeBatches, final IDataWriter dataWriter, final BufferedWriter writer, final ExtractMode mode) {
    if (activeBatches.size() > 0) {
        final List<OutgoingBatch> processedBatches = new ArrayList<OutgoingBatch>(activeBatches.size());
        Set<String> channelsProcessed = new HashSet<String>();
        long batchesSelectedAtMs = System.currentTimeMillis();
        OutgoingBatch currentBatch = null;
        ExecutorService executor = null;
        try {
            final boolean streamToFileEnabled = parameterService.is(ParameterConstants.STREAM_TO_FILE_ENABLED);
            long keepAliveMillis = parameterService.getLong(ParameterConstants.DATA_LOADER_SEND_ACK_KEEPALIVE);
            Node sourceNode = nodeService.findIdentity();
            final FutureExtractStatus status = new FutureExtractStatus();
            executor = Executors.newFixedThreadPool(1, new CustomizableThreadFactory(String.format("dataextractor-%s-%s", targetNode.getNodeGroupId(), targetNode.getNodeGroupId())));
            List<Future<FutureOutgoingBatch>> futures = new ArrayList<Future<FutureOutgoingBatch>>();
            processInfo.setBatchCount(activeBatches.size());
            for (int i = 0; i < activeBatches.size(); i++) {
                currentBatch = activeBatches.get(i);
                processInfo.setCurrentLoadId(currentBatch.getLoadId());
                processInfo.setDataCount(currentBatch.getDataEventCount());
                processInfo.setCurrentBatchId(currentBatch.getBatchId());
                channelsProcessed.add(currentBatch.getChannelId());
                currentBatch = requeryIfEnoughTimeHasPassed(batchesSelectedAtMs, currentBatch);
                processInfo.setStatus(ProcessInfo.Status.EXTRACTING);
                final OutgoingBatch extractBatch = currentBatch;
                Callable<FutureOutgoingBatch> callable = new Callable<FutureOutgoingBatch>() {

                    public FutureOutgoingBatch call() throws Exception {
                        return extractBatch(extractBatch, status, processInfo, targetNode, dataWriter, mode, activeBatches);
                    }
                };
                if (status.shouldExtractSkip) {
                    break;
                }
                futures.add(executor.submit(callable));
            }
            if (parameterService.is(ParameterConstants.SYNCHRONIZE_ALL_JOBS)) {
                executor.shutdown();
                boolean isProcessed = false;
                while (!isProcessed) {
                    try {
                        isProcessed = executor.awaitTermination(keepAliveMillis, TimeUnit.MILLISECONDS);
                    } catch (InterruptedException e) {
                        throw new RuntimeException(e);
                    }
                    if (!isProcessed) {
                        writeKeepAliveAck(writer, sourceNode, streamToFileEnabled);
                    }
                }
            }
            Iterator<OutgoingBatch> activeBatchIter = activeBatches.iterator();
            for (Future<FutureOutgoingBatch> future : futures) {
                currentBatch = activeBatchIter.next();
                boolean isProcessed = false;
                while (!isProcessed) {
                    try {
                        FutureOutgoingBatch extractBatch = future.get(keepAliveMillis, TimeUnit.MILLISECONDS);
                        currentBatch = extractBatch.getOutgoingBatch();
                        if (extractBatch.isExtractSkipped) {
                            break;
                        }
                        if (streamToFileEnabled || mode == ExtractMode.FOR_PAYLOAD_CLIENT) {
                            processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
                            processInfo.setCurrentLoadId(currentBatch.getLoadId());
                            boolean isRetry = extractBatch.isRetry() && extractBatch.getOutgoingBatch().getStatus() != OutgoingBatch.Status.IG;
                            currentBatch = sendOutgoingBatch(processInfo, targetNode, currentBatch, isRetry, dataWriter, writer, mode);
                        }
                        processedBatches.add(currentBatch);
                        isProcessed = true;
                        if (currentBatch.getStatus() != Status.OK) {
                            currentBatch.setLoadCount(currentBatch.getLoadCount() + 1);
                            changeBatchStatus(Status.LD, currentBatch, mode);
                        }
                    } catch (ExecutionException e) {
                        if (isNotBlank(e.getMessage()) && e.getMessage().contains("string truncation")) {
                            throw new RuntimeException("There is a good chance that the truncation error you are receiving is because contains_big_lobs on the '" + currentBatch.getChannelId() + "' channel needs to be turned on.", e.getCause() != null ? e.getCause() : e);
                        }
                        throw new RuntimeException(e.getCause() != null ? e.getCause() : e);
                    } catch (InterruptedException e) {
                        throw new RuntimeException(e);
                    } catch (TimeoutException e) {
                        writeKeepAliveAck(writer, sourceNode, streamToFileEnabled);
                    }
                }
            }
        } catch (RuntimeException e) {
            SQLException se = unwrapSqlException(e);
            if (currentBatch != null) {
                try {
                    /* Reread batch in case the ignore flag has been set */
                    currentBatch = outgoingBatchService.findOutgoingBatch(currentBatch.getBatchId(), currentBatch.getNodeId());
                    statisticManager.incrementDataExtractedErrors(currentBatch.getChannelId(), 1);
                    if (se != null) {
                        currentBatch.setSqlState(se.getSQLState());
                        currentBatch.setSqlCode(se.getErrorCode());
                        currentBatch.setSqlMessage(se.getMessage());
                    } else {
                        currentBatch.setSqlMessage(getRootMessage(e));
                    }
                    currentBatch.revertStatsOnError();
                    if (currentBatch.getStatus() != Status.IG && currentBatch.getStatus() != Status.OK) {
                        currentBatch.setStatus(Status.ER);
                        currentBatch.setErrorFlag(true);
                    }
                    outgoingBatchService.updateOutgoingBatch(currentBatch);
                } catch (Exception ex) {
                    log.error("Failed to update the outgoing batch status for failed batch {}", currentBatch, ex);
                } finally {
                    if (!isStreamClosedByClient(e)) {
                        if (e instanceof ProtocolException) {
                            IStagedResource resource = getStagedResource(currentBatch);
                            if (resource != null) {
                                resource.delete();
                            }
                        }
                        if (e.getCause() instanceof InterruptedException) {
                            log.info("Extract of batch {} was interrupted", currentBatch);
                        } else {
                            log.error("Failed to extract batch {}", currentBatch, e);
                        }
                    }
                    processInfo.setStatus(ProcessInfo.Status.ERROR);
                }
            } else {
                log.error("Could not log the outgoing batch status because the batch was null", e);
            }
        } finally {
            if (executor != null) {
                executor.shutdown();
            }
        }
        // Next, we update the node channel controls to the
        // current timestamp
        Calendar now = Calendar.getInstance();
        for (String channelProcessed : channelsProcessed) {
            NodeChannel nodeChannel = configurationService.getNodeChannel(channelProcessed, targetNode.getNodeId(), false);
            if (nodeChannel != null && nodeChannel.getExtractPeriodMillis() > 0) {
                nodeChannel.setLastExtractTime(now.getTime());
                configurationService.updateLastExtractTime(nodeChannel);
            }
        }
        return processedBatches;
    } else {
        return Collections.emptyList();
    }
}
Also used : CustomizableThreadFactory(org.jumpmind.util.CustomizableThreadFactory) SQLException(java.sql.SQLException) Node(org.jumpmind.symmetric.model.Node) ArrayList(java.util.ArrayList) Callable(java.util.concurrent.Callable) IStagedResource(org.jumpmind.symmetric.io.stage.IStagedResource) OutgoingBatch(org.jumpmind.symmetric.model.OutgoingBatch) ExecutionException(java.util.concurrent.ExecutionException) NodeChannel(org.jumpmind.symmetric.model.NodeChannel) HashSet(java.util.HashSet) TimeoutException(java.util.concurrent.TimeoutException) ProtocolException(org.jumpmind.symmetric.io.data.ProtocolException) Calendar(java.util.Calendar) TransformPoint(org.jumpmind.symmetric.io.data.transform.TransformPoint) CancellationException(java.util.concurrent.CancellationException) SymmetricException(org.jumpmind.symmetric.SymmetricException) SQLException(java.sql.SQLException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) ProtocolException(org.jumpmind.symmetric.io.data.ProtocolException) TimeoutException(java.util.concurrent.TimeoutException) IoException(org.jumpmind.exception.IoException) ExecutorService(java.util.concurrent.ExecutorService) Future(java.util.concurrent.Future)

Example 7 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class DataExtractorService method extractBatchRange.

public boolean extractBatchRange(Writer writer, String nodeId, Date startBatchTime, Date endBatchTime, String... channelIds) {
    boolean foundBatch = false;
    Node sourceNode = nodeService.findIdentity();
    OutgoingBatches batches = outgoingBatchService.getOutgoingBatchRange(nodeId, startBatchTime, endBatchTime, channelIds);
    List<OutgoingBatch> list = batches.getBatches();
    for (OutgoingBatch outgoingBatch : list) {
        Node targetNode = nodeService.findNode(nodeId);
        if (targetNode == null && Constants.UNROUTED_NODE_ID.equals(nodeId)) {
            targetNode = new Node();
            targetNode.setNodeId("-1");
        }
        if (targetNode != null) {
            IDataReader dataReader = new ExtractDataReader(symmetricDialect.getPlatform(), new SelectFromSymDataSource(outgoingBatch, sourceNode, targetNode, new ProcessInfo()));
            DataContext ctx = new DataContext();
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode);
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE, nodeService.findIdentity());
            new DataProcessor(dataReader, createTransformDataWriter(nodeService.findIdentity(), targetNode, new ProtocolDataWriter(nodeService.findIdentityNodeId(), writer, targetNode.requires13Compatiblity())), "extract range").process(ctx);
            foundBatch = true;
        }
    }
    return foundBatch;
}
Also used : IDataReader(org.jumpmind.symmetric.io.data.IDataReader) DataContext(org.jumpmind.symmetric.io.data.DataContext) ProtocolDataWriter(org.jumpmind.symmetric.io.data.writer.ProtocolDataWriter) Node(org.jumpmind.symmetric.model.Node) OutgoingBatches(org.jumpmind.symmetric.model.OutgoingBatches) OutgoingBatch(org.jumpmind.symmetric.model.OutgoingBatch) ProcessInfo(org.jumpmind.symmetric.model.ProcessInfo) DataProcessor(org.jumpmind.symmetric.io.data.DataProcessor) ExtractDataReader(org.jumpmind.symmetric.io.data.reader.ExtractDataReader)

Example 8 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class DataExtractorService method filterBatchesForExtraction.

private List<OutgoingBatch> filterBatchesForExtraction(OutgoingBatches batches, ChannelMap suspendIgnoreChannelsList) {
    if (parameterService.is(ParameterConstants.FILE_SYNC_ENABLE)) {
        List<Channel> fileSyncChannels = configurationService.getFileSyncChannels();
        for (Channel channel : fileSyncChannels) {
            batches.filterBatchesForChannel(channel);
        }
    }
    // We now have either our local suspend/ignore list, or the combined
    // remote send/ignore list and our local list (along with a
    // reservation, if we go this far...)
    // Now, we need to skip the suspended channels and ignore the
    // ignored ones by ultimately setting the status to ignored and
    // updating them.
    List<OutgoingBatch> ignoredBatches = batches.filterBatchesForChannels(suspendIgnoreChannelsList.getIgnoreChannels());
    // will be skipped in the future.
    for (OutgoingBatch batch : ignoredBatches) {
        batch.setStatus(OutgoingBatch.Status.OK);
        batch.incrementIgnoreCount();
        if (log.isDebugEnabled()) {
            log.debug("Batch {} is being ignored", batch.getBatchId());
        }
    }
    outgoingBatchService.updateOutgoingBatches(ignoredBatches);
    batches.filterBatchesForChannels(suspendIgnoreChannelsList.getSuspendChannels());
    // any other batches are loaded.
    if (parameterService.is(ParameterConstants.INITIAL_LOAD_BLOCK_CHANNELS, true)) {
        if (!batches.containsBatchesInError() && batches.containsLoadBatches()) {
            batches.removeNonLoadBatches();
        }
    }
    return batches.getBatches();
}
Also used : NodeChannel(org.jumpmind.symmetric.model.NodeChannel) Channel(org.jumpmind.symmetric.model.Channel) OutgoingBatch(org.jumpmind.symmetric.model.OutgoingBatch)

Example 9 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class DataExtractorService method extract.

public List<OutgoingBatch> extract(ProcessInfo processInfo, Node targetNode, String queue, IOutgoingTransport transport) {
    /*
         * make sure that data is routed before extracting if the route job is
         * not configured to start automatically
         */
    if (!parameterService.is(ParameterConstants.START_ROUTE_JOB) && parameterService.is(ParameterConstants.ROUTE_ON_EXTRACT)) {
        routerService.routeData(true);
    }
    OutgoingBatches batches = null;
    if (queue != null) {
        NodeGroupLinkAction defaultAction = configurationService.getNodeGroupLinkFor(nodeService.findIdentity().getNodeGroupId(), targetNode.getNodeGroupId(), false).getDataEventAction();
        ProcessInfoKey.ProcessType processType = processInfo.getKey().getProcessType();
        NodeGroupLinkAction action = null;
        if (processType.equals(ProcessInfoKey.ProcessType.PUSH_JOB)) {
            action = NodeGroupLinkAction.P;
        } else if (processType.equals(ProcessInfoKey.ProcessType.PULL_HANDLER)) {
            action = NodeGroupLinkAction.W;
        }
        batches = outgoingBatchService.getOutgoingBatches(targetNode.getNodeId(), queue, action, defaultAction, false);
    } else {
        batches = outgoingBatchService.getOutgoingBatches(targetNode.getNodeId(), false);
    }
    if (batches.containsBatches()) {
        ChannelMap channelMap = transport.getSuspendIgnoreChannelLists(configurationService, queue, targetNode);
        List<OutgoingBatch> activeBatches = filterBatchesForExtraction(batches, channelMap);
        if (activeBatches.size() > 0) {
            BufferedWriter writer = transport.openWriter();
            IDataWriter dataWriter = new ProtocolDataWriter(nodeService.findIdentityNodeId(), writer, targetNode.requires13Compatiblity());
            return extract(processInfo, targetNode, activeBatches, dataWriter, writer, ExtractMode.FOR_SYM_CLIENT);
        }
    }
    return Collections.emptyList();
}
Also used : ChannelMap(org.jumpmind.symmetric.model.ChannelMap) NodeGroupLinkAction(org.jumpmind.symmetric.model.NodeGroupLinkAction) ProtocolDataWriter(org.jumpmind.symmetric.io.data.writer.ProtocolDataWriter) ProcessType(org.jumpmind.symmetric.model.ProcessInfoKey.ProcessType) ProcessInfoKey(org.jumpmind.symmetric.model.ProcessInfoKey) OutgoingBatches(org.jumpmind.symmetric.model.OutgoingBatches) OutgoingBatch(org.jumpmind.symmetric.model.OutgoingBatch) BufferedWriter(java.io.BufferedWriter) BatchBufferedWriter(org.jumpmind.symmetric.transport.BatchBufferedWriter) IDataWriter(org.jumpmind.symmetric.io.data.IDataWriter)

Example 10 with OutgoingBatch

use of org.jumpmind.symmetric.model.OutgoingBatch in project symmetric-ds by JumpMind.

the class DataExtractorService method extractToPayload.

public List<OutgoingBatchWithPayload> extractToPayload(ProcessInfo processInfo, Node targetNode, PayloadType payloadType, boolean useJdbcTimestampFormat, boolean useUpsertStatements, boolean useDelimiterIdentifiers) {
    OutgoingBatches batches = outgoingBatchService.getOutgoingBatches(targetNode.getNodeId(), false);
    if (batches.containsBatches()) {
        ChannelMap channelMap = configurationService.getSuspendIgnoreChannelLists(targetNode.getNodeId());
        List<OutgoingBatch> activeBatches = filterBatchesForExtraction(batches, channelMap);
        if (activeBatches.size() > 0) {
            IDdlBuilder builder = DdlBuilderFactory.createDdlBuilder(targetNode.getDatabaseType());
            if (builder == null) {
                throw new IllegalStateException("Could not find a ddl builder registered for the database type of " + targetNode.getDatabaseType() + ".  Please check the database type setting for node '" + targetNode.getNodeId() + "'");
            }
            StructureDataWriter writer = new StructureDataWriter(symmetricDialect.getPlatform(), targetNode.getDatabaseType(), payloadType, useDelimiterIdentifiers, symmetricDialect.getBinaryEncoding(), useJdbcTimestampFormat, useUpsertStatements);
            List<OutgoingBatch> extractedBatches = extract(processInfo, targetNode, activeBatches, writer, null, ExtractMode.FOR_PAYLOAD_CLIENT);
            List<OutgoingBatchWithPayload> batchesWithPayload = new ArrayList<OutgoingBatchWithPayload>();
            for (OutgoingBatch batch : extractedBatches) {
                OutgoingBatchWithPayload batchWithPayload = new OutgoingBatchWithPayload(batch, payloadType);
                batchWithPayload.setPayload(writer.getPayloadMap().get(batch.getBatchId()));
                batchWithPayload.setPayloadType(payloadType);
                batchesWithPayload.add(batchWithPayload);
            }
            return batchesWithPayload;
        }
    }
    return Collections.emptyList();
}
Also used : ChannelMap(org.jumpmind.symmetric.model.ChannelMap) OutgoingBatchWithPayload(org.jumpmind.symmetric.model.OutgoingBatchWithPayload) IDdlBuilder(org.jumpmind.db.platform.IDdlBuilder) ArrayList(java.util.ArrayList) OutgoingBatches(org.jumpmind.symmetric.model.OutgoingBatches) OutgoingBatch(org.jumpmind.symmetric.model.OutgoingBatch) StructureDataWriter(org.jumpmind.symmetric.io.data.writer.StructureDataWriter)

Aggregations

OutgoingBatch (org.jumpmind.symmetric.model.OutgoingBatch)37 OutgoingBatches (org.jumpmind.symmetric.model.OutgoingBatches)12 ArrayList (java.util.ArrayList)10 ProcessInfo (org.jumpmind.symmetric.model.ProcessInfo)9 Node (org.jumpmind.symmetric.model.Node)8 IStagedResource (org.jumpmind.symmetric.io.stage.IStagedResource)6 ProcessInfoKey (org.jumpmind.symmetric.model.ProcessInfoKey)6 NodeChannel (org.jumpmind.symmetric.model.NodeChannel)5 SymmetricException (org.jumpmind.symmetric.SymmetricException)4 ProtocolDataWriter (org.jumpmind.symmetric.io.data.writer.ProtocolDataWriter)4 BatchAck (org.jumpmind.symmetric.model.BatchAck)4 Channel (org.jumpmind.symmetric.model.Channel)4 File (java.io.File)3 IOException (java.io.IOException)3 HashSet (java.util.HashSet)3 CancellationException (java.util.concurrent.CancellationException)3 IoException (org.jumpmind.exception.IoException)3 DataContext (org.jumpmind.symmetric.io.data.DataContext)3 ChannelMap (org.jumpmind.symmetric.model.ChannelMap)3 BufferedReader (java.io.BufferedReader)2