Examples with NodeChannel org.jumpmind.symmetric.model.NodeChannel used on opensource projects

Example 1 with NodeChannel

use of org.jumpmind.symmetric.model.NodeChannel in project symmetric-ds by JumpMind.

the class DataExtractorService method extract.

protected List<OutgoingBatch> extract(final ProcessInfo processInfo, final Node targetNode, final List<OutgoingBatch> activeBatches, final IDataWriter dataWriter, final BufferedWriter writer, final ExtractMode mode) {
    if (activeBatches.size() > 0) {
        final List<OutgoingBatch> processedBatches = new ArrayList<OutgoingBatch>(activeBatches.size());
        Set<String> channelsProcessed = new HashSet<String>();
        long batchesSelectedAtMs = System.currentTimeMillis();
        OutgoingBatch currentBatch = null;
        ExecutorService executor = null;
        try {
            final boolean streamToFileEnabled = parameterService.is(ParameterConstants.STREAM_TO_FILE_ENABLED);
            long keepAliveMillis = parameterService.getLong(ParameterConstants.DATA_LOADER_SEND_ACK_KEEPALIVE);
            Node sourceNode = nodeService.findIdentity();
            final FutureExtractStatus status = new FutureExtractStatus();
            executor = Executors.newFixedThreadPool(1, new CustomizableThreadFactory(String.format("dataextractor-%s-%s", targetNode.getNodeGroupId(), targetNode.getNodeGroupId())));
            List<Future<FutureOutgoingBatch>> futures = new ArrayList<Future<FutureOutgoingBatch>>();
            processInfo.setBatchCount(activeBatches.size());
            for (int i = 0; i < activeBatches.size(); i++) {
                currentBatch = activeBatches.get(i);
                processInfo.setCurrentLoadId(currentBatch.getLoadId());
                processInfo.setDataCount(currentBatch.getDataEventCount());
                processInfo.setCurrentBatchId(currentBatch.getBatchId());
                channelsProcessed.add(currentBatch.getChannelId());
                currentBatch = requeryIfEnoughTimeHasPassed(batchesSelectedAtMs, currentBatch);
                processInfo.setStatus(ProcessInfo.Status.EXTRACTING);
                final OutgoingBatch extractBatch = currentBatch;
                Callable<FutureOutgoingBatch> callable = new Callable<FutureOutgoingBatch>() {

                    public FutureOutgoingBatch call() throws Exception {
                        return extractBatch(extractBatch, status, processInfo, targetNode, dataWriter, mode, activeBatches);
                    }
                };
                if (status.shouldExtractSkip) {
                    break;
                }
                futures.add(executor.submit(callable));
            }
            if (parameterService.is(ParameterConstants.SYNCHRONIZE_ALL_JOBS)) {
                executor.shutdown();
                boolean isProcessed = false;
                while (!isProcessed) {
                    try {
                        isProcessed = executor.awaitTermination(keepAliveMillis, TimeUnit.MILLISECONDS);
                    } catch (InterruptedException e) {
                        throw new RuntimeException(e);
                    }
                    if (!isProcessed) {
                        writeKeepAliveAck(writer, sourceNode, streamToFileEnabled);
                    }
                }
            }
            Iterator<OutgoingBatch> activeBatchIter = activeBatches.iterator();
            for (Future<FutureOutgoingBatch> future : futures) {
                currentBatch = activeBatchIter.next();
                boolean isProcessed = false;
                while (!isProcessed) {
                    try {
                        FutureOutgoingBatch extractBatch = future.get(keepAliveMillis, TimeUnit.MILLISECONDS);
                        currentBatch = extractBatch.getOutgoingBatch();
                        if (extractBatch.isExtractSkipped) {
                            break;
                        }
                        if (streamToFileEnabled || mode == ExtractMode.FOR_PAYLOAD_CLIENT) {
                            processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
                            processInfo.setCurrentLoadId(currentBatch.getLoadId());
                            boolean isRetry = extractBatch.isRetry() && extractBatch.getOutgoingBatch().getStatus() != OutgoingBatch.Status.IG;
                            currentBatch = sendOutgoingBatch(processInfo, targetNode, currentBatch, isRetry, dataWriter, writer, mode);
                        }
                        processedBatches.add(currentBatch);
                        isProcessed = true;
                        if (currentBatch.getStatus() != Status.OK) {
                            currentBatch.setLoadCount(currentBatch.getLoadCount() + 1);
                            changeBatchStatus(Status.LD, currentBatch, mode);
                        }
                    } catch (ExecutionException e) {
                        if (isNotBlank(e.getMessage()) && e.getMessage().contains("string truncation")) {
                            throw new RuntimeException("There is a good chance that the truncation error you are receiving is because contains_big_lobs on the '" + currentBatch.getChannelId() + "' channel needs to be turned on.", e.getCause() != null ? e.getCause() : e);
                        }
                        throw new RuntimeException(e.getCause() != null ? e.getCause() : e);
                    } catch (InterruptedException e) {
                        throw new RuntimeException(e);
                    } catch (TimeoutException e) {
                        writeKeepAliveAck(writer, sourceNode, streamToFileEnabled);
                    }
                }
            }
        } catch (RuntimeException e) {
            SQLException se = unwrapSqlException(e);
            if (currentBatch != null) {
                try {
                    /* Reread batch in case the ignore flag has been set */
                    currentBatch = outgoingBatchService.findOutgoingBatch(currentBatch.getBatchId(), currentBatch.getNodeId());
                    statisticManager.incrementDataExtractedErrors(currentBatch.getChannelId(), 1);
                    if (se != null) {
                        currentBatch.setSqlState(se.getSQLState());
                        currentBatch.setSqlCode(se.getErrorCode());
                        currentBatch.setSqlMessage(se.getMessage());
                    } else {
                        currentBatch.setSqlMessage(getRootMessage(e));
                    }
                    currentBatch.revertStatsOnError();
                    if (currentBatch.getStatus() != Status.IG && currentBatch.getStatus() != Status.OK) {
                        currentBatch.setStatus(Status.ER);
                        currentBatch.setErrorFlag(true);
                    }
                    outgoingBatchService.updateOutgoingBatch(currentBatch);
                } catch (Exception ex) {
                    log.error("Failed to update the outgoing batch status for failed batch {}", currentBatch, ex);
                } finally {
                    if (!isStreamClosedByClient(e)) {
                        if (e instanceof ProtocolException) {
                            IStagedResource resource = getStagedResource(currentBatch);
                            if (resource != null) {
                                resource.delete();
                            }
                        }
                        if (e.getCause() instanceof InterruptedException) {
                            log.info("Extract of batch {} was interrupted", currentBatch);
                        } else {
                            log.error("Failed to extract batch {}", currentBatch, e);
                        }
                    }
                    processInfo.setStatus(ProcessInfo.Status.ERROR);
                }
            } else {
                log.error("Could not log the outgoing batch status because the batch was null", e);
            }
        } finally {
            if (executor != null) {
                executor.shutdown();
            }
        }
        // Next, we update the node channel controls to the
        // current timestamp
        Calendar now = Calendar.getInstance();
        for (String channelProcessed : channelsProcessed) {
            NodeChannel nodeChannel = configurationService.getNodeChannel(channelProcessed, targetNode.getNodeId(), false);
            if (nodeChannel != null && nodeChannel.getExtractPeriodMillis() > 0) {
                nodeChannel.setLastExtractTime(now.getTime());
                configurationService.updateLastExtractTime(nodeChannel);
            }
        }
        return processedBatches;
    } else {
        return Collections.emptyList();
    }
}

Example 2 with NodeChannel

use of org.jumpmind.symmetric.model.NodeChannel in project symmetric-ds by JumpMind.

the class MakeReservationUriHandler method isChannelEnabled.

protected boolean isChannelEnabled(String channelId, String nodeId) {
    NodeChannel nodeChannel = engine.getConfigurationService().getNodeChannel(channelId, false);
    boolean enabled = nodeChannel == null || (!nodeChannel.isSuspendEnabled() && !nodeChannel.isIgnoreEnabled() && nodeChannel.isEnabled());
    if (enabled) {
        nodeChannel = engine.getConfigurationService().getNodeChannel(channelId, nodeId, false);
        enabled = nodeChannel == null || (!nodeChannel.isSuspendEnabled() && !nodeChannel.isIgnoreEnabled() && nodeChannel.isEnabled());
    }
    return enabled;
}

Example 3 with NodeChannel

use of org.jumpmind.symmetric.model.NodeChannel in project symmetric-ds by JumpMind.

the class ConfigurationService method getSuspendIgnoreChannelLists.

public ChannelMap getSuspendIgnoreChannelLists(final String nodeId) {
    ChannelMap map = new ChannelMap();
    List<NodeChannel> ncs = getNodeChannels(nodeId, true);
    if (ncs != null) {
        for (NodeChannel nc : ncs) {
            if (nc.isSuspendEnabled()) {
                map.addSuspendChannels(nc.getChannelId());
            }
            if (nc.isIgnoreEnabled()) {
                map.addIgnoreChannels(nc.getChannelId());
            }
        }
    }
    return map;
}

Example 4 with NodeChannel

use of org.jumpmind.symmetric.model.NodeChannel in project symmetric-ds by JumpMind.

the class OutgoingBatchService method getOutgoingBatches.

/**
 * Select batches to process. Batches that are NOT in error will be returned
 * first. They will be ordered by batch id as the batches will have already
 * been created by {@link #buildOutgoingBatches(String)} in channel priority
 * order.
 */
public OutgoingBatches getOutgoingBatches(String nodeId, boolean includeDisabledChannels) {
    long ts = System.currentTimeMillis();
    final int maxNumberOfBatchesToSelect = parameterService.getInt(ParameterConstants.OUTGOING_BATCH_MAX_BATCHES_TO_SELECT, 1000);
    List<OutgoingBatch> list = (List<OutgoingBatch>) sqlTemplate.query(getSql("selectOutgoingBatchPrefixSql", "selectOutgoingBatchSql"), maxNumberOfBatchesToSelect, new OutgoingBatchMapper(includeDisabledChannels), new Object[] { nodeId, OutgoingBatch.Status.RQ.name(), OutgoingBatch.Status.NE.name(), OutgoingBatch.Status.QY.name(), OutgoingBatch.Status.SE.name(), OutgoingBatch.Status.LD.name(), OutgoingBatch.Status.ER.name(), OutgoingBatch.Status.IG.name() }, null);
    OutgoingBatches batches = new OutgoingBatches(list);
    List<NodeChannel> channels = new ArrayList<NodeChannel>(configurationService.getNodeChannels(nodeId, true));
    batches.sortChannels(channels);
    List<IOutgoingBatchFilter> filters = extensionService.getExtensionPointList(IOutgoingBatchFilter.class);
    List<OutgoingBatch> keepers = new ArrayList<OutgoingBatch>();
    for (NodeChannel channel : channels) {
        List<OutgoingBatch> batchesForChannel = getBatchesForChannelWindows(batches.getBatches(), nodeId, channel, configurationService.getNodeGroupChannelWindows(parameterService.getNodeGroupId(), channel.getChannelId()));
        if (filters != null) {
            for (IOutgoingBatchFilter filter : filters) {
                batchesForChannel = filter.filter(channel, batchesForChannel);
            }
        }
        if (parameterService.is(ParameterConstants.DATA_EXTRACTOR_ENABLED) || channel.getChannelId().equals(Constants.CHANNEL_CONFIG)) {
            keepers.addAll(batchesForChannel);
        }
    }
    batches.setBatches(keepers);
    long executeTimeInMs = System.currentTimeMillis() - ts;
    if (executeTimeInMs > Constants.LONG_OPERATION_THRESHOLD) {
        log.info("Selecting {} outgoing batch rows for node {} took {} ms", list.size(), nodeId, executeTimeInMs);
    }
    return batches;
}

Example 5 with NodeChannel

use of org.jumpmind.symmetric.model.NodeChannel in project symmetric-ds by JumpMind.

the class PushService method push.

public synchronized RemoteNodeStatuses push(boolean force) {
    IConfigurationService configurationService = engine.getConfigurationService();
    IOutgoingBatchService outgoingBatchService = engine.getOutgoingBatchService();
    INodeService nodeService = engine.getNodeService();
    IClusterService clusterService = engine.getClusterService();
    int availableThreadPairs = parameterService.getInt(ParameterConstants.PUSH_THREAD_COUNT_PER_SERVER);
    long minimumPeriodBetweenPushesMs = parameterService.getLong(ParameterConstants.PUSH_MINIMUM_PERIOD_MS, -1);
    RemoteNodeStatuses statuses = new RemoteNodeStatuses(configurationService.getChannels(false));
    Node identityNode = nodeService.findIdentity(false);
    if (identityNode != null && identityNode.isSyncEnabled()) {
        List<NodeHost> hosts = nodeService.findNodeHosts(identityNode.getNodeId());
        int clusterInstanceCount = hosts != null && hosts.size() > 0 ? hosts.size() : 1;
        NodeSecurity identitySecurity = nodeService.findNodeSecurity(identityNode.getNodeId());
        if (identitySecurity != null && (force || !clusterService.isInfiniteLocked(ClusterConstants.PUSH))) {
            Iterator<OutgoingBatchByNodeChannelCount> nodeChannels = outgoingBatchService.getOutgoingBatchByNodeChannelCount(availableThreadPairs * clusterInstanceCount, NodeGroupLinkAction.P, true).iterator();
            // based on percentage
            while (nodeChannels.hasNext() && pushWorkersWorking.size() < availableThreadPairs) {
                OutgoingBatchByNodeChannelCount batchCount = nodeChannels.next();
                String nodeId = batchCount.getNodeId();
                String channelId = batchCount.getChannelId();
                Node remoteNode = nodeService.findNode(nodeId);
                NodeChannel nodeChannel = configurationService.getNodeChannel(channelId, nodeId, false);
                if (nodeChannel != null && !nodeChannel.isFileSyncFlag() && !pushWorkersWorking.contains(nodeChannel)) {
                    boolean meetsMinimumTime = true;
                    // TODO error backoff logic
                    if (minimumPeriodBetweenPushesMs > 0 && nodeChannel.getLastExtractTime() != null && (System.currentTimeMillis() - nodeChannel.getLastExtractTime().getTime()) < minimumPeriodBetweenPushesMs) {
                        meetsMinimumTime = false;
                    }
                    if (meetsMinimumTime && clusterService.lockNodeChannel(ClusterConstants.PUSH, nodeId, channelId)) {
                        NodeChannelExtractForPushWorker worker = new NodeChannelExtractForPushWorker(remoteNode, identityNode, identitySecurity, nodeChannel, statuses.add(nodeId, channelId));
                        pushWorkersWorking.add(nodeChannel);
                        nodeChannelExtractForPushWorker.execute(worker);
                    }
                }
            }
        }
    }
    return statuses;
}