Examples with Channel org.jumpmind.symmetric.model.Channel used on opensource projects

Example 11 with Channel

use of org.jumpmind.symmetric.model.Channel in project symmetric-ds by JumpMind.

the class DataExtractorService method execute.

/**
     * This is a callback method used by the NodeCommunicationService that extracts an initial load
     * in the background.
     */
public void execute(NodeCommunication nodeCommunication, RemoteNodeStatus status) {
    if (!isApplicable(nodeCommunication, status)) {
        log.debug("{} failed isApplicable check and will not run.", this);
        return;
    }
    List<ExtractRequest> requests = getExtractRequestsForNode(nodeCommunication);
    long ts = System.currentTimeMillis();
    /*
         * Process extract requests until it has taken longer than 30 seconds, and then
         * allow the process to return so progress status can be seen.
         */
    for (int i = 0; i < requests.size() && (System.currentTimeMillis() - ts) <= Constants.LONG_OPERATION_THRESHOLD; i++) {
        ExtractRequest request = requests.get(i);
        Node identity = nodeService.findIdentity();
        Node targetNode = nodeService.findNode(nodeCommunication.getNodeId());
        log.info("Extracting batches for request {}. Starting at batch {}.  Ending at batch {}", new Object[] { request.getRequestId(), request.getStartBatchId(), request.getEndBatchId() });
        List<OutgoingBatch> batches = outgoingBatchService.getOutgoingBatchRange(request.getStartBatchId(), request.getEndBatchId()).getBatches();
        ProcessInfo processInfo = statisticManager.newProcessInfo(new ProcessInfoKey(identity.getNodeId(), nodeCommunication.getQueue(), nodeCommunication.getNodeId(), getProcessType()));
        processInfo.setBatchCount(batches.size());
        try {
            boolean areBatchesOk = true;
            /*
                 * check to see if batches have been OK'd by another reload
                 * request 
                 */
            for (OutgoingBatch outgoingBatch : batches) {
                if (outgoingBatch.getStatus() != Status.OK) {
                    areBatchesOk = false;
                    break;
                }
            }
            if (!areBatchesOk) {
                Channel channel = configurationService.getChannel(batches.get(0).getChannelId());
                /*
                     * "Trick" the extractor to extract one reload batch, but we
                     * will split it across the N batches when writing it
                     */
                OutgoingBatch firstBatch = batches.get(0);
                processInfo.setCurrentLoadId(firstBatch.getLoadId());
                IStagedResource resource = getStagedResource(firstBatch);
                if (resource != null && resource.exists() && resource.getState() != State.CREATE) {
                    resource.delete();
                }
                MultiBatchStagingWriter multiBatchStatingWriter = buildMultiBatchStagingWriter(request, identity, targetNode, batches, processInfo, channel);
                extractOutgoingBatch(processInfo, targetNode, multiBatchStatingWriter, firstBatch, false, false, ExtractMode.FOR_SYM_CLIENT);
                for (OutgoingBatch outgoingBatch : batches) {
                    resource = getStagedResource(outgoingBatch);
                    if (resource != null) {
                        resource.setState(State.DONE);
                    }
                }
            } else {
                log.info("Batches already had an OK status for request {}, batches {} to {}.  Not extracting", new Object[] { request.getRequestId(), request.getStartBatchId(), request.getEndBatchId() });
            }
            /*
                 * re-query the batches to see if they have been OK'd while
                 * extracting
                 */
            List<OutgoingBatch> checkBatches = outgoingBatchService.getOutgoingBatchRange(request.getStartBatchId(), request.getEndBatchId()).getBatches();
            areBatchesOk = true;
            /*
                 * check to see if batches have been OK'd by another reload
                 * request while extracting
                 */
            for (OutgoingBatch outgoingBatch : checkBatches) {
                if (outgoingBatch.getStatus() != Status.OK) {
                    areBatchesOk = false;
                    break;
                }
            }
            ISqlTransaction transaction = null;
            try {
                transaction = sqlTemplate.startSqlTransaction();
                updateExtractRequestStatus(transaction, request.getRequestId(), ExtractStatus.OK);
                if (!areBatchesOk) {
                    for (OutgoingBatch outgoingBatch : batches) {
                        if (!parameterService.is(ParameterConstants.INITIAL_LOAD_EXTRACT_AND_SEND_WHEN_STAGED, false)) {
                            outgoingBatch.setStatus(Status.NE);
                            outgoingBatchService.updateOutgoingBatch(transaction, outgoingBatch);
                        }
                    }
                } else {
                    log.info("Batches already had an OK status for request {}, batches {} to {}.  Not updating the status to NE", new Object[] { request.getRequestId(), request.getStartBatchId(), request.getEndBatchId() });
                }
                transaction.commit();
                log.info("Done extracting {} batches for request {}", (request.getEndBatchId() - request.getStartBatchId()) + 1, request.getRequestId());
            } catch (Error ex) {
                if (transaction != null) {
                    transaction.rollback();
                }
                throw ex;
            } catch (RuntimeException ex) {
                if (transaction != null) {
                    transaction.rollback();
                }
                throw ex;
            } finally {
                close(transaction);
            }
            processInfo.setStatus(org.jumpmind.symmetric.model.ProcessInfo.Status.OK);
        } catch (CancellationException ex) {
            log.info("Cancelled extract request {}. Starting at batch {}.  Ending at batch {}", new Object[] { request.getRequestId(), request.getStartBatchId(), request.getEndBatchId() });
            processInfo.setStatus(org.jumpmind.symmetric.model.ProcessInfo.Status.OK);
        } catch (RuntimeException ex) {
            log.debug("Failed to extract batches for request {}. Starting at batch {}.  Ending at batch {}", new Object[] { request.getRequestId(), request.getStartBatchId(), request.getEndBatchId() });
            processInfo.setStatus(org.jumpmind.symmetric.model.ProcessInfo.Status.ERROR);
            throw ex;
        }
    }
}

Example 12 with Channel

use of org.jumpmind.symmetric.model.Channel in project symmetric-ds by JumpMind.

the class DataExtractorService method changeBatchStatus.

protected final boolean changeBatchStatus(Status status, OutgoingBatch currentBatch, ExtractMode mode) {
    if (currentBatch.getStatus() != Status.IG) {
        currentBatch.setStatus(status);
    }
    if (mode != ExtractMode.EXTRACT_ONLY) {
        long batchStatusUpdateMillis = parameterService.getLong(ParameterConstants.OUTGOING_BATCH_UPDATE_STATUS_MILLIS);
        int batchStatusUpdateDataCount = parameterService.getInt(ParameterConstants.OUTGOING_BATCH_UPDATE_STATUS_DATA_COUNT);
        Channel channel = configurationService.getChannel(currentBatch.getChannelId());
        if (currentBatch.getStatus() == Status.RQ || currentBatch.getStatus() == Status.LD || currentBatch.getLastUpdatedTime() == null || System.currentTimeMillis() - batchStatusUpdateMillis >= currentBatch.getLastUpdatedTime().getTime() || channel.isReloadFlag() || currentBatch.getDataEventCount() > batchStatusUpdateDataCount) {
            outgoingBatchService.updateOutgoingBatch(currentBatch);
            return true;
        }
        return false;
    } else {
        return false;
    }
}

Example 13 with Channel

use of org.jumpmind.symmetric.model.Channel in project symmetric-ds by JumpMind.

the class ConfigurationService method initDefaultChannels.

@Override
public void initDefaultChannels() {
    if (defaultChannels != null) {
        clearCache();
        createDefaultChannels();
        List<NodeChannel> channels = getNodeChannels(false);
        for (Channel defaultChannel : defaultChannels.values()) {
            Channel channel = defaultChannel.findInList(channels);
            if (channel == null) {
                log.info("Auto-configuring {} channel", defaultChannel.getChannelId());
                saveChannel(defaultChannel, true);
            } else if (channel.getChannelId().equals(Constants.CHANNEL_RELOAD) && !channel.isReloadFlag()) {
                log.info("Setting reload flag on reload channel");
                channel.setReloadFlag(true);
                saveChannel(channel, true);
            } else if (channel.getChannelId().equals(Constants.CHANNEL_FILESYNC) && !channel.isFileSyncFlag()) {
                log.info("Setting file sync flag on file sync channel");
                channel.setFileSyncFlag(true);
                saveChannel(channel, true);
            } else if (channel.getChannelId().equals(Constants.CHANNEL_FILESYNC_RELOAD) && (!channel.isFileSyncFlag() || !channel.isReloadFlag())) {
                log.info("Setting reload and file sync flag on file sync reload channel");
                channel.setFileSyncFlag(true);
                saveChannel(channel, true);
            } else {
                log.debug("No need to create channel {}.  It already exists", defaultChannel.getChannelId());
            }
        }
        clearCache();
    }
}

Example 14 with Channel

use of org.jumpmind.symmetric.model.Channel in project symmetric-ds by JumpMind.

the class DataService method insertLoadBatchesForReload.

private void insertLoadBatchesForReload(Node targetNode, long loadId, String createBy, List<TriggerHistory> triggerHistories, Map<Integer, List<TriggerRouter>> triggerRoutersByHistoryId, boolean transactional, ISqlTransaction transaction, Map<String, TableReloadRequest> reloadRequests, ProcessInfo processInfo) {
    Map<String, Channel> channels = engine.getConfigurationService().getChannels(false);
    for (TriggerHistory triggerHistory : triggerHistories) {
        List<TriggerRouter> triggerRouters = triggerRoutersByHistoryId.get(triggerHistory.getTriggerHistoryId());
        processInfo.incrementCurrentDataCount();
        for (TriggerRouter triggerRouter : triggerRouters) {
            if (triggerRouter.getInitialLoadOrder() >= 0 && engine.getGroupletService().isTargetEnabled(triggerRouter, targetNode)) {
                String selectSql = null;
                if (reloadRequests != null) {
                    TableReloadRequest reloadRequest = reloadRequests.get(triggerRouter.getTriggerId() + triggerRouter.getRouterId());
                    selectSql = reloadRequest != null ? reloadRequest.getReloadSelect() : null;
                }
                if (StringUtils.isBlank(selectSql)) {
                    selectSql = StringUtils.isBlank(triggerRouter.getInitialLoadSelect()) ? Constants.ALWAYS_TRUE_CONDITION : triggerRouter.getInitialLoadSelect();
                }
                if (parameterService.is(ParameterConstants.INITIAL_LOAD_USE_EXTRACT_JOB)) {
                    Trigger trigger = triggerRouter.getTrigger();
                    String reloadChannel = getReloadChannelIdForTrigger(trigger, channels);
                    Channel channel = channels.get(reloadChannel);
                    Table table = platform.getTableFromCache(triggerHistory.getSourceCatalogName(), triggerHistory.getSourceSchemaName(), triggerHistory.getSourceTableName(), false);
                    processInfo.setCurrentTableName(table.getName());
                    int numberOfBatches = getNumberOfReloadBatches(table, triggerRouter, channel, targetNode, selectSql);
                    long startBatchId = -1;
                    long endBatchId = -1;
                    for (int i = 0; i < numberOfBatches; i++) {
                        // needs to grab the start and end batch id
                        endBatchId = insertReloadEvent(transaction, targetNode, triggerRouter, triggerHistory, selectSql, true, loadId, createBy, Status.RQ);
                        if (startBatchId == -1) {
                            startBatchId = endBatchId;
                        }
                    }
                    engine.getDataExtractorService().requestExtractRequest(transaction, targetNode.getNodeId(), channel.getQueue(), triggerRouter, startBatchId, endBatchId);
                } else {
                    insertReloadEvent(transaction, targetNode, triggerRouter, triggerHistory, selectSql, true, loadId, createBy, Status.NE);
                }
                if (!transactional) {
                    transaction.commit();
                }
            }
        }
    }
}

Example 15 with Channel

use of org.jumpmind.symmetric.model.Channel in project symmetric-ds by JumpMind.

the class DataService method getReloadChannelIdForTrigger.

private String getReloadChannelIdForTrigger(Trigger trigger, Map<String, Channel> channels) {
    String channelId = trigger != null ? trigger.getChannelId() : Constants.CHANNEL_DEFAULT;
    if (parameterService.is(ParameterConstants.INITIAL_LOAD_USE_RELOAD_CHANNEL)) {
        Channel normalChannel = channels.get(channelId);
        Channel reloadChannel = channels.get(trigger != null ? trigger.getReloadChannelId() : Constants.CHANNEL_RELOAD);
        if (normalChannel.isFileSyncFlag()) {
            if (reloadChannel != null && reloadChannel.isFileSyncFlag()) {
                channelId = reloadChannel.getChannelId();
            }
        } else {
            if (reloadChannel != null && reloadChannel.isReloadFlag()) {
                channelId = reloadChannel.getChannelId();
            } else {
                channelId = Constants.CHANNEL_RELOAD;
            }
        }
    }
    return channelId;
}