Examples with TableReloadRequest org.jumpmind.symmetric.model.TableReloadRequest used on opensource projects

Example 1 with TableReloadRequest

use of org.jumpmind.symmetric.model.TableReloadRequest in project symmetric-ds by JumpMind.

the class DataService method insertReloadEvents.

public void insertReloadEvents(Node targetNode, boolean reverse, List<TableReloadRequest> reloadRequests, ProcessInfo processInfo) {
    if (engine.getClusterService().lock(ClusterConstants.SYNCTRIGGERS)) {
        try {
            synchronized (engine.getTriggerRouterService()) {
                engine.getClusterService().lock(ClusterConstants.SYNCTRIGGERS);
                boolean isFullLoad = reloadRequests == null || (reloadRequests.size() == 1 && reloadRequests.get(0).isFullLoadRequest());
                if (!reverse) {
                    log.info("Queueing up " + (isFullLoad ? "an initial" : "a") + " load to node " + targetNode.getNodeId());
                } else {
                    log.info("Queueing up a reverse " + (isFullLoad ? "initial" : "") + " load to node " + targetNode.getNodeId());
                }
                /*
                     * Outgoing data events are pointless because we are
                     * reloading all data
                     */
                if (isFullLoad) {
                    engine.getOutgoingBatchService().markAllAsSentForNode(targetNode.getNodeId(), false);
                }
                INodeService nodeService = engine.getNodeService();
                ITriggerRouterService triggerRouterService = engine.getTriggerRouterService();
                Node sourceNode = nodeService.findIdentity();
                boolean transactional = parameterService.is(ParameterConstants.DATA_RELOAD_IS_BATCH_INSERT_TRANSACTIONAL);
                String nodeIdRecord = reverse ? nodeService.findIdentityNodeId() : targetNode.getNodeId();
                NodeSecurity nodeSecurity = nodeService.findNodeSecurity(nodeIdRecord);
                ISqlTransaction transaction = null;
                try {
                    transaction = platform.getSqlTemplate().startSqlTransaction();
                    long loadId = engine.getSequenceService().nextVal(transaction, Constants.SEQUENCE_OUTGOING_BATCH_LOAD_ID);
                    processInfo.setCurrentLoadId(loadId);
                    String createBy = reverse ? nodeSecurity.getRevInitialLoadCreateBy() : nodeSecurity.getInitialLoadCreateBy();
                    List<TriggerHistory> triggerHistories = new ArrayList<TriggerHistory>();
                    if (isFullLoad) {
                        triggerHistories = triggerRouterService.getActiveTriggerHistories();
                    } else {
                        for (TableReloadRequest reloadRequest : reloadRequests) {
                            triggerHistories.addAll(engine.getTriggerRouterService().getActiveTriggerHistories(new Trigger(reloadRequest.getTriggerId(), null)));
                        }
                    }
                    processInfo.setDataCount(triggerHistories.size());
                    Map<Integer, List<TriggerRouter>> triggerRoutersByHistoryId = triggerRouterService.fillTriggerRoutersByHistIdAndSortHist(sourceNode.getNodeGroupId(), targetNode.getNodeGroupId(), triggerHistories);
                    if (isFullLoad) {
                        callReloadListeners(true, targetNode, transactional, transaction, loadId);
                        insertCreateSchemaScriptPriorToReload(targetNode, nodeIdRecord, loadId, createBy, transactional, transaction);
                    }
                    Map<String, TableReloadRequest> mapReloadRequests = convertReloadListToMap(reloadRequests);
                    String symNodeSecurityReloadChannel = null;
                    try {
                        symNodeSecurityReloadChannel = triggerRoutersByHistoryId.get(triggerHistories.get(0).getTriggerHistoryId()).get(0).getTrigger().getReloadChannelId();
                    } catch (Exception e) {
                    }
                    if (isFullLoad || (reloadRequests != null && reloadRequests.size() > 0)) {
                        insertSqlEventsPriorToReload(targetNode, nodeIdRecord, loadId, createBy, transactional, transaction, reverse, triggerHistories, triggerRoutersByHistoryId, mapReloadRequests, isFullLoad, symNodeSecurityReloadChannel);
                    }
                    insertCreateBatchesForReload(targetNode, loadId, createBy, triggerHistories, triggerRoutersByHistoryId, transactional, transaction, mapReloadRequests);
                    insertDeleteBatchesForReload(targetNode, loadId, createBy, triggerHistories, triggerRoutersByHistoryId, transactional, transaction, mapReloadRequests);
                    insertSQLBatchesForReload(targetNode, loadId, createBy, triggerHistories, triggerRoutersByHistoryId, transactional, transaction, mapReloadRequests);
                    insertLoadBatchesForReload(targetNode, loadId, createBy, triggerHistories, triggerRoutersByHistoryId, transactional, transaction, mapReloadRequests, processInfo);
                    if (isFullLoad) {
                        String afterSql = parameterService.getString(reverse ? ParameterConstants.INITIAL_LOAD_REVERSE_AFTER_SQL : ParameterConstants.INITIAL_LOAD_AFTER_SQL);
                        if (isNotBlank(afterSql)) {
                            insertSqlEvent(transaction, targetNode, afterSql, true, loadId, createBy);
                        }
                    }
                    insertFileSyncBatchForReload(targetNode, loadId, createBy, transactional, transaction, processInfo);
                    if (isFullLoad) {
                        callReloadListeners(false, targetNode, transactional, transaction, loadId);
                        if (!reverse) {
                            nodeService.setInitialLoadEnabled(transaction, nodeIdRecord, false, false, loadId, createBy);
                        } else {
                            nodeService.setReverseInitialLoadEnabled(transaction, nodeIdRecord, false, false, loadId, createBy);
                        }
                    }
                    if (!Constants.DEPLOYMENT_TYPE_REST.equals(targetNode.getDeploymentType())) {
                        insertNodeSecurityUpdate(transaction, nodeIdRecord, targetNode.getNodeId(), true, loadId, createBy, symNodeSecurityReloadChannel);
                    }
                    engine.getStatisticManager().incrementNodesLoaded(1);
                    if (reloadRequests != null && reloadRequests.size() > 0) {
                        for (TableReloadRequest request : reloadRequests) {
                            transaction.prepareAndExecute(getSql("updateProcessedTableReloadRequest"), loadId, new Date(), request.getTargetNodeId(), request.getSourceNodeId(), request.getTriggerId(), request.getRouterId(), request.getCreateTime());
                        }
                        log.info("Table reload request(s) for load id " + loadId + " have been processed.");
                    }
                    transaction.commit();
                } catch (Error ex) {
                    if (transaction != null) {
                        transaction.rollback();
                    }
                    throw ex;
                } catch (RuntimeException ex) {
                    if (transaction != null) {
                        transaction.rollback();
                    }
                    throw ex;
                } finally {
                    close(transaction);
                }
                if (!reverse) {
                    /*
                         * Remove all incoming events for the node that we are
                         * starting a reload for
                         */
                    engine.getPurgeService().purgeAllIncomingEventsForNode(targetNode.getNodeId());
                }
            }
        } finally {
            engine.getClusterService().unlock(ClusterConstants.SYNCTRIGGERS);
        }
    } else {
        log.info("Not attempting to insert reload events because sync trigger is currently running");
    }
}

Example 2 with TableReloadRequest

use of org.jumpmind.symmetric.model.TableReloadRequest in project symmetric-ds by JumpMind.

the class RouterService method processTableRequestLoads.

public void processTableRequestLoads(Node source, ProcessInfo processInfo) {
    List<TableReloadRequest> loadsToProcess = engine.getDataService().getTableReloadRequestToProcess(source.getNodeId());
    if (loadsToProcess.size() > 0) {
        processInfo.setStatus(ProcessInfo.Status.CREATING);
        log.info("Found " + loadsToProcess.size() + " table reload requests to process.");
        gapDetector.setFullGapAnalysis(true);
        Map<String, List<TableReloadRequest>> requestsSplitByLoad = new HashMap<String, List<TableReloadRequest>>();
        for (TableReloadRequest load : loadsToProcess) {
            if (load.isFullLoadRequest() && isValidLoadTarget(load.getTargetNodeId())) {
                List<TableReloadRequest> fullLoad = new ArrayList<TableReloadRequest>();
                fullLoad.add(load);
                engine.getDataService().insertReloadEvents(engine.getNodeService().findNode(load.getTargetNodeId()), false, fullLoad, processInfo);
            } else {
                NodeSecurity targetNodeSecurity = engine.getNodeService().findNodeSecurity(load.getTargetNodeId());
                boolean registered = targetNodeSecurity != null && (targetNodeSecurity.getRegistrationTime() != null || targetNodeSecurity.getNodeId().equals(targetNodeSecurity.getCreatedAtNodeId()));
                if (registered) {
                    // Make loads unique to the target and create time
                    String key = load.getTargetNodeId() + "::" + load.getCreateTime().toString();
                    if (!requestsSplitByLoad.containsKey(key)) {
                        requestsSplitByLoad.put(key, new ArrayList<TableReloadRequest>());
                    }
                    requestsSplitByLoad.get(key).add(load);
                } else {
                    log.warn("There was a load queued up for '{}', but the node is not registered.  It is being ignored", load.getTargetNodeId());
                }
            }
        }
        for (Map.Entry<String, List<TableReloadRequest>> entry : requestsSplitByLoad.entrySet()) {
            engine.getDataService().insertReloadEvents(engine.getNodeService().findNode(entry.getKey().split("::")[0]), false, entry.getValue(), processInfo);
        }
    }
}

Example 3 with TableReloadRequest

use of org.jumpmind.symmetric.model.TableReloadRequest in project symmetric-ds by JumpMind.

the class DataService method insertLoadBatchesForReload.

private void insertLoadBatchesForReload(Node targetNode, long loadId, String createBy, List<TriggerHistory> triggerHistories, Map<Integer, List<TriggerRouter>> triggerRoutersByHistoryId, boolean transactional, ISqlTransaction transaction, Map<String, TableReloadRequest> reloadRequests, ProcessInfo processInfo) {
    Map<String, Channel> channels = engine.getConfigurationService().getChannels(false);
    for (TriggerHistory triggerHistory : triggerHistories) {
        List<TriggerRouter> triggerRouters = triggerRoutersByHistoryId.get(triggerHistory.getTriggerHistoryId());
        processInfo.incrementCurrentDataCount();
        for (TriggerRouter triggerRouter : triggerRouters) {
            if (triggerRouter.getInitialLoadOrder() >= 0 && engine.getGroupletService().isTargetEnabled(triggerRouter, targetNode)) {
                String selectSql = null;
                if (reloadRequests != null) {
                    TableReloadRequest reloadRequest = reloadRequests.get(triggerRouter.getTriggerId() + triggerRouter.getRouterId());
                    selectSql = reloadRequest != null ? reloadRequest.getReloadSelect() : null;
                }
                if (StringUtils.isBlank(selectSql)) {
                    selectSql = StringUtils.isBlank(triggerRouter.getInitialLoadSelect()) ? Constants.ALWAYS_TRUE_CONDITION : triggerRouter.getInitialLoadSelect();
                }
                if (parameterService.is(ParameterConstants.INITIAL_LOAD_USE_EXTRACT_JOB)) {
                    Trigger trigger = triggerRouter.getTrigger();
                    String reloadChannel = getReloadChannelIdForTrigger(trigger, channels);
                    Channel channel = channels.get(reloadChannel);
                    Table table = platform.getTableFromCache(triggerHistory.getSourceCatalogName(), triggerHistory.getSourceSchemaName(), triggerHistory.getSourceTableName(), false);
                    processInfo.setCurrentTableName(table.getName());
                    int numberOfBatches = getNumberOfReloadBatches(table, triggerRouter, channel, targetNode, selectSql);
                    long startBatchId = -1;
                    long endBatchId = -1;
                    for (int i = 0; i < numberOfBatches; i++) {
                        // needs to grab the start and end batch id
                        endBatchId = insertReloadEvent(transaction, targetNode, triggerRouter, triggerHistory, selectSql, true, loadId, createBy, Status.RQ);
                        if (startBatchId == -1) {
                            startBatchId = endBatchId;
                        }
                    }
                    engine.getDataExtractorService().requestExtractRequest(transaction, targetNode.getNodeId(), channel.getQueue(), triggerRouter, startBatchId, endBatchId);
                } else {
                    insertReloadEvent(transaction, targetNode, triggerRouter, triggerHistory, selectSql, true, loadId, createBy, Status.NE);
                }
                if (!transactional) {
                    transaction.commit();
                }
            }
        }
    }
}

Example 4 with TableReloadRequest

use of org.jumpmind.symmetric.model.TableReloadRequest in project symmetric-ds by JumpMind.

the class DataService method insertSQLBatchesForReload.

private void insertSQLBatchesForReload(Node targetNode, long loadId, String createBy, List<TriggerHistory> triggerHistories, Map<Integer, List<TriggerRouter>> triggerRoutersByHistoryId, boolean transactional, ISqlTransaction transaction, Map<String, TableReloadRequest> reloadRequests) {
    if (reloadRequests != null && reloadRequests.size() > 0) {
        int sqlEventsSent = 0;
        for (TriggerHistory triggerHistory : triggerHistories) {
            List<TriggerRouter> triggerRouters = triggerRoutersByHistoryId.get(triggerHistory.getTriggerHistoryId());
            TableReloadRequest currentRequest = reloadRequests.get(ParameterConstants.ALL + ParameterConstants.ALL);
            boolean fullLoad = currentRequest == null ? false : true;
            for (TriggerRouter triggerRouter : triggerRouters) {
                if (!fullLoad) {
                    currentRequest = reloadRequests.get(triggerRouter.getTriggerId() + triggerRouter.getRouterId());
                }
                //Check the before custom sql is present on the specific table reload request
                if (currentRequest != null && currentRequest.getBeforeCustomSql() != null && currentRequest.getBeforeCustomSql().length() > 0 && engine.getGroupletService().isTargetEnabled(triggerRouter, targetNode)) {
                    String tableName = triggerRouter.qualifiedTargetTableName(triggerHistory);
                    String formattedBeforeSql = String.format(currentRequest.getBeforeCustomSql(), tableName) + ";";
                    insertSqlEvent(transaction, triggerHistory, triggerRouter.getTrigger().getChannelId(), targetNode, formattedBeforeSql, true, loadId, createBy);
                    sqlEventsSent++;
                    if (!transactional) {
                        transaction.commit();
                    }
                }
            }
        }
        if (sqlEventsSent > 0) {
            log.info("Before sending load {} to target node {} SQL data events were sent for {} tables", new Object[] { loadId, targetNode, sqlEventsSent });
        }
    }
}

Example 5 with TableReloadRequest

use of org.jumpmind.symmetric.model.TableReloadRequest in project symmetric-ds by JumpMind.

the class DataService method insertSqlEventsPriorToReload.

private void insertSqlEventsPriorToReload(Node targetNode, String nodeIdRecord, long loadId, String createBy, boolean transactional, ISqlTransaction transaction, boolean reverse, List<TriggerHistory> triggerHistories, Map<Integer, List<TriggerRouter>> triggerRoutersByHistoryId, Map<String, TableReloadRequest> reloadRequests, boolean isFullLoad, String channelId) {
    if (!Constants.DEPLOYMENT_TYPE_REST.equals(targetNode.getDeploymentType())) {
        /*
                 * Insert node security so the client doing the initial load knows
                 * that an initial load is currently happening
                 */
        insertNodeSecurityUpdate(transaction, nodeIdRecord, targetNode.getNodeId(), true, loadId, createBy, channelId);
        if (isFullLoad) {
            /*
                     * Mark incoming batches as OK at the target node because we marked
                     * outgoing batches as OK at the source
                     */
            insertSqlEvent(transaction, targetNode, String.format("update %s_incoming_batch set status='OK', error_flag=0 where node_id='%s' and status != 'OK'", tablePrefix, engine.getNodeService().findIdentityNodeId()), true, loadId, createBy);
        }
    }
    if (!isFullLoad && reloadRequests != null && reloadRequests.size() > 0) {
        String beforeSql = "";
        int beforeSqlSent = 0;
        for (TriggerHistory triggerHistory : triggerHistories) {
            List<TriggerRouter> triggerRouters = triggerRoutersByHistoryId.get(triggerHistory.getTriggerHistoryId());
            for (TriggerRouter triggerRouter : triggerRouters) {
                String key = triggerRouter.getTriggerId() + triggerRouter.getRouterId();
                TableReloadRequest currentRequest = reloadRequests.get(key);
                if (currentRequest == null) {
                    throw new SymmetricException("Could not locate table reload request for key '" + key + "'. Available requests are: " + reloadRequests.keySet());
                }
                beforeSql = currentRequest.getBeforeCustomSql();
                if (isNotBlank(beforeSql)) {
                    String tableName = triggerRouter.qualifiedTargetTableName(triggerHistory);
                    String formattedBeforeSql = String.format(beforeSql, tableName) + ";";
                    insertSqlEvent(transaction, targetNode, formattedBeforeSql, true, loadId, createBy);
                    beforeSqlSent++;
                }
            }
        }
        if (beforeSqlSent > 0) {
            log.info("Before sending load {} to target node {} the before sql [{}] was sent for {} tables", new Object[] { loadId, targetNode, beforeSql, beforeSqlSent });
        }
    } else {
        String beforeSql = parameterService.getString(reverse ? ParameterConstants.INITIAL_LOAD_REVERSE_BEFORE_SQL : ParameterConstants.INITIAL_LOAD_BEFORE_SQL);
        if (isNotBlank(beforeSql)) {
            insertSqlEvent(transaction, targetNode, beforeSql, true, loadId, createBy);
        }
    }
}