Examples with NumberMapper org.jumpmind.db.sql.mapper.NumberMapper used on opensource projects

Example 1 with NumberMapper

use of org.jumpmind.db.sql.mapper.NumberMapper in project symmetric-ds by JumpMind.

the class DataGapDetector method beforeRouting.

/**
 * Always make sure sym_data_gap is up to date to make sure that we don't
 * dual route data.
 */
public void beforeRouting() {
    long printStats = System.currentTimeMillis();
    ProcessInfo processInfo = this.statisticManager.newProcessInfo(new ProcessInfoKey(nodeService.findIdentityNodeId(), null, ProcessType.GAP_DETECT));
    try {
        long ts = System.currentTimeMillis();
        processInfo.setStatus(Status.QUERYING);
        final List<DataGap> gaps = dataService.findDataGaps();
        long lastDataId = -1;
        final int dataIdIncrementBy = parameterService.getInt(ParameterConstants.DATA_ID_INCREMENT_BY);
        final long maxDataToSelect = parameterService.getInt(ParameterConstants.ROUTING_LARGEST_GAP_SIZE);
        long databaseTime = symmetricDialect.getDatabaseTime();
        int idsFilled = 0;
        int newGapsInserted = 0;
        int rangeChecked = 0;
        int gapsDeleted = 0;
        for (final DataGap dataGap : gaps) {
            final boolean lastGap = dataGap.equals(gaps.get(gaps.size() - 1));
            String sql = routerService.getSql("selectDistinctDataIdFromDataEventUsingGapsSql");
            ISqlTemplate sqlTemplate = symmetricDialect.getPlatform().getSqlTemplate();
            Object[] params = new Object[] { dataGap.getStartId(), dataGap.getEndId() };
            lastDataId = -1;
            processInfo.setStatus(Status.QUERYING);
            long queryForIdsTs = System.currentTimeMillis();
            List<Number> ids = sqlTemplate.query(sql, new NumberMapper(), params);
            if (System.currentTimeMillis() - queryForIdsTs > Constants.LONG_OPERATION_THRESHOLD) {
                log.info("It took longer than {}ms to run the following sql for gap from {} to {}.  {}", new Object[] { Constants.LONG_OPERATION_THRESHOLD, dataGap.getStartId(), dataGap.getEndId(), sql });
            }
            processInfo.setStatus(Status.PROCESSING);
            idsFilled += ids.size();
            rangeChecked += dataGap.getEndId() - dataGap.getStartId();
            ISqlTransaction transaction = null;
            try {
                transaction = sqlTemplate.startSqlTransaction();
                for (Number number : ids) {
                    long dataId = number.longValue();
                    processInfo.incrementCurrentDataCount();
                    if (lastDataId == -1 && dataGap.getStartId() + dataIdIncrementBy <= dataId) {
                        // there was a new gap at the start
                        dataService.insertDataGap(transaction, new DataGap(dataGap.getStartId(), dataId - 1));
                        newGapsInserted++;
                    } else if (lastDataId != -1 && lastDataId + dataIdIncrementBy != dataId && lastDataId != dataId) {
                        // found a gap somewhere in the existing gap
                        dataService.insertDataGap(transaction, new DataGap(lastDataId + 1, dataId - 1));
                        newGapsInserted++;
                    }
                    lastDataId = dataId;
                }
                /* if we found data in the gap */
                if (lastDataId != -1) {
                    if (!lastGap && lastDataId + dataIdIncrementBy <= dataGap.getEndId()) {
                        dataService.insertDataGap(transaction, new DataGap(lastDataId + dataIdIncrementBy, dataGap.getEndId()));
                        newGapsInserted++;
                    }
                    dataService.deleteDataGap(transaction, dataGap);
                    gapsDeleted++;
                /*
                     * if we did not find data in the gap and it was not the
                     * last gap
                     */
                } else if (!lastGap) {
                    if (dataService.countDataInRange(dataGap.getStartId() - 1, dataGap.getEndId() + 1) == 0) {
                        if (symmetricDialect.supportsTransactionViews()) {
                            long transactionViewClockSyncThresholdInMs = parameterService.getLong(ParameterConstants.DBDIALECT_ORACLE_TRANSACTION_VIEW_CLOCK_SYNC_THRESHOLD_MS, 60000);
                            Date createTime = dataService.findCreateTimeOfData(dataGap.getEndId() + 1);
                            if (createTime != null && !symmetricDialect.areDatabaseTransactionsPendingSince(createTime.getTime() + transactionViewClockSyncThresholdInMs)) {
                                if (dataService.countDataInRange(dataGap.getStartId() - 1, dataGap.getEndId() + 1) == 0) {
                                    if (dataGap.getStartId() == dataGap.getEndId()) {
                                        log.info("Found a gap in data_id at {}.  Skipping it because there are no pending transactions in the database", dataGap.getStartId());
                                    } else {
                                        log.info("Found a gap in data_id from {} to {}.  Skipping it because there are no pending transactions in the database", dataGap.getStartId(), dataGap.getEndId());
                                    }
                                    dataService.deleteDataGap(transaction, dataGap);
                                    gapsDeleted++;
                                }
                            }
                        } else if (isDataGapExpired(dataGap.getEndId() + 1, databaseTime)) {
                            if (dataGap.getStartId() == dataGap.getEndId()) {
                                log.info("Found a gap in data_id at {}.  Skipping it because the gap expired", dataGap.getStartId());
                            } else {
                                log.info("Found a gap in data_id from {} to {}.  Skipping it because the gap expired", dataGap.getStartId(), dataGap.getEndId());
                            }
                            dataService.deleteDataGap(transaction, dataGap);
                            gapsDeleted++;
                        }
                    }
                }
                if (System.currentTimeMillis() - printStats > 30000) {
                    log.info("The data gap detection process has been running for {}ms, detected {} rows that have been previously routed over a total gap range of {}, " + "inserted {} new gaps, and deleted {} gaps", new Object[] { System.currentTimeMillis() - ts, idsFilled, rangeChecked, newGapsInserted, gapsDeleted });
                    printStats = System.currentTimeMillis();
                }
                transaction.commit();
            } catch (Error ex) {
                if (transaction != null) {
                    transaction.rollback();
                }
                throw ex;
            } catch (RuntimeException ex) {
                if (transaction != null) {
                    transaction.rollback();
                }
                throw ex;
            } finally {
                if (transaction != null) {
                    transaction.close();
                }
            }
        }
        if (lastDataId != -1) {
            dataService.insertDataGap(new DataGap(lastDataId + 1, lastDataId + maxDataToSelect));
        }
        long updateTimeInMs = System.currentTimeMillis() - ts;
        if (updateTimeInMs > 10000) {
            log.info("Detecting gaps took {} ms", updateTimeInMs);
        }
        processInfo.setStatus(Status.OK);
    } catch (RuntimeException ex) {
        processInfo.setStatus(Status.ERROR);
        throw ex;
    }
}

Example 2 with NumberMapper

use of org.jumpmind.db.sql.mapper.NumberMapper in project symmetric-ds by JumpMind.

the class AcknowledgeService method ack.

public BatchAckResult ack(final BatchAck batch) {
    IRegistrationService registrationService = engine.getRegistrationService();
    IStagingManager stagingManager = engine.getStagingManager();
    IOutgoingBatchService outgoingBatchService = engine.getOutgoingBatchService();
    BatchAckResult result = new BatchAckResult(batch);
    for (IAcknowledgeEventListener listener : engine.getExtensionService().getExtensionPointList(IAcknowledgeEventListener.class)) {
        listener.onAcknowledgeEvent(batch);
    }
    if (batch.getBatchId() == Constants.VIRTUAL_BATCH_FOR_REGISTRATION) {
        if (batch.getStatus() == Status.OK) {
            registrationService.markNodeAsRegistered(batch.getNodeId());
        }
    } else {
        OutgoingBatch outgoingBatch = outgoingBatchService.findOutgoingBatch(batch.getBatchId(), batch.getNodeId());
        Status status = batch.getStatus();
        if (outgoingBatch != null) {
            // is OK.
            if (outgoingBatch.getStatus() != Status.OK && outgoingBatch.getStatus() != Status.IG) {
                outgoingBatch.setStatus(status);
                outgoingBatch.setErrorFlag(batch.getStatus() == Status.ER);
            } else {
                // clearing the error flag in case the user set the batch
                // status to OK
                Status oldStatus = outgoingBatch.getStatus();
                outgoingBatch.setStatus(Status.OK);
                outgoingBatch.setErrorFlag(false);
                log.info("Batch {} for {} was set to {}.  Updating the status to OK", new Object[] { batch.getBatchId(), batch.getNodeId(), oldStatus.name() });
            }
            if (batch.isIgnored()) {
                outgoingBatch.incrementIgnoreCount();
            }
            outgoingBatch.setNetworkMillis(batch.getNetworkMillis());
            outgoingBatch.setFilterMillis(batch.getFilterMillis());
            outgoingBatch.setLoadMillis(batch.getDatabaseMillis());
            outgoingBatch.setSqlCode(batch.getSqlCode());
            outgoingBatch.setSqlState(batch.getSqlState());
            outgoingBatch.setSqlMessage(batch.getSqlMessage());
            if (batch.getStatus() == Status.ER && batch.getErrorLine() != 0) {
                List<Number> ids = sqlTemplate.query(getSql("selectDataIdSql"), new NumberMapper(), outgoingBatch.getBatchId());
                if (ids.size() >= batch.getErrorLine()) {
                    outgoingBatch.setFailedDataId(ids.get((int) batch.getErrorLine() - 1).longValue());
                }
            }
            if (status == Status.ER) {
                log.error("The outgoing batch {} failed{}", outgoingBatch.getNodeBatchId(), batch.getSqlMessage() != null ? ". " + batch.getSqlMessage() : "");
                RouterStats routerStats = engine.getStatisticManager().getRouterStatsByBatch(batch.getBatchId());
                if (routerStats != null) {
                    log.info("Router stats for batch " + outgoingBatch.getBatchId() + ": " + routerStats.toString());
                }
            } else if (!outgoingBatch.isCommonFlag()) {
                IStagedResource stagingResource = stagingManager.find(Constants.STAGING_CATEGORY_OUTGOING, outgoingBatch.getNodeId(), outgoingBatch.getBatchId());
                if (stagingResource != null) {
                    stagingResource.setState(State.DONE);
                }
            }
            outgoingBatchService.updateOutgoingBatch(outgoingBatch);
            if (status == Status.OK) {
                Channel channel = engine.getConfigurationService().getChannel(outgoingBatch.getChannelId());
                if (channel != null && channel.isFileSyncFlag()) {
                    /* Acknowledge the file_sync in case the file needs deleted. */
                    engine.getFileSyncService().acknowledgeFiles(outgoingBatch);
                }
                engine.getStatisticManager().removeRouterStatsByBatch(batch.getBatchId());
            }
        } else {
            log.error("Could not find batch {}-{} to acknowledge as {}", new Object[] { batch.getNodeId(), batch.getBatchId(), status.name() });
            result.setOk(false);
        }
    }
    return result;
}