Examples with DataGap org.jumpmind.symmetric.model.DataGap used on opensource projects

Example 11 with DataGap

use of org.jumpmind.symmetric.model.DataGap in project symmetric-ds by JumpMind.

the class DataServiceTest method testFindDataGaps2.

@Test
public void testFindDataGaps2() throws Exception {
    final List<DataGap> gaps1 = new ArrayList<DataGap>();
    gaps1.add(new DataGap(30953884, 80953883));
    gaps1.add(new DataGap(30953883, 80953883));
    when(sqlTemplate.queryForLong(Matchers.anyString())).thenReturn(0L);
    @SuppressWarnings("unchecked") ISqlRowMapper<DataGap> anyMapper = (ISqlRowMapper<DataGap>) Matchers.anyObject();
    when(sqlTemplate.query(Matchers.anyString(), anyMapper, (Object[]) Matchers.anyVararg())).thenReturn(gaps1);
    dataService.findDataGaps();
    verifyNoMoreInteractions(sqlTransaction);
}

Example 12 with DataGap

use of org.jumpmind.symmetric.model.DataGap in project symmetric-ds by JumpMind.

the class DataGapFastDetector method afterRouting.

/**
     * Always make sure sym_data_gap is up to date to make sure that we don't
     * dual route data.
     */
public void afterRouting() {
    ProcessInfo processInfo = this.statisticManager.newProcessInfo(new ProcessInfoKey(nodeService.findIdentityNodeId(), null, ProcessType.GAP_DETECT));
    processInfo.setStatus(Status.PROCESSING);
    long printStats = System.currentTimeMillis();
    long gapTimoutInMs = parameterService.getLong(ParameterConstants.ROUTING_STALE_DATA_ID_GAP_TIME);
    final int dataIdIncrementBy = parameterService.getInt(ParameterConstants.DATA_ID_INCREMENT_BY);
    Date currentDate = new Date(routingStartTime);
    boolean isBusyExpire = false;
    long lastBusyExpireRunTime = getLastBusyExpireRunTime();
    if (!isAllDataRead) {
        if (lastBusyExpireRunTime == 0) {
            setLastBusyExpireRunTime(System.currentTimeMillis());
        } else {
            long busyExpireMillis = parameterService.getLong(ParameterConstants.ROUTING_STALE_GAP_BUSY_EXPIRE_TIME);
            isBusyExpire = System.currentTimeMillis() - lastBusyExpireRunTime >= busyExpireMillis;
        }
    } else if (lastBusyExpireRunTime != 0) {
        setLastBusyExpireRunTime(0);
    }
    try {
        long ts = System.currentTimeMillis();
        long lastDataId = -1;
        int dataIdCount = 0;
        int rangeChecked = 0;
        int expireChecked = 0;
        gapsAll.addAll(gaps);
        Map<DataGap, List<Long>> dataIdMap = getDataIdMap();
        if (System.currentTimeMillis() - ts > 30000) {
            log.info("It took {}ms to map {} data IDs into {} gaps", new Object[] { System.currentTimeMillis() - ts, dataIds.size(), gaps.size() });
        }
        for (final DataGap dataGap : gaps) {
            final boolean lastGap = dataGap.equals(gaps.get(gaps.size() - 1));
            lastDataId = -1;
            List<Long> ids = dataIdMap.get(dataGap);
            dataIdCount += ids.size();
            rangeChecked += dataGap.getEndId() - dataGap.getStartId();
            // if we found data in the gap
            if (ids.size() > 0) {
                gapsDeleted.add(dataGap);
                gapsAll.remove(dataGap);
            // if we did not find data in the gap and it was not the last gap
            } else if (!lastGap && (isAllDataRead || isBusyExpire)) {
                Date createTime = dataGap.getCreateTime();
                boolean isExpired = false;
                if (supportsTransactionViews) {
                    isExpired = createTime != null && (createTime.getTime() < earliestTransactionTime || earliestTransactionTime == 0);
                } else {
                    isExpired = createTime != null && routingStartTime - createTime.getTime() > gapTimoutInMs;
                }
                if (isExpired) {
                    boolean isGapEmpty = false;
                    if (!isAllDataRead) {
                        isGapEmpty = dataService.countDataInRange(dataGap.getStartId() - 1, dataGap.getEndId() + 1) == 0;
                        expireChecked++;
                    }
                    if (isAllDataRead || isGapEmpty) {
                        if (log.isDebugEnabled()) {
                            if (dataGap.getStartId() == dataGap.getEndId()) {
                                log.debug("Found a gap in data_id at {}.  Skipping it because " + (supportsTransactionViews ? "there are no pending transactions" : "the gap expired"), dataGap.getStartId());
                            } else {
                                log.debug("Found a gap in data_id from {} to {}.  Skipping it because " + (supportsTransactionViews ? "there are no pending transactions" : "the gap expired"), dataGap.getStartId(), dataGap.getEndId());
                            }
                        }
                        gapsDeleted.add(dataGap);
                        gapsAll.remove(dataGap);
                    }
                }
            }
            for (Number number : ids) {
                long dataId = number.longValue();
                processInfo.incrementCurrentDataCount();
                if (lastDataId == -1 && dataGap.getStartId() + dataIdIncrementBy <= dataId) {
                    // there was a new gap at the start
                    addDataGap(new DataGap(dataGap.getStartId(), dataId - 1, currentDate));
                } else if (lastDataId != -1 && lastDataId + dataIdIncrementBy != dataId && lastDataId != dataId) {
                    // found a gap somewhere in the existing gap
                    addDataGap(new DataGap(lastDataId + 1, dataId - 1, currentDate));
                }
                lastDataId = dataId;
            }
            // if we found data in the gap
            if (lastDataId != -1 && !lastGap && lastDataId + dataIdIncrementBy <= dataGap.getEndId()) {
                addDataGap(new DataGap(lastDataId + dataIdIncrementBy, dataGap.getEndId(), currentDate));
            }
            if (System.currentTimeMillis() - printStats > 30000) {
                log.info("The data gap detection has been running for {}ms, detected {} rows over a gap range of {}, " + "found {} new gaps, found old {} gaps, and checked data in {} gaps", new Object[] { System.currentTimeMillis() - ts, dataIdCount, rangeChecked, gapsAdded.size(), gapsDeleted.size(), expireChecked });
                printStats = System.currentTimeMillis();
            }
        }
        if (lastDataId != -1) {
            DataGap newGap = new DataGap(lastDataId + 1, lastDataId + maxDataToSelect, currentDate);
            if (addDataGap(newGap)) {
                log.debug("Inserting new last data gap: {}", newGap);
            }
        }
        printStats = saveDataGaps(ts, printStats);
        setFullGapAnalysis(false);
        if (isBusyExpire) {
            setLastBusyExpireRunTime(System.currentTimeMillis());
        }
        long updateTimeInMs = System.currentTimeMillis() - ts;
        if (updateTimeInMs > 10000) {
            log.info("Detecting gaps took {} ms", updateTimeInMs);
        }
        processInfo.setStatus(Status.OK);
    } catch (RuntimeException ex) {
        processInfo.setStatus(Status.ERROR);
        throw ex;
    }
}

Example 13 with DataGap

use of org.jumpmind.symmetric.model.DataGap in project symmetric-ds by JumpMind.

the class DataGapFastDetector method insertDataGaps.

protected long insertDataGaps(ISqlTransaction transaction, long ts, long printStats) {
    int counter = 0;
    for (DataGap dataGap : gapsAdded) {
        dataService.insertDataGap(transaction, dataGap);
        counter++;
        if (System.currentTimeMillis() - printStats > 30000) {
            log.info("The data gap detection has been running for {}ms, inserted {} of {} new gaps", new Object[] { System.currentTimeMillis() - ts, counter, gapsDeleted.size() });
            printStats = System.currentTimeMillis();
        }
    }
    return printStats;
}

Example 14 with DataGap

use of org.jumpmind.symmetric.model.DataGap in project symmetric-ds by JumpMind.

the class DataGapFastDetector method saveDataGaps.

protected long saveDataGaps(long ts, long printStats) {
    ISqlTemplate sqlTemplate = symmetricDialect.getPlatform().getSqlTemplate();
    int totalGapChanges = gapsDeleted.size() + gapsAdded.size();
    if (totalGapChanges > 0) {
        ISqlTransaction transaction = null;
        gaps = new ArrayList<DataGap>(gapsAll);
        Collections.sort(gaps);
        try {
            transaction = sqlTemplate.startSqlTransaction();
            int maxGapChanges = parameterService.getInt(ParameterConstants.ROUTING_MAX_GAP_CHANGES);
            if (!parameterService.is(ParameterConstants.CLUSTER_LOCKING_ENABLED) && (totalGapChanges > maxGapChanges || useInMemoryGaps)) {
                dataService.deleteAllDataGaps(transaction);
                if (useInMemoryGaps && totalGapChanges <= maxGapChanges) {
                    log.info("There are {} data gap changes, which is within the max of {}, so switching to database", totalGapChanges, maxGapChanges);
                    useInMemoryGaps = false;
                    printStats = insertDataGaps(transaction, ts, printStats);
                } else {
                    if (!useInMemoryGaps) {
                        log.info("There are {} data gap changes, which exceeds the max of {}, so switching to in-memory", totalGapChanges, maxGapChanges);
                        useInMemoryGaps = true;
                    }
                    DataGap newGap = new DataGap(gaps.get(0).getStartId(), gaps.get(gaps.size() - 1).getEndId());
                    dataService.insertDataGap(transaction, newGap);
                }
            } else {
                printStats = deleteDataGaps(transaction, ts, printStats);
                printStats = insertDataGaps(transaction, ts, printStats);
            }
            transaction.commit();
        } catch (Error ex) {
            if (transaction != null) {
                transaction.rollback();
            }
            throw ex;
        } catch (RuntimeException ex) {
            if (transaction != null) {
                transaction.rollback();
            }
            throw ex;
        } finally {
            if (transaction != null) {
                transaction.close();
            }
        }
    }
    return printStats;
}

Example 15 with DataGap

use of org.jumpmind.symmetric.model.DataGap in project symmetric-ds by JumpMind.

the class DataGapFastDetector method queryDataIdMap.

protected void queryDataIdMap() {
    String sql = routerService.getSql("selectDistinctDataIdFromDataEventUsingGapsSql");
    ISqlTemplate sqlTemplate = symmetricDialect.getPlatform().getSqlTemplate();
    for (DataGap dataGap : gaps) {
        long queryForIdsTs = System.currentTimeMillis();
        Object[] params = new Object[] { dataGap.getStartId(), dataGap.getEndId() };
        List<Long> ids = sqlTemplate.query(sql, this, params);
        dataIds.addAll(ids);
        if (System.currentTimeMillis() - queryForIdsTs > Constants.LONG_OPERATION_THRESHOLD) {
            log.info("It took longer than {}ms to run the following sql for gap from {} to {}.  {}", new Object[] { Constants.LONG_OPERATION_THRESHOLD, dataGap.getStartId(), dataGap.getEndId(), sql });
        }
    }
}