Examples with DataGap org.jumpmind.symmetric.model.DataGap used on opensource projects

Example 16 with DataGap

use of org.jumpmind.symmetric.model.DataGap in project symmetric-ds by JumpMind.

the class RouterService method getReadyChannels.

protected Set<String> getReadyChannels() {
    List<DataGap> dataGaps = gapDetector.getDataGaps();
    int dataIdSqlType = engine.getSymmetricDialect().getSqlTypeForIds();
    int numberOfGapsToQualify = parameterService.getInt(ParameterConstants.ROUTING_MAX_GAPS_TO_QUALIFY_IN_SQL, 100);
    int maxGapsBeforeGreaterThanQuery = parameterService.getInt(ParameterConstants.ROUTING_DATA_READER_THRESHOLD_GAPS_TO_USE_GREATER_QUERY, 100);
    String sql;
    Object[] args;
    int[] types;
    if (maxGapsBeforeGreaterThanQuery > 0 && dataGaps.size() > maxGapsBeforeGreaterThanQuery) {
        sql = getSql("selectChannelsUsingStartDataId");
        args = new Object[] { dataGaps.get(0).getStartId() };
        types = new int[] { dataIdSqlType };
    } else {
        sql = qualifyUsingDataGaps(dataGaps, numberOfGapsToQualify, getSql("selectChannelsUsingGapsSql"));
        int numberOfArgs = 2 * (numberOfGapsToQualify < dataGaps.size() ? numberOfGapsToQualify : dataGaps.size());
        args = new Object[numberOfArgs];
        types = new int[numberOfArgs];
        for (int i = 0; i < numberOfGapsToQualify && i < dataGaps.size(); i++) {
            DataGap gap = dataGaps.get(i);
            args[i * 2] = gap.getStartId();
            types[i * 2] = dataIdSqlType;
            if ((i + 1) == numberOfGapsToQualify && (i + 1) < dataGaps.size()) {
                args[i * 2 + 1] = dataGaps.get(dataGaps.size() - 1).getEndId();
            } else {
                args[i * 2 + 1] = gap.getEndId();
            }
            types[i * 2 + 1] = dataIdSqlType;
        }
    }
    final Set<String> readyChannels = new HashSet<String>();
    sqlTemplate.query(sql, new ISqlRowMapper<String>() {

        public String mapRow(Row row) {
            readyChannels.add(row.getString("channel_id"));
            return null;
        }
    }, args, types);
    return readyChannels;
}

Example 17 with DataGap

use of org.jumpmind.symmetric.model.DataGap in project symmetric-ds by JumpMind.

the class DataGapDetectorTest method testGapsOverlap.

@Test
public void testGapsOverlap() throws Exception {
    List<Long> dataIds = new ArrayList<Long>();
    List<DataGap> dataGaps = new ArrayList<DataGap>();
    dataGaps.add(new DataGap(30953883, 80953883));
    dataGaps.add(new DataGap(30953884, 80953883));
    runGapDetector(dataGaps, dataIds, true);
    verify(dataService).findDataGaps();
    verify(dataService).deleteDataGap(sqlTransaction, new DataGap(30953884, 80953883));
    verifyNoMoreInteractions(dataService);
}

Example 18 with DataGap

use of org.jumpmind.symmetric.model.DataGap in project symmetric-ds by JumpMind.

the class DataGapDetectorTest method testGapBusyExpireNoRun.

@Test
public void testGapBusyExpireNoRun() throws Exception {
    List<DataGap> dataGaps = new ArrayList<DataGap>();
    dataGaps.add(new DataGap(3, 3));
    dataGaps.add(new DataGap(5, 6));
    dataGaps.add(new DataGap(7, 50000006));
    when(symmetricDialect.getDatabaseTime()).thenReturn(System.currentTimeMillis() + 60001L);
    when(contextService.getLong(ContextConstants.ROUTING_LAST_BUSY_EXPIRE_RUN_TIME)).thenReturn(System.currentTimeMillis());
    runGapDetector(dataGaps, new ArrayList<Long>(), false);
    verify(dataService).findDataGaps();
    verifyNoMoreInteractions(dataService);
}

Example 19 with DataGap

use of org.jumpmind.symmetric.model.DataGap in project symmetric-ds by JumpMind.

the class SnapshotUtil method writeRuntimeStats.

protected static void writeRuntimeStats(ISymmetricEngine engine, File tmpDir) {
    FileOutputStream fos = null;
    try {
        fos = new FileOutputStream(new File(tmpDir, "runtime-stats.properties"));
        Properties runtimeProperties = new Properties() {

            private static final long serialVersionUID = 1L;

            public synchronized Enumeration<Object> keys() {
                return Collections.enumeration(new TreeSet<Object>(super.keySet()));
            }
        };
        DataSource dataSource = engine.getDatabasePlatform().getDataSource();
        if (dataSource instanceof BasicDataSource) {
            BasicDataSource dbcp = (BasicDataSource) dataSource;
            runtimeProperties.setProperty("connections.idle", String.valueOf(dbcp.getNumIdle()));
            runtimeProperties.setProperty("connections.used", String.valueOf(dbcp.getNumActive()));
            runtimeProperties.setProperty("connections.max", String.valueOf(dbcp.getMaxActive()));
        }
        Runtime rt = Runtime.getRuntime();
        runtimeProperties.setProperty("memory.free", String.valueOf(rt.freeMemory()));
        runtimeProperties.setProperty("memory.used", String.valueOf(rt.totalMemory() - rt.freeMemory()));
        runtimeProperties.setProperty("memory.max", String.valueOf(rt.maxMemory()));
        List<MemoryPoolMXBean> memoryPools = new ArrayList<MemoryPoolMXBean>(ManagementFactory.getMemoryPoolMXBeans());
        long usedHeapMemory = 0;
        for (MemoryPoolMXBean memoryPool : memoryPools) {
            if (memoryPool.getType().equals(MemoryType.HEAP)) {
                MemoryUsage memoryUsage = memoryPool.getCollectionUsage();
                runtimeProperties.setProperty("memory.heap." + memoryPool.getName().toLowerCase().replaceAll(" ", "."), Long.toString(memoryUsage.getUsed()));
                usedHeapMemory += memoryUsage.getUsed();
            }
        }
        runtimeProperties.setProperty("memory.heap.total", Long.toString(usedHeapMemory));
        OperatingSystemMXBean osBean = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
        runtimeProperties.setProperty("os.name", System.getProperty("os.name") + " (" + System.getProperty("os.arch") + ")");
        runtimeProperties.setProperty("os.processors", String.valueOf(osBean.getAvailableProcessors()));
        runtimeProperties.setProperty("os.load.average", String.valueOf(osBean.getSystemLoadAverage()));
        runtimeProperties.setProperty("engine.is.started", Boolean.toString(engine.isStarted()));
        runtimeProperties.setProperty("engine.last.restart", engine.getLastRestartTime().toString());
        runtimeProperties.setProperty("time.server", new Date().toString());
        runtimeProperties.setProperty("time.database", new Date(engine.getSymmetricDialect().getDatabaseTime()).toString());
        runtimeProperties.setProperty("batch.unrouted.data.count", Long.toString(engine.getRouterService().getUnroutedDataCount()));
        runtimeProperties.setProperty("batch.outgoing.errors.count", Long.toString(engine.getOutgoingBatchService().countOutgoingBatchesInError()));
        runtimeProperties.setProperty("batch.outgoing.tosend.count", Long.toString(engine.getOutgoingBatchService().countOutgoingBatchesUnsent()));
        runtimeProperties.setProperty("batch.incoming.errors.count", Long.toString(engine.getIncomingBatchService().countIncomingBatchesInError()));
        List<DataGap> gaps = engine.getDataService().findDataGapsByStatus(DataGap.Status.GP);
        runtimeProperties.setProperty("data.gap.count", Long.toString(gaps.size()));
        if (gaps.size() > 0) {
            runtimeProperties.setProperty("data.gap.start.id", Long.toString(gaps.get(0).getStartId()));
            runtimeProperties.setProperty("data.gap.end.id", Long.toString(gaps.get(gaps.size() - 1).getEndId()));
        }
        runtimeProperties.setProperty("data.id.min", Long.toString(engine.getDataService().findMinDataId()));
        runtimeProperties.setProperty("data.id.max", Long.toString(engine.getDataService().findMaxDataId()));
        runtimeProperties.put("jvm.title", Runtime.class.getPackage().getImplementationTitle());
        runtimeProperties.put("jvm.vendor", Runtime.class.getPackage().getImplementationVendor());
        runtimeProperties.put("jvm.version", Runtime.class.getPackage().getImplementationVersion());
        RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean();
        List<String> arguments = runtimeMxBean.getInputArguments();
        runtimeProperties.setProperty("jvm.arguments", arguments.toString());
        runtimeProperties.store(fos, "runtime-stats.properties");
    } catch (Exception e) {
        log.warn("Failed to export runtime-stats information", e);
    } finally {
        IOUtils.closeQuietly(fos);
    }
}

Example 20 with DataGap

use of org.jumpmind.symmetric.model.DataGap in project symmetric-ds by JumpMind.

the class DataGapFastDetector method fixOverlappingGaps.

protected void fixOverlappingGaps(List<DataGap> gaps, ProcessInfo processInfo) {
    try {
        ISqlTransaction transaction = null;
        log.debug("Looking for overlapping gaps");
        try {
            ISqlTemplate sqlTemplate = symmetricDialect.getPlatform().getSqlTemplate();
            transaction = sqlTemplate.startSqlTransaction();
            DataGap prevGap = null, lastGap = null;
            for (int i = 0; i < gaps.size(); i++) {
                DataGap curGap = gaps.get(i);
                if (lastGap != null) {
                    log.warn("Removing gap found after last gap: " + curGap);
                    dataService.deleteDataGap(transaction, curGap);
                    gaps.remove(i--);
                } else {
                    if (lastGap == null && curGap.gapSize() >= maxDataToSelect - 1) {
                        lastGap = curGap;
                    }
                    if (prevGap != null) {
                        if (prevGap.overlaps(curGap)) {
                            log.warn("Removing overlapping gaps: " + prevGap + ", " + curGap);
                            dataService.deleteDataGap(transaction, prevGap);
                            dataService.deleteDataGap(transaction, curGap);
                            DataGap newGap = null;
                            if (curGap.equals(lastGap)) {
                                newGap = new DataGap(prevGap.getStartId(), prevGap.getStartId() + maxDataToSelect - 1);
                            } else {
                                newGap = new DataGap(prevGap.getStartId(), prevGap.getEndId() > curGap.getEndId() ? prevGap.getEndId() : curGap.getEndId());
                            }
                            log.warn("Inserting new gap to fix overlap: " + newGap);
                            dataService.insertDataGap(transaction, newGap);
                            gaps.remove(i--);
                            gaps.set(i, newGap);
                            curGap = newGap;
                        }
                    }
                }
                prevGap = curGap;
            }
            transaction.commit();
        } catch (Error ex) {
            if (transaction != null) {
                transaction.rollback();
            }
            throw ex;
        } catch (RuntimeException ex) {
            if (transaction != null) {
                transaction.rollback();
            }
            throw ex;
        } finally {
            if (transaction != null) {
                transaction.close();
            }
        }
    } catch (RuntimeException ex) {
        processInfo.setStatus(Status.ERROR);
        throw ex;
    }
}