Examples with IDataToRouteReader org.jumpmind.symmetric.route.IDataToRouteReader used on opensource projects

Example 1 with IDataToRouteReader

use of org.jumpmind.symmetric.route.IDataToRouteReader in project symmetric-ds by JumpMind.

the class RouterService method selectDataAndRoute.

/**
     * Pre-read data and fill up a queue so we can peek ahead to see if we have
     * crossed a database transaction boundary. Then route each {@link Data}
     * while continuing to keep the queue filled until the result set is
     * entirely read.
     * 
     * @param conn
     *            The connection to use for selecting the data.
     * @param context
     *            The current context of the routing process
     */
protected int selectDataAndRoute(ProcessInfo processInfo, ChannelRouterContext context) throws InterruptedException {
    IDataToRouteReader reader = startReading(context);
    Data data = null;
    Data nextData = null;
    int totalDataCount = 0;
    int totalDataEventCount = 0;
    int statsDataCount = 0;
    int statsDataEventCount = 0;
    final int maxNumberOfEventsBeforeFlush = parameterService.getInt(ParameterConstants.ROUTING_FLUSH_JDBC_BATCH_SIZE);
    try {
        long ts = System.currentTimeMillis();
        long startTime = System.currentTimeMillis();
        nextData = reader.take();
        do {
            if (nextData != null) {
                data = nextData;
                nextData = reader.take();
                if (data != null) {
                    processInfo.setCurrentTableName(data.getTableName());
                    processInfo.incrementCurrentDataCount();
                    boolean atTransactionBoundary = false;
                    if (nextData != null) {
                        String nextTxId = nextData.getTransactionId();
                        atTransactionBoundary = nextTxId == null || !nextTxId.equals(data.getTransactionId());
                    }
                    context.setEncountedTransactionBoundary(atTransactionBoundary);
                    statsDataCount++;
                    totalDataCount++;
                    int dataEventsInserted = routeData(processInfo, data, context);
                    statsDataEventCount += dataEventsInserted;
                    totalDataEventCount += dataEventsInserted;
                    long insertTs = System.currentTimeMillis();
                    try {
                        if (maxNumberOfEventsBeforeFlush <= context.getDataEventList().size() || context.isNeedsCommitted()) {
                            engine.getDataService().insertDataEvents(context.getSqlTransaction(), context.getDataEventList());
                            context.clearDataEventsList();
                        }
                        if (context.isNeedsCommitted()) {
                            completeBatchesAndCommit(context);
                        }
                    } finally {
                        context.incrementStat(System.currentTimeMillis() - insertTs, ChannelRouterContext.STAT_INSERT_DATA_EVENTS_MS);
                        if (statsDataCount > StatisticConstants.FLUSH_SIZE_ROUTER_DATA) {
                            engine.getStatisticManager().incrementDataRouted(context.getChannel().getChannelId(), statsDataCount);
                            statsDataCount = 0;
                            engine.getStatisticManager().incrementDataEventInserted(context.getChannel().getChannelId(), statsDataEventCount);
                            statsDataEventCount = 0;
                        }
                    }
                    long routeTs = System.currentTimeMillis() - ts;
                    if (routeTs > 60000 && context != null) {
                        log.info("Routing for channel '{}' has been processing for {} seconds. The following stats have been gathered: " + "totalDataRoutedCount={}, totalDataEventCount={}, startDataId={}, endDataId={}, dataReadCount={}, peekAheadFillCount={}, transactions={}, dataGaps={}", new Object[] { context.getChannel().getChannelId(), ((System.currentTimeMillis() - startTime) / 1000), totalDataCount, totalDataEventCount, context.getStartDataId(), context.getEndDataId(), context.getDataReadCount(), context.getPeekAheadFillCount(), context.getTransactions().toString(), context.getDataGaps().toString() });
                        ts = System.currentTimeMillis();
                    }
                    context.setLastDataProcessed(data);
                }
            } else {
                data = null;
            }
        } while (data != null);
        long routeTime = System.currentTimeMillis() - startTime;
        if (routeTime > 60000 && context != null) {
            log.info("Done routing for channel '{}' which took {} seconds", new Object[] { context.getChannel().getChannelId(), ((System.currentTimeMillis() - startTime) / 1000) });
            ts = System.currentTimeMillis();
        }
    } finally {
        reader.setReading(false);
        if (statsDataCount > 0) {
            engine.getStatisticManager().incrementDataRouted(context.getChannel().getChannelId(), statsDataCount);
        }
        if (statsDataEventCount > 0) {
            engine.getStatisticManager().incrementDataEventInserted(context.getChannel().getChannelId(), statsDataEventCount);
        }
    }
    context.incrementStat(totalDataCount, ChannelRouterContext.STAT_DATA_ROUTED_COUNT);
    return totalDataEventCount;
}

Example 2 with IDataToRouteReader

use of org.jumpmind.symmetric.route.IDataToRouteReader in project symmetric-ds by JumpMind.

the class RouterService method startReading.

protected IDataToRouteReader startReading(ChannelRouterContext context) {
    IDataToRouteReader reader = new DataGapRouteReader(context, engine);
    if (parameterService.is(ParameterConstants.SYNCHRONIZE_ALL_JOBS)) {
        reader.run();
    } else {
        if (readThread == null) {
            readThread = Executors.newCachedThreadPool(new ThreadFactory() {

                final AtomicInteger threadNumber = new AtomicInteger(1);

                final String namePrefix = parameterService.getEngineName().toLowerCase() + "-router-reader-";

                public Thread newThread(Runnable r) {
                    Thread t = new Thread(r);
                    t.setName(namePrefix + threadNumber.getAndIncrement());
                    if (t.isDaemon()) {
                        t.setDaemon(false);
                    }
                    if (t.getPriority() != Thread.NORM_PRIORITY) {
                        t.setPriority(Thread.NORM_PRIORITY);
                    }
                    return t;
                }
            });
        }
        readThread.execute(reader);
    }
    return reader;
}