Examples with Data org.jumpmind.symmetric.model.Data used on opensource projects

Example 36 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class RouterService method routeDataForChannel.

protected int routeDataForChannel(ProcessInfo processInfo, final NodeChannel nodeChannel, final Node sourceNode) {
    ChannelRouterContext context = null;
    long ts = System.currentTimeMillis();
    int dataCount = -1;
    try {
        List<TriggerRouter> triggerRouters = engine.getTriggerRouterService().getTriggerRouters(false);
        boolean producesCommonBatches = producesCommonBatches(nodeChannel.getChannel(), parameterService.getNodeGroupId(), triggerRouters);
        boolean onlyDefaultRoutersAssigned = onlyDefaultRoutersAssigned(nodeChannel.getChannel(), parameterService.getNodeGroupId(), triggerRouters);
        context = new ChannelRouterContext(sourceNode.getNodeId(), nodeChannel, symmetricDialect.getPlatform().getSqlTemplate().startSqlTransaction());
        context.setProduceCommonBatches(producesCommonBatches);
        context.setOnlyDefaultRoutersAssigned(onlyDefaultRoutersAssigned);
        context.setDataGaps(gapDetector.getDataGaps());
        dataCount = selectDataAndRoute(processInfo, context);
        return dataCount;
    } catch (DelayRoutingException ex) {
        log.info("The routing process for the {} channel is being delayed.  {}", nodeChannel.getChannelId(), isNotBlank(ex.getMessage()) ? ex.getMessage() : "");
        if (context != null) {
            context.rollback();
        }
        return 0;
    } catch (InterruptedException ex) {
        log.warn("The routing process was interrupted.  Rolling back changes");
        if (context != null) {
            context.rollback();
        }
        return 0;
    } catch (SyntaxParsingException ex) {
        log.error(String.format("Failed to route and batch data on '%s' channel due to an invalid router expression", nodeChannel.getChannelId()), ex);
        if (context != null) {
            context.rollback();
        }
        return 0;
    } catch (Throwable ex) {
        log.error(String.format("Failed to route and batch data on '%s' channel", nodeChannel.getChannelId()), ex);
        if (context != null) {
            context.rollback();
        }
        return 0;
    } finally {
        try {
            if (dataCount > 0) {
                long insertTs = System.currentTimeMillis();
                engine.getDataService().insertDataEvents(context.getSqlTransaction(), context.getDataEventList());
                context.clearDataEventsList();
                completeBatchesAndCommit(context);
                gapDetector.addDataIds(context.getDataIds());
                gapDetector.setIsAllDataRead(context.getDataIds().size() < context.getChannel().getMaxDataToRoute());
                context.incrementStat(System.currentTimeMillis() - insertTs, ChannelRouterContext.STAT_INSERT_DATA_EVENTS_MS);
                if (parameterService.is(ParameterConstants.ROUTING_COLLECT_STATS_UNROUTED)) {
                    Data lastDataProcessed = context.getLastDataProcessed();
                    if (lastDataProcessed != null && lastDataProcessed.getDataId() > 0) {
                        String channelId = nodeChannel.getChannelId();
                        long queryTs = System.currentTimeMillis();
                        long dataLeftToRoute = sqlTemplate.queryForInt(getSql("selectUnroutedCountForChannelSql"), channelId, lastDataProcessed.getDataId());
                        queryTs = System.currentTimeMillis() - queryTs;
                        if (queryTs > Constants.LONG_OPERATION_THRESHOLD) {
                            log.warn("Unrouted query for channel {} took longer than expected. The query took {} ms.", channelId, queryTs);
                        }
                        engine.getStatisticManager().setDataUnRouted(channelId, dataLeftToRoute);
                    }
                }
            }
        } catch (Exception e) {
            if (context != null) {
                context.rollback();
            }
            log.error("", e);
        } finally {
            long totalTime = System.currentTimeMillis() - ts;
            context.incrementStat(totalTime, ChannelRouterContext.STAT_ROUTE_TOTAL_TIME);
            context.logStats(log, totalTime);
            context.cleanup();
        }
    }
}

Example 37 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class DataGapRouteReaderTest method testTransactionalChannelTwoTransactionsRouted.

@Test
public void testTransactionalChannelTwoTransactionsRouted() throws Exception {
    nodeChannel.setBatchAlgorithm(TransactionalBatchAlgorithm.NAME);
    nodeChannel.setMaxDataToRoute(100);
    when(parameterService.getInt(ParameterConstants.ROUTING_PEEK_AHEAD_WINDOW)).thenReturn(100);
    List<DataGap> dataGaps = new ArrayList<DataGap>();
    dataGaps.add(new DataGap(0, Long.MAX_VALUE));
    List<Data> data = new ArrayList<Data>();
    data.add(new Data(1, null, null, null, TABLE1, null, null, null, TRAN1, null));
    data.add(new Data(2, null, null, null, TABLE1, null, null, null, TRAN1, null));
    data.add(new Data(3, null, null, null, TABLE1, null, null, null, TRAN2, null));
    data.add(new Data(4, null, null, null, TABLE1, null, null, null, TRAN2, null));
    data.add(new Data(5, null, null, null, TABLE1, null, null, null, TRAN1, null));
    data.add(new Data(6, null, null, null, TABLE1, null, null, null, TRAN1, null));
    ISqlRowMapper<Data> mapper = any();
    when(sqlTemplate.queryForCursor((String) any(), mapper, (Object[]) any(), (int[]) any())).thenReturn(new ListReadCursor(data));
    DataGapRouteReader dataGapRouteReader = buildReader(50, dataGaps);
    dataGapRouteReader.execute();
    BlockingQueue<Data> queue = dataGapRouteReader.getDataQueue();
    assertEquals(7, queue.size());
    Iterator<Data> iter = queue.iterator();
    int index = 0;
    long[] ids = { 1, 2, 5, 6, 3, 4, -1 };
    while (iter.hasNext()) {
        Data d = iter.next();
        assertEquals(ids[index], d.getDataId());
        index++;
    }
}

Example 38 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class DataGapRouteReaderTest method testTransactionalChannelReachMaxPeekAheadSizeThreshold.

@Test
public void testTransactionalChannelReachMaxPeekAheadSizeThreshold() throws Exception {
    nodeChannel.setBatchAlgorithm(TransactionalBatchAlgorithm.NAME);
    nodeChannel.setMaxDataToRoute(100);
    when(parameterService.getInt(ParameterConstants.ROUTING_PEEK_AHEAD_WINDOW)).thenReturn(100);
    List<DataGap> dataGaps = new ArrayList<DataGap>();
    dataGaps.add(new DataGap(0, Long.MAX_VALUE));
    List<Data> data = new ArrayList<Data>();
    data.add(new Data(1, null, null, null, TABLE1, null, null, null, TRAN1, null));
    data.add(new Data(2, null, null, null, TABLE1, null, null, null, TRAN1, null));
    data.add(new Data(3, null, null, null, TABLE1, null, null, null, TRAN2, null));
    data.add(new Data(4, null, null, null, TABLE1, null, null, null, TRAN1, null));
    data.add(new Data(5, null, null, null, TABLE1, null, null, null, TRAN2, null));
    data.add(new Data(6, null, null, null, TABLE1, null, null, null, TRAN1, null));
    data.add(new Data(7, null, null, null, TABLE1, null, null, null, TRAN2, null));
    ISqlRowMapper<Data> mapper = any();
    when(sqlTemplate.queryForCursor((String) any(), mapper, (Object[]) any(), (int[]) any())).thenReturn(new ListReadCursor(data));
    DataGapRouteReader dataGapRouteReader = buildReader(0, dataGaps);
    dataGapRouteReader.execute();
    BlockingQueue<Data> queue = dataGapRouteReader.getDataQueue();
    assertEquals(5, queue.size());
    Iterator<Data> iter = queue.iterator();
    int index = 0;
    long[] ids = { 1, 2, 4, 6, -1 };
    while (iter.hasNext()) {
        Data d = iter.next();
        assertEquals(ids[index], d.getDataId());
        index++;
    }
}

Example 39 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class ColumnMatchDataRouterTest method testExpressionNotEqualsNodeId.

@Test
public void testExpressionNotEqualsNodeId() {
    ColumnMatchDataRouter router = new ColumnMatchDataRouter();
    SimpleRouterContext routingContext = new SimpleRouterContext();
    HashSet<Node> nodes = new HashSet<Node>();
    nodes.add(new Node("100", "client"));
    nodes.add(new Node("200", "client"));
    nodes.add(new Node("300", "client"));
    TriggerHistory triggerHist = new TriggerHistory("mytable", "ID", "ID,NODE_ID,COLUMN2");
    Data data = new Data();
    data.setDataId(1);
    data.setDataEventType(DataEventType.INSERT);
    data.setRowData("1,100,Super Dooper");
    data.setTriggerHistory(triggerHist);
    Table table = new Table();
    NodeChannel nodeChannel = new NodeChannel();
    Router route = new Router();
    route.setRouterExpression("NODE_ID != :NODE_ID");
    route.setRouterId("route1");
    DataMetaData dataMetaData = new DataMetaData(data, table, route, nodeChannel);
    Set<String> result = router.routeToNodes(routingContext, dataMetaData, nodes, false, false, null);
    assertEquals(2, result.size());
    assertEquals(true, result.contains("200"));
    assertEquals(true, result.contains("300"));
}

Example 40 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class ConfigurationChangedDataRouterTest method buildDataMetaData.

protected DataMetaData buildDataMetaData(String tableName, String nodeId) {
    Data data = new Data();
    data.setTableName(tableName);
    data.setDataEventType(DataEventType.UPDATE);
    data.setTriggerHistory(new TriggerHistory(tableName, "NODE_ID", "NODE_ID"));
    data.setPkData(nodeId);
    data.setRowData(nodeId);
    return new DataMetaData(data, new Table(tableName), null, null);
}