Examples with Data org.jumpmind.symmetric.model.Data used on opensource projects

Example 1 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class DataGapRouteReader method fillPeekAheadQueue.

protected boolean fillPeekAheadQueue(List<Data> peekAheadQueue, int peekAheadCount, ISqlReadCursor<Data> cursor) throws SQLException {
    boolean moreData = true;
    int dataCount = 0;
    long ts = System.currentTimeMillis();
    Data data = null;
    boolean isFirstRead = context.getStartDataId() == 0;
    while (reading && dataCount < peekAheadCount) {
        data = cursor.next();
        if (data != null) {
            if (process(data)) {
                peekAheadQueue.add(data);
                peekAheadSizeInBytes += data.getSizeInBytes();
                dataCount++;
                context.incrementStat(System.currentTimeMillis() - ts, ChannelRouterContext.STAT_READ_DATA_MS);
            } else {
                context.incrementStat(System.currentTimeMillis() - ts, ChannelRouterContext.STAT_REREAD_DATA_MS);
            }
            if (isFirstRead) {
                context.setStartDataId(data.getDataId());
                isFirstRead = false;
            }
            context.setEndDataId(data.getDataId());
            ts = System.currentTimeMillis();
        } else {
            moreData = false;
            break;
        }
    }
    context.incrementDataReadCount(dataCount);
    context.incrementPeekAheadFillCount(1);
    int size = peekAheadQueue.size();
    if (context.getMaxPeekAheadQueueSize() < size) {
        context.setMaxPeekAheadQueueSize(size);
    }
    return moreData && reading;
}

Example 2 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class DataExtractorService method extractConfigurationStandalone.

/**
     * Extract the SymmetricDS configuration for the passed in {@link Node}.
     */
public void extractConfigurationStandalone(Node targetNode, Writer writer, String... tablesToExclude) {
    Node sourceNode = nodeService.findIdentity();
    if (targetNode != null && sourceNode != null) {
        Batch batch = new Batch(BatchType.EXTRACT, Constants.VIRTUAL_BATCH_FOR_REGISTRATION, Constants.CHANNEL_CONFIG, symmetricDialect.getBinaryEncoding(), sourceNode.getNodeId(), targetNode.getNodeId(), false);
        NodeGroupLink nodeGroupLink = new NodeGroupLink(parameterService.getNodeGroupId(), targetNode.getNodeGroupId());
        List<TriggerRouter> triggerRouters = triggerRouterService.buildTriggerRoutersForSymmetricTables(StringUtils.isBlank(targetNode.getSymmetricVersion()) ? Version.version() : targetNode.getSymmetricVersion(), nodeGroupLink, tablesToExclude);
        List<SelectFromTableEvent> initialLoadEvents = new ArrayList<SelectFromTableEvent>(triggerRouters.size() * 2);
        for (int i = triggerRouters.size() - 1; i >= 0; i--) {
            TriggerRouter triggerRouter = triggerRouters.get(i);
            String channelId = triggerRouter.getTrigger().getChannelId();
            if (Constants.CHANNEL_CONFIG.equals(channelId) || Constants.CHANNEL_HEARTBEAT.equals(channelId)) {
                if (filter(targetNode, triggerRouter.getTrigger().getSourceTableName())) {
                    TriggerHistory triggerHistory = triggerRouterService.getNewestTriggerHistoryForTrigger(triggerRouter.getTrigger().getTriggerId(), null, null, triggerRouter.getTrigger().getSourceTableName());
                    if (triggerHistory == null) {
                        Trigger trigger = triggerRouter.getTrigger();
                        Table table = symmetricDialect.getPlatform().getTableFromCache(trigger.getSourceCatalogName(), trigger.getSourceSchemaName(), trigger.getSourceTableName(), false);
                        if (table == null) {
                            throw new IllegalStateException("Could not find a required table: " + triggerRouter.getTrigger().getSourceTableName());
                        }
                        triggerHistory = new TriggerHistory(table, triggerRouter.getTrigger(), symmetricDialect.getTriggerTemplate());
                        triggerHistory.setTriggerHistoryId(Integer.MAX_VALUE - i);
                    }
                    StringBuilder sql = new StringBuilder(symmetricDialect.createPurgeSqlFor(targetNode, triggerRouter, triggerHistory));
                    addPurgeCriteriaToConfigurationTables(triggerRouter.getTrigger().getSourceTableName(), sql);
                    String sourceTable = triggerHistory.getSourceTableName();
                    Data data = new Data(1, null, sql.toString(), DataEventType.SQL, sourceTable, null, triggerHistory, triggerRouter.getTrigger().getChannelId(), null, null);
                    data.putAttribute(Data.ATTRIBUTE_ROUTER_ID, triggerRouter.getRouter().getRouterId());
                    initialLoadEvents.add(new SelectFromTableEvent(data));
                }
            }
        }
        for (int i = 0; i < triggerRouters.size(); i++) {
            TriggerRouter triggerRouter = triggerRouters.get(i);
            String channelId = triggerRouter.getTrigger().getChannelId();
            if (Constants.CHANNEL_CONFIG.equals(channelId) || Constants.CHANNEL_HEARTBEAT.equals(channelId)) {
                if (filter(targetNode, triggerRouter.getTrigger().getSourceTableName())) {
                    TriggerHistory triggerHistory = triggerRouterService.getNewestTriggerHistoryForTrigger(triggerRouter.getTrigger().getTriggerId(), null, null, null);
                    if (triggerHistory == null) {
                        Trigger trigger = triggerRouter.getTrigger();
                        triggerHistory = new TriggerHistory(symmetricDialect.getPlatform().getTableFromCache(trigger.getSourceCatalogName(), trigger.getSourceSchemaName(), trigger.getSourceTableName(), false), trigger, symmetricDialect.getTriggerTemplate());
                        triggerHistory.setTriggerHistoryId(Integer.MAX_VALUE - i);
                    }
                    Table table = symmetricDialect.getPlatform().getTableFromCache(triggerHistory.getSourceCatalogName(), triggerHistory.getSourceSchemaName(), triggerHistory.getSourceTableName(), false);
                    String initialLoadSql = "1=1 order by ";
                    String quote = symmetricDialect.getPlatform().getDdlBuilder().getDatabaseInfo().getDelimiterToken();
                    Column[] pkColumns = table.getPrimaryKeyColumns();
                    for (int j = 0; j < pkColumns.length; j++) {
                        if (j > 0) {
                            initialLoadSql += ", ";
                        }
                        initialLoadSql += quote + pkColumns[j].getName() + quote;
                    }
                    if (!triggerRouter.getTrigger().getSourceTableName().endsWith(TableConstants.SYM_NODE_IDENTITY)) {
                        initialLoadEvents.add(new SelectFromTableEvent(targetNode, triggerRouter, triggerHistory, initialLoadSql));
                    } else {
                        Data data = new Data(1, null, targetNode.getNodeId(), DataEventType.INSERT, triggerHistory.getSourceTableName(), null, triggerHistory, triggerRouter.getTrigger().getChannelId(), null, null);
                        initialLoadEvents.add(new SelectFromTableEvent(data));
                    }
                }
            }
        }
        SelectFromTableSource source = new SelectFromTableSource(batch, initialLoadEvents);
        ExtractDataReader dataReader = new ExtractDataReader(this.symmetricDialect.getPlatform(), source);
        ProtocolDataWriter dataWriter = new ProtocolDataWriter(nodeService.findIdentityNodeId(), writer, targetNode.requires13Compatiblity());
        DataProcessor processor = new DataProcessor(dataReader, dataWriter, "configuration extract");
        DataContext ctx = new DataContext();
        ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode);
        ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE, sourceNode);
        processor.process(ctx);
        if (triggerRouters.size() == 0) {
            log.error("{} attempted registration, but was sent an empty configuration", targetNode);
        }
    }
}

Example 3 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class DataGapRouteReaderTest method testTransactionalOrderingWithGaps.

@SuppressWarnings("unchecked")
@Test
public void testTransactionalOrderingWithGaps() throws Exception {
    nodeChannel.setBatchAlgorithm(DefaultBatchAlgorithm.NAME);
    nodeChannel.setMaxDataToRoute(100);
    when(parameterService.getInt(ParameterConstants.ROUTING_PEEK_AHEAD_WINDOW)).thenReturn(2);
    List<DataGap> dataGaps = new ArrayList<DataGap>();
    dataGaps.add(new DataGap(0, 3));
    dataGaps.add(new DataGap(4, Long.MAX_VALUE));
    List<Data> data = new ArrayList<Data>();
    data.add(new Data(1, null, null, null, TABLE1, null, null, null, TRAN1, null));
    data.add(new Data(2, null, null, null, TABLE1, null, null, null, TRAN1, null));
    data.add(new Data(3, null, null, null, TABLE1, null, null, null, TRAN2, null));
    data.add(new Data(4, null, null, null, TABLE1, null, null, null, TRAN1, null));
    data.add(new Data(5, null, null, null, TABLE1, null, null, null, TRAN2, null));
    when(sqlTemplate.queryForCursor((String) any(), (ISqlRowMapper<Data>) any(), (Object[]) any(), (int[]) any())).thenReturn(new ListReadCursor(data));
    DataGapRouteReader dataGapRouteReader = buildReader(50, dataGaps);
    dataGapRouteReader.execute();
    BlockingQueue<Data> queue = dataGapRouteReader.getDataQueue();
    assertEquals(6, queue.size());
    Iterator<Data> iter = queue.iterator();
    int index = 0;
    long[] ids = { 1, 2, 4, 3, 5, -1 };
    while (iter.hasNext()) {
        Data d = iter.next();
        assertEquals(ids[index], d.getDataId());
        index++;
    }
}

Example 4 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class DataGapRouteReaderTest method testNonTransactionalChannelMaxDataToRoute.

@Test
public void testNonTransactionalChannelMaxDataToRoute() throws Exception {
    nodeChannel.setBatchAlgorithm(NonTransactionalBatchAlgorithm.NAME);
    nodeChannel.setMaxDataToRoute(3);
    when(parameterService.getInt(ParameterConstants.ROUTING_PEEK_AHEAD_WINDOW)).thenReturn(2);
    List<DataGap> dataGaps = new ArrayList<DataGap>();
    dataGaps.add(new DataGap(0, Long.MAX_VALUE));
    List<Data> data = new ArrayList<Data>();
    data.add(new Data(1, null, null, null, TABLE1, null, null, null, TRAN1, null));
    data.add(new Data(2, null, null, null, TABLE1, null, null, null, TRAN1, null));
    data.add(new Data(3, null, null, null, TABLE1, null, null, null, TRAN2, null));
    data.add(new Data(4, null, null, null, TABLE1, null, null, null, TRAN1, null));
    data.add(new Data(5, null, null, null, TABLE1, null, null, null, TRAN2, null));
    ISqlRowMapper<Data> mapper = any();
    when(sqlTemplate.queryForCursor((String) any(), mapper, (Object[]) any(), (int[]) any())).thenReturn(new ListReadCursor(data));
    DataGapRouteReader dataGapRouteReader = buildReader(50, dataGaps);
    dataGapRouteReader.execute();
    BlockingQueue<Data> queue = dataGapRouteReader.getDataQueue();
    assertEquals(4, queue.size());
    Iterator<Data> iter = queue.iterator();
    int index = 0;
    long[] ids = { 1, 2, 4, -1 };
    while (iter.hasNext()) {
        Data d = iter.next();
        assertEquals(ids[index], d.getDataId());
        index++;
    }
}

Example 5 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class DataGapRouteReaderTest method testDontPeekAheadWhenPeekAheadQueueIsAlreadyFull.

@Test
public void testDontPeekAheadWhenPeekAheadQueueIsAlreadyFull() throws Exception {
    nodeChannel.setBatchAlgorithm(TransactionalBatchAlgorithm.NAME);
    nodeChannel.setMaxDataToRoute(100);
    when(parameterService.getInt(ParameterConstants.ROUTING_PEEK_AHEAD_WINDOW)).thenReturn(5);
    List<DataGap> dataGaps = new ArrayList<DataGap>();
    dataGaps.add(new DataGap(0, Long.MAX_VALUE));
    List<Data> data = new ArrayList<Data>();
    for (int i = 0; i < 100; i++) {
        data.add(new Data(i, null, null, null, TABLE1, null, null, null, Integer.toString(i % 2 == 0 ? i : i - 1), null));
    }
    ISqlRowMapper<Data> mapper = any();
    when(sqlTemplate.queryForCursor((String) any(), mapper, (Object[]) any(), (int[]) any())).thenReturn(new ListReadCursor(data));
    DataGapRouteReader dataGapRouteReader = buildReader(100, dataGaps);
    dataGapRouteReader.execute();
    /*
         * Test that the peek ahead queue size doesn't get bigger than twice the peek ahead size
         */
    assertEquals(10, dataGapRouteReader.context.getMaxPeekAheadQueueSize());
    assertEquals(21, dataGapRouteReader.context.getPeekAheadFillCount());
    BlockingQueue<Data> queue = dataGapRouteReader.getDataQueue();
    assertEquals(101, queue.size());
    Iterator<Data> iter = queue.iterator();
    int index = 0;
    while (iter.hasNext()) {
        Data d = iter.next();
        assertEquals(index < 100 ? index : -1, d.getDataId());
        index++;
    }
}