Examples with Data org.jumpmind.symmetric.model.Data used on opensource projects

Example 16 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class RouterService method selectDataAndRoute.

/**
     * Pre-read data and fill up a queue so we can peek ahead to see if we have
     * crossed a database transaction boundary. Then route each {@link Data}
     * while continuing to keep the queue filled until the result set is
     * entirely read.
     * 
     * @param conn
     *            The connection to use for selecting the data.
     * @param context
     *            The current context of the routing process
     */
protected int selectDataAndRoute(ProcessInfo processInfo, ChannelRouterContext context) throws InterruptedException {
    IDataToRouteReader reader = startReading(context);
    Data data = null;
    Data nextData = null;
    int totalDataCount = 0;
    int totalDataEventCount = 0;
    int statsDataCount = 0;
    int statsDataEventCount = 0;
    final int maxNumberOfEventsBeforeFlush = parameterService.getInt(ParameterConstants.ROUTING_FLUSH_JDBC_BATCH_SIZE);
    try {
        long ts = System.currentTimeMillis();
        long startTime = System.currentTimeMillis();
        nextData = reader.take();
        do {
            if (nextData != null) {
                data = nextData;
                nextData = reader.take();
                if (data != null) {
                    processInfo.setCurrentTableName(data.getTableName());
                    processInfo.incrementCurrentDataCount();
                    boolean atTransactionBoundary = false;
                    if (nextData != null) {
                        String nextTxId = nextData.getTransactionId();
                        atTransactionBoundary = nextTxId == null || !nextTxId.equals(data.getTransactionId());
                    }
                    context.setEncountedTransactionBoundary(atTransactionBoundary);
                    statsDataCount++;
                    totalDataCount++;
                    int dataEventsInserted = routeData(processInfo, data, context);
                    statsDataEventCount += dataEventsInserted;
                    totalDataEventCount += dataEventsInserted;
                    long insertTs = System.currentTimeMillis();
                    try {
                        if (maxNumberOfEventsBeforeFlush <= context.getDataEventList().size() || context.isNeedsCommitted()) {
                            engine.getDataService().insertDataEvents(context.getSqlTransaction(), context.getDataEventList());
                            context.clearDataEventsList();
                        }
                        if (context.isNeedsCommitted()) {
                            completeBatchesAndCommit(context);
                        }
                    } finally {
                        context.incrementStat(System.currentTimeMillis() - insertTs, ChannelRouterContext.STAT_INSERT_DATA_EVENTS_MS);
                        if (statsDataCount > StatisticConstants.FLUSH_SIZE_ROUTER_DATA) {
                            engine.getStatisticManager().incrementDataRouted(context.getChannel().getChannelId(), statsDataCount);
                            statsDataCount = 0;
                            engine.getStatisticManager().incrementDataEventInserted(context.getChannel().getChannelId(), statsDataEventCount);
                            statsDataEventCount = 0;
                        }
                    }
                    long routeTs = System.currentTimeMillis() - ts;
                    if (routeTs > 60000 && context != null) {
                        log.info("Routing for channel '{}' has been processing for {} seconds. The following stats have been gathered: " + "totalDataRoutedCount={}, totalDataEventCount={}, startDataId={}, endDataId={}, dataReadCount={}, peekAheadFillCount={}, transactions={}, dataGaps={}", new Object[] { context.getChannel().getChannelId(), ((System.currentTimeMillis() - startTime) / 1000), totalDataCount, totalDataEventCount, context.getStartDataId(), context.getEndDataId(), context.getDataReadCount(), context.getPeekAheadFillCount(), context.getTransactions().toString(), context.getDataGaps().toString() });
                        ts = System.currentTimeMillis();
                    }
                    context.setLastDataProcessed(data);
                }
            } else {
                data = null;
            }
        } while (data != null);
        long routeTime = System.currentTimeMillis() - startTime;
        if (routeTime > 60000 && context != null) {
            log.info("Done routing for channel '{}' which took {} seconds", new Object[] { context.getChannel().getChannelId(), ((System.currentTimeMillis() - startTime) / 1000) });
            ts = System.currentTimeMillis();
        }
    } finally {
        reader.setReading(false);
        if (statsDataCount > 0) {
            engine.getStatisticManager().incrementDataRouted(context.getChannel().getChannelId(), statsDataCount);
        }
        if (statsDataEventCount > 0) {
            engine.getStatisticManager().incrementDataEventInserted(context.getChannel().getChannelId(), statsDataEventCount);
        }
    }
    context.incrementStat(totalDataCount, ChannelRouterContext.STAT_DATA_ROUTED_COUNT);
    return totalDataEventCount;
}

Example 17 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class AbstractFileParsingRouter method deleteFileIfNecessary.

public void deleteFileIfNecessary(DataMetaData dataMetaData) {
    Data data = dataMetaData.getData();
    Table snapshotTable = dataMetaData.getTable();
    if (data.getDataEventType() == DataEventType.INSERT || data.getDataEventType() == DataEventType.UPDATE) {
        List<File> filesToDelete = new ArrayList<File>();
        Map<String, String> columnData = data.toColumnNameValuePairs(snapshotTable.getColumnNames(), CsvData.ROW_DATA);
        FileSnapshot fileSnapshot = new FileSnapshot();
        fileSnapshot.setTriggerId(columnData.get("TRIGGER_ID"));
        fileSnapshot.setRouterId(columnData.get("ROUTER_ID"));
        fileSnapshot.setFileModifiedTime(Long.parseLong(columnData.get("FILE_MODIFIED_TIME")));
        fileSnapshot.setFileName(columnData.get("FILE_NAME"));
        fileSnapshot.setRelativeDir(columnData.get("RELATIVE_DIR"));
        fileSnapshot.setLastEventType(LastEventType.fromCode(columnData.get("LAST_EVENT_TYPE")));
        FileTriggerRouter triggerRouter = getEngine().getFileSyncService().getFileTriggerRouter(fileSnapshot.getTriggerId(), fileSnapshot.getRouterId());
        if (triggerRouter != null) {
            FileTrigger fileTrigger = triggerRouter.getFileTrigger();
            if (fileTrigger.isDeleteAfterSync()) {
                File file = fileTrigger.createSourceFile(fileSnapshot);
                if (!file.isDirectory()) {
                    filesToDelete.add(file);
                    if (fileTrigger.isSyncOnCtlFile()) {
                        File ctlFile = getEngine().getFileSyncService().getControleFile(file);
                        filesToDelete.add(ctlFile);
                    }
                }
            } else if (getEngine().getParameterService().is(ParameterConstants.FILE_SYNC_DELETE_CTL_FILE_AFTER_SYNC, false)) {
                File file = fileTrigger.createSourceFile(fileSnapshot);
                if (!file.isDirectory()) {
                    if (fileTrigger.isSyncOnCtlFile()) {
                        File ctlFile = getEngine().getFileSyncService().getControleFile(file);
                        filesToDelete.add(ctlFile);
                    }
                }
            }
        }
        if (filesToDelete != null && filesToDelete.size() > 0) {
            for (File file : filesToDelete) {
                if (file != null && file.exists()) {
                    log.debug("Deleting the '{}' file", file.getAbsolutePath());
                    boolean deleted = FileUtils.deleteQuietly(file);
                    if (!deleted) {
                        log.warn("Failed to 'delete on sync' the {} file", file.getAbsolutePath());
                    }
                }
                file = null;
            }
            filesToDelete = null;
        }
    }
}

Example 18 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class AbstractFileParsingRouter method routeToNodes.

@Override
public Set<String> routeToNodes(SimpleRouterContext context, DataMetaData dataMetaData, Set<Node> nodes, boolean initialLoad, boolean initialLoadSelectUsed, TriggerRouter triggerRouter) {
    Map<String, String> newData = getNewDataAsString(null, dataMetaData, getEngine().getSymmetricDialect());
    String targetTableName = dataMetaData.getRouter().getTargetTableName();
    String fileName = newData.get("FILE_NAME");
    String relativeDir = newData.get("RELATIVE_DIR");
    String triggerId = newData.get("TRIGGER_ID");
    String lastEventType = newData.get("LAST_EVENT_TYPE");
    String routerExpression = dataMetaData.getRouter().getRouterExpression();
    String channelId = "default";
    String filePath = relativeDir + "/" + fileName;
    IContextService contextService = getEngine().getContextService();
    if (lastEventType.equals(DataEventType.DELETE.toString())) {
        log.debug("File deleted (" + filePath + "), cleaning up context value.");
        contextService.delete(filePath);
    } else {
        if (routerExpression != null) {
            String[] keyValues = routerExpression.split(",");
            if (keyValues.length > 0) {
                for (int i = 0; i < keyValues.length; i++) {
                    String[] keyValue = keyValues[i].split("=");
                    if (keyValue.length > 1) {
                        if (ROUTER_EXPRESSION_CHANNEL_KEY.equals(keyValue[0])) {
                            channelId = keyValue[1];
                        }
                    }
                }
            }
        }
        if (triggerId != null) {
            String baseDir = getEngine().getFileSyncService().getFileTrigger(triggerId).getBaseDir();
            File file = createSourceFile(baseDir, relativeDir, fileName);
            Integer lineNumber = contextService.getString(filePath) == null ? 0 : new Integer(contextService.getString(filePath));
            List<String> dataRows = parse(file, lineNumber);
            String columnNames = getColumnNames();
            String nodeList = buildNodeList(nodes);
            String externalData = new StringBuilder(EXTERNAL_DATA_TRIGGER_KEY).append("=").append(triggerId).append(",").append(EXTERNAL_DATA_ROUTER_KEY).append("=").append(dataMetaData.getRouter().getRouterId()).append(",").append(EXTERNAL_DATA_FILE_DATA_ID).append("=").append(dataMetaData.getData().getDataId()).toString();
            for (String row : dataRows) {
                Data data = new Data();
                data.setChannelId(channelId);
                data.setDataEventType(DataEventType.INSERT);
                data.setRowData(row);
                data.setTableName(targetTableName);
                data.setNodeList(nodeList);
                data.setTriggerHistory(getTriggerHistory(targetTableName, columnNames));
                data.setExternalData(externalData);
                data.setDataId(getEngine().getDataService().insertData(data));
                lineNumber++;
            }
            if (!dataRows.isEmpty()) {
                try {
                    contextService.save(filePath, lineNumber.toString());
                    deleteFileIfNecessary(dataMetaData);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }
    return new HashSet<String>();
}

Example 19 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class ColumnMatchDataRouterTest method testExpressionExternalData.

@Test
public void testExpressionExternalData() {
    ColumnMatchDataRouter router = new ColumnMatchDataRouter();
    SimpleRouterContext routingContext = new SimpleRouterContext();
    HashSet<Node> nodes = new HashSet<Node>();
    nodes.add(new Node("100", "client"));
    nodes.add(new Node("200", "client"));
    nodes.add(new Node("300", "client"));
    TriggerHistory triggerHist = new TriggerHistory("mytable", "ID", "ID,NODE_ID,COLUMN2");
    Data data = new Data();
    data.setDataId(1);
    data.setDataEventType(DataEventType.INSERT);
    data.setRowData("1,100,Super Dooper");
    data.setExternalData("100");
    data.setTriggerHistory(triggerHist);
    Table table = new Table();
    NodeChannel nodeChannel = new NodeChannel();
    Router route = new Router();
    route.setRouterExpression("EXTERNAL_DATA = :NODE_ID");
    route.setRouterId("route1");
    DataMetaData dataMetaData = new DataMetaData(data, table, route, nodeChannel);
    Set<String> result = router.routeToNodes(routingContext, dataMetaData, nodes, false, false, null);
    assertEquals(1, result.size());
    assertEquals(true, result.contains("100"));
}

Example 20 with Data

use of org.jumpmind.symmetric.model.Data in project symmetric-ds by JumpMind.

the class ColumnMatchDataRouterTest method testExpressionNotEqualsNull.

@Test
public void testExpressionNotEqualsNull() {
    ColumnMatchDataRouter router = new ColumnMatchDataRouter();
    SimpleRouterContext routingContext = new SimpleRouterContext();
    HashSet<Node> nodes = new HashSet<Node>();
    nodes.add(new Node("100", "client"));
    nodes.add(new Node("200", "client"));
    TriggerHistory triggerHist = new TriggerHistory("mytable", "ID", "ID,NODE_ID,COLUMN2");
    Data data = new Data();
    data.setDataId(1);
    data.setDataEventType(DataEventType.INSERT);
    data.setRowData("1,100,");
    data.setTriggerHistory(triggerHist);
    Table table = new Table();
    NodeChannel nodeChannel = new NodeChannel();
    Router route = new Router();
    route.setRouterExpression("COLUMN2 != NULL");
    route.setRouterId("route1");
    DataMetaData dataMetaData = new DataMetaData(data, table, route, nodeChannel);
    Set<String> result = router.routeToNodes(routingContext, dataMetaData, nodes, false, false, null);
    assertEquals(0, result.size());
}