Examples with NodeChannel org.jumpmind.symmetric.model.NodeChannel used on opensource projects

Example 6 with NodeChannel

use of org.jumpmind.symmetric.model.NodeChannel in project symmetric-ds by JumpMind.

the class AbstractRouterServiceTest method testColumnMatchTransactionalOnlyRoutingToNode1.

public void testColumnMatchTransactionalOnlyRoutingToNode1() {
    resetBatches();
    TriggerRouter trigger1 = getTestRoutingTableTrigger(TEST_TABLE_1);
    trigger1.getRouter().setRouterType("column");
    trigger1.getRouter().setRouterExpression("ROUTING_VARCHAR=:NODE_ID");
    getTriggerRouterService().saveTriggerRouter(trigger1);
    getTriggerRouterService().syncTriggers();
    NodeChannel testChannel = getConfigurationService().getNodeChannel(TestConstants.TEST_CHANNEL_ID, false);
    testChannel.setMaxBatchToSend(10000);
    testChannel.setBatchAlgorithm("transactional");
    getConfigurationService().saveChannel(testChannel, true);
    // should be 51 batches for table 1
    insert(TEST_TABLE_1, 500, true);
    insert(TEST_TABLE_1, 50, false);
    getRouterService().routeData(true);
    final int EXPECTED_BATCHES = getDbDialect().supportsTransactionId() ? 51 : 550;
    OutgoingBatches batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_1.getNodeId(), false);
    filterForChannels(batches, testChannel);
    Assert.assertEquals(EXPECTED_BATCHES, batches.getBatches().size());
    Assert.assertEquals(EXPECTED_BATCHES, countBatchesForChannel(batches, testChannel));
    batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_2.getNodeId(), false);
    filterForChannels(batches, testChannel);
    // Node 2 has sync disabled
    Assert.assertEquals(0, batches.getBatches().size());
    batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_3.getNodeId(), false);
    filterForChannels(batches, testChannel);
    // Batch was targeted only at node 1
    Assert.assertEquals(0, batches.getBatches().size());
    resetBatches();
    Assert.assertEquals(0, countBatchesForChannel(getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_1.getNodeId(), false), testChannel));
    execute("delete from " + TEST_TABLE_1, null);
    Assert.assertEquals(0, countBatchesForChannel(getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_1.getNodeId(), false), testChannel));
    getRouterService().routeData(true);
    Assert.assertEquals(getDbDialect().supportsTransactionId() ? 1 : 705, countBatchesForChannel(getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_1.getNodeId(), false), testChannel));
    resetBatches();
}

Example 7 with NodeChannel

use of org.jumpmind.symmetric.model.NodeChannel in project symmetric-ds by JumpMind.

the class AbstractRouterServiceTest method testBshTransactionalRoutingOnUpdate.

public void testBshTransactionalRoutingOnUpdate() {
    resetBatches();
    TriggerRouter trigger1 = getTestRoutingTableTrigger(TEST_TABLE_1);
    trigger1.getRouter().setRouterType("bsh");
    trigger1.getRouter().setRouterExpression("targetNodes.add(ROUTING_VARCHAR); targetNodes.add(OLD_ROUTING_VARCHAR);");
    getTriggerRouterService().saveTriggerRouter(trigger1);
    getTriggerRouterService().syncTriggers();
    NodeChannel testChannel = getConfigurationService().getNodeChannel(TestConstants.TEST_CHANNEL_ID, false);
    testChannel.setMaxBatchToSend(1000);
    testChannel.setMaxBatchSize(5);
    testChannel.setBatchAlgorithm("transactional");
    getConfigurationService().saveChannel(testChannel, true);
    long ts = System.currentTimeMillis();
    ISqlTransaction transaction = null;
    int count = 0;
    try {
        transaction = getSqlTemplate().startSqlTransaction();
        count = transaction.prepareAndExecute(String.format("update %s set routing_varchar=?", TEST_TABLE_1), NODE_GROUP_NODE_3.getNodeId());
        transaction.commit();
    } finally {
        transaction.close();
    }
    logger.info("Just recorded a change to " + count + " rows in " + TEST_TABLE_1 + " in " + (System.currentTimeMillis() - ts) + "ms");
    ts = System.currentTimeMillis();
    getRouterService().routeData(true);
    logger.info("Just routed " + count + " rows in " + TEST_TABLE_1 + " in " + (System.currentTimeMillis() - ts) + "ms");
    OutgoingBatches batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_1.getNodeId(), false);
    filterForChannels(batches, testChannel);
    Assert.assertEquals(getDbDialect().supportsTransactionId() ? 1 : 510, batches.getBatches().size());
    Assert.assertEquals(getDbDialect().supportsTransactionId() ? count : 1, (int) batches.getBatches().get(0).getDataEventCount());
    batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_2.getNodeId(), false);
    filterForChannels(batches, testChannel);
    // Node 2 has sync disabled
    Assert.assertEquals(0, batches.getBatches().size());
    batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_3.getNodeId(), false);
    filterForChannels(batches, testChannel);
    Assert.assertEquals(getDbDialect().supportsTransactionId() ? 1 : 510, batches.getBatches().size());
    Assert.assertEquals(getDbDialect().supportsTransactionId() ? count : 1, (int) batches.getBatches().get(0).getDataEventCount());
    resetBatches();
}

Example 8 with NodeChannel

use of org.jumpmind.symmetric.model.NodeChannel in project symmetric-ds by JumpMind.

the class AbstractRouterServiceTest method testBshRoutingDeletesToNode3.

public void testBshRoutingDeletesToNode3() {
    resetBatches();
    TriggerRouter trigger1 = getTestRoutingTableTrigger(TEST_TABLE_1);
    trigger1.getRouter().setRouterType("bsh");
    trigger1.getRouter().setRouterExpression("targetNodes.add(ROUTING_VARCHAR); if (OLD_ROUTING_VARCHAR != void) { targetNodes.add(OLD_ROUTING_VARCHAR); }");
    getTriggerRouterService().saveTriggerRouter(trigger1);
    getTriggerRouterService().syncTriggers();
    NodeChannel testChannel = getConfigurationService().getNodeChannel(TestConstants.TEST_CHANNEL_ID, false);
    testChannel.setMaxBatchToSend(10000);
    final int MAX_BATCH_SIZE = 100;
    testChannel.setMaxBatchSize(MAX_BATCH_SIZE);
    testChannel.setBatchAlgorithm("nontransactional");
    getConfigurationService().saveChannel(testChannel, true);
    int count = getSqlTemplate().update(String.format("delete from %s", TEST_TABLE_1));
    getRouterService().routeData(true);
    OutgoingBatches batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_3.getNodeId(), false);
    filterForChannels(batches, testChannel);
    Assert.assertEquals(count / MAX_BATCH_SIZE + (count % MAX_BATCH_SIZE > 0 ? 1 : 0), batches.getBatches().size());
    batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_2.getNodeId(), false);
    // Node 2 has sync disabled
    Assert.assertEquals(0, batches.getBatches().size());
    batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_1.getNodeId(), false);
    filterForChannels(batches, testChannel);
    // Batch was targeted only at node 3
    Assert.assertEquals(0, batches.getBatches().size());
    resetBatches();
}

Example 9 with NodeChannel

use of org.jumpmind.symmetric.model.NodeChannel in project symmetric-ds by JumpMind.

the class AbstractRouterServiceTest method testColumnMatchOnNotNull.

public void testColumnMatchOnNotNull() {
    NodeChannel testChannel = getConfigurationService().getNodeChannel(TestConstants.TEST_CHANNEL_ID, false);
    TriggerRouter trigger = getTestRoutingTableTrigger(TEST_TABLE_1);
    trigger.getRouter().setRouterType("column");
    trigger.getRouter().setRouterExpression("ROUTING_VARCHAR!=NULL");
    getTriggerRouterService().saveTriggerRouter(trigger);
    getTriggerRouterService().syncTriggers();
    resetBatches();
    update(TEST_TABLE_1, "Not Routed");
    Assert.assertEquals(0, countBatchesForChannel(getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_1.getNodeId(), false), testChannel));
    getRouterService().routeData(true);
    Assert.assertEquals(1, countBatchesForChannel(getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_1.getNodeId(), false), testChannel));
    resetBatches();
    update(TEST_TABLE_1, null);
    Assert.assertEquals(0, countBatchesForChannel(getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_1.getNodeId(), false), testChannel));
    getRouterService().routeData(true);
    Assert.assertEquals(0, countBatchesForChannel(getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_1.getNodeId(), false), testChannel));
}

Example 10 with NodeChannel

use of org.jumpmind.symmetric.model.NodeChannel in project symmetric-ds by JumpMind.

the class DataExtractorService method extract.

protected List<OutgoingBatch> extract(final ProcessInfo processInfo, final Node targetNode, final List<OutgoingBatch> activeBatches, final IDataWriter dataWriter, final BufferedWriter writer, final ExtractMode mode) {
    if (activeBatches.size() > 0) {
        final List<OutgoingBatch> processedBatches = new ArrayList<OutgoingBatch>(activeBatches.size());
        Set<String> channelsProcessed = new HashSet<String>();
        long batchesSelectedAtMs = System.currentTimeMillis();
        OutgoingBatch currentBatch = null;
        ExecutorService executor = null;
        try {
            final boolean streamToFileEnabled = parameterService.is(ParameterConstants.STREAM_TO_FILE_ENABLED);
            long keepAliveMillis = parameterService.getLong(ParameterConstants.DATA_LOADER_SEND_ACK_KEEPALIVE);
            Node sourceNode = nodeService.findIdentity();
            final FutureExtractStatus status = new FutureExtractStatus();
            executor = Executors.newFixedThreadPool(1, new CustomizableThreadFactory(String.format("dataextractor-%s-%s", targetNode.getNodeGroupId(), targetNode.getNodeGroupId())));
            List<Future<FutureOutgoingBatch>> futures = new ArrayList<Future<FutureOutgoingBatch>>();
            processInfo.setBatchCount(activeBatches.size());
            for (int i = 0; i < activeBatches.size(); i++) {
                currentBatch = activeBatches.get(i);
                processInfo.setCurrentLoadId(currentBatch.getLoadId());
                processInfo.setDataCount(currentBatch.getDataEventCount());
                processInfo.setCurrentBatchId(currentBatch.getBatchId());
                channelsProcessed.add(currentBatch.getChannelId());
                currentBatch = requeryIfEnoughTimeHasPassed(batchesSelectedAtMs, currentBatch);
                processInfo.setStatus(ProcessInfo.Status.EXTRACTING);
                final OutgoingBatch extractBatch = currentBatch;
                Callable<FutureOutgoingBatch> callable = new Callable<FutureOutgoingBatch>() {

                    public FutureOutgoingBatch call() throws Exception {
                        return extractBatch(extractBatch, status, processInfo, targetNode, dataWriter, mode, activeBatches);
                    }
                };
                if (status.shouldExtractSkip) {
                    break;
                }
                futures.add(executor.submit(callable));
            }
            if (parameterService.is(ParameterConstants.SYNCHRONIZE_ALL_JOBS)) {
                executor.shutdown();
                boolean isProcessed = false;
                while (!isProcessed) {
                    try {
                        isProcessed = executor.awaitTermination(keepAliveMillis, TimeUnit.MILLISECONDS);
                    } catch (InterruptedException e) {
                        throw new RuntimeException(e);
                    }
                    if (!isProcessed) {
                        writeKeepAliveAck(writer, sourceNode, streamToFileEnabled);
                    }
                }
            }
            Iterator<OutgoingBatch> activeBatchIter = activeBatches.iterator();
            for (Future<FutureOutgoingBatch> future : futures) {
                currentBatch = activeBatchIter.next();
                boolean isProcessed = false;
                while (!isProcessed) {
                    try {
                        FutureOutgoingBatch extractBatch = future.get(keepAliveMillis, TimeUnit.MILLISECONDS);
                        currentBatch = extractBatch.getOutgoingBatch();
                        if (extractBatch.isExtractSkipped) {
                            break;
                        }
                        if (streamToFileEnabled || mode == ExtractMode.FOR_PAYLOAD_CLIENT) {
                            processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
                            processInfo.setCurrentLoadId(currentBatch.getLoadId());
                            boolean isRetry = extractBatch.isRetry() && extractBatch.getOutgoingBatch().getStatus() != OutgoingBatch.Status.IG;
                            currentBatch = sendOutgoingBatch(processInfo, targetNode, currentBatch, isRetry, dataWriter, writer, mode);
                        }
                        processedBatches.add(currentBatch);
                        isProcessed = true;
                        if (currentBatch.getStatus() != Status.OK) {
                            currentBatch.setLoadCount(currentBatch.getLoadCount() + 1);
                            changeBatchStatus(Status.LD, currentBatch, mode);
                        }
                    } catch (ExecutionException e) {
                        if (isNotBlank(e.getMessage()) && e.getMessage().contains("string truncation")) {
                            throw new RuntimeException("There is a good chance that the truncation error you are receiving is because contains_big_lobs on the '" + currentBatch.getChannelId() + "' channel needs to be turned on.", e.getCause() != null ? e.getCause() : e);
                        }
                        throw new RuntimeException(e.getCause() != null ? e.getCause() : e);
                    } catch (InterruptedException e) {
                        throw new RuntimeException(e);
                    } catch (TimeoutException e) {
                        writeKeepAliveAck(writer, sourceNode, streamToFileEnabled);
                    }
                }
            }
        } catch (RuntimeException e) {
            SQLException se = unwrapSqlException(e);
            if (currentBatch != null) {
                try {
                    /* Reread batch in case the ignore flag has been set */
                    currentBatch = outgoingBatchService.findOutgoingBatch(currentBatch.getBatchId(), currentBatch.getNodeId());
                    statisticManager.incrementDataExtractedErrors(currentBatch.getChannelId(), 1);
                    if (se != null) {
                        currentBatch.setSqlState(se.getSQLState());
                        currentBatch.setSqlCode(se.getErrorCode());
                        currentBatch.setSqlMessage(se.getMessage());
                    } else {
                        currentBatch.setSqlMessage(getRootMessage(e));
                    }
                    currentBatch.revertStatsOnError();
                    if (currentBatch.getStatus() != Status.IG && currentBatch.getStatus() != Status.OK) {
                        currentBatch.setStatus(Status.ER);
                        currentBatch.setErrorFlag(true);
                    }
                    outgoingBatchService.updateOutgoingBatch(currentBatch);
                } catch (Exception ex) {
                    log.error("Failed to update the outgoing batch status for failed batch {}", currentBatch, ex);
                } finally {
                    if (!isStreamClosedByClient(e)) {
                        if (e instanceof ProtocolException) {
                            IStagedResource resource = getStagedResource(currentBatch);
                            if (resource != null) {
                                resource.delete();
                            }
                        }
                        if (e.getCause() instanceof InterruptedException) {
                            log.info("Extract of batch {} was interrupted", currentBatch);
                        } else {
                            log.error("Failed to extract batch {}", currentBatch, e);
                        }
                    }
                    processInfo.setStatus(ProcessInfo.Status.ERROR);
                }
            } else {
                log.error("Could not log the outgoing batch status because the batch was null", e);
            }
        } finally {
            if (executor != null) {
                executor.shutdown();
            }
        }
        // Next, we update the node channel controls to the
        // current timestamp
        Calendar now = Calendar.getInstance();
        for (String channelProcessed : channelsProcessed) {
            NodeChannel nodeChannel = configurationService.getNodeChannel(channelProcessed, targetNode.getNodeId(), false);
            if (nodeChannel != null && nodeChannel.getExtractPeriodMillis() > 0) {
                nodeChannel.setLastExtractTime(now.getTime());
                configurationService.updateLastExtractTime(nodeChannel);
            }
        }
        return processedBatches;
    } else {
        return Collections.emptyList();
    }
}