Examples with OutgoingBatches org.jumpmind.symmetric.model.OutgoingBatches used on opensource projects

Example 26 with OutgoingBatches

use of org.jumpmind.symmetric.model.OutgoingBatches in project symmetric-ds by JumpMind.

the class AbstractRouterServiceTest method testMultiChannelRoutingToEveryone.

public void testMultiChannelRoutingToEveryone() {
    resetBatches();
    TriggerRouter trigger1 = getTestRoutingTableTrigger(TEST_TABLE_1);
    getTriggerRouterService().saveTriggerRouter(trigger1);
    TriggerRouter trigger2 = getTestRoutingTableTrigger(TEST_TABLE_2);
    getTriggerRouterService().saveTriggerRouter(trigger2);
    getTriggerRouterService().syncTriggers();
    NodeChannel testChannel = getConfigurationService().getNodeChannel(TestConstants.TEST_CHANNEL_ID, false);
    NodeChannel otherChannel = getConfigurationService().getNodeChannel(TestConstants.TEST_CHANNEL_ID_OTHER, false);
    Assert.assertEquals(50, testChannel.getMaxBatchSize());
    Assert.assertEquals(1, otherChannel.getMaxBatchSize());
    /*
         * Should be 1 batch for table 1 on the testchannel w/ max batch size of
         * 50
         */
    insert(TEST_TABLE_1, 5, false);
    /* this should generate 15 batches because the max batch size is 1 */
    insert(TEST_TABLE_2, 15, false);
    insert(TEST_TABLE_1, 50, true);
    getRouterService().routeData(true);
    final int EXPECTED_BATCHES = getDbDialect().supportsTransactionId() ? 16 : 17;
    OutgoingBatches batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_1.getNodeId(), false);
    filterForChannels(batches, testChannel, otherChannel);
    Assert.assertEquals(EXPECTED_BATCHES, batches.getBatches().size());
    Assert.assertEquals(getDbDialect().supportsTransactionId() ? 1 : 2, countBatchesForChannel(batches, testChannel));
    Assert.assertEquals(15, countBatchesForChannel(batches, otherChannel));
    batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_2.getNodeId(), false);
    filterForChannels(batches, testChannel, otherChannel);
    // Node 2 has sync disabled
    Assert.assertEquals(0, batches.getBatches().size());
    batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_3.getNodeId(), false);
    filterForChannels(batches, testChannel, otherChannel);
    Assert.assertEquals(EXPECTED_BATCHES, batches.getBatches().size());
    resetBatches();
    // should be 2 batches for table 1 on the testchannel w/ max batch size
    // of 50
    insert(TEST_TABLE_1, 50, false);
    // this should generate 1 batches because the max batch size is 1, but
    // the batch is transactional
    insert(TEST_TABLE_2, 15, true);
    insert(TEST_TABLE_1, 50, false);
    getRouterService().routeData(true);
    batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_1.getNodeId(), false);
    filterForChannels(batches, testChannel, otherChannel);
    Assert.assertEquals(getDbDialect().supportsTransactionId() ? 3 : 17, batches.getBatches().size());
    Assert.assertEquals(2, countBatchesForChannel(batches, testChannel));
    Assert.assertEquals(getDbDialect().supportsTransactionId() ? 1 : 15, countBatchesForChannel(batches, otherChannel));
}

Example 27 with OutgoingBatches

use of org.jumpmind.symmetric.model.OutgoingBatches in project symmetric-ds by JumpMind.

the class AbstractRouterServiceTest method testSubSelectNonTransactionalRoutingToNode1.

public void testSubSelectNonTransactionalRoutingToNode1() {
    resetBatches();
    TriggerRouter trigger1 = getTestRoutingTableTrigger(TEST_TABLE_1);
    trigger1.getRouter().setRouterType("subselect");
    trigger1.getRouter().setRouterExpression("c.node_id=:ROUTING_VARCHAR");
    getTriggerRouterService().saveTriggerRouter(trigger1);
    getTriggerRouterService().syncTriggers();
    NodeChannel testChannel = getConfigurationService().getNodeChannel(TestConstants.TEST_CHANNEL_ID, false);
    testChannel.setMaxBatchToSend(1000);
    testChannel.setMaxBatchSize(5);
    testChannel.setBatchAlgorithm("nontransactional");
    getConfigurationService().saveChannel(testChannel, true);
    // should be 100 batches for table 1, even though we committed the
    // changes as part of a transaction
    insert(TEST_TABLE_1, 500, true);
    getRouterService().routeData(true);
    final int EXPECTED_BATCHES = 100;
    OutgoingBatches batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_1.getNodeId(), false);
    filterForChannels(batches, testChannel);
    Assert.assertEquals(EXPECTED_BATCHES, batches.getBatches().size());
    Assert.assertEquals(EXPECTED_BATCHES, countBatchesForChannel(batches, testChannel));
    batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_2.getNodeId(), false);
    filterForChannels(batches, testChannel);
    // Node 2 has sync disabled
    Assert.assertEquals(0, batches.getBatches().size());
    batches = getOutgoingBatchService().getOutgoingBatches(NODE_GROUP_NODE_3.getNodeId(), false);
    filterForChannels(batches, testChannel);
    // Batch was targeted only at node 1
    Assert.assertEquals(0, batches.getBatches().size());
    resetBatches();
}

Example 28 with OutgoingBatches

use of org.jumpmind.symmetric.model.OutgoingBatches in project symmetric-ds by JumpMind.

the class FileSyncService method sendFiles.

public List<OutgoingBatch> sendFiles(ProcessInfo processInfo, Node targetNode, IOutgoingTransport outgoingTransport) {
    List<OutgoingBatch> processedBatches = new ArrayList<OutgoingBatch>();
    List<OutgoingBatch> batchesToProcess = new ArrayList<OutgoingBatch>();
    List<Channel> fileSyncChannels = engine.getConfigurationService().getFileSyncChannels();
    for (Channel channel : fileSyncChannels) {
        OutgoingBatches batches = engine.getOutgoingBatchService().getOutgoingBatches(targetNode.getNodeId(), false);
        batchesToProcess.addAll(batches.filterBatchesForChannel(channel));
    }
    OutgoingBatch currentBatch = null;
    IStagingManager stagingManager = engine.getStagingManager();
    long memoryThresholdInBytes = parameterService.getLong(ParameterConstants.STREAM_TO_FILE_THRESHOLD);
    IStagedResource stagedResource = stagingManager.create(memoryThresholdInBytes, Constants.STAGING_CATEGORY_OUTGOING, processInfo.getSourceNodeId(), targetNode.getNodeId(), "filesync.zip");
    try {
        long maxBytesToSync = parameterService.getLong(ParameterConstants.TRANSPORT_MAX_BYTES_TO_SYNC);
        FileSyncZipDataWriter dataWriter = new FileSyncZipDataWriter(maxBytesToSync, this, engine.getNodeService(), stagedResource);
        try {
            for (int i = 0; i < batchesToProcess.size(); i++) {
                currentBatch = batchesToProcess.get(i);
                processInfo.incrementBatchCount();
                processInfo.setCurrentBatchId(currentBatch.getBatchId());
                ((DataExtractorService) engine.getDataExtractorService()).extractOutgoingBatch(processInfo, targetNode, dataWriter, currentBatch, false, true, DataExtractorService.ExtractMode.FOR_SYM_CLIENT);
                processedBatches.add(currentBatch);
                /*
                     * check to see if max bytes to sync has been reached and
                     * stop processing batches
                     */
                if (dataWriter.readyToSend()) {
                    break;
                }
            }
        } finally {
            dataWriter.finish();
        }
        processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
        for (int i = 0; i < batchesToProcess.size(); i++) {
            batchesToProcess.get(i).setStatus(Status.SE);
        }
        engine.getOutgoingBatchService().updateOutgoingBatches(batchesToProcess);
        try {
            if (stagedResource.exists()) {
                InputStream is = stagedResource.getInputStream();
                try {
                    OutputStream os = outgoingTransport.openStream();
                    IOUtils.copy(is, os);
                    os.flush();
                } catch (IOException e) {
                    throw new IoException(e);
                }
            }
            for (int i = 0; i < batchesToProcess.size(); i++) {
                batchesToProcess.get(i).setStatus(Status.LD);
            }
            engine.getOutgoingBatchService().updateOutgoingBatches(batchesToProcess);
        } finally {
            stagedResource.close();
        }
    } catch (RuntimeException e) {
        if (currentBatch != null) {
            engine.getStatisticManager().incrementDataExtractedErrors(currentBatch.getChannelId(), 1);
            currentBatch.setSqlMessage(getRootMessage(e));
            currentBatch.revertStatsOnError();
            if (currentBatch.getStatus() != Status.IG) {
                currentBatch.setStatus(Status.ER);
            }
            currentBatch.setErrorFlag(true);
            engine.getOutgoingBatchService().updateOutgoingBatch(currentBatch);
            if (isStreamClosedByClient(e)) {
                log.warn("Failed to extract batch {}.  The stream was closed by the client.  The error was: {}", currentBatch, getRootMessage(e));
            } else {
                log.error("Failed to extract batch {}", currentBatch, e);
            }
        } else {
            log.error("Could not log the outgoing batch status because the batch was null", e);
        }
        throw e;
    } finally {
        if (stagedResource != null) {
            stagedResource.delete();
        }
    }
    return processedBatches;
}

Example 29 with OutgoingBatches

use of org.jumpmind.symmetric.model.OutgoingBatches in project symmetric-ds by JumpMind.

the class SimpleIntegrationTest method test29SyncShellCommand.

@Test(timeout = 120000)
public void test29SyncShellCommand() throws Exception {
    logTestRunning();
    IDataService rootDataService = getServer().getDataService();
    IOutgoingBatchService rootOutgoingBatchService = getServer().getOutgoingBatchService();
    testFlag = false;
    String scriptData = "org.jumpmind.symmetric.test.SimpleIntegrationTest.testFlag=true;";
    rootDataService.sendScript(TestConstants.TEST_CLIENT_EXTERNAL_ID, scriptData, false);
    clientPull();
    OutgoingBatches batches = rootOutgoingBatchService.getOutgoingBatches(TestConstants.TEST_CLIENT_NODE.getNodeId(), false);
    assertEquals(0, batches.countBatches(true));
    assertTrue("Expected the testFlag static variable to have been set to true", testFlag);
}

Example 30 with OutgoingBatches

use of org.jumpmind.symmetric.model.OutgoingBatches in project symmetric-ds by JumpMind.

the class SimpleIntegrationTest method test14SuspendIgnorePullRemoteLocalComboBatches.

@Test(timeout = 120000)
public void test14SuspendIgnorePullRemoteLocalComboBatches() throws Exception {
    // test suspend / ignore with remote database specifying the suspends
    // and ignores
    logTestRunning();
    Date date = DateUtils.parseDate("2009-09-30", new String[] { "yyyy-MM-dd" });
    Order order = new Order("442", 100, "C", date);
    serverTestService.insertOrder(order);
    clientPull();
    IOutgoingBatchService rootOutgoingBatchService = getServer().getOutgoingBatchService();
    OutgoingBatches batches = rootOutgoingBatchService.getOutgoingBatches(TestConstants.TEST_CLIENT_NODE.getNodeId(), false);
    assertNotNull(clientTestService.getOrder(order.getOrderId()));
    assertEquals("There should be no outgoing batches", 0, batches.getBatches().size());
    IConfigurationService rootConfigurationService = getServer().getConfigurationService();
    IConfigurationService clientConfigurationService = getClient().getConfigurationService();
    // suspend on remote
    NodeChannel c = rootConfigurationService.getNodeChannel(TestConstants.TEST_CHANNEL_ID, TestConstants.TEST_CLIENT_EXTERNAL_ID, false);
    c.setSuspendEnabled(true);
    rootConfigurationService.saveNodeChannel(c, true);
    // ignore on local
    c = clientConfigurationService.getNodeChannel(TestConstants.TEST_CHANNEL_ID, TestConstants.TEST_CLIENT_EXTERNAL_ID, false);
    c.setIgnoreEnabled(true);
    clientConfigurationService.saveNodeChannel(c, true);
    order = new Order("443", 100, "C", date);
    serverTestService.insertOrder(order);
    clientPull();
    batches = rootOutgoingBatchService.getOutgoingBatches(TestConstants.TEST_CLIENT_NODE.getNodeId(), false);
    assertEquals("There should be no outgoing batches", 0, batches.getBatches().size());
    assertNull(clientTestService.getOrder(order.getOrderId()));
    // ignore on remote
    c = rootConfigurationService.getNodeChannel(TestConstants.TEST_CHANNEL_ID, TestConstants.TEST_CLIENT_EXTERNAL_ID, false);
    c.setIgnoreEnabled(true);
    c.setSuspendEnabled(false);
    rootConfigurationService.saveNodeChannel(c, true);
    // suspend on local
    c = clientConfigurationService.getNodeChannel(TestConstants.TEST_CHANNEL_ID, TestConstants.TEST_CLIENT_EXTERNAL_ID, false);
    c.setIgnoreEnabled(false);
    c.setSuspendEnabled(true);
    clientConfigurationService.saveNodeChannel(c, true);
    order = new Order("444", 100, "C", date);
    clientPull();
    batches = rootOutgoingBatchService.getOutgoingBatches(TestConstants.TEST_CLIENT_NODE.getNodeId(), false);
    assertEquals("There should be no outgoing batches", 0, batches.getBatches().size());
    assertNull(clientTestService.getOrder(order.getOrderId()));
    // Cleanup!
    c = rootConfigurationService.getNodeChannel(TestConstants.TEST_CHANNEL_ID, TestConstants.TEST_CLIENT_EXTERNAL_ID, false);
    c.setSuspendEnabled(false);
    c.setIgnoreEnabled(false);
    rootConfigurationService.saveNodeChannel(c, true);
    c = clientConfigurationService.getNodeChannel(TestConstants.TEST_CHANNEL_ID, TestConstants.TEST_CLIENT_EXTERNAL_ID, false);
    c.setSuspendEnabled(false);
    c.setIgnoreEnabled(false);
    clientConfigurationService.saveNodeChannel(c, true);
    clientPull();
}