Examples with IncomingBatch org.jumpmind.symmetric.model.IncomingBatch used on opensource projects

Example 16 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class DataLoaderService method loadDataFromOfflineTransport.

public List<IncomingBatch> loadDataFromOfflineTransport(Node remote, RemoteNodeStatus status, IIncomingTransport transport) throws IOException {
    Node local = nodeService.findIdentity();
    ProcessInfo processInfo = statisticManager.newProcessInfo(new ProcessInfoKey(remote.getNodeId(), local.getNodeId(), ProcessType.OFFLINE_PULL));
    List<IncomingBatch> list = null;
    try {
        list = loadDataFromTransport(processInfo, remote, transport, null);
        if (list.size() > 0) {
            processInfo.setStatus(ProcessInfo.Status.ACKING);
            status.updateIncomingStatus(list);
        }
        if (containsError(list)) {
            processInfo.setStatus(ProcessInfo.Status.ERROR);
        } else {
            processInfo.setStatus(ProcessInfo.Status.OK);
        }
    } catch (RuntimeException e) {
        processInfo.setStatus(ProcessInfo.Status.ERROR);
        throw e;
    } catch (IOException e) {
        processInfo.setStatus(ProcessInfo.Status.ERROR);
        throw e;
    }
    return list;
}

Example 17 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class DataLoaderService method loadDataFromPull.

public void loadDataFromPull(Node remote, RemoteNodeStatus status) throws IOException {
    Node local = nodeService.findIdentity();
    if (local == null) {
        local = new Node(this.parameterService, symmetricDialect);
    }
    try {
        NodeSecurity localSecurity = nodeService.findNodeSecurity(local.getNodeId(), true);
        IIncomingTransport transport = null;
        boolean isRegisterTransport = false;
        if (remote != null && localSecurity != null) {
            Map<String, String> requestProperties = new HashMap<String, String>();
            ChannelMap suspendIgnoreChannels = configurationService.getSuspendIgnoreChannelLists();
            requestProperties.put(WebConstants.SUSPENDED_CHANNELS, suspendIgnoreChannels.getSuspendChannelsAsString());
            requestProperties.put(WebConstants.IGNORED_CHANNELS, suspendIgnoreChannels.getIgnoreChannelsAsString());
            requestProperties.put(WebConstants.THREAD_CHANNEL, status.getChannelId());
            transport = transportManager.getPullTransport(remote, local, localSecurity.getNodePassword(), requestProperties, parameterService.getRegistrationUrl());
        } else {
            transport = transportManager.getRegisterTransport(local, parameterService.getRegistrationUrl());
            log.info("Using registration URL of {}", transport.getUrl());
            List<INodeRegistrationListener> registrationListeners = extensionService.getExtensionPointList(INodeRegistrationListener.class);
            for (INodeRegistrationListener l : registrationListeners) {
                l.registrationUrlUpdated(transport.getUrl());
            }
            remote = new Node();
            remote.setSyncUrl(parameterService.getRegistrationUrl());
            isRegisterTransport = true;
        }
        ProcessInfo processInfo = statisticManager.newProcessInfo(new ProcessInfoKey(remote.getNodeId(), status.getChannelId(), local.getNodeId(), ProcessType.PULL_JOB));
        try {
            List<IncomingBatch> list = loadDataFromTransport(processInfo, remote, transport, null);
            if (list.size() > 0) {
                processInfo.setStatus(ProcessInfo.Status.ACKING);
                status.updateIncomingStatus(list);
                local = nodeService.findIdentity();
                if (local != null) {
                    localSecurity = nodeService.findNodeSecurity(local.getNodeId(), !isRegisterTransport);
                    if (StringUtils.isNotBlank(transport.getRedirectionUrl())) {
                        /*
                             * We were redirected for the pull, we need to
                             * redirect for the ack
                             */
                        String url = transport.getRedirectionUrl();
                        int index = url.indexOf("/registration?");
                        if (index >= 0) {
                            url = url.substring(0, index);
                        }
                        log.info("Setting the sync url for ack to: {}", url);
                        remote.setSyncUrl(url);
                    }
                    sendAck(remote, local, localSecurity, list, transportManager);
                }
            }
            if (containsError(list)) {
                processInfo.setStatus(ProcessInfo.Status.ERROR);
            } else {
                processInfo.setStatus(ProcessInfo.Status.OK);
            }
            updateBatchToSendCount(remote, transport);
        } catch (RuntimeException e) {
            processInfo.setStatus(ProcessInfo.Status.ERROR);
            throw e;
        } catch (IOException e) {
            processInfo.setStatus(ProcessInfo.Status.ERROR);
            throw e;
        }
    } catch (RegistrationRequiredException e) {
        if (StringUtils.isBlank(remote.getSyncUrl()) || remote.getSyncUrl().equals(parameterService.getRegistrationUrl())) {
            log.warn("Node information missing on the server.  Attempting to re-register remote.getSyncUrl()={}", remote.getSyncUrl());
            loadDataFromPull(null, status);
            nodeService.findIdentity(false);
        } else {
            log.warn("Failed to pull data from node '{}'. It probably is missing a node security record for '{}'.", remote.getNodeId(), local.getNodeId());
        }
    } catch (MalformedURLException e) {
        if (remote != null) {
            log.error("Could not connect to the {} node's transport because of a bad URL: '{}' {}", remote.getNodeId(), remote.getSyncUrl(), e);
        } else {
            log.error("", e);
        }
        throw e;
    }
}

Example 18 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class DataLoaderService method loadDataFromConfig.

public void loadDataFromConfig(Node remote, RemoteNodeStatus status, boolean force) throws IOException {
    if (engine.getParameterService().isRegistrationServer()) {
        return;
    }
    Node local = nodeService.findIdentity();
    try {
        NodeSecurity localSecurity = nodeService.findNodeSecurity(local.getNodeId(), true);
        String configVersion = force ? "" : local.getConfigVersion();
        IIncomingTransport transport = engine.getTransportManager().getConfigTransport(remote, local, localSecurity.getNodePassword(), Version.version(), configVersion, remote.getSyncUrl());
        ProcessInfo processInfo = statisticManager.newProcessInfo(new ProcessInfoKey(remote.getNodeId(), Constants.CHANNEL_CONFIG, local.getNodeId(), ProcessType.PULL_CONFIG_JOB));
        try {
            log.info("Requesting current configuration {symmetricVersion={}, configVersion={}}", Version.version(), local.getConfigVersion());
            List<IncomingBatch> list = loadDataFromTransport(processInfo, remote, transport, null);
            if (containsError(list)) {
                processInfo.setStatus(ProcessInfo.Status.ERROR);
            } else {
                if (list.size() > 0) {
                    status.updateIncomingStatus(list);
                    local.setConfigVersion(Version.version());
                    nodeService.save(local);
                }
                processInfo.setStatus(ProcessInfo.Status.OK);
            }
        } catch (RuntimeException e) {
            processInfo.setStatus(ProcessInfo.Status.ERROR);
            throw e;
        } catch (IOException e) {
            processInfo.setStatus(ProcessInfo.Status.ERROR);
            throw e;
        }
    } catch (RegistrationRequiredException e) {
        log.warn("Failed to pull configuration from node '{}'. It probably is missing a node security record for '{}'.", remote.getNodeId(), local.getNodeId());
    } catch (MalformedURLException e) {
        log.error("Could not connect to the {} node's transport because of a bad URL: '{}' {}", remote.getNodeId(), remote.getSyncUrl(), e);
        throw e;
    }
}

Example 19 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class AbstractTransportManager method getAcknowledgementData.

protected String getAcknowledgementData(boolean requires13Format, String nodeId, List<IncomingBatch> list) throws IOException {
    StringBuilder builder = new StringBuilder();
    if (!requires13Format) {
        for (IncomingBatch batch : list) {
            long batchId = batch.getBatchId();
            Object value = null;
            if (batch.getStatus() == Status.OK) {
                value = WebConstants.ACK_BATCH_OK;
            } else if (batch.getStatus() == Status.RS) {
                value = WebConstants.ACK_BATCH_RESEND;
            } else {
                value = batch.getFailedRowNumber();
            }
            append(builder, WebConstants.ACK_BATCH_NAME + batch.getBatchId(), value);
            append(builder, WebConstants.ACK_NODE_ID + batchId, nodeId);
            append(builder, WebConstants.ACK_NETWORK_MILLIS + batchId, batch.getNetworkMillis());
            append(builder, WebConstants.ACK_FILTER_MILLIS + batchId, batch.getFilterMillis());
            append(builder, WebConstants.ACK_DATABASE_MILLIS + batchId, batch.getDatabaseMillis());
            append(builder, WebConstants.ACK_BYTE_COUNT + batchId, batch.getByteCount());
            if (batch.getIgnoreCount() > 0) {
                append(builder, WebConstants.ACK_IGNORE_COUNT + batchId, batch.getIgnoreCount());
            }
            if (batch.getStatus() == Status.ER) {
                append(builder, WebConstants.ACK_SQL_STATE + batchId, batch.getSqlState());
                append(builder, WebConstants.ACK_SQL_CODE + batchId, batch.getSqlCode());
                append(builder, WebConstants.ACK_SQL_MESSAGE + batchId, batch.getSqlMessage());
            }
        }
    } else {
        for (IncomingBatch batch : list) {
            Object value = null;
            if (batch.getStatus() == Status.OK || batch.getStatus() == Status.IG) {
                value = WebConstants.ACK_BATCH_OK;
            } else {
                value = batch.getFailedRowNumber();
            }
            append(builder, WebConstants.ACK_BATCH_NAME + batch.getBatchId(), value);
        }
    }
    return builder.toString();
}

Example 20 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class AbstractDataLoaderServiceTest method test04SqlStatistics.

@Test
public void test04SqlStatistics() throws Exception {
    Level old = setLoggingLevelForTest(Level.OFF);
    String[] insertValues = new String[TEST_COLUMNS.length];
    insertValues[2] = insertValues[4] = "sql stat test";
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    CsvWriter writer = getWriter(out);
    writer.writeRecord(new String[] { CsvConstants.NODEID, TestConstants.TEST_CLIENT_EXTERNAL_ID });
    writer.writeRecord(new String[] { CsvConstants.CHANNEL, TestConstants.TEST_CHANNEL_ID });
    String nextBatchId = getNextBatchId();
    writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId });
    writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
    // Clean insert
    String firstId = getNextId();
    insertValues[0] = firstId;
    writer.write(CsvConstants.INSERT);
    writer.writeRecord(insertValues, true);
    // Clean insert
    String secondId = getNextId();
    insertValues[0] = secondId;
    writer.write(CsvConstants.INSERT);
    writer.writeRecord(insertValues, true);
    String thirdId = getNextId();
    insertValues[0] = thirdId;
    // date column ...
    insertValues[5] = "This is a very long string that will fail upon insert into the database.";
    writer.write(CsvConstants.INSERT);
    writer.writeRecord(insertValues, true);
    writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId });
    writer.close();
    load(out);
    IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID);
    assertNotNull(batch);
    assertEquals(batch.getStatus(), IncomingBatch.Status.ER, "Wrong status. " + printDatabase());
    assertEquals(batch.getFailedRowNumber(), 3l, "Wrong failed row number. " + printDatabase());
    Assert.assertEquals("Wrong byte count: " + batch.getByteCount() + ". " + printDatabase(), 394l, batch.getByteCount());
    assertEquals(batch.getStatementCount(), 3l, "Wrong statement count. " + printDatabase());
    assertEquals(batch.getFallbackInsertCount(), 0l, "Wrong fallback insert count. " + printDatabase());
    assertEquals(batch.getFallbackUpdateCount(), 0l, "Wrong fallback update count. " + printDatabase());
    assertEquals(batch.getMissingDeleteCount(), 0l, "Wrong missing delete count. " + printDatabase());
    assertNull(batch.getSqlState(), "Sql state should be null. " + printDatabase());
    assertNotNull(batch.getSqlMessage(), "Sql message should not be null. " + printDatabase());
    setLoggingLevelForTest(old);
}