Examples with IIncomingBatchService org.jumpmind.symmetric.service.IIncomingBatchService used on opensource projects

Example 1 with IIncomingBatchService

use of org.jumpmind.symmetric.service.IIncomingBatchService in project symmetric-ds by JumpMind.

the class FileSyncService method processZip.

protected List<IncomingBatch> processZip(InputStream is, String sourceNodeId, ProcessInfo processInfo) throws IOException {
    File unzipDir = new File(parameterService.getTempDirectory(), String.format("filesync_incoming/%s/%s", engine.getNodeService().findIdentityNodeId(), sourceNodeId));
    FileUtils.deleteDirectory(unzipDir);
    unzipDir.mkdirs();
    AppUtils.unzip(is, unzipDir);
    Set<Long> batchIds = new TreeSet<Long>();
    String[] files = unzipDir.list(DirectoryFileFilter.INSTANCE);
    if (files != null) {
        for (int i = 0; i < files.length; i++) {
            try {
                batchIds.add(Long.parseLong(files[i]));
            } catch (NumberFormatException e) {
                log.error("Unexpected directory name.  Expected a number representing a batch id.  Instead the directory was named '{}'", files[i]);
            }
        }
    }
    List<IncomingBatch> batchesProcessed = new ArrayList<IncomingBatch>();
    IIncomingBatchService incomingBatchService = engine.getIncomingBatchService();
    processInfo.setStatus(ProcessInfo.Status.LOADING);
    for (Long batchId : batchIds) {
        processInfo.setCurrentBatchId(batchId);
        processInfo.incrementBatchCount();
        File batchDir = new File(unzipDir, Long.toString(batchId));
        IncomingBatch incomingBatch = new IncomingBatch();
        File batchInfo = new File(batchDir, "batch-info.txt");
        if (batchInfo.exists()) {
            List<String> info = FileUtils.readLines(batchInfo);
            if (info != null && info.size() > 0) {
                incomingBatch.setChannelId(info.get(0).trim());
            } else {
                incomingBatch.setChannelId(Constants.CHANNEL_FILESYNC);
            }
        } else {
            incomingBatch.setChannelId(Constants.CHANNEL_FILESYNC);
        }
        incomingBatch.setBatchId(batchId);
        incomingBatch.setStatus(IncomingBatch.Status.LD);
        incomingBatch.setNodeId(sourceNodeId);
        incomingBatch.setByteCount(FileUtils.sizeOfDirectory(batchDir));
        batchesProcessed.add(incomingBatch);
        if (incomingBatchService.acquireIncomingBatch(incomingBatch)) {
            File syncScript = new File(batchDir, "sync.bsh");
            if (syncScript.exists()) {
                String script = FileUtils.readFileToString(syncScript);
                Interpreter interpreter = new Interpreter();
                boolean isLocked = false;
                try {
                    interpreter.set("log", log);
                    interpreter.set("batchDir", batchDir.getAbsolutePath().replace('\\', '/'));
                    interpreter.set("engine", engine);
                    interpreter.set("sourceNodeId", sourceNodeId);
                    long waitMillis = getParameterService().getLong(ParameterConstants.FILE_SYNC_LOCK_WAIT_MS);
                    log.debug("The {} node is attempting to get shared lock for to update incoming status", sourceNodeId);
                    isLocked = engine.getClusterService().lock(ClusterConstants.FILE_SYNC_SHARED, ClusterConstants.TYPE_SHARED, waitMillis);
                    if (isLocked) {
                        log.debug("The {} node got a shared file sync lock", sourceNodeId);
                        @SuppressWarnings("unchecked") Map<String, String> filesToEventType = (Map<String, String>) interpreter.eval(script);
                        updateFileIncoming(sourceNodeId, filesToEventType);
                        incomingBatch.setStatementCount(filesToEventType != null ? filesToEventType.size() : 0);
                    } else {
                        throw new RuntimeException("Could not obtain file sync shared lock within " + waitMillis + " millis");
                    }
                    incomingBatch.setStatus(IncomingBatch.Status.OK);
                    if (incomingBatchService.isRecordOkBatchesEnabled()) {
                        incomingBatchService.updateIncomingBatch(incomingBatch);
                    } else if (incomingBatch.isRetry()) {
                        incomingBatchService.deleteIncomingBatch(incomingBatch);
                    }
                } catch (Throwable ex) {
                    if (ex instanceof TargetError) {
                        Throwable target = ((TargetError) ex).getTarget();
                        if (target != null) {
                            ex = target;
                        }
                    } else if (ex instanceof EvalError) {
                        log.error("Failed to evalulate the script:\n{}", script);
                    }
                    if (ex instanceof FileConflictException) {
                        log.error(ex.getMessage() + ".  Failed to process file sync batch " + batchId);
                    } else {
                        log.error("Failed to process file sync batch " + batchId, ex);
                    }
                    incomingBatch.setErrorFlag(true);
                    incomingBatch.setStatus(IncomingBatch.Status.ER);
                    incomingBatch.setSqlMessage(ex.getMessage());
                    if (incomingBatchService.isRecordOkBatchesEnabled() || incomingBatch.isRetry()) {
                        incomingBatchService.updateIncomingBatch(incomingBatch);
                    } else {
                        incomingBatchService.insertIncomingBatch(incomingBatch);
                    }
                    processInfo.setStatus(ProcessInfo.Status.ERROR);
                    break;
                } finally {
                    log.debug("The {} node is done processing file sync files", sourceNodeId);
                    if (isLocked) {
                        engine.getClusterService().unlock(ClusterConstants.FILE_SYNC_SHARED, ClusterConstants.TYPE_SHARED);
                    }
                }
            } else {
                log.error("Could not find the sync.bsh script for batch {}", batchId);
            }
        }
    }
    return batchesProcessed;
}

Example 2 with IIncomingBatchService

use of org.jumpmind.symmetric.service.IIncomingBatchService in project symmetric-ds by JumpMind.

the class SimpleIntegrationTest method test03InitialLoad.

@Test(timeout = 120000)
public void test03InitialLoad() {
    logTestRunning();
    serverTestService.insertIntoTestUseStreamLob(100, "test_use_stream_lob", THIS_IS_A_TEST);
    serverTestService.insertIntoTestUseStreamLob(100, "test_use_capture_lob", THIS_IS_A_TEST);
    Customer customer = new Customer(301, "Linus", true, "42 Blanket Street", "Santa Claus", "IN", 90009, new Date(), new Date(), THIS_IS_A_TEST, BINARY_DATA);
    serverTestService.insertCustomer(customer);
    serverTestService.insertIntoTestTriggerTable(new Object[] { 1, "wow", "mom" });
    serverTestService.insertIntoTestTriggerTable(new Object[] { 2, "mom", "wow" });
    INodeService rootNodeService = getServer().getNodeService();
    INodeService clientNodeService = getClient().getNodeService();
    String nodeId = rootNodeService.findNodeByExternalId(TestConstants.TEST_CLIENT_NODE_GROUP, TestConstants.TEST_CLIENT_EXTERNAL_ID).getNodeId();
    getServer().reloadNode(nodeId, "test");
    IOutgoingBatchService rootOutgoingBatchService = getServer().getOutgoingBatchService();
    assertFalse(rootOutgoingBatchService.isInitialLoadComplete(nodeId));
    assertTrue(rootNodeService.findNodeSecurity(TestConstants.TEST_CLIENT_EXTERNAL_ID).isInitialLoadEnabled());
    do {
        clientPull();
    } while (!rootOutgoingBatchService.isInitialLoadComplete(nodeId));
    assertFalse(rootNodeService.findNodeSecurity(TestConstants.TEST_CLIENT_EXTERNAL_ID).isInitialLoadEnabled());
    NodeSecurity clientNodeSecurity = clientNodeService.findNodeSecurity(TestConstants.TEST_CLIENT_EXTERNAL_ID);
    assertFalse(clientNodeSecurity.isInitialLoadEnabled());
    assertNotNull(clientNodeSecurity.getInitialLoadTime());
    IIncomingBatchService clientIncomingBatchService = getClient().getIncomingBatchService();
    assertEquals("The initial load errored out." + printRootAndClientDatabases(), 0, clientIncomingBatchService.countIncomingBatchesInError());
    assertEquals("test_triggers_table on the client did not contain the expected number of rows", 2, clientTestService.countTestTriggersTable());
    assertEquals("test_customer on the client did not contain the expected number of rows", 2, clientTestService.count("test_customer"));
    assertEquals("Initial load was not successful according to the client", NodeStatus.DATA_LOAD_COMPLETED, clientNodeService.getNodeStatus());
    assertEquals("Initial load was not successful accordign to the root", false, rootNodeService.findNodeSecurity(TestConstants.TEST_CLIENT_EXTERNAL_ID).isInitialLoadEnabled());
    clientTestService.assertTestBlobIsInDatabase(100, "test_use_capture_lob", THIS_IS_A_TEST);
    clientTestService.assertTestBlobIsInDatabase(100, "test_use_stream_lob", THIS_IS_A_TEST);
}