Examples with IncomingBatch org.jumpmind.symmetric.model.IncomingBatch used on opensource projects

Example 11 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class DataLoaderService method loadDataFromPush.

/**
     * Load database from input stream and write acknowledgment to output
     * stream. This is used for a "push" request with a response of an
     * acknowledgment.
     */
public void loadDataFromPush(Node sourceNode, String channelId, InputStream in, OutputStream out) throws IOException {
    Node local = nodeService.findIdentity();
    if (local != null) {
        ProcessInfo processInfo = statisticManager.newProcessInfo(new ProcessInfoKey(sourceNode.getNodeId(), channelId, local.getNodeId(), ProcessInfoKey.ProcessType.PUSH_HANDLER));
        try {
            List<IncomingBatch> batchList = loadDataFromTransport(processInfo, sourceNode, new InternalIncomingTransport(in), out);
            logDataReceivedFromPush(sourceNode, batchList);
            NodeSecurity security = nodeService.findNodeSecurity(local.getNodeId());
            processInfo.setStatus(ProcessInfo.Status.ACKING);
            transportManager.writeAcknowledgement(out, sourceNode, batchList, local, security != null ? security.getNodePassword() : null);
            if (containsError(batchList)) {
                processInfo.setStatus(ProcessInfo.Status.ERROR);
            } else {
                processInfo.setStatus(ProcessInfo.Status.OK);
            }
        } catch (Exception e) {
            processInfo.setStatus(ProcessInfo.Status.ERROR);
            if (e instanceof RuntimeException) {
                throw (RuntimeException) e;
            } else if (e instanceof IOException) {
                throw (IOException) e;
            }
            throw new RuntimeException(e);
        }
    } else {
        throw new SymmetricException("Could not load data because the node is not registered");
    }
}

Example 12 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class DataLoaderService method loadDataFromTransport.

/**
     * Load database from input stream and return a list of batch statuses. This
     * is used for a pull request that responds with data, and the
     * acknowledgment is sent later.
     */
protected List<IncomingBatch> loadDataFromTransport(final ProcessInfo processInfo, final Node sourceNode, IIncomingTransport transport, OutputStream out) throws IOException {
    final ManageIncomingBatchListener listener = new ManageIncomingBatchListener();
    final DataContext ctx = new DataContext();
    Throwable error = null;
    try {
        Node targetNode = nodeService.findIdentity();
        ctx.put(Constants.DATA_CONTEXT_ENGINE, engine);
        if (targetNode != null) {
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode);
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_ID, targetNode.getNodeId());
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_GROUP_ID, targetNode.getNodeGroupId());
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_EXTERNAL_ID, targetNode.getExternalId());
        }
        if (sourceNode != null) {
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE, sourceNode);
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_ID, sourceNode.getNodeId());
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_GROUP_ID, sourceNode.getNodeGroupId());
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_EXTERNAL_ID, sourceNode.getExternalId());
        }
        for (ILoadSyncLifecycleListener l : extensionService.getExtensionPointList(ILoadSyncLifecycleListener.class)) {
            l.syncStarted(ctx);
        }
        long memoryThresholdInBytes = parameterService.getLong(ParameterConstants.STREAM_TO_FILE_THRESHOLD);
        String targetNodeId = nodeService.findIdentityNodeId();
        if (parameterService.is(ParameterConstants.STREAM_TO_FILE_ENABLED)) {
            processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
            ExecutorService executor = Executors.newFixedThreadPool(1, new CustomizableThreadFactory(String.format("dataloader-%s-%s", sourceNode.getNodeGroupId(), sourceNode.getNodeId())));
            LoadIntoDatabaseOnArrivalListener loadListener = new LoadIntoDatabaseOnArrivalListener(processInfo, sourceNode.getNodeId(), listener, executor);
            new SimpleStagingDataWriter(transport.openReader(), stagingManager, Constants.STAGING_CATEGORY_INCOMING, memoryThresholdInBytes, BatchType.LOAD, targetNodeId, ctx, loadListener).process();
            /* Previously submitted tasks will still be executed */
            executor.shutdown();
            OutputStreamWriter outWriter = null;
            if (out != null) {
                outWriter = new OutputStreamWriter(out, IoConstants.ENCODING);
                long keepAliveMillis = parameterService.getLong(ParameterConstants.DATA_LOADER_SEND_ACK_KEEPALIVE);
                while (!executor.awaitTermination(keepAliveMillis, TimeUnit.MILLISECONDS)) {
                    outWriter.write("1=1&");
                    outWriter.flush();
                }
            } else {
                executor.awaitTermination(12, TimeUnit.HOURS);
            }
            loadListener.isDone();
        } else {
            DataProcessor processor = new DataProcessor(new ProtocolDataReader(BatchType.LOAD, targetNodeId, transport.openReader()), null, listener, "data load") {

                @Override
                protected IDataWriter chooseDataWriter(Batch batch) {
                    return buildDataWriter(processInfo, sourceNode.getNodeId(), batch.getChannelId(), batch.getBatchId(), ((ManageIncomingBatchListener) listener).getCurrentBatch().isRetry());
                }
            };
            processor.process(ctx);
        }
    } catch (Throwable ex) {
        error = ex;
        logAndRethrow(ex);
    } finally {
        transport.close();
        for (ILoadSyncLifecycleListener l : extensionService.getExtensionPointList(ILoadSyncLifecycleListener.class)) {
            l.syncEnded(ctx, listener.getBatchesProcessed(), error);
        }
    }
    return listener.getBatchesProcessed();
}

Example 13 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class FileSyncService method loadFilesFromPush.

public void loadFilesFromPush(String nodeId, InputStream in, OutputStream out) {
    INodeService nodeService = engine.getNodeService();
    Node local = nodeService.findIdentity();
    Node sourceNode = nodeService.findNode(nodeId, true);
    if (local != null && sourceNode != null) {
        ProcessInfo processInfo = engine.getStatisticManager().newProcessInfo(new ProcessInfoKey(nodeId, local.getNodeId(), ProcessInfoKey.ProcessType.FILE_SYNC_PUSH_HANDLER));
        try {
            List<IncomingBatch> list = processZip(in, nodeId, processInfo);
            NodeSecurity security = nodeService.findNodeSecurity(local.getNodeId(), true);
            processInfo.setStatus(ProcessInfo.Status.ACKING);
            engine.getTransportManager().writeAcknowledgement(out, sourceNode, list, local, security != null ? security.getNodePassword() : null);
            processInfo.setStatus(ProcessInfo.Status.OK);
        } catch (Throwable e) {
            processInfo.setStatus(ProcessInfo.Status.ERROR);
            if (e instanceof IOException) {
                throw new IoException((IOException) e);
            } else if (e instanceof RuntimeException) {
                throw (RuntimeException) e;
            } else {
                throw new RuntimeException(e);
            }
        }
    } else {
        throw new SymmetricException("Could not load data because the node is not registered");
    }
}

Example 14 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class FileSyncService method processZip.

protected List<IncomingBatch> processZip(InputStream is, String sourceNodeId, ProcessInfo processInfo) throws IOException {
    File unzipDir = new File(parameterService.getTempDirectory(), String.format("filesync_incoming/%s/%s", engine.getNodeService().findIdentityNodeId(), sourceNodeId));
    FileUtils.deleteDirectory(unzipDir);
    unzipDir.mkdirs();
    try {
        AppUtils.unzip(is, unzipDir);
    } catch (IoException ex) {
        if (ex.toString().contains("EOFException")) {
        // This happens on Android, when there is an empty zip.
        //log.debug("Caught exception while unzipping.", ex);
        } else {
            throw ex;
        }
    }
    Set<Long> batchIds = new TreeSet<Long>();
    String[] files = unzipDir.list(DirectoryFileFilter.INSTANCE);
    if (files != null) {
        for (int i = 0; i < files.length; i++) {
            try {
                batchIds.add(Long.parseLong(files[i]));
            } catch (NumberFormatException e) {
                log.error("Unexpected directory name.  Expected a number representing a batch id.  Instead the directory was named '{}'", files[i]);
            }
        }
    }
    List<IncomingBatch> batchesProcessed = new ArrayList<IncomingBatch>();
    IIncomingBatchService incomingBatchService = engine.getIncomingBatchService();
    processInfo.setStatus(ProcessInfo.Status.LOADING);
    for (Long batchId : batchIds) {
        processInfo.setCurrentBatchId(batchId);
        processInfo.incrementBatchCount();
        File batchDir = new File(unzipDir, Long.toString(batchId));
        IncomingBatch incomingBatch = new IncomingBatch();
        File batchInfo = new File(batchDir, "batch-info.txt");
        if (batchInfo.exists()) {
            List<String> info = FileUtils.readLines(batchInfo);
            if (info != null && info.size() > 0) {
                incomingBatch.setChannelId(info.get(0).trim());
            } else {
                incomingBatch.setChannelId(Constants.CHANNEL_FILESYNC);
            }
        } else {
            incomingBatch.setChannelId(Constants.CHANNEL_FILESYNC);
        }
        incomingBatch.setBatchId(batchId);
        incomingBatch.setStatus(IncomingBatch.Status.LD);
        incomingBatch.setNodeId(sourceNodeId);
        incomingBatch.setByteCount(FileUtils.sizeOfDirectory(batchDir));
        batchesProcessed.add(incomingBatch);
        if (incomingBatchService.acquireIncomingBatch(incomingBatch)) {
            File syncScript = new File(batchDir, "sync.bsh");
            if (syncScript.exists()) {
                String script = FileUtils.readFileToString(syncScript);
                Interpreter interpreter = new Interpreter();
                boolean isLocked = false;
                try {
                    setInterpreterVariables(engine, sourceNodeId, batchDir, interpreter);
                    long waitMillis = getParameterService().getLong(ParameterConstants.FILE_SYNC_LOCK_WAIT_MS);
                    log.debug("The {} node is attempting to get shared lock for to update incoming status", sourceNodeId);
                    isLocked = engine.getClusterService().lock(ClusterConstants.FILE_SYNC_SHARED, ClusterConstants.TYPE_SHARED, waitMillis);
                    if (isLocked) {
                        log.debug("The {} node got a shared file sync lock", sourceNodeId);
                        @SuppressWarnings("unchecked") Map<String, String> filesToEventType = (Map<String, String>) interpreter.eval(script);
                        if (engine.getParameterService().is(ParameterConstants.FILE_SYNC_PREVENT_PING_BACK)) {
                            updateFileIncoming(sourceNodeId, filesToEventType);
                        }
                        incomingBatch.setStatementCount(filesToEventType != null ? filesToEventType.size() : 0);
                    } else {
                        throw new RuntimeException("Could not obtain file sync shared lock within " + waitMillis + " millis");
                    }
                    incomingBatch.setStatus(IncomingBatch.Status.OK);
                    if (incomingBatchService.isRecordOkBatchesEnabled()) {
                        incomingBatchService.updateIncomingBatch(incomingBatch);
                    } else if (incomingBatch.isRetry()) {
                        incomingBatchService.deleteIncomingBatch(incomingBatch);
                    }
                } catch (Throwable ex) {
                    if (ex instanceof TargetError) {
                        Throwable target = ((TargetError) ex).getTarget();
                        if (target != null) {
                            ex = target;
                        }
                    } else if (ex instanceof EvalError) {
                        log.error("Failed to evalulate the script:\n{}", script);
                    }
                    if (ex instanceof FileConflictException) {
                        log.error(ex.getMessage() + ".  Failed to process file sync batch " + batchId);
                    } else {
                        log.error("Failed to process file sync batch " + batchId, ex);
                    }
                    incomingBatch.setErrorFlag(true);
                    incomingBatch.setStatus(IncomingBatch.Status.ER);
                    incomingBatch.setSqlMessage(ex.getMessage());
                    if (incomingBatchService.isRecordOkBatchesEnabled() || incomingBatch.isRetry()) {
                        incomingBatchService.updateIncomingBatch(incomingBatch);
                    } else {
                        incomingBatchService.insertIncomingBatch(incomingBatch);
                    }
                    processInfo.setStatus(ProcessInfo.Status.ERROR);
                    break;
                } finally {
                    log.debug("The {} node is done processing file sync files", sourceNodeId);
                    if (isLocked) {
                        engine.getClusterService().unlock(ClusterConstants.FILE_SYNC_SHARED, ClusterConstants.TYPE_SHARED);
                    }
                }
            } else {
                log.error("Could not find the sync.bsh script for batch {}", batchId);
            }
        }
    }
    return batchesProcessed;
}

Example 15 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class FileSyncService method pullFilesFromNode.

protected void pullFilesFromNode(NodeCommunication nodeCommunication, RemoteNodeStatus status, Node identity, NodeSecurity security) {
    IIncomingTransport transport = null;
    ProcessInfo processInfo = engine.getStatisticManager().newProcessInfo(new ProcessInfoKey(nodeCommunication.getNodeId(), identity.getNodeId(), ProcessType.FILE_SYNC_PULL_JOB));
    try {
        processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
        ITransportManager transportManager;
        if (!engine.getParameterService().is(ParameterConstants.NODE_OFFLINE)) {
            transportManager = engine.getTransportManager();
            transport = transportManager.getFilePullTransport(nodeCommunication.getNode(), identity, security.getNodePassword(), null, parameterService.getRegistrationUrl());
        } else {
            transportManager = ((AbstractSymmetricEngine) engine).getOfflineTransportManager();
            transport = transportManager.getFilePullTransport(nodeCommunication.getNode(), identity, security.getNodePassword(), null, parameterService.getRegistrationUrl());
        }
        List<IncomingBatch> batchesProcessed = processZip(transport.openStream(), nodeCommunication.getNodeId(), processInfo);
        if (batchesProcessed.size() > 0) {
            processInfo.setStatus(ProcessInfo.Status.ACKING);
            status.updateIncomingStatus(batchesProcessed);
            sendAck(nodeCommunication.getNode(), identity, security, batchesProcessed, transportManager);
        }
        if (!status.failed() && batchesProcessed.size() > 0) {
            log.info("Pull files received from {}.  {} files and {} batches were processed", new Object[] { nodeCommunication.getNodeId(), status.getDataProcessed(), status.getBatchesProcessed() });
        } else if (status.failed()) {
            log.info("There was a failure while pulling files from {}.  {} files and {} batches were processed", new Object[] { nodeCommunication.getNodeId(), status.getDataProcessed(), status.getBatchesProcessed() });
        }
    } catch (NoContentException noContentEx) {
        log.debug("Server reported no batches. " + noContentEx);
    } catch (Exception e) {
        fireOffline(e, nodeCommunication.getNode(), status);
    } finally {
        if (transport != null) {
            transport.close();
            if (processInfo.getStatus() != ProcessInfo.Status.ERROR) {
                processInfo.setStatus(ProcessInfo.Status.OK);
            }
            if (transport instanceof FileIncomingTransport) {
                ((FileIncomingTransport) transport).complete(!status.failed());
            }
        }
    }
}