Examples with IDataLoaderService org.jumpmind.symmetric.service.IDataLoaderService used on opensource projects

Example 1 with IDataLoaderService

use of org.jumpmind.symmetric.service.IDataLoaderService in project symmetric-ds by JumpMind.

the class SymmetricAdmin method importBatch.

private void importBatch(CommandLine line, List<String> args) throws Exception {
    IDataLoaderService service = getSymmetricEngine().getDataLoaderService();
    InputStream in = null;
    if (args.size() == 0) {
        in = System.in;
    } else {
        in = new FileInputStream(args.get(0));
    }
    service.loadDataFromPush(getSymmetricEngine().getNodeService().findIdentity(), in, System.out);
    System.out.flush();
    in.close();
}

Example 2 with IDataLoaderService

use of org.jumpmind.symmetric.service.IDataLoaderService in project symmetric-ds by JumpMind.

the class PushUriHandler method handle.

public void handle(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
    String nodeId = ServletUtils.getParameter(req, WebConstants.NODE_ID);
    String channelId = getChannelId(req);
    log.info("About to service push request for {}", nodeId);
    IStagingManager stagingManager = engine.getStagingManager();
    IDataLoaderService dataLoaderService = engine.getDataLoaderService();
    INodeService nodeService = engine.getNodeService();
    IStatisticManager statisticManager = engine.getStatisticManager();
    String identityNodeId = nodeService.findIdentityNodeId();
    ProcessInfo processInfo = statisticManager.newProcessInfo(new ProcessInfoKey(nodeId, identityNodeId, ProcessType.TRANSFER_FROM, channelId));
    BufferedReader reader = null;
    BufferedWriter writer = null;
    DataLoaderWorker worker = null;
    try {
        Node sourceNode = engine.getNodeService().findNode(nodeId);
        processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
        reader = new BufferedReader(new InputStreamReader(createInputStream(req)));
        long streamToFileThreshold = parameterService.getLong(ParameterConstants.STREAM_TO_FILE_THRESHOLD);
        String line = reader.readLine();
        StringBuilder batchPrefix = new StringBuilder();
        Long batchId = null;
        while (line != null) {
            if (line.startsWith(CsvConstants.BATCH)) {
                batchId = getBatchId(line);
                IStagedResource resource = stagingManager.create(streamToFileThreshold, Constants.STAGING_CATEGORY_INCOMING, nodeId, batchId);
                writer = resource.getWriter();
                writer.write(batchPrefix.toString());
            } else if (line.startsWith(CsvConstants.COMMIT)) {
                writer.write(line);
                writer.close();
                writer = null;
                if (worker == null) {
                    worker = dataLoaderService.createDataLoaderWorker(ProcessType.LOAD_FROM_PUSH, channelId, sourceNode);
                }
                worker.queueUpLoad(new IncomingBatch(batchId, nodeId));
                batchId = null;
            }
            if (batchId == null) {
                batchPrefix.append(line).append("\n");
            } else if (writer != null) {
                writer.write(line);
                writer.write("\n");
            }
            line = reader.readLine();
        }
        processInfo.setStatus(ProcessInfo.Status.OK);
    } catch (RuntimeException ex) {
        processInfo.setStatus(ProcessInfo.Status.ERROR);
        throw ex;
    } finally {
        IOUtils.closeQuietly(reader);
        IOUtils.closeQuietly(writer);
    }
    PrintWriter resWriter = res.getWriter();
    if (worker != null) {
        worker.queueUpLoad(new DataLoaderService.EOM());
        while (!worker.isComplete()) {
            String status = "done";
            IncomingBatch batch = worker.waitForNextBatchToComplete();
            if (batch == null) {
                status = "in progress";
                batch = worker.getCurrentlyLoading();
            }
            if (batch != null && !(batch instanceof DataLoaderService.EOM)) {
                ArrayList<IncomingBatch> list = new ArrayList<IncomingBatch>(1);
                list.add(batch);
                log.info("sending {} ack ... for {}", status, batch);
                // TODO 13 support
                resWriter.write(engine.getTransportManager().getAcknowledgementData(false, identityNodeId, list));
                resWriter.write("\n");
                resWriter.flush();
            }
        }
    }
    res.flushBuffer();
    log.debug("Done servicing push request for {}", nodeId);
}

Example 3 with IDataLoaderService

use of org.jumpmind.symmetric.service.IDataLoaderService in project symmetric-ds by JumpMind.

the class AbstractDataLoaderServiceTest method getDataLoaderService.

protected IDataLoaderService getDataLoaderService() {
    DataLoaderService dataLoaderService = (DataLoaderService) getSymmetricEngine().getDataLoaderService();
    dataLoaderService.setTransportManager(transportManager);
    return dataLoaderService;
}

Example 4 with IDataLoaderService

use of org.jumpmind.symmetric.service.IDataLoaderService in project symmetric-ds by JumpMind.

the class AbstractTest method loadConfigAtRegistrationServer.

/**
 * Loads configuration in the format of classname.csv at the registration
 * server
 */
protected void loadConfigAtRegistrationServer() throws Exception {
    ISymmetricEngine regEngine = getRegServer().getEngine();
    IDataLoaderService dataLoaderService = regEngine.getDataLoaderService();
    boolean inError = false;
    String fileName = getClass().getSimpleName() + ".csv";
    log.info("Loading " + fileName + " on " + regEngine.getEngineName());
    InputStream is = getClass().getResourceAsStream(fileName);
    assertNotNull("Could not find configuration as a resource", is);
    List<IncomingBatch> batches = dataLoaderService.loadDataBatch(IOUtils.toString(is));
    for (IncomingBatch batch : batches) {
        if (batch.getStatus() == Status.ER) {
            inError = true;
        }
    }
    assertFalse("Failed to load configuration", inError);
}

Example 5 with IDataLoaderService

use of org.jumpmind.symmetric.service.IDataLoaderService in project symmetric-ds by JumpMind.

the class RestService method loadProfileImpl.

private void loadProfileImpl(ISymmetricEngine engine, MultipartFile file) {
    IDataLoaderService dataLoaderService = engine.getDataLoaderService();
    boolean inError = false;
    try {
        String content = new String(file.getBytes());
        List<IncomingBatch> batches = dataLoaderService.loadDataBatch(content);
        for (IncomingBatch batch : batches) {
            if (batch.getStatus() == Status.ER) {
                inError = true;
            }
        }
    } catch (Exception e) {
        inError = true;
    }
    if (inError) {
        throw new InternalServerErrorException();
    }
}