Examples with Batch org.jumpmind.symmetric.io.data.Batch used on opensource projects

Example 1 with Batch

use of org.jumpmind.symmetric.io.data.Batch in project symmetric-ds by JumpMind.

the class DataLoaderService method loadDataFromTransport.

/**
     * Load database from input stream and return a list of batch statuses. This
     * is used for a pull request that responds with data, and the
     * acknowledgment is sent later.
     */
protected List<IncomingBatch> loadDataFromTransport(final ProcessInfo processInfo, final Node sourceNode, IIncomingTransport transport, OutputStream out) throws IOException {
    final ManageIncomingBatchListener listener = new ManageIncomingBatchListener();
    final DataContext ctx = new DataContext();
    Throwable error = null;
    try {
        Node targetNode = nodeService.findIdentity();
        ctx.put(Constants.DATA_CONTEXT_ENGINE, engine);
        if (targetNode != null) {
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode);
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_ID, targetNode.getNodeId());
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_GROUP_ID, targetNode.getNodeGroupId());
            ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_EXTERNAL_ID, targetNode.getExternalId());
        }
        if (sourceNode != null) {
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE, sourceNode);
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_ID, sourceNode.getNodeId());
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_GROUP_ID, sourceNode.getNodeGroupId());
            ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_EXTERNAL_ID, sourceNode.getExternalId());
        }
        for (ILoadSyncLifecycleListener l : extensionService.getExtensionPointList(ILoadSyncLifecycleListener.class)) {
            l.syncStarted(ctx);
        }
        long memoryThresholdInBytes = parameterService.getLong(ParameterConstants.STREAM_TO_FILE_THRESHOLD);
        String targetNodeId = nodeService.findIdentityNodeId();
        if (parameterService.is(ParameterConstants.STREAM_TO_FILE_ENABLED)) {
            processInfo.setStatus(ProcessInfo.Status.TRANSFERRING);
            ExecutorService executor = Executors.newFixedThreadPool(1, new CustomizableThreadFactory(String.format("dataloader-%s-%s", sourceNode.getNodeGroupId(), sourceNode.getNodeId())));
            LoadIntoDatabaseOnArrivalListener loadListener = new LoadIntoDatabaseOnArrivalListener(processInfo, sourceNode.getNodeId(), listener, executor);
            new SimpleStagingDataWriter(transport.openReader(), stagingManager, Constants.STAGING_CATEGORY_INCOMING, memoryThresholdInBytes, BatchType.LOAD, targetNodeId, ctx, loadListener).process();
            /* Previously submitted tasks will still be executed */
            executor.shutdown();
            OutputStreamWriter outWriter = null;
            if (out != null) {
                outWriter = new OutputStreamWriter(out, IoConstants.ENCODING);
                long keepAliveMillis = parameterService.getLong(ParameterConstants.DATA_LOADER_SEND_ACK_KEEPALIVE);
                while (!executor.awaitTermination(keepAliveMillis, TimeUnit.MILLISECONDS)) {
                    outWriter.write("1=1&");
                    outWriter.flush();
                }
            } else {
                executor.awaitTermination(12, TimeUnit.HOURS);
            }
            loadListener.isDone();
        } else {
            DataProcessor processor = new DataProcessor(new ProtocolDataReader(BatchType.LOAD, targetNodeId, transport.openReader()), null, listener, "data load") {

                @Override
                protected IDataWriter chooseDataWriter(Batch batch) {
                    return buildDataWriter(processInfo, sourceNode.getNodeId(), batch.getChannelId(), batch.getBatchId(), ((ManageIncomingBatchListener) listener).getCurrentBatch().isRetry());
                }
            };
            processor.process(ctx);
        }
    } catch (Throwable ex) {
        error = ex;
        logAndRethrow(ex);
    } finally {
        transport.close();
        for (ILoadSyncLifecycleListener l : extensionService.getExtensionPointList(ILoadSyncLifecycleListener.class)) {
            l.syncEnded(ctx, listener.getBatchesProcessed(), error);
        }
    }
    return listener.getBatchesProcessed();
}

Example 2 with Batch

use of org.jumpmind.symmetric.io.data.Batch in project symmetric-ds by JumpMind.

the class MultiBatchStagingWriter method startNewBatch.

protected void startNewBatch() {
    this.nextBatch();
    long memoryThresholdInBytes = this.dataExtractorService.parameterService.getLong(ParameterConstants.STREAM_TO_FILE_THRESHOLD);
    this.currentDataWriter = buildWriter(memoryThresholdInBytes);
    this.batch = new Batch(BatchType.EXTRACT, outgoingBatch.getBatchId(), outgoingBatch.getChannelId(), this.dataExtractorService.symmetricDialect.getBinaryEncoding(), sourceNodeId, outgoingBatch.getNodeId(), false);
    this.currentDataWriter.open(context);
    this.currentDataWriter.start(batch);
    processInfo.incrementBatchCount();
    if (table == null) {
        throw new SymmetricException("'table' cannot null while starting new batch.  Batch: " + outgoingBatch + ". Check trigger/router configs.");
    }
    this.currentDataWriter.start(table);
}

Example 3 with Batch

use of org.jumpmind.symmetric.io.data.Batch in project symmetric-ds by JumpMind.

the class SimpleStagingDataWriter method process.

public void process() throws IOException {
    String catalogLine = null, schemaLine = null, nodeLine = null, binaryLine = null, channelLine = null;
    TableLine tableLine = null;
    Map<TableLine, TableLine> syncTableLines = new HashMap<TableLine, TableLine>();
    Map<TableLine, TableLine> batchTableLines = new HashMap<TableLine, TableLine>();
    IStagedResource resource = null;
    String line = null;
    long startTime = System.currentTimeMillis(), ts = startTime, lineCount = 0;
    while (reader.readRecord()) {
        line = reader.getRawRecord();
        if (line.startsWith(CsvConstants.CATALOG)) {
            catalogLine = line;
            writeLine(line);
        } else if (line.startsWith(CsvConstants.SCHEMA)) {
            schemaLine = line;
            writeLine(line);
        } else if (line.startsWith(CsvConstants.TABLE)) {
            tableLine = new TableLine(catalogLine, schemaLine, line);
            TableLine batchTableLine = batchTableLines.get(tableLine);
            if (batchTableLine != null) {
                tableLine = batchTableLine;
                writeLine(line);
            } else {
                TableLine syncTableLine = syncTableLines.get(tableLine);
                if (syncTableLine != null) {
                    tableLine = syncTableLine;
                    writeLine(tableLine.catalogLine);
                    writeLine(tableLine.schemaLine);
                    writeLine(line);
                    writeLine(tableLine.keysLine);
                    writeLine(tableLine.columnsLine);
                } else {
                    syncTableLines.put(tableLine, tableLine);
                    batchTableLines.put(tableLine, tableLine);
                    writeLine(line);
                }
            }
        } else if (line.startsWith(CsvConstants.KEYS)) {
            tableLine.keysLine = line;
            writeLine(line);
        } else if (line.startsWith(CsvConstants.COLUMNS)) {
            tableLine.columnsLine = line;
            writeLine(line);
        } else if (line.startsWith(CsvConstants.BATCH)) {
            batch = new Batch(batchType, Long.parseLong(getArgLine(line)), getArgLine(channelLine), getBinaryEncoding(binaryLine), getArgLine(nodeLine), targetNodeId, false);
            String location = batch.getStagedLocation();
            resource = stagingManager.find(category, location, batch.getBatchId());
            if (resource == null || resource.getState() == State.DONE) {
                log.debug("Creating staged resource for batch {}", batch.getNodeBatchId());
                resource = stagingManager.create(category, location, batch.getBatchId());
            }
            writer = resource.getWriter(memoryThresholdInBytes);
            writeLine(nodeLine);
            writeLine(binaryLine);
            writeLine(channelLine);
            writeLine(line);
            if (listeners != null) {
                for (IProtocolDataWriterListener listener : listeners) {
                    listener.start(context, batch);
                }
            }
        } else if (line.startsWith(CsvConstants.COMMIT)) {
            if (writer != null) {
                writeLine(line);
                resource.close();
                resource.setState(State.DONE);
                writer = null;
            }
            batchTableLines.clear();
            if (listeners != null) {
                for (IProtocolDataWriterListener listener : listeners) {
                    listener.end(context, batch, resource);
                }
            }
        } else if (line.startsWith(CsvConstants.RETRY)) {
            batch = new Batch(batchType, Long.parseLong(getArgLine(line)), getArgLine(channelLine), getBinaryEncoding(binaryLine), getArgLine(nodeLine), targetNodeId, false);
            String location = batch.getStagedLocation();
            resource = stagingManager.find(category, location, batch.getBatchId());
            if (resource == null || resource.getState() == State.CREATE) {
                resource = null;
                writer = null;
            }
            if (listeners != null) {
                for (IProtocolDataWriterListener listener : listeners) {
                    listener.start(context, batch);
                }
            }
        } else if (line.startsWith(CsvConstants.NODEID)) {
            nodeLine = line;
        } else if (line.startsWith(CsvConstants.BINARY)) {
            binaryLine = line;
        } else if (line.startsWith(CsvConstants.CHANNEL)) {
            channelLine = line;
        } else {
            if (writer == null) {
                throw new IllegalStateException("Invalid batch data was received: " + line);
            }
            TableLine batchLine = batchTableLines.get(tableLine);
            if (batchLine == null || (batchLine != null && batchLine.columnsLine == null)) {
                TableLine syncLine = syncTableLines.get(tableLine);
                if (syncLine != null) {
                    log.debug("Injecting keys and columns to be backwards compatible");
                    if (batchLine == null) {
                        batchLine = syncLine;
                        batchTableLines.put(batchLine, batchLine);
                        writeLine(batchLine.tableLine);
                    }
                    batchLine.keysLine = syncLine.keysLine;
                    writeLine(syncLine.keysLine);
                    batchLine.columnsLine = syncLine.columnsLine;
                    writeLine(syncLine.columnsLine);
                }
            }
            int size = line.length();
            if (size > MAX_WRITE_LENGTH) {
                log.debug("Exceeded max line length with {}", size);
                for (int i = 0; i < size; i = i + MAX_WRITE_LENGTH) {
                    int end = i + MAX_WRITE_LENGTH;
                    writer.append(line, i, end < size ? end : size);
                }
                writer.append("\n");
            } else {
                writeLine(line);
            }
        }
        lineCount++;
        if (System.currentTimeMillis() - ts > 60000) {
            log.info("Batch '{}', for node '{}', for process 'transfer to stage' has been processing for {} seconds.  The following stats have been gathered: {}", new Object[] { (batch != null ? batch.getBatchId() : 0), (batch != null ? batch.getTargetNodeId() : ""), (System.currentTimeMillis() - startTime) / 1000, "LINES=" + lineCount + ", BYTES=" + ((resource == null) ? 0 : resource.getSize()) });
            ts = System.currentTimeMillis();
        }
    }
}

Example 4 with Batch

use of org.jumpmind.symmetric.io.data.Batch in project symmetric-ds by JumpMind.

the class AbstractWriterTest method writeData.

protected long writeData(IDataWriter writer, TableCsvData... datas) {
    this.lastDataWriterUsed = writer;
    DataContext context = new DataContext();
    writer.open(context);
    try {
        for (TableCsvData tableCsvData : datas) {
            Batch batch = new Batch(BatchType.LOAD, getNextBatchId(), "default", BinaryEncoding.BASE64, "00000", "00001", false);
            try {
                writer.start(batch);
                if (writer.start(tableCsvData.table)) {
                    for (CsvData d : tableCsvData.data) {
                        writer.write(d);
                    }
                    writer.end(tableCsvData.table);
                }
                writer.end(batch, false);
            } catch (IgnoreBatchException ex) {
                writer.end(batch, false);
            } catch (Exception ex) {
                writer.end(batch, true);
                if (!isErrorExpected()) {
                    if (ex instanceof RuntimeException) {
                        throw (RuntimeException) ex;
                    } else {
                        throw new RuntimeException(ex);
                    }
                }
            }
        }
    } finally {
        writer.close();
    }
    long statementCount = 0;
    Collection<Statistics> stats = writer.getStatistics().values();
    for (Statistics statistics : stats) {
        statementCount += statistics.get(DataWriterStatisticConstants.STATEMENTCOUNT);
    }
    return statementCount;
}

Example 5 with Batch

use of org.jumpmind.symmetric.io.data.Batch in project symmetric-ds by JumpMind.

the class ProtocolDataReaderTest method testTableContextSwitch.

@Test
public void testTableContextSwitch() {
    String nodeId = "1";
    long batchId = 1;
    String channelId = "test";
    StringBuilder builder = beginCsv(nodeId);
    beginBatch(builder, batchId, channelId);
    putTableN(builder, 1, true);
    putInsert(builder, 4);
    putTableN(builder, 2, true);
    putInsert(builder, 4);
    putTableN(builder, 1, false);
    putInsert(builder, 2);
    putTableN(builder, 2, false);
    putInsert(builder, 2);
    endCsv(builder);
    ProtocolDataReader reader = new ProtocolDataReader(BatchType.LOAD, "test", builder);
    DataContext ctx = new DataContext(reader);
    reader.open(ctx);
    Batch batch = reader.nextBatch();
    assertNotNull(batch);
    Table table = reader.nextTable();
    assertNotNull(table);
    assertEquals(2, table.getColumnCount());
    assertEquals(1, table.getPrimaryKeyColumnCount());
    assertEquals("test1", table.getName());
    int dataCount = 0;
    while (reader.nextData() != null) {
        dataCount++;
    }
    assertEquals(4, dataCount);
    table = reader.nextTable();
    assertNotNull(table);
    assertEquals(2, table.getColumnCount());
    assertEquals(1, table.getPrimaryKeyColumnCount());
    assertEquals("test2", table.getName());
    dataCount = 0;
    while (reader.nextData() != null) {
        dataCount++;
    }
    assertEquals(4, dataCount);
    table = reader.nextTable();
    assertNotNull(table);
    assertEquals(2, table.getColumnCount());
    assertEquals(1, table.getPrimaryKeyColumnCount());
    assertEquals("test1", table.getName());
    dataCount = 0;
    while (reader.nextData() != null) {
        dataCount++;
    }
    assertEquals(2, dataCount);
    table = reader.nextTable();
    assertNotNull(table);
    assertEquals(2, table.getColumnCount());
    assertEquals(1, table.getPrimaryKeyColumnCount());
    assertEquals("test2", table.getName());
    dataCount = 0;
    while (reader.nextData() != null) {
        dataCount++;
    }
    assertEquals(2, dataCount);
}