Examples with DataProcessor org.jumpmind.symmetric.io.data.DataProcessor used on opensource projects

Example 6 with DataProcessor

use of org.jumpmind.symmetric.io.data.DataProcessor in project symmetric-ds by JumpMind.

the class XmlDataReaderTest method testNilDataElement.

@Test
public void testNilDataElement() {
    XmlDataReader reader = new XmlDataReader(getClass().getResourceAsStream("xmldatareadertest1.xml"));
    TestableDataWriter writer = new TestableDataWriter();
    DataProcessor processor = new DataProcessor(reader, writer, "test");
    processor.process();
    List<CsvData> dataRead = writer.getDatas();
    assertEquals(4, dataRead.size());
    Map<String, String> data1 = dataRead.get(1).toColumnNameValuePairs(writer.getLastTableRead().getColumnNames(), CsvData.ROW_DATA);
    assertEquals("1", data1.get("id"));
    assertEquals("A", data1.get("my_value"));
    Map<String, String> data2 = dataRead.get(2).toColumnNameValuePairs(writer.getLastTableRead().getColumnNames(), CsvData.ROW_DATA);
    assertEquals("2", data2.get("id"));
    assertEquals(null, data2.get("my_value"));
}

Example 7 with DataProcessor

use of org.jumpmind.symmetric.io.data.DataProcessor in project symmetric-ds by JumpMind.

the class DataExtractorService method extractBatchRange.

public boolean extractBatchRange(Writer writer, String nodeId, long startBatchId, long endBatchId) {
    boolean foundBatch = false;
    Node sourceNode = nodeService.findIdentity();
    for (long batchId = startBatchId; batchId <= endBatchId; batchId++) {
        OutgoingBatch batch = outgoingBatchService.findOutgoingBatch(batchId, nodeId);
        if (batch != null) {
            Node targetNode = nodeService.findNode(nodeId);
            if (targetNode == null && Constants.UNROUTED_NODE_ID.equals(nodeId)) {
                targetNode = new Node();
                targetNode.setNodeId("-1");
            }
            if (targetNode != null) {
                IDataReader dataReader = new ExtractDataReader(symmetricDialect.getPlatform(), new SelectFromSymDataSource(batch, sourceNode, targetNode, new ProcessInfo()));
                DataContext ctx = new DataContext();
                ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode);
                ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE, nodeService.findIdentity());
                new DataProcessor(dataReader, createTransformDataWriter(nodeService.findIdentity(), targetNode, new ProtocolDataWriter(nodeService.findIdentityNodeId(), writer, targetNode.requires13Compatiblity())), "extract range").process(ctx);
                foundBatch = true;
            }
        }
    }
    return foundBatch;
}

Example 8 with DataProcessor

use of org.jumpmind.symmetric.io.data.DataProcessor in project symmetric-ds by JumpMind.

the class StagingDataWriterTest method readThenWrite.

public void readThenWrite(long threshold) throws Exception {
    InputStreamReader is = new InputStreamReader(getClass().getResourceAsStream("FileCsvDataWriterTest.1.csv"));
    String origCsv = IOUtils.toString(is);
    is.close();
    StagingManager stagingManager = new StagingManager(DIR.getAbsolutePath());
    ProtocolDataReader reader = new ProtocolDataReader(BatchType.LOAD, "test", origCsv);
    StagingDataWriter writer = new StagingDataWriter(threshold, false, "aaa", "test", stagingManager, new BatchListener());
    DataProcessor processor = new DataProcessor(reader, writer, "test");
    processor.process(new DataContext());
    assertEquals(1, batchesWritten.size());
    assertEquals(convertEol(origCsv), convertEol(batchesWritten.get(0)));
    IStagedResource resource = (IStagedResource) stagingManager.find("test", "aaa", 1);
    assertNotNull(resource);
    if (threshold > origCsv.length()) {
        assertFalse(resource.getFile().exists());
    } else {
        assertTrue(resource.getFile().exists());
    }
    resource.delete();
    assertFalse(resource.getFile().exists());
}