Examples with IncomingBatch org.jumpmind.symmetric.model.IncomingBatch used on opensource projects

Example 1 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class IncomingBatchService method markIncomingBatchesOk.

public void markIncomingBatchesOk(String nodeId) {
    List<IncomingBatch> batches = listIncomingBatchesInErrorFor(nodeId);
    for (IncomingBatch incomingBatch : batches) {
        if (isRecordOkBatchesEnabled()) {
            incomingBatch.setErrorFlag(false);
            incomingBatch.setStatus(Status.OK);
            updateIncomingBatch(incomingBatch);
        } else {
            deleteIncomingBatch(incomingBatch);
        }
    }
}

Example 2 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class IncomingBatchService method acquireIncomingBatch.

public boolean acquireIncomingBatch(IncomingBatch batch) {
    boolean okayToProcess = true;
    if (batch.isPersistable()) {
        IncomingBatch existingBatch = null;
        if (isRecordOkBatchesEnabled()) {
            try {
                insertIncomingBatch(batch);
            } catch (UniqueKeyException e) {
                batch.setRetry(true);
                existingBatch = findIncomingBatch(batch.getBatchId(), batch.getNodeId());
            }
        } else {
            existingBatch = findIncomingBatch(batch.getBatchId(), batch.getNodeId());
            if (existingBatch != null) {
                batch.setRetry(true);
            }
        }
        if (batch.isRetry()) {
            if (existingBatch.getStatus() == Status.ER || existingBatch.getStatus() == Status.LD || existingBatch.getStatus() == Status.RS || !parameterService.is(ParameterConstants.INCOMING_BATCH_SKIP_DUPLICATE_BATCHES_ENABLED)) {
                okayToProcess = true;
                existingBatch.setStatus(Status.LD);
                log.info("Retrying batch {}", batch.getNodeBatchId());
            } else if (existingBatch.getStatus() == Status.IG) {
                okayToProcess = false;
                batch.setStatus(Status.OK);
                batch.incrementIgnoreCount();
                existingBatch.setStatus(Status.OK);
                existingBatch.incrementIgnoreCount();
                log.info("Ignoring batch {}", batch.getNodeBatchId());
            } else {
                okayToProcess = false;
                batch.setStatus(existingBatch.getStatus());
                batch.setByteCount(existingBatch.getByteCount());
                batch.setDatabaseMillis(existingBatch.getDatabaseMillis());
                batch.setNetworkMillis(existingBatch.getNetworkMillis());
                batch.setFilterMillis(existingBatch.getFilterMillis());
                batch.setSkipCount(existingBatch.getSkipCount() + 1);
                batch.setStatementCount(existingBatch.getStatementCount());
                existingBatch.setSkipCount(existingBatch.getSkipCount() + 1);
                log.info("Skipping batch {}", batch.getNodeBatchId());
            }
            updateIncomingBatch(existingBatch);
        }
    }
    return okayToProcess;
}

Example 3 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class AbstractDataLoaderServiceTest method test01IncomingBatch.

@Test
public void test01IncomingBatch() throws Exception {
    String[] insertValues = new String[TEST_COLUMNS.length];
    insertValues[2] = insertValues[4] = "incoming test";
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    CsvWriter writer = getWriter(out);
    writer.writeRecord(new String[] { CsvConstants.NODEID, TestConstants.TEST_CLIENT_EXTERNAL_ID });
    writer.writeRecord(new String[] { CsvConstants.CHANNEL, TestConstants.TEST_CHANNEL_ID });
    String nextBatchId = getNextBatchId();
    writer.writeRecord(new String[] { CsvConstants.BATCH, nextBatchId });
    writeTable(writer, TEST_TABLE, TEST_KEYS, TEST_COLUMNS);
    insertValues[0] = getNextId();
    writer.write(CsvConstants.INSERT);
    writer.writeRecord(insertValues, true);
    writer.writeRecord(new String[] { CsvConstants.COMMIT, nextBatchId });
    writer.close();
    load(out);
    IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID);
    assertEquals(batch.getStatus(), IncomingBatch.Status.OK, "Wrong status. " + printDatabase());
    assertEquals(batch.getChannelId(), TestConstants.TEST_CHANNEL_ID, "Wrong channel. " + printDatabase());
}

Example 4 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class AbstractDataLoaderServiceTest method test06ErrorWhileSkip.

@Test
public void test06ErrorWhileSkip() throws Exception {
    Level old = setLoggingLevelForTest(Level.OFF);
    String[] values = { getNextId(), "string2", "string not null2", "char2", "char not null2", "2007-01-02 00:00:00.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "0.474" };
    testSimple(CsvConstants.INSERT, values, values);
    assertEquals(findIncomingBatchStatus(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID), IncomingBatch.Status.OK, "Wrong status");
    IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID);
    assertNotNull(batch);
    assertEquals(batch.getStatus(), IncomingBatch.Status.OK, "Wrong status");
    assertEquals(batch.getFailedRowNumber(), 0l, "Wrong failed row number");
    assertEquals(batch.getStatementCount(), 1l, "Wrong statement count");
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    CsvWriter writer = getWriter(out);
    writer.writeRecord(new String[] { CsvConstants.NODEID, TestConstants.TEST_CLIENT_EXTERNAL_ID });
    writer.writeRecord(new String[] { CsvConstants.CHANNEL, TestConstants.TEST_CHANNEL_ID });
    writer.writeRecord(new String[] { CsvConstants.BATCH, getBatchId() });
    writer.write(CsvConstants.KEYS);
    writer.writeRecord(TEST_KEYS);
    writer.writeRecord(new String[] { CsvConstants.COMMIT, getBatchId() });
    writer.close();
    // Pause a moment to guarantee our batch comes back in time order
    Thread.sleep(10);
    load(out);
    assertEquals(findIncomingBatchStatus(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID), IncomingBatch.Status.OK, "Wrong status");
    setLoggingLevelForTest(old);
}

Example 5 with IncomingBatch

use of org.jumpmind.symmetric.model.IncomingBatch in project symmetric-ds by JumpMind.

the class AbstractDataLoaderServiceTest method test05SkippingResentBatch.

@Test
public void test05SkippingResentBatch() throws Exception {
    String[] values = { getNextId(), "resend string", "resend string not null", "resend char", "resend char not null", "2007-01-25 00:00:00.000", "2007-01-25 01:01:01.000", "0", "7", "10.10", "0.474" };
    getNextBatchId();
    for (long i = 0; i < 7; i++) {
        batchId--;
        testSimple(CsvConstants.INSERT, values, values);
        assertEquals(findIncomingBatchStatus(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID), IncomingBatch.Status.OK, "Wrong status");
        IncomingBatch batch = getIncomingBatchService().findIncomingBatch(batchId, TestConstants.TEST_CLIENT_EXTERNAL_ID);
        assertNotNull(batch);
        assertEquals(batch.getStatus(), IncomingBatch.Status.OK, "Wrong status");
        assertEquals(batch.getSkipCount(), i);
        assertEquals(batch.getFailedRowNumber(), 0l, "Wrong failed row number");
        assertEquals(batch.getStatementCount(), 1l, "Wrong statement count");
        assertEquals(batch.getFallbackInsertCount(), 0l, "Wrong fallback insert count");
        assertEquals(batch.getFallbackUpdateCount(), 0l, "Wrong fallback update count");
        // pause to make sure we get a different start time on the incoming
        // batch batch
        Thread.sleep(10);
    }
}