Examples with BootstrapEventProcessResult com.linkedin.databus.bootstrap.api.BootstrapEventProcessResult used on opensource projects

Example 1 with BootstrapEventProcessResult

use of com.linkedin.databus.bootstrap.api.BootstrapEventProcessResult in project databus by linkedin.

the class BootstrapProcessor method streamOutRows.

private boolean streamOutRows(Checkpoint ckpt, ResultSet rs, BootstrapEventCallback callback, long maxRowsPerFetch) throws SQLException, BootstrapProcessingException {
    BootstrapEventProcessResult result = null;
    long windowScn = Long.MIN_VALUE;
    // Number or rows returned in the result-set so far
    long numRowsReadFromDb = 0;
    while (rs.next()) {
        numRowsReadFromDb++;
        long rid = rs.getLong(1);
        result = callback.onEvent(rs, _curStatsCollector);
        if ((result.isClientBufferLimitExceeded()) || (result.isError())) {
            // row if it is not sent due to client buffer space limitation.
            break;
        }
        if (DbusClientMode.BOOTSTRAP_SNAPSHOT == ckpt.getConsumptionMode()) {
            ckpt.onSnapshotEvent(rid);
        } else if (DbusClientMode.BOOTSTRAP_CATCHUP == ckpt.getConsumptionMode()) {
            windowScn = rs.getLong(3);
            ckpt.onCatchupEvent(windowScn, rid);
        } else {
            String errMsg = "The checkpoint received by bootstrap server is neither SNAPSHOT nor CATCHUP" + ckpt;
            LOG.error(errMsg);
            throw new RuntimeException(errMsg);
        }
    }
    if (numRowsReadFromDb > maxRowsPerFetch) {
        String errMsg = "Number of rows read from DB = " + numRowsReadFromDb + " are greater than sepcfied maxRowsPerFetch = " + maxRowsPerFetch;
        LOG.error(errMsg);
        throw new RuntimeException(errMsg);
    }
    // Sends checkpoint to client if prescribed conditions are met
    writeCkptIfAppropriate(result, callback, numRowsReadFromDb, ckpt, rs.getStatement().toString());
    // Computes whether or not "a bootstrap phase" has completed
    boolean isPhaseCompleted = computeIsPhaseCompleted(result, ckpt, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    // TBD : Isn't this too late to have the check here ?
    if (DbusClientMode.BOOTSTRAP_CATCHUP == ckpt.getConsumptionMode() && ckpt.getBootstrapTargetScn() < windowScn) {
        throw new RuntimeException("Events with higher windowscn delivered: bootstrapTargetScn=" + ckpt.getBootstrapTargetScn() + " event windowscn=" + windowScn);
    }
    return isPhaseCompleted;
}

Example 2 with BootstrapEventProcessResult

use of com.linkedin.databus.bootstrap.api.BootstrapEventProcessResult in project databus by linkedin.

the class TestBootstrapProcessor method testIsPhaseCompletedFlag.

@Test
public void testIsPhaseCompletedFlag() throws Exception {
    BootstrapProcessor bp = new BootstrapProcessor();
    long processedRowCount = 1;
    // order of args : processedRowCount, isLimitExceeded, isDropped, isError
    BootstrapEventProcessResult result_ok = new BootstrapEventProcessResult(processedRowCount, false, false, false);
    BootstrapEventProcessResult result_err = new BootstrapEventProcessResult(processedRowCount, false, false, true);
    BootstrapEventProcessResult result_ile = new BootstrapEventProcessResult(processedRowCount, true, false, false);
    Checkpoint ckpt_ss = new Checkpoint("{\"consumption_mode\":\"BOOTSTRAP_SNAPSHOT\", \"bootstrap_since_scn\":0," + "\"bootstrap_start_scn\":1000,\"bootstrap_target_scn\":2000,\"bootstrap_catchup_source_index\":0," + "\"bootstrap_snapshot_source_index\":1}");
    ckpt_ss.assertCheckpoint();
    Checkpoint ckpt_cu = new Checkpoint("{\"consumption_mode\":\"BOOTSTRAP_CATCHUP\", \"bootstrap_since_scn\":0," + "\"bootstrap_start_scn\":1000,\"bootstrap_target_scn\":2000,\"bootstrap_catchup_source_index\":1," + "\"bootstrap_snapshot_source_index\":1}");
    ckpt_cu.assertCheckpoint();
    long numRowsReadFromDb = 1;
    long maxRowsPerFetch = 2;
    long windowScn = 1;
    // result is null, phaseCompleted == false ( irrespective of snapshot / catchup phase )
    numRowsReadFromDb = 0;
    boolean pc = bp.computeIsPhaseCompleted(null, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertTrue(pc);
    pc = bp.computeIsPhaseCompleted(null, ckpt_cu, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertTrue(pc);
    numRowsReadFromDb = 1;
    // numRowsReadFromDb < maxRowsPerFetch, in SNAPSHOT mode
    pc = bp.computeIsPhaseCompleted(result_ok, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertTrue(pc);
    // Same as above, but result.isError == true ( overrides all other conditions )
    pc = bp.computeIsPhaseCompleted(result_err, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // Same as above, but result.isLimitExceeded == true ( overrides all other conditions )
    pc = bp.computeIsPhaseCompleted(result_ile, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // numRowsReadFromDb == maxRowsPerFetch, in SNAPSHOT mode
    numRowsReadFromDb = 2;
    pc = bp.computeIsPhaseCompleted(result_ok, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // Same as above, but result.isError == true
    pc = bp.computeIsPhaseCompleted(result_err, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // Same as above, but result.isLimitExceeded == true
    pc = bp.computeIsPhaseCompleted(result_ile, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    numRowsReadFromDb = 1;
    // numRowsReadFromDb < maxRowsPerFetch, in CATCHUP mode, windowScn != bootstrap_target_scn
    pc = bp.computeIsPhaseCompleted(result_ok, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertTrue(pc);
    windowScn = ckpt_cu.getBootstrapTargetScn();
    pc = bp.computeIsPhaseCompleted(result_ok, ckpt_cu, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertTrue(pc);
    // Same as above, but result.isError == true ( overrides all other conditions )
    pc = bp.computeIsPhaseCompleted(result_err, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // Same as above, but result.isLimitExceeded == true ( overrides result being null )
    pc = bp.computeIsPhaseCompleted(result_ile, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // numRowsReadFromDb == maxRowsPerFetch, in CATCHUP mode
    numRowsReadFromDb = 2;
    // not equal to bootstrap_target_scn
    windowScn = 10;
    pc = bp.computeIsPhaseCompleted(result_ok, ckpt_cu, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // equal to bootstrap_target_scn, but does not matter still
    windowScn = ckpt_cu.getBootstrapTargetScn();
    pc = bp.computeIsPhaseCompleted(result_ok, ckpt_cu, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // Same as above, but result.isError == true
    pc = bp.computeIsPhaseCompleted(result_err, ckpt_cu, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // Same as above, but result.isLimitExceeded == true
    pc = bp.computeIsPhaseCompleted(result_ile, ckpt_cu, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
}

Example 3 with BootstrapEventProcessResult

use of com.linkedin.databus.bootstrap.api.BootstrapEventProcessResult in project databus by linkedin.

the class EventProcessor method onEvent.

public BootstrapEventProcessResult onEvent(ResultSet rs, DbusEventsStatisticsCollector statsCollector) throws BootstrapProcessingException {
    try {
        rIds.add(rs.getLong(1));
        sequences.add(rs.getLong(2));
        srcKeys.add(rs.getString(3));
        values.add(rs.getBytes(4));
    } catch (Exception ex) {
        throw new BootstrapProcessingException(ex);
    }
    return new BootstrapEventProcessResult(1, false, false);
}

Example 4 with BootstrapEventProcessResult

use of com.linkedin.databus.bootstrap.api.BootstrapEventProcessResult in project databus by linkedin.

the class BootstrapEventWriter method onEvent.

@Override
public BootstrapEventProcessResult onEvent(ResultSet rs, DbusEventsStatisticsCollector statsCollector) throws BootstrapProcessingException {
    long rid = -1;
    boolean exceededBufferLimit = false;
    boolean dropped = true;
    try {
        if (null == _event) {
            ByteBuffer tmpBuffer = ByteBuffer.wrap(rs.getBytes(4));
            if (_debug) {
                LOG.debug("BUFFER SIZE:" + tmpBuffer.limit());
            }
            _event = _eventFactory.createReadOnlyDbusEventFromBuffer(tmpBuffer, tmpBuffer.position());
        } else {
            ByteBuffer tmpBuffer = ByteBuffer.wrap(rs.getBytes(4));
            if (_debug) {
                LOG.debug("Resized BUFFER SIZE:" + tmpBuffer.limit());
            }
            _event = _event.reset(tmpBuffer, 0);
        }
        if (_debug) {
            LOG.debug("Event fetched: " + _event.size() + " for source:" + _event.srcId());
        }
        if (!_event.isValid()) {
            LOG.error("got an error event :" + _event.toString());
            return BootstrapEventProcessResult.getFailedEventProcessingResult(_numRowsWritten);
        }
        rid = rs.getLong(1);
        if (_debug) {
            LOG.debug("sending: " + _event.getDbusEventKey() + " " + _event.sequence());
            LOG.debug("event size:" + _event.size());
        }
        if ((null == _filter) || (_filter.allow(_event))) {
            if (_debug) {
                if (null != _filter) {
                    LOG.debug("Event :" + _event.getDbusEventKey() + " passed filter check !!");
                }
            }
            // client has enough space for this event
            if (_bytesSent + _event.size() < _clientFreeBufferSize) {
                int sentBytes = _event.writeTo(_writeChannel, _encoding);
                // On exception, sentBytes are set to 0
                if (0 >= sentBytes) {
                    // to avoid successive write failures !!
                    return BootstrapEventProcessResult.getFailedEventProcessingResult(_numRowsWritten);
                }
                _bytesSent += sentBytes;
                // tracks processed Rows only
                _numRowsWritten++;
                dropped = false;
                if (_debug) {
                    LOG.debug("SENT " + _bytesSent);
                }
                if (null != statsCollector) {
                    statsCollector.registerDataEvent(_event);
                    if (_debug) {
                        LOG.debug("Stats NumEvents :" + statsCollector.getTotalStats().getNumDataEvents());
                    }
                }
            } else {
                exceededBufferLimit = true;
                _sizeOfPendingEvent = _event.size();
                LOG.info("Terminating batch with max. size of " + _clientFreeBufferSize + "; Bytes sent in the current batch is " + _bytesSent + "; Rows processed in the batch is " + _numRowsWritten + ((_numRowsWritten <= 0) ? ", Pending Event Size is : " + _sizeOfPendingEvent : ""));
            }
        } else {
            if (null != statsCollector) {
                statsCollector.registerDataEventFiltered(_event);
                if (_debug) {
                    LOG.debug("Stats NumFilteredEvents :" + statsCollector.getTotalStats().getNumDataEventsFiltered());
                }
                if (_debug) {
                    LOG.debug("Event :" + _event.getDbusEventKey() + " failed filter check !!");
                }
            }
        }
    } catch (SQLException e) {
        LOG.error("SQLException encountered while sending to client row " + rid);
        throw new BootstrapProcessingException(e);
    }
    return new BootstrapEventProcessResult(_numRowsWritten, exceededBufferLimit, dropped);
}

Example 5 with BootstrapEventProcessResult

use of com.linkedin.databus.bootstrap.api.BootstrapEventProcessResult in project databus by linkedin.

the class TestBootstrapProcessor method testCkptWriteLogic.

@Test
public void testCkptWriteLogic() throws Exception {
    BootstrapProcessor bp = new BootstrapProcessor();
    long processedRowCount = 1;
    // order of args : processedRowCount, isLimitExceeded, isDropped, isError
    BootstrapEventProcessResult result_ok = new BootstrapEventProcessResult(processedRowCount, false, false, false);
    BootstrapEventProcessResult result_zero_ok = new BootstrapEventProcessResult(0, false, false, false);
    BootstrapEventProcessResult result_err = new BootstrapEventProcessResult(processedRowCount, false, false, true);
    BootstrapEventProcessResult result_zero_ile = new BootstrapEventProcessResult(0, true, false, false);
    Checkpoint ckpt_ss = new Checkpoint("{\"consumption_mode\":\"BOOTSTRAP_SNAPSHOT\", \"bootstrap_since_scn\":0," + "\"bootstrap_start_scn\":1000,\"bootstrap_target_scn\":2000,\"bootstrap_catchup_source_index\":0," + "\"bootstrap_snapshot_source_index\":1}");
    ckpt_ss.assertCheckpoint();
    long numRowsReadFromDb = 1;
    BootstrapEventCallback callback1 = EasyMock.createMock(BootstrapEventCallback.class);
    EasyMock.replay(callback1);
    BootstrapEventCallback callback2 = EasyMock.createMock(BootstrapEventCallback.class);
    callback2.onCheckpointEvent(ckpt_ss, null);
    EasyMock.replay(callback2);
    BootstrapEventCallback callback3 = EasyMock.createMock(BootstrapEventCallback.class);
    callback3.onCheckpointEvent(ckpt_ss, null);
    EasyMock.replay(callback3);
    // No checkpoint when result is null
    bp.writeCkptIfAppropriate(null, callback1, numRowsReadFromDb, ckpt_ss, "test");
    // No checkpoint when result is error
    bp.writeCkptIfAppropriate(result_err, callback1, numRowsReadFromDb, ckpt_ss, "test");
    // numRowsWritten == 1, must checkpoint
    bp.writeCkptIfAppropriate(result_ok, callback2, numRowsReadFromDb, ckpt_ss, "test");
    // numRowsWritten == 0, must checkpoint as numRowsReadFromDb > 0
    bp.writeCkptIfAppropriate(result_zero_ok, callback3, numRowsReadFromDb, ckpt_ss, "test");
    // numRowsWritten == 0, must have checkpointed as numRowsReadFromDb > 0. However result is client buffer exceeded
    // So .. sorry to disappoint but no checkpoint as we want that pending_event_header
    bp.writeCkptIfAppropriate(result_zero_ile, callback1, numRowsReadFromDb, ckpt_ss, "test");
    // result != null, numRowsWritten == 0, numRowsReadFromDb == 0. We expect a RuntimeException here
    try {
        bp.writeCkptIfAppropriate(result_zero_ok, callback2, 0, ckpt_ss, "test");
        Assert.fail();
    } catch (RuntimeException e) {
    }
}