Examples with Checkpoint com.linkedin.databus.core.Checkpoint used on opensource projects

Example 96 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestBootstrapProcessor method testCkptWriteLogic.

@Test
public void testCkptWriteLogic() throws Exception {
    BootstrapProcessor bp = new BootstrapProcessor();
    long processedRowCount = 1;
    // order of args : processedRowCount, isLimitExceeded, isDropped, isError
    BootstrapEventProcessResult result_ok = new BootstrapEventProcessResult(processedRowCount, false, false, false);
    BootstrapEventProcessResult result_zero_ok = new BootstrapEventProcessResult(0, false, false, false);
    BootstrapEventProcessResult result_err = new BootstrapEventProcessResult(processedRowCount, false, false, true);
    BootstrapEventProcessResult result_zero_ile = new BootstrapEventProcessResult(0, true, false, false);
    Checkpoint ckpt_ss = new Checkpoint("{\"consumption_mode\":\"BOOTSTRAP_SNAPSHOT\", \"bootstrap_since_scn\":0," + "\"bootstrap_start_scn\":1000,\"bootstrap_target_scn\":2000,\"bootstrap_catchup_source_index\":0," + "\"bootstrap_snapshot_source_index\":1}");
    ckpt_ss.assertCheckpoint();
    long numRowsReadFromDb = 1;
    BootstrapEventCallback callback1 = EasyMock.createMock(BootstrapEventCallback.class);
    EasyMock.replay(callback1);
    BootstrapEventCallback callback2 = EasyMock.createMock(BootstrapEventCallback.class);
    callback2.onCheckpointEvent(ckpt_ss, null);
    EasyMock.replay(callback2);
    BootstrapEventCallback callback3 = EasyMock.createMock(BootstrapEventCallback.class);
    callback3.onCheckpointEvent(ckpt_ss, null);
    EasyMock.replay(callback3);
    // No checkpoint when result is null
    bp.writeCkptIfAppropriate(null, callback1, numRowsReadFromDb, ckpt_ss, "test");
    // No checkpoint when result is error
    bp.writeCkptIfAppropriate(result_err, callback1, numRowsReadFromDb, ckpt_ss, "test");
    // numRowsWritten == 1, must checkpoint
    bp.writeCkptIfAppropriate(result_ok, callback2, numRowsReadFromDb, ckpt_ss, "test");
    // numRowsWritten == 0, must checkpoint as numRowsReadFromDb > 0
    bp.writeCkptIfAppropriate(result_zero_ok, callback3, numRowsReadFromDb, ckpt_ss, "test");
    // numRowsWritten == 0, must have checkpointed as numRowsReadFromDb > 0. However result is client buffer exceeded
    // So .. sorry to disappoint but no checkpoint as we want that pending_event_header
    bp.writeCkptIfAppropriate(result_zero_ile, callback1, numRowsReadFromDb, ckpt_ss, "test");
    // result != null, numRowsWritten == 0, numRowsReadFromDb == 0. We expect a RuntimeException here
    try {
        bp.writeCkptIfAppropriate(result_zero_ok, callback2, 0, ckpt_ss, "test");
        Assert.fail();
    } catch (RuntimeException e) {
    }
}

Example 97 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestBootstrap method testBootstrapProcessor.

@Test
public void testBootstrapProcessor() throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException, BootstrapProcessingException, DatabusException, BootstrapDatabaseTooOldException, BootstrapDBException {
    EventProcessor processorCallback = new EventProcessor();
    BootstrapConfig config = new BootstrapConfig();
    BootstrapReadOnlyConfig staticConfig = config.build();
    BootstrapServerConfig configBuilder = new BootstrapServerConfig();
    configBuilder.setEnableMinScnCheck(false);
    BootstrapServerStaticConfig staticServerConfig = configBuilder.build();
    BootstrapProcessor processor = new BootstrapProcessor(staticServerConfig, null);
    String sourceName = "TestBootstrap.testBootstrapProcessor.events";
    // Create the tables for all the sources before starting up the threads
    BootstrapConn _bootstrapConn = new BootstrapConn();
    boolean autoCommit = true;
    _bootstrapConn.initBootstrapConn(autoCommit, staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBName());
    BootstrapDBMetaDataDAO dao = new BootstrapDBMetaDataDAO(_bootstrapConn, staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBName(), autoCommit);
    SourceStatusInfo srcStatusInfo = dao.getSrcIdStatusFromDB(sourceName, false);
    if (srcStatusInfo.getSrcId() >= 0) {
        dao.dropSourceInDB(srcStatusInfo.getSrcId());
    }
    dao.addNewSourceInDB(sourceName, BootstrapProducerStatus.ACTIVE);
    srcStatusInfo = dao.getSrcIdStatusFromDB(sourceName, false);
    setBootstrapLoginfoTable(_bootstrapConn, 0, Long.MAX_VALUE);
    //insert one row
    insertOneSnapshotEvent(dao, srcStatusInfo.getSrcId(), Long.MAX_VALUE - 10, Long.MAX_VALUE - 100, "check", "test_payload_data");
    BootstrapCheckpointHandler bstCheckpointHandler = new BootstrapCheckpointHandler(sourceName);
    Checkpoint c = bstCheckpointHandler.createInitialBootstrapCheckpoint(null, 0L);
    c.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    c.setBootstrapStartScn(Long.MAX_VALUE - 10);
    c.setSnapshotOffset(Long.MAX_VALUE - 1000);
    // System.out.println("Snapshot");
    processor.streamSnapShotRows(c, processorCallback);
    Assert.assertEquals(processorCallback.getrIds().size(), 1);
    Assert.assertEquals(Long.MAX_VALUE - 10, processorCallback.getrIds().get(0).longValue());
    Assert.assertEquals(Long.MAX_VALUE - 100, processorCallback.getSequences().get(0).longValue());
    Assert.assertEquals("check", processorCallback.getSrcKeys().get(0));
    Assert.assertEquals("test_payload_data", new String(processorCallback.getValues().get(0)));
    processorCallback.reset();
    c.setSnapshotOffset(Checkpoint.FULLY_CONSUMED_WINDOW_OFFSET);
    c.setBootstrapTargetScn(c.getBootstrapStartScn().longValue());
    bstCheckpointHandler.advanceAfterTargetScn(c);
    bstCheckpointHandler.advanceAfterSnapshotPhase(c);
    //		  c.setConsumptionMode(DbusClientMode.BOOTSTRAP_CATCHUP);
    //		  c.setCatchupSource(sourceName);
    //		  c.setWindowScn(0L);
    //		  c.setWindowOffset(0);
    // System.out.println("Catchup");
    boolean phaseCompleted = processor.streamCatchupRows(c, processorCallback);
    Assert.assertEquals(true, phaseCompleted);
}

Example 98 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class DatabusBootstrapProducer method validateAndRepairBootstrapDBCheckpoint.

/**
   *
   *
   * Compares the Checkpoint and bootstrap_producer_state's SCN to check against
   * the possibility of gap in event consumption!! If gap is found, makes
   * best-effort to repair it.
   *
   * Three Scenario that should be allowed
   *
   * 1. Bootstrap Producer started after seeding In this case checkpoint SCN
   * should not be greater than producer SCN. 2. Bootstrap Producer started
   * after adding sources without seeding In this case the producer SCN is "-1".
   * The checkpoint "could" be empty 3. Bootstrap Producer bounced In this case
   * checkpoint SCN should not be greater than producer SCN.
   *
   * @throws SQLException
   * @throws BootstrapDBException
   *           if there is a gap and cannot be repaired
   * @throws IOException
   *           `
   */
private void validateAndRepairBootstrapDBCheckpoint() throws SQLException, BootstrapDBException, IOException {
    LOG.info("Validating bootstrap DB checkpoints !!");
    for (List<DatabusSubscription> subsList : _relayGroups.keySet()) {
        List<String> sourceNames = DatabusSubscription.getStrList(subsList);
        long scn = -1;
        try {
            scn = _dbDao.getMinWindowSCNFromStateTable(sourceNames, "bootstrap_producer_state");
        } catch (BootstrapDBException ex) {
            LOG.error("Got exception while trying to fetch SCN from bootstrap_producer_state for sources :" + sourceNames, ex);
            throw ex;
        }
        CheckpointPersistenceProvider provider = getCheckpointPersistenceProvider();
        Checkpoint cp = provider.loadCheckpoint(sourceNames);
        LOG.info("Bootstrap Producer SCN :" + scn + ", Checkpoint :" + cp);
        if (null != cp) {
            if (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION) {
                // Bootstrapping bootstrap Producer not yet supported !!
                String msg = "Bootstrap Producer starting from non-online consumption mode for sources :" + sourceNames + ", Ckpt :" + cp;
                LOG.error(msg);
                throw new BootstrapDBException(msg);
            } else {
                String msg = null;
                if (((cp.getWindowScn() > scn) && (scn > -1)) || ((cp.getWindowScn() < scn) && (scn > -1))) {
                    if (((cp.getWindowScn() > scn) && (scn > -1)))
                        LOG.warn("Non-Empty checkpint. Bootstrap Producer is at SCN:" + scn + ", while checkpoint is :" + cp + ", Could result in gap in event consumption. Repairing ckpt !!");
                    else
                        LOG.info("Non-Empty checkpoint. Bootstrap Producer is at SCN:" + scn + ", while checkpoint is :" + cp + ", Copying producer Scn to checkpoint !!");
                    cp.setWindowScn(scn);
                    cp.setWindowOffset(-1);
                    try {
                        provider.removeCheckpoint(sourceNames);
                        provider.storeCheckpoint(sourceNames, cp);
                        // Check if persisted properly
                        cp = provider.loadCheckpoint(sourceNames);
                        if ((null == cp) || (cp.getWindowScn() != scn) || (cp.getWindowOffset() != -1) || (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION)) {
                            msg = "Unable to repair and store the new checkpoint (" + cp + ") to make it same as producer SCN (" + scn + ") !!";
                            LOG.fatal(msg);
                            throw new BootstrapDBException(msg);
                        }
                    } catch (IOException ex) {
                        msg = "Unable to repair and store the new checkpoint (" + cp + ") to make it same as producer SCN (" + scn + ") !!";
                        LOG.fatal(msg, ex);
                        throw new BootstrapDBException(msg);
                    }
                }
            }
        } else {
            /**
         * Currently since bootstrapping is not available, a null ckpt would
         * result in flexible checkpoint and could result in gap !!
         */
            if (scn > -1) {
                String msg = "Empty checkpoint. Bootstrap Producer SCN is at SCN:" + scn + ", while checkpoint is null !! Could result in gap in event consumption. Repairing ckpt !!";
                LOG.warn(msg);
                cp = new Checkpoint();
                cp.setWindowScn(scn);
                cp.setWindowOffset(-1);
                cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
                try {
                    provider.removeCheckpoint(sourceNames);
                    provider.storeCheckpoint(sourceNames, cp);
                    // Check if persisted properly
                    cp = provider.loadCheckpoint(sourceNames);
                    if ((null == cp) || (cp.getWindowScn() != scn) || (cp.getWindowOffset() != -1) || (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION)) {
                        LOG.fatal("Unable to repair and store the checkpoint (" + cp + ") to make it same as producer SCN (" + scn + ") !!");
                        throw new BootstrapDBException(msg);
                    }
                } catch (IOException ex) {
                    msg = "Unable to repair and store the checkpoint (" + cp + ") to make it same as producer SCN (" + scn + ") !!";
                    LOG.fatal(msg, ex);
                    throw new BootstrapDBException(msg);
                }
            }
        }
    }
    LOG.info("Validating bootstrap DB checkpoints done successfully!!");
}

Example 99 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class BootstrapProcessor method createCatchupStatement.

// TODO: DDSDBUS-345 : Bootstrap Serving might be incorrect when multiple bootstrap
// servers are serving
private PreparedStatement createCatchupStatement(int srcId, int logId, Checkpoint currState) throws SQLException {
    Connection conn = _dbDao.getBootstrapConn().getDBConn();
    String catchupTab = "log_" + srcId + "_" + logId;
    PreparedStatement stmt = null;
    String catchUpString = getCatchupSQLString(catchupTab, currState.getCatchupSource());
    long offset = -1;
    try {
        stmt = conn.prepareStatement(catchUpString);
        offset = currState.getWindowOffset();
        int i = 1;
        stmt.setLong(i++, offset);
        stmt.setLong(i++, currState.getBootstrapStartScn());
        stmt.setLong(i++, currState.getBootstrapTargetScn());
        stmt.setLong(i++, currState.getBootstrapSinceScn());
        stmt.setLong(i++, _maxCatchupRowsPerFetch);
    } catch (SQLException ex) {
        DBHelper.close(stmt);
    }
    LOG.info("Catchup SQL String: " + catchUpString + ", " + offset + ", " + currState.getBootstrapStartScn() + " , " + currState.getBootstrapTargetScn() + " , " + currState.getBootstrapSinceScn() + " , " + _maxCatchupRowsPerFetch);
    return stmt;
}

Example 100 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class BootstrapRequestProcessor method doProcess.

/*
   * (non-Javadoc)
   *
   * @see
   * com.linkedin.databus.container.request.RequestProcessor#process(com.linkedin.databus
   * .container.request.DatabusRequest)
   */
@Override
protected DatabusRequest doProcess(DatabusRequest request) throws IOException, RequestProcessingException {
    BootstrapProcessor processor = null;
    BootstrapHttpStatsCollector bootstrapStatsCollector = _bootstrapServer.getBootstrapStatsCollector();
    long startTime = System.currentTimeMillis();
    boolean isDebug = LOG.isDebugEnabled();
    try {
        try {
            String threadName = Thread.currentThread().getName();
            DbusEventsStatisticsCollector threadCollector = _bootstrapServer.getOutBoundStatsCollectors().getStatsCollector(threadName);
            if (null == threadCollector) {
                threadCollector = new DbusEventsStatisticsCollector(_bootstrapServer.getContainerStaticConfig().getId(), threadName, true, false, _bootstrapServer.getMbeanServer());
                StatsCollectors<DbusEventsStatisticsCollector> ds = _bootstrapServer.getOutBoundStatsCollectors();
                ds.addStatsCollector(threadName, threadCollector);
            }
            processor = new BootstrapProcessor(_config, threadCollector);
        } catch (Exception e) {
            if (null != bootstrapStatsCollector) {
                bootstrapStatsCollector.registerErrBootstrap();
            }
            throw new RequestProcessingException(e);
        }
        DatabusComponentStatus componentStatus = _componentStatus.getStatusSnapshot();
        if (!componentStatus.isRunningStatus()) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrBootstrap();
            throw new RequestProcessingException(componentStatus.getMessage());
        }
        String partitionInfoString = request.getParams().getProperty(PARTITION_INFO_PARAM);
        DbusKeyFilter keyFilter = null;
        if ((null != partitionInfoString) && (!partitionInfoString.isEmpty())) {
            try {
                keyFilter = KeyFilterConfigJSONFactory.parseDbusKeyFilter(partitionInfoString);
                if (isDebug)
                    LOG.debug("ServerSideFilter is :" + keyFilter);
            } catch (Exception ex) {
                String msg = "Unable to parse partitionInfo from request. PartitionInfo was :" + partitionInfoString;
                LOG.error(msg, ex);
                throw new RequestProcessingException(msg, ex);
            }
        }
        String outputFormat = request.getParams().getProperty(OUTPUT_PARAM);
        Encoding enc = Encoding.BINARY;
        if (null != outputFormat) {
            try {
                enc = Encoding.valueOf(outputFormat.toUpperCase());
            } catch (Exception ex) {
                LOG.error("Unable to find the output format for bootstrap request for " + outputFormat + ". Using Binary!!", ex);
            }
        }
        processor.setKeyFilter(keyFilter);
        String checkpointString = request.getRequiredStringParam(CHECKPOINT_PARAM);
        int bufferMarginSpace = DEFAULT_BUFFER_MARGIN_SPACE;
        if (null != _serverHostPort) {
            bufferMarginSpace = Math.max(bufferMarginSpace, (_serverHostPort.length() + Checkpoint.BOOTSTRAP_SERVER_INFO.length() + DEFAULT_JSON_OVERHEAD_BYTES));
        }
        int clientFreeBufferSize = request.getRequiredIntParam(BATCHSIZE_PARAM) - checkpointString.length() - bufferMarginSpace;
        BootstrapEventWriter writer = null;
        if (_config.getPredicatePushDown())
            writer = createEventWriter(request, clientFreeBufferSize, null, enc);
        else
            writer = createEventWriter(request, clientFreeBufferSize, keyFilter, enc);
        Checkpoint cp = new Checkpoint(checkpointString);
        DbusClientMode consumptionMode = cp.getConsumptionMode();
        LOG.info("Bootstrap request received: " + "fetchSize=" + clientFreeBufferSize + ", consumptionMode=" + consumptionMode + ", checkpoint=" + checkpointString + ", predicatePushDown=" + _config.getPredicatePushDown());
        try {
            boolean phaseCompleted = false;
            switch(consumptionMode) {
                case BOOTSTRAP_SNAPSHOT:
                    phaseCompleted = processor.streamSnapShotRows(new Checkpoint(checkpointString), writer);
                    break;
                case BOOTSTRAP_CATCHUP:
                    phaseCompleted = processor.streamCatchupRows(new Checkpoint(checkpointString), writer);
                    break;
                default:
                    if (null != bootstrapStatsCollector)
                        bootstrapStatsCollector.registerErrBootstrap();
                    throw new RequestProcessingException("Unexpected mode: " + consumptionMode);
            }
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerBootStrapReq(cp, System.currentTimeMillis() - startTime, clientFreeBufferSize);
            if (writer.getNumRowsWritten() == 0 && writer.getSizeOfPendingEvent() > 0) {
                // Append a header to indicate to the client that we do have at least one event to
                // send, but it is too large to fit into client's offered buffer.
                request.getResponseContent().addMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE, writer.getSizeOfPendingEvent());
                if (isDebug) {
                    LOG.debug("Returning 0 events but have pending event of size " + writer.getSizeOfPendingEvent());
                }
            }
            if (phaseCompleted) {
                request.getResponseContent().setMetadata(BootstrapProcessor.PHASE_COMPLETED_HEADER_NAME, BootstrapProcessor.PHASE_COMPLETED_HEADER_TRUE);
            }
        } catch (BootstrapDatabaseTooOldException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrDatabaseTooOld();
            LOG.error("Bootstrap database is too old!", e);
            throw new RequestProcessingException(e);
        } catch (BootstrapDBException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrBootstrap();
            throw new RequestProcessingException(e);
        } catch (SQLException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrSqlException();
            throw new RequestProcessingException(e);
        } catch (BootstrapProcessingException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrBootstrap();
            throw new RequestProcessingException(e);
        }
    } finally {
        if (null != processor)
            processor.shutdown();
    }
    return request;
}