Examples with DbusClientMode com.linkedin.databus.core.DbusClientMode used on opensource projects

Example 1 with DbusClientMode

use of com.linkedin.databus.core.DbusClientMode in project databus by linkedin.

the class RelayPullThread method doBootstrapComplete.

private void doBootstrapComplete(BootstrapResultMessage bootstrapResultMessage) {
    Checkpoint cp = bootstrapResultMessage.getBootstrapCheckpoint();
    // we must have persisted some checkpoint at the end of the bootstrap phase
    if (null == cp) {
        bootstrapResultMessage.switchToBootstrapFailed(new RuntimeException("No persistent checkpoint found at the end of bootstrap!"));
        doBootstrapFailed(bootstrapResultMessage);
    } else {
        _currentState.setRelayFellOff(false);
        DbusClientMode consumptionMode = cp.getConsumptionMode();
        _log.info("Bootstrap completed: " + "Consumption Mode=" + consumptionMode + " startScn=" + cp.getBootstrapStartScn() + " targetScn=" + cp.getBootstrapTargetScn() + " sinceScn=" + cp.getBootstrapSinceScn() + " windowScn=" + cp.getWindowScn());
        UnifiedClientStats unifiedClientStats = _sourcesConn.getUnifiedClientStats();
        if (unifiedClientStats != null) {
            // should always be false, but just in case of weird edge cases:
            boolean isBootstrapping = (consumptionMode == DbusClientMode.BOOTSTRAP_SNAPSHOT || consumptionMode == DbusClientMode.BOOTSTRAP_CATCHUP);
            unifiedClientStats.setBootstrappingState(isBootstrapping);
        }
        // clear Bootstrap scns for future bootstraps
        cp.resetBootstrap();
        if (toTearConnAfterHandlingResponse()) {
            tearConnectionAndEnqueuePickServer();
        } else {
            _currentState.switchToRequestStream(cp);
            enqueueMessage(_currentState);
        }
    }
}

Example 2 with DbusClientMode

use of com.linkedin.databus.core.DbusClientMode in project databus by linkedin.

the class RelayDispatcher method processSysEvent.

@Override
protected boolean processSysEvent(DispatcherState curState, DbusEvent event) {
    boolean success = true;
    if (event.isCheckpointMessage()) {
        Checkpoint ckpt = null;
        try {
            ckpt = DbusEventUtils.getCheckpointFromEvent(event);
            DbusClientMode bootstrapMode = ckpt.getConsumptionMode();
            if (bootstrapMode != DbusClientMode.ONLINE_CONSUMPTION) {
                if (_bootstrapPuller == null) {
                    _log.error("Checkpoint specifies that the consumer is bootstrapping, but bootstrapPuller is not present (Is bootstrap disabled ?)");
                    return false;
                }
                ckpt.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
                if (curState.getStateId() != DispatcherState.StateId.EXPECT_EVENT_WINDOW) {
                    _log.warn("The current state of the dispatcher is NOT as expected (" + DispatcherState.StateId.EXPECT_EVENT_WINDOW.name() + "). State prior to this: " + curState.getStateId().name());
                //Fixing bug that caused TestRelayBootstrapSwitch to fail; no apparent need to rollback
                //curState.switchToRollback();
                //doRollback(curState);
                }
                curState.getEventsIterator().getEventBuffer().clear();
                curState.resetIterators();
                curState.switchToExpectEventWindow();
                _bootstrapPuller.enqueueMessage(LifecycleMessage.createStartMessage());
                _log.info("Switching to bootstrap mode");
            } else {
                success = super.processSysEvent(curState, event);
            }
        } catch (Exception e) {
            DbusPrettyLogUtils.logExceptionAtError("Internal error processing a system event", e, _log);
            success = false;
        }
    } else if (event.isSCNRegressMessage()) {
        SCNRegressMessage message = DbusEventUtils.getSCNRegressFromEvent(event);
        _log.info("Switching relays, some of the events maybe replayed. The Checkpoint to which the client with regress: " + message);
        curState.setSCNRegress(true);
        curState.switchToExpectEventWindow();
    //enqueueMessage(curState);
    } else {
        success = super.processSysEvent(curState, event);
    }
    return success;
}

Example 3 with DbusClientMode

use of com.linkedin.databus.core.DbusClientMode in project databus by linkedin.

the class BootstrapRequestProcessor method doProcess.

/*
   * (non-Javadoc)
   *
   * @see
   * com.linkedin.databus.container.request.RequestProcessor#process(com.linkedin.databus
   * .container.request.DatabusRequest)
   */
@Override
protected DatabusRequest doProcess(DatabusRequest request) throws IOException, RequestProcessingException {
    BootstrapProcessor processor = null;
    BootstrapHttpStatsCollector bootstrapStatsCollector = _bootstrapServer.getBootstrapStatsCollector();
    long startTime = System.currentTimeMillis();
    boolean isDebug = LOG.isDebugEnabled();
    try {
        try {
            String threadName = Thread.currentThread().getName();
            DbusEventsStatisticsCollector threadCollector = _bootstrapServer.getOutBoundStatsCollectors().getStatsCollector(threadName);
            if (null == threadCollector) {
                threadCollector = new DbusEventsStatisticsCollector(_bootstrapServer.getContainerStaticConfig().getId(), threadName, true, false, _bootstrapServer.getMbeanServer());
                StatsCollectors<DbusEventsStatisticsCollector> ds = _bootstrapServer.getOutBoundStatsCollectors();
                ds.addStatsCollector(threadName, threadCollector);
            }
            processor = new BootstrapProcessor(_config, threadCollector);
        } catch (Exception e) {
            if (null != bootstrapStatsCollector) {
                bootstrapStatsCollector.registerErrBootstrap();
            }
            throw new RequestProcessingException(e);
        }
        DatabusComponentStatus componentStatus = _componentStatus.getStatusSnapshot();
        if (!componentStatus.isRunningStatus()) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrBootstrap();
            throw new RequestProcessingException(componentStatus.getMessage());
        }
        String partitionInfoString = request.getParams().getProperty(PARTITION_INFO_PARAM);
        DbusKeyFilter keyFilter = null;
        if ((null != partitionInfoString) && (!partitionInfoString.isEmpty())) {
            try {
                keyFilter = KeyFilterConfigJSONFactory.parseDbusKeyFilter(partitionInfoString);
                if (isDebug)
                    LOG.debug("ServerSideFilter is :" + keyFilter);
            } catch (Exception ex) {
                String msg = "Unable to parse partitionInfo from request. PartitionInfo was :" + partitionInfoString;
                LOG.error(msg, ex);
                throw new RequestProcessingException(msg, ex);
            }
        }
        String outputFormat = request.getParams().getProperty(OUTPUT_PARAM);
        Encoding enc = Encoding.BINARY;
        if (null != outputFormat) {
            try {
                enc = Encoding.valueOf(outputFormat.toUpperCase());
            } catch (Exception ex) {
                LOG.error("Unable to find the output format for bootstrap request for " + outputFormat + ". Using Binary!!", ex);
            }
        }
        processor.setKeyFilter(keyFilter);
        String checkpointString = request.getRequiredStringParam(CHECKPOINT_PARAM);
        int bufferMarginSpace = DEFAULT_BUFFER_MARGIN_SPACE;
        if (null != _serverHostPort) {
            bufferMarginSpace = Math.max(bufferMarginSpace, (_serverHostPort.length() + Checkpoint.BOOTSTRAP_SERVER_INFO.length() + DEFAULT_JSON_OVERHEAD_BYTES));
        }
        int clientFreeBufferSize = request.getRequiredIntParam(BATCHSIZE_PARAM) - checkpointString.length() - bufferMarginSpace;
        BootstrapEventWriter writer = null;
        if (_config.getPredicatePushDown())
            writer = createEventWriter(request, clientFreeBufferSize, null, enc);
        else
            writer = createEventWriter(request, clientFreeBufferSize, keyFilter, enc);
        Checkpoint cp = new Checkpoint(checkpointString);
        DbusClientMode consumptionMode = cp.getConsumptionMode();
        LOG.info("Bootstrap request received: " + "fetchSize=" + clientFreeBufferSize + ", consumptionMode=" + consumptionMode + ", checkpoint=" + checkpointString + ", predicatePushDown=" + _config.getPredicatePushDown());
        try {
            boolean phaseCompleted = false;
            switch(consumptionMode) {
                case BOOTSTRAP_SNAPSHOT:
                    phaseCompleted = processor.streamSnapShotRows(new Checkpoint(checkpointString), writer);
                    break;
                case BOOTSTRAP_CATCHUP:
                    phaseCompleted = processor.streamCatchupRows(new Checkpoint(checkpointString), writer);
                    break;
                default:
                    if (null != bootstrapStatsCollector)
                        bootstrapStatsCollector.registerErrBootstrap();
                    throw new RequestProcessingException("Unexpected mode: " + consumptionMode);
            }
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerBootStrapReq(cp, System.currentTimeMillis() - startTime, clientFreeBufferSize);
            if (writer.getNumRowsWritten() == 0 && writer.getSizeOfPendingEvent() > 0) {
                // Append a header to indicate to the client that we do have at least one event to
                // send, but it is too large to fit into client's offered buffer.
                request.getResponseContent().addMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE, writer.getSizeOfPendingEvent());
                if (isDebug) {
                    LOG.debug("Returning 0 events but have pending event of size " + writer.getSizeOfPendingEvent());
                }
            }
            if (phaseCompleted) {
                request.getResponseContent().setMetadata(BootstrapProcessor.PHASE_COMPLETED_HEADER_NAME, BootstrapProcessor.PHASE_COMPLETED_HEADER_TRUE);
            }
        } catch (BootstrapDatabaseTooOldException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrDatabaseTooOld();
            LOG.error("Bootstrap database is too old!", e);
            throw new RequestProcessingException(e);
        } catch (BootstrapDBException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrBootstrap();
            throw new RequestProcessingException(e);
        } catch (SQLException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrSqlException();
            throw new RequestProcessingException(e);
        } catch (BootstrapProcessingException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrBootstrap();
            throw new RequestProcessingException(e);
        }
    } finally {
        if (null != processor)
            processor.shutdown();
    }
    return request;
}

Example 4 with DbusClientMode

use of com.linkedin.databus.core.DbusClientMode in project databus by linkedin.

the class DbusBootstrapHttpStats method registerBootStrapReq.

@Override
public void registerBootStrapReq(Checkpoint cp, long latency, long size) {
    if (!_enabled.get())
        return;
    Lock writeLock = acquireWriteLock();
    try {
        _event.numReqBootstrap++;
        DbusClientMode mode = cp.getConsumptionMode();
        if (mode == DbusClientMode.BOOTSTRAP_CATCHUP) {
            _event.latencyCatchup += latency;
            _event.numReqCatchup++;
        } else if (mode == DbusClientMode.BOOTSTRAP_SNAPSHOT) {
            _event.numReqSnapshot++;
            _event.latencySnapshot += latency;
        }
        _event.maxBootstrapSCN = maxValue(cp.getWindowScn(), _event.maxBootstrapSCN);
        _event.minBootstrapSCN = minValue(cp.getWindowScn(), _event.minBootstrapSCN);
        _event.sizeBatch += size;
    } finally {
        releaseLock(writeLock);
    }
}