Examples with BootstrapDBException com.linkedin.databus2.core.container.request.BootstrapDBException used on opensource projects

Example 1 with BootstrapDBException

use of com.linkedin.databus2.core.container.request.BootstrapDBException in project databus by linkedin.

the class RemoteExceptionHandler method getException.

public Throwable getException(ChunkedBodyReadableByteChannel readChannel) {
    Throwable remoteException = null;
    String err = getExceptionName(readChannel);
    if (null != err) {
        // in theory, we shall be reading the actual exception from the read channel.
        if (err.equalsIgnoreCase(ScnNotFoundException.class.getName())) {
            remoteException = new ScnNotFoundException();
        } else if (err.equalsIgnoreCase(BootstrapDatabaseTooOldException.class.getName())) {
            remoteException = new BootstrapDatabaseTooOldException();
        } else if (err.equalsIgnoreCase(PullerRetriesExhaustedException.class.getName())) {
            remoteException = new PullerRetriesExhaustedException();
        } else if (err.equalsIgnoreCase(BootstrapDatabaseTooYoungException.class.getName())) {
            remoteException = new BootstrapDatabaseTooYoungException();
        } else if (err.equalsIgnoreCase(BootstrapDBException.class.getName())) {
            remoteException = new BootstrapDBException();
        } else if (err.equalsIgnoreCase(SQLException.class.getName())) {
            remoteException = new SQLException();
        } else {
            LOG.error("Unexpected remote error received: " + err);
        }
        LOG.info("Remote exception received: " + remoteException);
    }
    return remoteException;
}

Example 2 with BootstrapDBException

use of com.linkedin.databus2.core.container.request.BootstrapDBException in project databus by linkedin.

the class DatabusBootstrapProducer method validateAndRepairBootstrapDBCheckpoint.

/**
   *
   *
   * Compares the Checkpoint and bootstrap_producer_state's SCN to check against
   * the possibility of gap in event consumption!! If gap is found, makes
   * best-effort to repair it.
   *
   * Three Scenario that should be allowed
   *
   * 1. Bootstrap Producer started after seeding In this case checkpoint SCN
   * should not be greater than producer SCN. 2. Bootstrap Producer started
   * after adding sources without seeding In this case the producer SCN is "-1".
   * The checkpoint "could" be empty 3. Bootstrap Producer bounced In this case
   * checkpoint SCN should not be greater than producer SCN.
   *
   * @throws SQLException
   * @throws BootstrapDBException
   *           if there is a gap and cannot be repaired
   * @throws IOException
   *           `
   */
private void validateAndRepairBootstrapDBCheckpoint() throws SQLException, BootstrapDBException, IOException {
    LOG.info("Validating bootstrap DB checkpoints !!");
    for (List<DatabusSubscription> subsList : _relayGroups.keySet()) {
        List<String> sourceNames = DatabusSubscription.getStrList(subsList);
        long scn = -1;
        try {
            scn = _dbDao.getMinWindowSCNFromStateTable(sourceNames, "bootstrap_producer_state");
        } catch (BootstrapDBException ex) {
            LOG.error("Got exception while trying to fetch SCN from bootstrap_producer_state for sources :" + sourceNames, ex);
            throw ex;
        }
        CheckpointPersistenceProvider provider = getCheckpointPersistenceProvider();
        Checkpoint cp = provider.loadCheckpoint(sourceNames);
        LOG.info("Bootstrap Producer SCN :" + scn + ", Checkpoint :" + cp);
        if (null != cp) {
            if (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION) {
                // Bootstrapping bootstrap Producer not yet supported !!
                String msg = "Bootstrap Producer starting from non-online consumption mode for sources :" + sourceNames + ", Ckpt :" + cp;
                LOG.error(msg);
                throw new BootstrapDBException(msg);
            } else {
                String msg = null;
                if (((cp.getWindowScn() > scn) && (scn > -1)) || ((cp.getWindowScn() < scn) && (scn > -1))) {
                    if (((cp.getWindowScn() > scn) && (scn > -1)))
                        LOG.warn("Non-Empty checkpint. Bootstrap Producer is at SCN:" + scn + ", while checkpoint is :" + cp + ", Could result in gap in event consumption. Repairing ckpt !!");
                    else
                        LOG.info("Non-Empty checkpoint. Bootstrap Producer is at SCN:" + scn + ", while checkpoint is :" + cp + ", Copying producer Scn to checkpoint !!");
                    cp.setWindowScn(scn);
                    cp.setWindowOffset(-1);
                    try {
                        provider.removeCheckpoint(sourceNames);
                        provider.storeCheckpoint(sourceNames, cp);
                        // Check if persisted properly
                        cp = provider.loadCheckpoint(sourceNames);
                        if ((null == cp) || (cp.getWindowScn() != scn) || (cp.getWindowOffset() != -1) || (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION)) {
                            msg = "Unable to repair and store the new checkpoint (" + cp + ") to make it same as producer SCN (" + scn + ") !!";
                            LOG.fatal(msg);
                            throw new BootstrapDBException(msg);
                        }
                    } catch (IOException ex) {
                        msg = "Unable to repair and store the new checkpoint (" + cp + ") to make it same as producer SCN (" + scn + ") !!";
                        LOG.fatal(msg, ex);
                        throw new BootstrapDBException(msg);
                    }
                }
            }
        } else {
            /**
         * Currently since bootstrapping is not available, a null ckpt would
         * result in flexible checkpoint and could result in gap !!
         */
            if (scn > -1) {
                String msg = "Empty checkpoint. Bootstrap Producer SCN is at SCN:" + scn + ", while checkpoint is null !! Could result in gap in event consumption. Repairing ckpt !!";
                LOG.warn(msg);
                cp = new Checkpoint();
                cp.setWindowScn(scn);
                cp.setWindowOffset(-1);
                cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
                try {
                    provider.removeCheckpoint(sourceNames);
                    provider.storeCheckpoint(sourceNames, cp);
                    // Check if persisted properly
                    cp = provider.loadCheckpoint(sourceNames);
                    if ((null == cp) || (cp.getWindowScn() != scn) || (cp.getWindowOffset() != -1) || (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION)) {
                        LOG.fatal("Unable to repair and store the checkpoint (" + cp + ") to make it same as producer SCN (" + scn + ") !!");
                        throw new BootstrapDBException(msg);
                    }
                } catch (IOException ex) {
                    msg = "Unable to repair and store the checkpoint (" + cp + ") to make it same as producer SCN (" + scn + ") !!";
                    LOG.fatal(msg, ex);
                    throw new BootstrapDBException(msg);
                }
            }
        }
    }
    LOG.info("Validating bootstrap DB checkpoints done successfully!!");
}

Example 3 with BootstrapDBException

use of com.linkedin.databus2.core.container.request.BootstrapDBException in project databus by linkedin.

the class BootstrapRequestProcessor method doProcess.

/*
   * (non-Javadoc)
   *
   * @see
   * com.linkedin.databus.container.request.RequestProcessor#process(com.linkedin.databus
   * .container.request.DatabusRequest)
   */
@Override
protected DatabusRequest doProcess(DatabusRequest request) throws IOException, RequestProcessingException {
    BootstrapProcessor processor = null;
    BootstrapHttpStatsCollector bootstrapStatsCollector = _bootstrapServer.getBootstrapStatsCollector();
    long startTime = System.currentTimeMillis();
    boolean isDebug = LOG.isDebugEnabled();
    try {
        try {
            String threadName = Thread.currentThread().getName();
            DbusEventsStatisticsCollector threadCollector = _bootstrapServer.getOutBoundStatsCollectors().getStatsCollector(threadName);
            if (null == threadCollector) {
                threadCollector = new DbusEventsStatisticsCollector(_bootstrapServer.getContainerStaticConfig().getId(), threadName, true, false, _bootstrapServer.getMbeanServer());
                StatsCollectors<DbusEventsStatisticsCollector> ds = _bootstrapServer.getOutBoundStatsCollectors();
                ds.addStatsCollector(threadName, threadCollector);
            }
            processor = new BootstrapProcessor(_config, threadCollector);
        } catch (Exception e) {
            if (null != bootstrapStatsCollector) {
                bootstrapStatsCollector.registerErrBootstrap();
            }
            throw new RequestProcessingException(e);
        }
        DatabusComponentStatus componentStatus = _componentStatus.getStatusSnapshot();
        if (!componentStatus.isRunningStatus()) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrBootstrap();
            throw new RequestProcessingException(componentStatus.getMessage());
        }
        String partitionInfoString = request.getParams().getProperty(PARTITION_INFO_PARAM);
        DbusKeyFilter keyFilter = null;
        if ((null != partitionInfoString) && (!partitionInfoString.isEmpty())) {
            try {
                keyFilter = KeyFilterConfigJSONFactory.parseDbusKeyFilter(partitionInfoString);
                if (isDebug)
                    LOG.debug("ServerSideFilter is :" + keyFilter);
            } catch (Exception ex) {
                String msg = "Unable to parse partitionInfo from request. PartitionInfo was :" + partitionInfoString;
                LOG.error(msg, ex);
                throw new RequestProcessingException(msg, ex);
            }
        }
        String outputFormat = request.getParams().getProperty(OUTPUT_PARAM);
        Encoding enc = Encoding.BINARY;
        if (null != outputFormat) {
            try {
                enc = Encoding.valueOf(outputFormat.toUpperCase());
            } catch (Exception ex) {
                LOG.error("Unable to find the output format for bootstrap request for " + outputFormat + ". Using Binary!!", ex);
            }
        }
        processor.setKeyFilter(keyFilter);
        String checkpointString = request.getRequiredStringParam(CHECKPOINT_PARAM);
        int bufferMarginSpace = DEFAULT_BUFFER_MARGIN_SPACE;
        if (null != _serverHostPort) {
            bufferMarginSpace = Math.max(bufferMarginSpace, (_serverHostPort.length() + Checkpoint.BOOTSTRAP_SERVER_INFO.length() + DEFAULT_JSON_OVERHEAD_BYTES));
        }
        int clientFreeBufferSize = request.getRequiredIntParam(BATCHSIZE_PARAM) - checkpointString.length() - bufferMarginSpace;
        BootstrapEventWriter writer = null;
        if (_config.getPredicatePushDown())
            writer = createEventWriter(request, clientFreeBufferSize, null, enc);
        else
            writer = createEventWriter(request, clientFreeBufferSize, keyFilter, enc);
        Checkpoint cp = new Checkpoint(checkpointString);
        DbusClientMode consumptionMode = cp.getConsumptionMode();
        LOG.info("Bootstrap request received: " + "fetchSize=" + clientFreeBufferSize + ", consumptionMode=" + consumptionMode + ", checkpoint=" + checkpointString + ", predicatePushDown=" + _config.getPredicatePushDown());
        try {
            boolean phaseCompleted = false;
            switch(consumptionMode) {
                case BOOTSTRAP_SNAPSHOT:
                    phaseCompleted = processor.streamSnapShotRows(new Checkpoint(checkpointString), writer);
                    break;
                case BOOTSTRAP_CATCHUP:
                    phaseCompleted = processor.streamCatchupRows(new Checkpoint(checkpointString), writer);
                    break;
                default:
                    if (null != bootstrapStatsCollector)
                        bootstrapStatsCollector.registerErrBootstrap();
                    throw new RequestProcessingException("Unexpected mode: " + consumptionMode);
            }
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerBootStrapReq(cp, System.currentTimeMillis() - startTime, clientFreeBufferSize);
            if (writer.getNumRowsWritten() == 0 && writer.getSizeOfPendingEvent() > 0) {
                // Append a header to indicate to the client that we do have at least one event to
                // send, but it is too large to fit into client's offered buffer.
                request.getResponseContent().addMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE, writer.getSizeOfPendingEvent());
                if (isDebug) {
                    LOG.debug("Returning 0 events but have pending event of size " + writer.getSizeOfPendingEvent());
                }
            }
            if (phaseCompleted) {
                request.getResponseContent().setMetadata(BootstrapProcessor.PHASE_COMPLETED_HEADER_NAME, BootstrapProcessor.PHASE_COMPLETED_HEADER_TRUE);
            }
        } catch (BootstrapDatabaseTooOldException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrDatabaseTooOld();
            LOG.error("Bootstrap database is too old!", e);
            throw new RequestProcessingException(e);
        } catch (BootstrapDBException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrBootstrap();
            throw new RequestProcessingException(e);
        } catch (SQLException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrSqlException();
            throw new RequestProcessingException(e);
        } catch (BootstrapProcessingException e) {
            if (null != bootstrapStatsCollector)
                bootstrapStatsCollector.registerErrBootstrap();
            throw new RequestProcessingException(e);
        }
    } finally {
        if (null != processor)
            processor.shutdown();
    }
    return request;
}