Examples with CheckpointPersistenceProvider com.linkedin.databus.client.pub.CheckpointPersistenceProvider used on opensource projects

Example 1 with CheckpointPersistenceProvider

use of com.linkedin.databus.client.pub.CheckpointPersistenceProvider in project databus by linkedin.

the class DatabusHttpClientImpl method initializeRelayConnections.

private synchronized void initializeRelayConnections() {
    for (List<DatabusSubscription> subsList : _relayGroups.keySet()) {
        List<String> sourcesStrList = DatabusSubscription.getStrList(subsList);
        List<DatabusV2ConsumerRegistration> relayConsumers = getRelayGroupStreamConsumers().get(subsList);
        //nothing to do
        if (null == relayConsumers || 0 == relayConsumers.size())
            continue;
        try {
            DatabusSourcesConnection.StaticConfig connConfig = getClientStaticConfig().getConnection(sourcesStrList);
            if (null == connConfig) {
                connConfig = getClientStaticConfig().getConnectionDefaults();
            }
            // make sure we have the right policy.
            if (!connConfig.getEventBuffer().isEnableScnIndex() && connConfig.getEventBuffer().getQueuePolicy() != DbusEventBuffer.QueuePolicy.BLOCK_ON_WRITE) {
                throw new InvalidConfigException("If SCN index is disabled, queue policy must be BLOCK_ON_WRITE");
            }
            CheckpointPersistenceProvider cpPersistenceProvder = getCheckpointPersistenceProvider();
            if (null != cpPersistenceProvder && getClientStaticConfig().getCheckpointPersistence().isClearBeforeUse()) {
                cpPersistenceProvder.removeCheckpoint(sourcesStrList);
            }
            ServerInfo server0 = _relayGroups.get(subsList).iterator().next();
            ArrayList<DatabusV2ConsumerRegistration> bstConsumersRegs = new ArrayList<DatabusV2ConsumerRegistration>();
            for (List<DatabusSubscription> bstSubSourcesList : getRelayGroupBootstrapConsumers().keySet()) {
                List<DatabusV2ConsumerRegistration> bstRegsistrations = getRelayGroupBootstrapConsumers().get(bstSubSourcesList);
                for (DatabusV2ConsumerRegistration bstConsumerReg : bstRegsistrations) {
                    if (server0.supportsSources(bstConsumerReg.getSources())) {
                        bstConsumersRegs.add(bstConsumerReg);
                    }
                }
            }
            DbusEventBuffer eventBuffer = connConfig.getEventBuffer().getOrCreateEventBuffer(_eventFactory);
            eventBuffer.setDropOldEvents(true);
            eventBuffer.start(0);
            DbusEventBuffer bootstrapBuffer = null;
            // create bootstrap only if it is enabled
            if (_clientStaticConfig.getRuntime().getBootstrap().isEnabled()) {
                bootstrapBuffer = new DbusEventBuffer(connConfig.getEventBuffer());
                bootstrapBuffer.setDropOldEvents(false);
                bootstrapBuffer.start(0);
            }
            LOG.info("The sourcesList is " + sourcesStrList);
            LOG.info("The relayGroupStreamConsumers is " + getRelayGroupStreamConsumers().get(subsList));
            Set<ServerInfo> relays = _relayGroups.get(subsList);
            Set<ServerInfo> bootstrapServices = _bootstrapGroups.get(subsList);
            String statsCollectorName = generateSubsStatsName(sourcesStrList);
            int ownerId = getContainerStaticConfig().getId();
            _bootstrapEventsStats.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".inbound.bs", true, false, getMbeanServer()));
            _inBoundStatsCollectors.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".inbound", true, false, getMbeanServer()));
            _outBoundStatsCollectors.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".outbound", true, false, getMbeanServer()));
            _consumerStatsCollectors.addStatsCollector(statsCollectorName, new ConsumerCallbackStats(ownerId, statsCollectorName + ".inbound.cons", statsCollectorName + ".inbound.cons", true, false, null, getMbeanServer()));
            _bsConsumerStatsCollectors.addStatsCollector(statsCollectorName, new ConsumerCallbackStats(ownerId, statsCollectorName + ".inbound.bs.cons", statsCollectorName + ".inbound.bs.cons", true, false, null, getMbeanServer()));
            _unifiedClientStatsCollectors.addStatsCollector(statsCollectorName, new UnifiedClientStats(ownerId, statsCollectorName + ".inbound.unified.cons", statsCollectorName + ".inbound.unified.cons", true, false, _clientStaticConfig.getPullerThreadDeadnessThresholdMs(), null, getMbeanServer()));
            ConnectionStateFactory connStateFactory = new ConnectionStateFactory(DatabusSubscription.getStrList(subsList));
            DatabusSourcesConnection newConn = new DatabusSourcesConnection(connConfig, subsList, relays, bootstrapServices, relayConsumers, //_relayGroupBootstrapConsumers.get(sourcesList),
            bstConsumersRegs, eventBuffer, bootstrapBuffer, getDefaultExecutorService(), getContainerStatsCollector(), _inBoundStatsCollectors.getStatsCollector(statsCollectorName), _bootstrapEventsStats.getStatsCollector(statsCollectorName), _consumerStatsCollectors.getStatsCollector(statsCollectorName), _bsConsumerStatsCollectors.getStatsCollector(statsCollectorName), _unifiedClientStatsCollectors.getStatsCollector(statsCollectorName), getCheckpointPersistenceProvider(), getRelayConnFactory(), getBootstrapConnFactory(), getHttpStatsCollector(), null, this, _eventFactory, connStateFactory);
            newConn.start();
            _relayConnections.add(newConn);
        } catch (Exception e) {
            LOG.error("connection initialization issue for source(s):" + subsList + "; please check your configuration", e);
        }
    }
    if (0 == _relayConnections.size()) {
        LOG.warn("No connections specified");
    }
}

Example 2 with CheckpointPersistenceProvider

use of com.linkedin.databus.client.pub.CheckpointPersistenceProvider in project databus by linkedin.

the class DatabusV2ClusterRegistrationImpl method activateOnePartition.

/**
   * Callback to activate one partition that was added
   * @param partition
   * @throws DatabusException
   */
private synchronized void activateOnePartition(DbusPartitionInfo partition) throws DatabusClientException {
    _log.info("Trying to activate partition :" + partition);
    try {
        if (regMap.containsKey(partition)) {
            _log.info("Partition (" + partition + ") is already added and is currently in state : " + regMap.get(partition).getState() + " skipping !!");
            return;
        }
        // Call factories to get consumer callbacks and server-side filter config.
        Collection<DatabusCombinedConsumer> consumers = _consumerFactory.createPartitionedConsumers(_clusterInfo, partition);
        DbusKeyCompositeFilterConfig filterConfig = null;
        if (_serverSideFilterFactory != null)
            filterConfig = _serverSideFilterFactory.createServerSideFilter(_clusterInfo, partition);
        if ((null == consumers) || (consumers.isEmpty())) {
            _log.error("ConsumerFactory for cluster (" + _clusterInfo + ") returned null or empty consumers ");
            throw new DatabusClientException("ConsumerFactory for cluster (" + _clusterInfo + ") returned null or empty consumers");
        }
        // Create Registration
        RegistrationId id = new RegistrationId(_id + "-" + partition.getPartitionId());
        CheckpointPersistenceProvider ckptProvider = createCheckpointPersistenceProvider(partition);
        DatabusV2RegistrationImpl reg = createChildRegistration(id, _client, ckptProvider);
        reg.addDatabusConsumers(consumers);
        String[] srcs = new String[_sources.size()];
        srcs = _sources.toArray(srcs);
        reg.addSubscriptions(srcs);
        regMap.put(partition, reg);
        reg.onRegister();
        // Add Server-Side Filter
        if (null != filterConfig)
            reg.withServerSideFilter(filterConfig);
        // Notify Listener
        if (null != _partitionListener)
            _partitionListener.onAddPartition(partition, reg);
        // Start the registration
        try {
            reg.start();
        } catch (DatabusClientException e) {
            _log.error("Got exception while starting the registration for partition (" + partition + ")", e);
            throw e;
        }
        //Add partition Specific metrics to cluster-merge
        _relayEventStatsMerger.addStatsCollector(id.getId(), (DbusEventsStatisticsCollector) reg.getRelayEventStats());
        _bootstrapEventStatsMerger.addStatsCollector(id.getId(), (DbusEventsStatisticsCollector) reg.getBootstrapEventStats());
        _relayCallbackStatsMerger.addStatsCollector(id.getId(), (ConsumerCallbackStats) reg.getRelayCallbackStats());
        _bootstrapCallbackStatsMerger.addStatsCollector(id.getId(), (ConsumerCallbackStats) reg.getBootstrapCallbackStats());
        _log.info("Partition (" + partition + ") started !!");
    } catch (DatabusException e) {
        _log.error("Got exception while activating partition(" + partition + ")", e);
        throw new DatabusClientException(e);
    } catch (ClusterCheckpointException e) {
        _log.error("Got exception while activating partition(" + partition + ")", e);
        throw new DatabusClientException(e);
    }
}

Example 3 with CheckpointPersistenceProvider

use of com.linkedin.databus.client.pub.CheckpointPersistenceProvider in project databus by linkedin.

the class CheckpointSerializerMain method main.

public static void main(String[] args) throws Exception {
    parseArgs(args);
    PatternLayout defaultLayout = new PatternLayout("%d{ISO8601} +%r [%t] (%p) {%c{1}} %m%n");
    ConsoleAppender defaultAppender = new ConsoleAppender(defaultLayout);
    Logger.getRootLogger().removeAllAppenders();
    Logger.getRootLogger().addAppender(defaultAppender);
    Logger.getRootLogger().setLevel(Level.INFO);
    Logger.getRootLogger().info("NOTE. This tool works only with V2/V1 checkpoints");
    CheckpointPersistenceProvider cp3 = null;
    if (null != _cp3Props) {
        CheckpointPersistenceStaticConfigBuilder cp3ConfBuilder = new CheckpointPersistenceStaticConfigBuilder();
        ConfigLoader<CheckpointPersistenceStaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.CheckpointPersistenceStaticConfig>(_propPrefix, cp3ConfBuilder);
        configLoader.loadConfig(_cp3Props);
        CheckpointPersistenceStaticConfig cp3Conf = cp3ConfBuilder.build();
        if (cp3Conf.getType() != CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM) {
            throw new RuntimeException("don't know what to do with cp3 type:" + cp3Conf.getType());
        }
        cp3 = new FileSystemCheckpointPersistenceProvider(cp3Conf.getFileSystem(), 2);
    } else if (null != _clientProps) {
        DatabusHttpClientImpl.Config clientConfBuilder = new DatabusHttpClientImpl.Config();
        ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>(_propPrefix, clientConfBuilder);
        configLoader.loadConfig(_clientProps);
        DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build();
        if (clientConf.getCheckpointPersistence().getType() != CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM) {
            throw new RuntimeException("don't know what to do with cp3 type:" + clientConf.getCheckpointPersistence().getType());
        }
        cp3 = new FileSystemCheckpointPersistenceProvider(clientConf.getCheckpointPersistence().getFileSystem(), 2);
    }
    List<String> sourceList = Arrays.asList(_sources);
    Checkpoint cpOld = null != cp3 ? cp3.loadCheckpoint(sourceList) : new Checkpoint();
    Checkpoint cpNew;
    if (Action.PRINT == _action) {
        cpNew = updateCheckpoint(cpOld);
    } else if (Action.CHANGE == _action) {
        cpNew = updateCheckpoint(cpOld);
        cp3.storeCheckpoint(sourceList, cpNew);
        //reread as a sanity check
        cpNew = cp3.loadCheckpoint(sourceList);
    } else if (Action.DELETE == _action) {
        cp3.removeCheckpoint(sourceList);
        cpNew = cp3.loadCheckpoint(sourceList);
    } else {
        throw new RuntimeException("don't know what to do with action: " + _action);
    }
    if (null != cpOld)
        System.out.println("old: " + cpOld.toString());
    else
        System.out.println("old: null");
    if (null != cpNew)
        System.out.println("new: " + cpNew.toString());
    else
        System.out.println("new: null");
}

Example 4 with CheckpointPersistenceProvider

use of com.linkedin.databus.client.pub.CheckpointPersistenceProvider in project databus by linkedin.

the class DatabusBootstrapProducer method validateAndRepairBootstrapDBCheckpoint.

/**
   *
   *
   * Compares the Checkpoint and bootstrap_producer_state's SCN to check against
   * the possibility of gap in event consumption!! If gap is found, makes
   * best-effort to repair it.
   *
   * Three Scenario that should be allowed
   *
   * 1. Bootstrap Producer started after seeding In this case checkpoint SCN
   * should not be greater than producer SCN. 2. Bootstrap Producer started
   * after adding sources without seeding In this case the producer SCN is "-1".
   * The checkpoint "could" be empty 3. Bootstrap Producer bounced In this case
   * checkpoint SCN should not be greater than producer SCN.
   *
   * @throws SQLException
   * @throws BootstrapDBException
   *           if there is a gap and cannot be repaired
   * @throws IOException
   *           `
   */
private void validateAndRepairBootstrapDBCheckpoint() throws SQLException, BootstrapDBException, IOException {
    LOG.info("Validating bootstrap DB checkpoints !!");
    for (List<DatabusSubscription> subsList : _relayGroups.keySet()) {
        List<String> sourceNames = DatabusSubscription.getStrList(subsList);
        long scn = -1;
        try {
            scn = _dbDao.getMinWindowSCNFromStateTable(sourceNames, "bootstrap_producer_state");
        } catch (BootstrapDBException ex) {
            LOG.error("Got exception while trying to fetch SCN from bootstrap_producer_state for sources :" + sourceNames, ex);
            throw ex;
        }
        CheckpointPersistenceProvider provider = getCheckpointPersistenceProvider();
        Checkpoint cp = provider.loadCheckpoint(sourceNames);
        LOG.info("Bootstrap Producer SCN :" + scn + ", Checkpoint :" + cp);
        if (null != cp) {
            if (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION) {
                // Bootstrapping bootstrap Producer not yet supported !!
                String msg = "Bootstrap Producer starting from non-online consumption mode for sources :" + sourceNames + ", Ckpt :" + cp;
                LOG.error(msg);
                throw new BootstrapDBException(msg);
            } else {
                String msg = null;
                if (((cp.getWindowScn() > scn) && (scn > -1)) || ((cp.getWindowScn() < scn) && (scn > -1))) {
                    if (((cp.getWindowScn() > scn) && (scn > -1)))
                        LOG.warn("Non-Empty checkpint. Bootstrap Producer is at SCN:" + scn + ", while checkpoint is :" + cp + ", Could result in gap in event consumption. Repairing ckpt !!");
                    else
                        LOG.info("Non-Empty checkpoint. Bootstrap Producer is at SCN:" + scn + ", while checkpoint is :" + cp + ", Copying producer Scn to checkpoint !!");
                    cp.setWindowScn(scn);
                    cp.setWindowOffset(-1);
                    try {
                        provider.removeCheckpoint(sourceNames);
                        provider.storeCheckpoint(sourceNames, cp);
                        // Check if persisted properly
                        cp = provider.loadCheckpoint(sourceNames);
                        if ((null == cp) || (cp.getWindowScn() != scn) || (cp.getWindowOffset() != -1) || (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION)) {
                            msg = "Unable to repair and store the new checkpoint (" + cp + ") to make it same as producer SCN (" + scn + ") !!";
                            LOG.fatal(msg);
                            throw new BootstrapDBException(msg);
                        }
                    } catch (IOException ex) {
                        msg = "Unable to repair and store the new checkpoint (" + cp + ") to make it same as producer SCN (" + scn + ") !!";
                        LOG.fatal(msg, ex);
                        throw new BootstrapDBException(msg);
                    }
                }
            }
        } else {
            /**
         * Currently since bootstrapping is not available, a null ckpt would
         * result in flexible checkpoint and could result in gap !!
         */
            if (scn > -1) {
                String msg = "Empty checkpoint. Bootstrap Producer SCN is at SCN:" + scn + ", while checkpoint is null !! Could result in gap in event consumption. Repairing ckpt !!";
                LOG.warn(msg);
                cp = new Checkpoint();
                cp.setWindowScn(scn);
                cp.setWindowOffset(-1);
                cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
                try {
                    provider.removeCheckpoint(sourceNames);
                    provider.storeCheckpoint(sourceNames, cp);
                    // Check if persisted properly
                    cp = provider.loadCheckpoint(sourceNames);
                    if ((null == cp) || (cp.getWindowScn() != scn) || (cp.getWindowOffset() != -1) || (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION)) {
                        LOG.fatal("Unable to repair and store the checkpoint (" + cp + ") to make it same as producer SCN (" + scn + ") !!");
                        throw new BootstrapDBException(msg);
                    }
                } catch (IOException ex) {
                    msg = "Unable to repair and store the checkpoint (" + cp + ") to make it same as producer SCN (" + scn + ") !!";
                    LOG.fatal(msg, ex);
                    throw new BootstrapDBException(msg);
                }
            }
        }
    }
    LOG.info("Validating bootstrap DB checkpoints done successfully!!");
}