Examples with DatabusSubscription com.linkedin.databus.core.data_model.DatabusSubscription used on opensource projects

Example 16 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class TestGenericDispatcher method testMultiWindowsHappyPath.

@Test(groups = { "small", "functional" })
public void testMultiWindowsHappyPath() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testMultiWindowsHappyPath");
    //log.setLevel(Level.DEBUG);
    log.info("start");
    int source1EventsNum = 3;
    int source2EventsNum = 5;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    int windowsNum = 3;
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    int curEventNum = 1;
    for (int w = 0; w < windowsNum; ++w) {
        eventsBuf.startEvents();
        initBufferWithEvents(eventsBuf, curEventNum, source1EventsNum, (short) 1, keyCounts, srcidCounts);
        curEventNum += source1EventsNum;
        initBufferWithEvents(eventsBuf, curEventNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
        curEventNum += source2EventsNum;
        eventsBuf.endEvents(100L * (w + 1), null);
    }
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(new StateVerifyingStreamConsumer(null), keyCounts, srcidCounts);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher);
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    for (long i = 1; i < curEventNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
    }
    assertEquals("correct amount of callbacks for srcid 1", windowsNum * source1EventsNum, srcidCounts.get((short) 1).intValue());
    assertEquals("correct amount of callbacks for srcid 2", windowsNum * source2EventsNum, srcidCounts.get((short) 2).intValue());
    verifyNoLocks(null, eventsBuf);
    log.info("end\n");
}

Example 17 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class TestGenericDispatcher method testRollbackFailure.

@Test(groups = { "small", "functional" })
public /**
     * Test a relatively big window which forces a checkpoint which fails. Then trigger
     * a rollback. The dispatcher should shutdown. */
void testRollbackFailure() throws InvalidConfigException {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testRollbackFailure");
    //log.setLevel(Level.DEBUG);
    log.info("start");
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 100, (short) 1, null, null);
    eventsBuf.endEvents(100L, null);
    RollbackFailingConsumer mockConsumer = new RollbackFailingConsumer(new StateVerifyingStreamConsumer(null), LOG, 2);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(mockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    DatabusSourcesConnection.Config connCfgBuilder = new DatabusSourcesConnection.Config();
    connCfgBuilder.setConsumerParallelism(1);
    connCfgBuilder.setEventBuffer(_generic100KBufferConfig);
    connCfgBuilder.setFreeBufferThreshold(10000);
    connCfgBuilder.setConsumerTimeBudgetMs(1000);
    connCfgBuilder.setCheckpointThresholdPct(5.0);
    connCfgBuilder.getDispatcherRetries().setMaxRetryNum(10);
    DatabusSourcesConnection.StaticConfig connStaticCfg = connCfgBuilder.build();
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", connStaticCfg, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher);
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    try {
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                LOG.info(dispatcher.getStatus().getStatus().toString());
                LOG.info(String.valueOf(dispatcher.getStatus().getStatus() == DatabusComponentStatus.Status.SHUTDOWN));
                return dispatcher.getStatus().getStatus() == DatabusComponentStatus.Status.SHUTDOWN;
            }
        }, "dispatcher shtudown", 50000, LOG);
    } finally {
        dispatcher.shutdown();
    }
    verifyNoLocks(null, eventsBuf);
    log.info("end\n");
}

Example 18 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class TestGenericDispatcher method testMetadataSchema.

@Test
public void testMetadataSchema() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testMetadataSchema");
    //log.setLevel(Level.DEBUG);
    log.info("start");
    int source1EventsNum = 2;
    int source2EventsNum = 2;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    final StateVerifyingStreamConsumer svsConsumer = new StateVerifyingStreamConsumer(null);
    //svsConsumer.getLog().setLevel(Level.DEBUG);
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(svsConsumer, keyCounts, srcidCounts);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher, "testMetadataSchema-dispatcher");
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    //add meta data schema
    byte[] crc32 = { 0x01, 0x02, 0x03, 0x04 };
    List<RegisterResponseMetadataEntry> lMeta = new ArrayList<RegisterResponseMetadataEntry>();
    lMeta.add(new RegisterResponseMetadataEntry((short) 1, META1_SCHEMA_STR, crc32));
    lMeta.add(new RegisterResponseMetadataEntry((short) 2, META2_SCHEMA_STR, crc32));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap, lMeta));
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L, null);
    //check standard execution of callbacks
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    for (long i = 1; i <= source1EventsNum + source2EventsNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
    }
    assertEquals("incorrect amount of callbacks for srcid 1", source1EventsNum, srcidCounts.get((short) 1).intValue());
    assertEquals("incorrect amount of callbacks for srcid 2", source2EventsNum, srcidCounts.get((short) 2).intValue());
    //check metadata schemas
    EventCountingConsumer myCons = (EventCountingConsumer) mockConsumer;
    VersionedSchema metadataSchema = myCons.getMetadataSchema();
    Assert.assertTrue(null != metadataSchema);
    log.info("Metadata VersionedSchema = " + metadataSchema);
    Assert.assertEquals(metadataSchema.getVersion(), 2);
    Assert.assertEquals(metadataSchema.getSchemaBaseName(), SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE);
    verifyNoLocks(log, eventsBuf);
    log.info("end\n");
}

Example 19 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class TestDatabusSourcesConnection method testLoggerNameV2_1.

@Test
public void testLoggerNameV2_1() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException {
    String source = "com.linkedin.events.dbName.tableName";
    List<DatabusSubscription> ds = DatabusSubscription.createSubscriptionList(Arrays.asList(source));
    DatabusSourcesConnection dsc = DatabusSourcesConnection.createDatabusSourcesConnectionForTesting();
    String partName = DatabusSubscription.getPrettyNameForListOfSubscriptions(ds);
    Assert.assertEquals(partName, "dbName_tableName");
    Class<?>[] arg1 = new Class<?>[] { List.class, String.class };
    Method constructPrettyNameForLogging = DatabusSourcesConnection.class.getDeclaredMethod("constructPrettyNameForLogging", arg1);
    constructPrettyNameForLogging.setAccessible(true);
    Object prettyName = constructPrettyNameForLogging.invoke(dsc, ds, "test_1234");
    Assert.assertEquals(prettyName, "dbName_tableName_test_1234");
}

Example 20 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class RelayEventProducer method createDatabusSourcesConnection.

public static DatabusSourcesConnection createDatabusSourcesConnection(String producerName, int id, String serverName, String subscriptionString, DatabusCombinedConsumer consumer, long internalBufferMaxSize, int largestEventSize, long consumerTimeoutMs, long pollIntervalMs, long connTimeoutMs, int consumerParallelism, boolean blockingBuffer, DatabusClientNettyThreadPools nettyThreadPools, int noEventsTimeoutSec, int maxEventVersion, int initReadBufferSize) throws InvalidConfigException {
    // the assumption here is that the list of subscriptions will become the
    // list of sources hosted by the relay
    Set<ServerInfo> relayServices = createServerInfo(serverName, subscriptionString);
    // null bootstrapService
    Set<ServerInfo> bootstrapServices = null;
    // create subscription objects based on what is required by subscription
    String[] subscriptionList = subscriptionString.split(",");
    List<DatabusSubscription> subsList = DatabusSubscription.createSubscriptionList(Arrays.asList(subscriptionList));
    List<String> sourcesStrList = DatabusSubscription.getStrList(subsList);
    LOG.info("The sourcesList is " + sourcesStrList);
    // create registration objects with consumers
    List<DatabusV2ConsumerRegistration> relayConsumers = createDatabusV2ConsumerRegistration(consumer, sourcesStrList);
    List<DatabusV2ConsumerRegistration> bstConsumers = null;
    // setup sources connection config
    DatabusSourcesConnection.Config confBuilder = new DatabusSourcesConnection.Config();
    confBuilder.setId(id);
    // consume whatever is in relay
    confBuilder.setConsumeCurrent(true);
    //this is set to false as the behaviour is to read the latest SCN when SCN is not found, the buffer isn't cleared
    //as such , so a possibility of gaps in events arises. What we want ideally is to clear existing buffer and then consume from latest SCN
    confBuilder.setReadLatestScnOnError(false);
    // 10s max consumer timeout
    confBuilder.setConsumerTimeBudgetMs(consumerTimeoutMs);
    // poll interval in ms and infinite retry;
    confBuilder.getPullerRetries().setMaxRetryNum(-1);
    confBuilder.getPullerRetries().setInitSleep(pollIntervalMs);
    confBuilder.setConsumerParallelism(consumerParallelism);
    confBuilder.getDispatcherRetries().setMaxRetryNum(1);
    // internal buffer conf
    DbusEventBuffer.Config bufferConf = new DbusEventBuffer.Config();
    bufferConf.setMaxSize(internalBufferMaxSize);
    bufferConf.setAllocationPolicy("DIRECT_MEMORY");
    if (initReadBufferSize > 0) {
        bufferConf.setAverageEventSize(initReadBufferSize);
    }
    bufferConf.setMaxEventSize(largestEventSize);
    //client buffer's scn index- not used
    bufferConf.setScnIndexSize(64 * 1024);
    String queuePolicy = blockingBuffer ? "BLOCK_ON_WRITE" : "OVERWRITE_ON_WRITE";
    bufferConf.setQueuePolicy(queuePolicy);
    //get appropriate checkpointThresholdPct
    double newCkptPct = confBuilder.computeSafeCheckpointThresholdPct(bufferConf);
    if (newCkptPct < 5.0 || newCkptPct > 95.0) {
        LOG.warn("Not setting required checkpointThresholdPct : " + newCkptPct + "to  accommodate largestEventSize= " + largestEventSize + " in buffer of size " + bufferConf.getMaxSize());
        if (newCkptPct <= 0.0) {
            //unlikely to happen: if it does retain default
            newCkptPct = confBuilder.getCheckpointThresholdPct();
        }
        if (newCkptPct < 5.0) {
            newCkptPct = 5.0;
        } else if (newCkptPct > 95.0) {
            newCkptPct = 95.0;
        }
    }
    LOG.info("Setting checkpointThresholdPct:" + newCkptPct);
    confBuilder.setCheckpointThresholdPct(newCkptPct);
    confBuilder.setEventBuffer(bufferConf);
    confBuilder.setNoEventsConnectionResetTimeSec(noEventsTimeoutSec);
    DatabusSourcesConnection.StaticConfig connConfig = confBuilder.build();
    // internal buffers of databus client library
    DbusEventBuffer buffer = new DbusEventBuffer(connConfig.getEventBuffer());
    buffer.start(0);
    DbusEventBuffer bootstrapBuffer = null;
    // Create threadpools and netty managers
    // read - write timeout in ms
    long readTimeoutMs = connTimeoutMs;
    long writeTimeoutMs = connTimeoutMs;
    long bstReadTimeoutMs = connTimeoutMs;
    int protocolVersion = 2;
    // connection factory
    NettyHttpConnectionFactory defaultConnFactory = new NettyHttpConnectionFactory(nettyThreadPools.getBossExecutorService(), nettyThreadPools.getIoExecutorService(), null, nettyThreadPools.getTimer(), writeTimeoutMs, readTimeoutMs, bstReadTimeoutMs, protocolVersion, maxEventVersion, nettyThreadPools.getChannelGroup());
    // Create Thread pool for consumer threads
    int maxThreadsNum = 1;
    int keepAliveMs = 1000;
    ThreadPoolExecutor defaultExecutorService = new OrderedMemoryAwareThreadPoolExecutor(maxThreadsNum, 0, 0, keepAliveMs, TimeUnit.MILLISECONDS);
    ConsumerCallbackStats relayConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.cons", producerName + ".inbound.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
    ConsumerCallbackStats bootstrapConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.bs.cons", producerName + ".inbound.bs.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
    UnifiedClientStats unifiedClientStats = new UnifiedClientStats(id, producerName + ".inbound.unified.cons", producerName + ".inbound.unified.cons", true, false, UnifiedClientStats.DEFAULT_DEADNESS_THRESHOLD_MS, null, ManagementFactory.getPlatformMBeanServer());
    DatabusRelayConnectionFactory relayConnFactory = defaultConnFactory;
    DatabusBootstrapConnectionFactory bootstrapConnFactory = defaultConnFactory;
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sourcesStrList);
    DatabusSourcesConnection conn = new DatabusSourcesConnection(connConfig, subsList, relayServices, bootstrapServices, relayConsumers, bstConsumers, buffer, bootstrapBuffer, defaultExecutorService, // getContainerStatsCollector(),
    null, // getInboundEventStatisticsCollector(),
    null, // getBootstrapEventsStatsCollector(),
    null, // relay callback stats
    relayConsumerStats, // bootstrap callback stats
    bootstrapConsumerStats, // combined relay/bootstrap callback stats
    unifiedClientStats, // getCheckpointPersistenceProvider(),
    null, relayConnFactory, bootstrapConnFactory, // getHttpStatsCollector(),
    null, // RegistrationId
    null, null, new DbusEventV2Factory(), // TODO Get the ref to factory from HttpRelay.
    connStateFactory);
    return conn;
}