Examples with DatabusStreamConsumer com.linkedin.databus.client.pub.DatabusStreamConsumer used on opensource projects

Example 11 with DatabusStreamConsumer

use of com.linkedin.databus.client.pub.DatabusStreamConsumer in project databus by linkedin.

the class TestGenericDispatcher method testOneWindowHappyPath.

@Test(groups = { "small", "functional" })
public void testOneWindowHappyPath() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testOneWindowHappyPath");
    //log.setLevel(Level.DEBUG);
    log.info("start");
    int source1EventsNum = 2;
    int source2EventsNum = 2;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    final StateVerifyingStreamConsumer svsConsumer = new StateVerifyingStreamConsumer(null);
    //svsConsumer.getLog().setLevel(Level.DEBUG);
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(svsConsumer, keyCounts, srcidCounts);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    dispatcher.setSchemaIdCheck(false);
    Thread dispatcherThread = new Thread(dispatcher, "testOneWindowHappyPath-dispatcher");
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L, null);
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    Checkpoint cp = dispatcher.getDispatcherState().getLastSuccessfulCheckpoint();
    Assert.assertEquals(Checkpoint.FULLY_CONSUMED_WINDOW_OFFSET, cp.getWindowOffset());
    Assert.assertEquals(cp.getWindowScn(), cp.getPrevScn());
    for (long i = 1; i <= source1EventsNum + source2EventsNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
    }
    assertEquals("correct amount of callbacks for srcid 1", source1EventsNum, srcidCounts.get((short) 1).intValue());
    assertEquals("correct amount of callbacks for srcid 2", source2EventsNum, srcidCounts.get((short) 2).intValue());
    verifyNoLocks(log, eventsBuf);
    log.info("end\n");
}

Example 12 with DatabusStreamConsumer

use of com.linkedin.databus.client.pub.DatabusStreamConsumer in project databus by linkedin.

the class TestGenericDispatcher method testMultiWindowsHappyPath.

@Test(groups = { "small", "functional" })
public void testMultiWindowsHappyPath() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testMultiWindowsHappyPath");
    //log.setLevel(Level.DEBUG);
    log.info("start");
    int source1EventsNum = 3;
    int source2EventsNum = 5;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    int windowsNum = 3;
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    int curEventNum = 1;
    for (int w = 0; w < windowsNum; ++w) {
        eventsBuf.startEvents();
        initBufferWithEvents(eventsBuf, curEventNum, source1EventsNum, (short) 1, keyCounts, srcidCounts);
        curEventNum += source1EventsNum;
        initBufferWithEvents(eventsBuf, curEventNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
        curEventNum += source2EventsNum;
        eventsBuf.endEvents(100L * (w + 1), null);
    }
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(new StateVerifyingStreamConsumer(null), keyCounts, srcidCounts);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher);
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    for (long i = 1; i < curEventNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
    }
    assertEquals("correct amount of callbacks for srcid 1", windowsNum * source1EventsNum, srcidCounts.get((short) 1).intValue());
    assertEquals("correct amount of callbacks for srcid 2", windowsNum * source2EventsNum, srcidCounts.get((short) 2).intValue());
    verifyNoLocks(null, eventsBuf);
    log.info("end\n");
}

Example 13 with DatabusStreamConsumer

use of com.linkedin.databus.client.pub.DatabusStreamConsumer in project databus by linkedin.

the class TestGenericDispatcher method testMetadataSchema.

@Test
public void testMetadataSchema() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testMetadataSchema");
    //log.setLevel(Level.DEBUG);
    log.info("start");
    int source1EventsNum = 2;
    int source2EventsNum = 2;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    final StateVerifyingStreamConsumer svsConsumer = new StateVerifyingStreamConsumer(null);
    //svsConsumer.getLog().setLevel(Level.DEBUG);
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(svsConsumer, keyCounts, srcidCounts);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher, "testMetadataSchema-dispatcher");
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    //add meta data schema
    byte[] crc32 = { 0x01, 0x02, 0x03, 0x04 };
    List<RegisterResponseMetadataEntry> lMeta = new ArrayList<RegisterResponseMetadataEntry>();
    lMeta.add(new RegisterResponseMetadataEntry((short) 1, META1_SCHEMA_STR, crc32));
    lMeta.add(new RegisterResponseMetadataEntry((short) 2, META2_SCHEMA_STR, crc32));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap, lMeta));
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L, null);
    //check standard execution of callbacks
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    for (long i = 1; i <= source1EventsNum + source2EventsNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
    }
    assertEquals("incorrect amount of callbacks for srcid 1", source1EventsNum, srcidCounts.get((short) 1).intValue());
    assertEquals("incorrect amount of callbacks for srcid 2", source2EventsNum, srcidCounts.get((short) 2).intValue());
    //check metadata schemas
    EventCountingConsumer myCons = (EventCountingConsumer) mockConsumer;
    VersionedSchema metadataSchema = myCons.getMetadataSchema();
    Assert.assertTrue(null != metadataSchema);
    log.info("Metadata VersionedSchema = " + metadataSchema);
    Assert.assertEquals(metadataSchema.getVersion(), 2);
    Assert.assertEquals(metadataSchema.getSchemaBaseName(), SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE);
    verifyNoLocks(log, eventsBuf);
    log.info("end\n");
}

Example 14 with DatabusStreamConsumer

use of com.linkedin.databus.client.pub.DatabusStreamConsumer in project databus by linkedin.

the class DatabusHttpClientImpl method registerDatabusStreamListener.

/**
   *
   * @param listeners
   * @param sources
   * @param filterConfig
   * @throws DatabusClientException
   */
public synchronized void registerDatabusStreamListener(DatabusStreamConsumer[] listeners, List<String> sources, DbusKeyCompositeFilterConfig filterConfig) throws DatabusClientException {
    List<DatabusStreamConsumer> listenersList = Arrays.asList(listeners);
    List<SelectingDatabusCombinedConsumer> sdccListenersList = SelectingDatabusCombinedConsumerFactory.convertListOfStreamConsumers(listenersList);
    List<DatabusCombinedConsumer> dccListenersList = new ArrayList<DatabusCombinedConsumer>();
    for (SelectingDatabusCombinedConsumer sdcc : sdccListenersList) {
        dccListenersList.add(sdcc);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(dccListenersList, sources, filterConfig);
    registerDatabusListener(consumerReg, _relayGroups, getRelayGroupStreamConsumers(), DatabusSubscription.createSubscriptionList(sources));
}

Example 15 with DatabusStreamConsumer

use of com.linkedin.databus.client.pub.DatabusStreamConsumer in project databus by linkedin.

the class TestMultiConsumerCallback method test3GroupedStreamConsumersHappyPath.

@Test(groups = { "small", "functional" })
public void test3GroupedStreamConsumersHappyPath() {
    LOG.info("\n\nstarting test3GroupedStreamConsumersHappyPath()");
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 1, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 2, 2, (short) 3, keyCounts);
    eventsBuf.endEvents(100L);
    DatabusStreamConsumer mockConsumer1 = EasyMock.createStrictMock("consumer1", DatabusStreamConsumer.class);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    DatabusStreamConsumer mockConsumer2 = EasyMock.createStrictMock("consumer2", DatabusStreamConsumer.class);
    EasyMock.makeThreadSafe(mockConsumer2, true);
    DatabusStreamConsumer mockConsumer3 = EasyMock.createStrictMock("consumer3", DatabusStreamConsumer.class);
    EasyMock.makeThreadSafe(mockConsumer3, true);
    DatabusCombinedConsumer sdccMockConsumer1 = new SelectingDatabusCombinedConsumer(mockConsumer1);
    DatabusCombinedConsumer sdccMockConsumer2 = new SelectingDatabusCombinedConsumer(mockConsumer2);
    DatabusCombinedConsumer sdccMockConsumer3 = new SelectingDatabusCombinedConsumer(mockConsumer3);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg1 = new DatabusV2ConsumerRegistration(Arrays.asList(sdccMockConsumer1, sdccMockConsumer2, sdccMockConsumer3), sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg1);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    assert iter.hasNext() : "unable to read event";
    DbusEvent event1 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event2 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event3 = iter.next();
    initMockStreamConsumer3OptEventFullLifecycle(mockConsumer1, event1, event2, event3, keyCounts);
    initMockStreamConsumer3OptEventFullLifecycle(mockConsumer2, event1, event2, event3, keyCounts);
    initMockStreamConsumer3OptEventFullLifecycle(mockConsumer3, event1, event2, event3, keyCounts);
    assert3EventFullLifecycle(callback, event1, event2, event3);
    EasyMock.verify(mockConsumer1);
    EasyMock.verify(mockConsumer2);
    EasyMock.verify(mockConsumer3);
    assert (keyCounts.get(1L).get() + keyCounts.get(2L).get() + keyCounts.get(3L).get()) == 3 : "invalid number of calls: " + keyCounts.get(1L).get() + "," + keyCounts.get(2L).get() + "," + keyCounts.get(3L).get();
}