Examples with DatabusStreamConsumer com.linkedin.databus.client.pub.DatabusStreamConsumer used on opensource projects

Example 21 with DatabusStreamConsumer

use of com.linkedin.databus.client.pub.DatabusStreamConsumer in project databus by linkedin.

the class TestGenericDispatcher method testOneWindowTwoIndependentConsumersHappyPath.

@Test(groups = { "small", "functional" })
public void testOneWindowTwoIndependentConsumersHappyPath() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testOneWindowTwoIndependentConsumersHappyPath");
    log.setLevel(Level.INFO);
    log.info("start");
    final Level saveLevel = Logger.getLogger("com.linkedin.databus.client").getLevel();
    //Logger.getLogger("com.linkedin.databus.client").setLevel(Level.DEBUG);
    final int source1EventsNum = 2;
    final int source2EventsNum = 2;
    final Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    final Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L);
    Hashtable<Long, AtomicInteger> keyCounts2 = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts2 = new Hashtable<Short, AtomicInteger>();
    for (Long key : keyCounts.keySet()) {
        keyCounts2.put(key, new AtomicInteger(0));
    }
    for (Short srcid : srcidCounts.keySet()) {
        srcidCounts2.put(srcid, new AtomicInteger(0));
    }
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(new StateVerifyingStreamConsumer(null), keyCounts, srcidCounts);
    DatabusStreamConsumer mockConsumer2 = new EventCountingConsumer(new StateVerifyingStreamConsumer(null), keyCounts2, srcidCounts2);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    SelectingDatabusCombinedConsumer sdccMockConsumer2 = new SelectingDatabusCombinedConsumer(mockConsumer2);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    DatabusV2ConsumerRegistration consumer2Reg = new DatabusV2ConsumerRegistration(sdccMockConsumer2, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg, consumer2Reg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher);
    dispatcherThread.setDaemon(true);
    log.info("starting dispatcher thread");
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    log.info("starting event dispatch");
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            return null != dispatcher.getDispatcherState().getEventsIterator() && !dispatcher.getDispatcherState().getEventsIterator().hasNext();
        }
    }, "all events processed", 5000, log);
    dispatcher.shutdown();
    log.info("all events processed");
    for (long i = 1; i <= source1EventsNum + source2EventsNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts2.get(i).intValue());
    }
    assertEquals("correct amount of callbacks for srcid 1", source1EventsNum, srcidCounts.get((short) 1).intValue());
    assertEquals("correct amount of callbacks for srcid 2", source2EventsNum, srcidCounts.get((short) 2).intValue());
    assertEquals("correct amount of callbacks for srcid 1", source1EventsNum, srcidCounts2.get((short) 1).intValue());
    assertEquals("correct amount of callbacks for srcid 2", source2EventsNum, srcidCounts2.get((short) 2).intValue());
    verifyNoLocks(null, eventsBuf);
    Logger.getLogger("com.linkedin.databus.client").setLevel(saveLevel);
    log.info("end\n");
}

Example 22 with DatabusStreamConsumer

use of com.linkedin.databus.client.pub.DatabusStreamConsumer in project databus by linkedin.

the class TestGenericDispatcher method testLargeWindowCheckpointFrequency.

@Test(groups = { "small", "functional" })
public void testLargeWindowCheckpointFrequency() throws Exception {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testLargeWindowCheckpointFrequency");
    log.info("start");
    /* Consumer creation */
    int timeTakenForEventInMs = 1;
    DatabusStreamConsumer tConsumer = new TimeoutTestConsumer(timeTakenForEventInMs);
    DatabusCombinedConsumer sdccTConsumer = new SelectingDatabusCombinedConsumer(tConsumer);
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    short srcId = 1;
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
    schemaMap.put(1L, l1);
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    List<String> sources = new ArrayList<String>();
    for (int i = 1; i <= 1; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccTConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    /* Source configuration */
    double thresholdChkptPct = 50.0;
    DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
    conf.setCheckpointThresholdPct(thresholdChkptPct);
    DatabusSourcesConnection.StaticConfig connConfig = conf.build();
    /* Generate events **/
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add(srcId);
    int numEvents = 100;
    int payloadSize = 20;
    int maxWindowSize = 80;
    DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
    Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, srcTestEvents) > 0);
    int size = 0;
    for (DbusEvent e : srcTestEvents) {
        if (!e.isControlMessage()) {
            size = e.size();
            break;
        }
    }
    //make buffer large enough to hold data
    int producerBufferSize = (int) (numEvents * size * 1.1);
    int individualBufferSize = producerBufferSize;
    int indexSize = producerBufferSize / 10;
    int stagingBufferSize = producerBufferSize;
    /*Event Buffer creation */
    final TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    /* Generic Dispatcher creation */
    TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("freqCkpt", connConfig, subs, new InMemoryPersistenceProvider(), dataEventsBuffer, mConsumer, true);
    /* Launch writer */
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, null);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    tEmitter.join();
    /* Launch dispatcher */
    Thread tDispatcher = new Thread(dispatcher);
    tDispatcher.start();
    /* Now initialize this damn state machine */
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    Thread.sleep(2000);
    System.out.println("Number of checkpoints = " + dispatcher.getNumCheckPoints());
    Assert.assertTrue(dispatcher.getNumCheckPoints() == 3);
    dispatcher.shutdown();
    verifyNoLocks(null, dataEventsBuffer);
    log.info("end\n");
}

Example 23 with DatabusStreamConsumer

use of com.linkedin.databus.client.pub.DatabusStreamConsumer in project databus by linkedin.

the class TestGenericDispatcher method testTwoWindowEventIndependentConsumersCallbackFailure.

@Test(groups = { "small", "functional" })
public void testTwoWindowEventIndependentConsumersCallbackFailure() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testTwoWindowEventIndependentConsumersCallbackFailure");
    log.info("start");
    int source1EventsNum = 4;
    int source2EventsNum = 5;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(200L);
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(new StateVerifyingStreamConsumer(new DataSourceFailingStreamConsumer("source2")), keyCounts, srcidCounts);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    Hashtable<Long, AtomicInteger> keyCounts2 = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts2 = new Hashtable<Short, AtomicInteger>();
    for (Long key : keyCounts.keySet()) {
        keyCounts2.put(key, new AtomicInteger(0));
    }
    for (Short srcid : srcidCounts.keySet()) {
        srcidCounts2.put(srcid, new AtomicInteger(0));
    }
    DatabusStreamConsumer mockConsumer2 = new EventCountingConsumer(new StateVerifyingStreamConsumer(null), keyCounts2, srcidCounts2);
    SelectingDatabusCombinedConsumer sdccMockConsumer2 = new SelectingDatabusCombinedConsumer(mockConsumer2);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    DatabusV2ConsumerRegistration consumerReg2 = new DatabusV2ConsumerRegistration(sdccMockConsumer2, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg, consumerReg2);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher);
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    for (long i = 1; i <= source1EventsNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts2.get(i).intValue());
    }
    for (long i = source2EventsNum + 1; i <= source1EventsNum + source2EventsNum; ++i) {
        assert keyCounts.get(1L + source1EventsNum).intValue() == 0 : "correct amount of callbacks for key " + i + ":" + keyCounts.get(i).intValue();
    }
    verifyNoLocks(null, eventsBuf);
    log.info("end\n");
}