Examples with DatabusV2ConsumerRegistration com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration used on opensource projects

Example 11 with DatabusV2ConsumerRegistration

use of com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration in project databus by linkedin.

the class RelayEventProducer method createDatabusV2ConsumerRegistration.

static List<DatabusV2ConsumerRegistration> createDatabusV2ConsumerRegistration(DatabusCombinedConsumer consumer, List<String> sourcesStrList) {
    ArrayList<DatabusV2ConsumerRegistration> regs = new ArrayList<DatabusV2ConsumerRegistration>();
    // No server side filtering just as yet
    regs.add(new DatabusV2ConsumerRegistration(consumer, sourcesStrList, null));
    return regs;
}

Example 12 with DatabusV2ConsumerRegistration

use of com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration in project databus by linkedin.

the class DatabusHttpClientImpl method registerDatabusStreamListener.

/**
   *
   * @param listeners
   * @param sources
   * @param filterConfig
   * @throws DatabusClientException
   */
public synchronized void registerDatabusStreamListener(DatabusStreamConsumer[] listeners, List<String> sources, DbusKeyCompositeFilterConfig filterConfig) throws DatabusClientException {
    List<DatabusStreamConsumer> listenersList = Arrays.asList(listeners);
    List<SelectingDatabusCombinedConsumer> sdccListenersList = SelectingDatabusCombinedConsumerFactory.convertListOfStreamConsumers(listenersList);
    List<DatabusCombinedConsumer> dccListenersList = new ArrayList<DatabusCombinedConsumer>();
    for (SelectingDatabusCombinedConsumer sdcc : sdccListenersList) {
        dccListenersList.add(sdcc);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(dccListenersList, sources, filterConfig);
    registerDatabusListener(consumerReg, _relayGroups, getRelayGroupStreamConsumers(), DatabusSubscription.createSubscriptionList(sources));
}

Example 13 with DatabusV2ConsumerRegistration

use of com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration in project databus by linkedin.

the class TestGenericDispatcher method testTwoWindowEventCallbackFailure.

@Test(groups = { "small", "functional" })
public void testTwoWindowEventCallbackFailure() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testTwoWindowEventCallbackFailure");
    log.info("start");
    int source1EventsNum = 2;
    int source2EventsNum = 2;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(200L);
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(new StateVerifyingStreamConsumer(new DataEventFailingStreamConsumer((short) 2)), keyCounts, srcidCounts);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher);
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    for (long i = 1; i <= source1EventsNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
    }
    for (long i = source2EventsNum + 1; i <= source1EventsNum + source2EventsNum; ++i) {
        assert keyCounts.get(1L + source1EventsNum).intValue() > 1 : "correct amount of callbacks for key " + i + ":" + keyCounts.get(i).intValue();
    }
    verifyNoLocks(null, eventsBuf);
    log.info("end\n");
}

Example 14 with DatabusV2ConsumerRegistration

use of com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration in project databus by linkedin.

the class TestGenericDispatcher method testOneWindowTwoGroupedConsumersHappyPath.

@Test(groups = { "small", "functional" })
public void testOneWindowTwoGroupedConsumersHappyPath() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testOneWindowTwoGroupedConsumersHappyPath");
    log.info("start");
    int source1EventsNum = 2;
    int source2EventsNum = 2;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L);
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(new StateVerifyingStreamConsumer(null), keyCounts, srcidCounts);
    DatabusStreamConsumer mockConsumer2 = new EventCountingConsumer(new StateVerifyingStreamConsumer(null), keyCounts, srcidCounts);
    DatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    DatabusCombinedConsumer sdccMockConsumer2 = new SelectingDatabusCombinedConsumer(mockConsumer2);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(Arrays.asList(sdccMockConsumer, sdccMockConsumer2), sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher);
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    for (long i = 1; i <= source1EventsNum + source2EventsNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
    }
    assertEquals("correct amount of callbacks for srcid 1", source1EventsNum, srcidCounts.get((short) 1).intValue());
    assertEquals("correct amount of callbacks for srcid 2", source2EventsNum, srcidCounts.get((short) 2).intValue());
    verifyNoLocks(null, eventsBuf);
    log.info("end\n");
}

Example 15 with DatabusV2ConsumerRegistration

use of com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration in project databus by linkedin.

the class TestGenericDispatcher method runPartialWindowCheckpointPersistence.

/**
     *
     * @param numEvents : number of events in buffer
     * @param maxWindowSize : window size expressed as number of events
     * @param numFailWindow : nth end-of-window that will fail
     * @throws Exception
     */
void runPartialWindowCheckpointPersistence(int numEvents, int maxWindowSize, int numFailWindow) throws Exception {
    /* Experiment setup */
    int payloadSize = 20;
    int numCheckpoints = numEvents / maxWindowSize;
    /* Consumer creation */
    //setup consumer to fail on data callback at the nth event
    int timeTakenForDataEventInMs = 1;
    int timeTakenForControlEventInMs = 1;
    int numFailCheckpointEvent = 0;
    int numFailDataEvent = 0;
    int numFailEndWindow = numFailWindow;
    int numFailures = 1;
    //fail at the specified window; no retries; so the dispatcher should stop having written one window; but having checkpointed the other
    //thanks to a very small checkpoint frequency threshold
    TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForDataEventInMs, timeTakenForControlEventInMs, numFailCheckpointEvent, numFailDataEvent, numFailEndWindow, numFailures);
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    short srcId = 1;
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
    schemaMap.put(1L, l1);
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    List<String> sources = new ArrayList<String>();
    for (int i = 1; i <= 1; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    long consumerTimeBudgetMs = 60 * 1000;
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    //Single threaded execution of consumer
    MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    /* Generate events **/
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    Vector<Short> srcIdList = new Vector<Short>();
    srcIdList.add(srcId);
    DbusEventGenerator evGen = new DbusEventGenerator(15000, srcIdList);
    //Assumption: generates events with  non-decreasing timestamps
    Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, true, srcTestEvents) > 0);
    int totalSize = 0;
    int maxSize = 0;
    for (DbusEvent e : srcTestEvents) {
        totalSize += e.size();
        maxSize = (e.size() > maxSize) ? e.size() : maxSize;
    }
    /* Source configuration */
    double thresholdChkptPct = 5.0;
    DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
    conf.setCheckpointThresholdPct(thresholdChkptPct);
    conf.getDispatcherRetries().setMaxRetryNum(0);
    conf.setFreeBufferThreshold(maxSize);
    conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
    int freeBufferThreshold = conf.getFreeBufferThreshold();
    DatabusSourcesConnection.StaticConfig connConfig = conf.build();
    //make buffer large enough to hold data; the control events are large that contain checkpoints
    int producerBufferSize = totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
    int individualBufferSize = producerBufferSize;
    int indexSize = producerBufferSize / 10;
    int stagingBufferSize = producerBufferSize;
    /*Event Buffer creation */
    TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    /* Generic Dispatcher creation */
    InMemoryPersistenceProvider cpPersister = new InMemoryPersistenceProvider();
    TestDispatcher<DatabusCombinedConsumer> dispatcher = new TestDispatcher<DatabusCombinedConsumer>("OnlinePartialWindowCheckpointPersistence", connConfig, subs, cpPersister, dataEventsBuffer, mConsumer, true);
    /* Launch writer */
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, 0, null);
    Thread tEmitter = new Thread(eventProducer);
    //be generous ; use worst case for num control events
    long waitTimeMs = (numEvents * timeTakenForDataEventInMs + numEvents * timeTakenForControlEventInMs) * 4;
    tEmitter.start();
    tEmitter.join(waitTimeMs);
    /* Launch dispatcher */
    Thread tDispatcher = new Thread(dispatcher);
    tDispatcher.start();
    /* Now initialize this  state machine */
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    //wait for dispatcher to finish reading the events;
    tDispatcher.join(waitTimeMs);
    Assert.assertFalse(tEmitter.isAlive());
    Assert.assertFalse(tDispatcher.isAlive());
    LOG.info("tConsumer: " + tConsumer);
    HashMap<List<String>, Checkpoint> cps = cpPersister.getCheckpoints();
    for (Map.Entry<List<String>, Checkpoint> i : cps.entrySet()) {
        Checkpoint cp = i.getValue();
        LOG.info("checkpoint=" + cp);
        Assert.assertEquals(cp.getWindowOffset().longValue(), -1L);
        //check if lastSeenCheckpoint by consumer is higher than scn persisted
        Assert.assertTrue(tConsumer.getLastSeenCheckpointScn() > cp.getWindowScn());
        //the latest event seen should be newer (or at least as new) as the checkpoint
        Assert.assertTrue(tConsumer.getLastTsInNanosOfEvent() >= tConsumer.getLastTsInNanosOfWindow());
        if (tConsumer.getLastSeenWindowScn() > 0) {
            Assert.assertEquals(cp.getWindowScn(), tConsumer.getLastSeenWindowScn());
            //check if the timestamp in checkpoint is the same as checkpoint of last completed window (ts of last event of the window)
            Assert.assertEquals(tConsumer.getLastTsInNanosOfWindow(), cp.getTsNsecs());
        } else {
            //not even one window was processed before error; expect uninitialized timestamp
            Assert.assertEquals(Checkpoint.UNSET_TS_NSECS, cp.getTsNsecs());
        }
    }
}