Examples with DatabusStreamConsumer com.linkedin.databus.client.pub.DatabusStreamConsumer used on opensource projects

Example 1 with DatabusStreamConsumer

use of com.linkedin.databus.client.pub.DatabusStreamConsumer in project databus by linkedin.

the class IntegratedDummyDatabusConsumer method initConn.

public void initConn(List<String> sources) throws IOException, InvalidConfigException, DatabusClientException, DatabusException {
    StringBuilder sourcesString = new StringBuilder();
    boolean firstSrc = true;
    for (String source : sources) {
        if (!firstSrc)
            sourcesString.append(",");
        firstSrc = false;
        sourcesString.append(source);
    }
    _fileBasedCallback.init();
    ArrayList<DatabusBootstrapConsumer> bootstrapCallbacks = new ArrayList<DatabusBootstrapConsumer>();
    bootstrapCallbacks.add(this);
    ArrayList<DatabusStreamConsumer> streamCallbacks = new ArrayList<DatabusStreamConsumer>();
    streamCallbacks.add(this);
    DatabusHttpClientImpl.Config clientConfigBuilder = new DatabusHttpClientImpl.Config();
    clientConfigBuilder.getContainer().getJmx().setJmxServicePort(5555);
    clientConfigBuilder.getContainer().setId(545454);
    clientConfigBuilder.getContainer().setHttpPort(8082);
    clientConfigBuilder.getCheckpointPersistence().setType(ProviderType.FILE_SYSTEM.toString());
    clientConfigBuilder.getCheckpointPersistence().getFileSystem().setRootDirectory("./integratedconsumer-checkpoints");
    clientConfigBuilder.getCheckpointPersistence().setClearBeforeUse(true);
    clientConfigBuilder.getRuntime().getBootstrap().setEnabled(true);
    DatabusSourcesConnection.Config srcDefaultConfig = new DatabusSourcesConnection.Config();
    srcDefaultConfig.setFreeBufferThreshold((int) (_maxEventBufferSize * 0.05));
    srcDefaultConfig.setCheckpointThresholdPct(80);
    // 60 sec before retries (unless disabled)
    srcDefaultConfig.setConsumerTimeBudgetMs(_useConsumerTimeout ? 60000 : 0);
    // max of 3 retries
    srcDefaultConfig.getDispatcherRetries().setMaxRetryNum(3);
    clientConfigBuilder.setConnectionDefaults(srcDefaultConfig);
    DbusEventBuffer.Config eventBufferConfig = clientConfigBuilder.getConnectionDefaults().getEventBuffer();
    eventBufferConfig.setMaxSize(_maxEventBufferSize);
    eventBufferConfig.setAverageEventSize(_maxReadBufferSize);
    // TODO: the following shall be used once we can set eventbuffer for bootstrap through the config builder (DDSDBUS-82)
    // For now, bootstrap buffer will use the same config as relay buffer.
    //clientConfigBuilder.getConnectionDefaults().
    DatabusHttpClientImpl.StaticConfig clientConfig = clientConfigBuilder.build();
    ServerInfoBuilder relayBuilder = clientConfig.getRuntime().getRelay("1");
    relayBuilder.setName("DefaultRelay");
    relayBuilder.setHost("localhost");
    relayBuilder.setPort(9000);
    relayBuilder.setSources(sourcesString.toString());
    ServerInfoBuilder bootstrapBuilder = clientConfig.getRuntime().getBootstrap().getService("2");
    bootstrapBuilder.setName("DefaultBootstrapServices");
    bootstrapBuilder.setHost("localhost");
    bootstrapBuilder.setPort(6060);
    bootstrapBuilder.setSources(sourcesString.toString());
    _dbusClient = new DatabusHttpClientImpl(clientConfig);
    _dbusClient.registerDatabusStreamListener(this, sources, null);
    _dbusClient.registerDatabusBootstrapListener(this, sources, null);
    // add pause processor
    try {
        _dbusClient.getProcessorRegistry().register(ConsumerPauseRequestProcessor.COMMAND_NAME, new ConsumerPauseRequestProcessor(null, this));
    } catch (ProcessorRegistrationConflictException e) {
        LOG.error("Failed to register " + ConsumerPauseRequestProcessor.COMMAND_NAME);
    }
}

Example 2 with DatabusStreamConsumer

use of com.linkedin.databus.client.pub.DatabusStreamConsumer in project databus by linkedin.

the class TestConsumerRegistration method testMultipleConsumers.

@Test
public void testMultipleConsumers() throws Exception {
    DatabusCombinedConsumer logConsumer1 = new LoggingConsumer();
    DatabusCombinedConsumer logConsumer2 = new LoggingConsumer();
    List<DatabusCombinedConsumer> lcs = new ArrayList<DatabusCombinedConsumer>();
    lcs.add(logConsumer1);
    lcs.add(logConsumer2);
    List<String> sources = new ArrayList<String>();
    ConsumerRegistration consumerReg = new ConsumerRegistration(lcs, sources, null);
    DatabusStreamConsumer cons = consumerReg.getConsumer();
    boolean condition = logConsumer1.equals(cons) || logConsumer2.equals(cons);
    Assert.assertEquals(condition, true);
    return;
}

Example 3 with DatabusStreamConsumer

use of com.linkedin.databus.client.pub.DatabusStreamConsumer in project databus by linkedin.

the class TestMultiConsumerCallback method test1StreamConsumerHappyPath.

@Test(groups = { "small", "functional" })
public void test1StreamConsumerHappyPath() {
    LOG.info("\n\nstarting test1StreamConsumerHappyPath()");
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 1, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 2, 2, (short) 3, keyCounts);
    eventsBuf.endEvents(100L);
    DatabusStreamConsumer mockConsumer = EasyMock.createStrictMock(DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 60000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    assert iter.hasNext() : "unable to read event";
    DbusEvent event1 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event2 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event3 = iter.next();
    initMockStreamConsumer3EventFullLifecycle(mockConsumer, event1, event2, event3, keyCounts);
    assert3EventFullLifecycle(callback, event1, event2, event3);
    EasyMock.verify(mockConsumer);
    assert keyCounts.get(1L).get() == 1 : "invalid number of event(1) calls: " + keyCounts.get(1L).get();
    assert keyCounts.get(2L).get() == 1 : "invalid number of event(2) calls:" + keyCounts.get(2L).get();
    assert keyCounts.get(3L).get() == 1 : "invalid number of event(3) calls:" + keyCounts.get(3L).get();
}

Example 4 with DatabusStreamConsumer

use of com.linkedin.databus.client.pub.DatabusStreamConsumer in project databus by linkedin.

the class TestMultiConsumerCallback method test1StreamConsumerCallFailure.

@Test(groups = { "small", "functional" })
public void test1StreamConsumerCallFailure() {
    LOG.info("\n\nstarting test1StreamConsumerCallFailure()");
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 1, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 2, 2, (short) 3, keyCounts);
    eventsBuf.endEvents(100L);
    DatabusStreamConsumer mockConsumer = EasyMock.createStrictMock(DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    assert iter.hasNext() : "unable to read event";
    DbusEvent event1 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event2 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event3 = iter.next();
    initMockFailingStreamConsumer3EventFullLifecycle(mockConsumer, event1, event2, event3, keyCounts);
    assert3EventFullLifecycleWithFailure(callback, event1, event2, event3);
    EasyMock.verify(mockConsumer);
    assert keyCounts.get(1L).get() == 1 : "invalid number of event(1) calls: " + keyCounts.get(1L).get();
    assert keyCounts.get(2L).get() == 1 : "invalid number of event(2) calls:" + keyCounts.get(2L).get();
    assert keyCounts.get(3L).get() == 1 : "invalid number of event(3) calls:" + keyCounts.get(3L).get();
}

Example 5 with DatabusStreamConsumer

use of com.linkedin.databus.client.pub.DatabusStreamConsumer in project databus by linkedin.

the class TestMultiConsumerCallback method testConsumersWithException.

private void testConsumersWithException(Throwable exception) {
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 1, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 2, 2, (short) 3, keyCounts);
    eventsBuf.endEvents(100L);
    DatabusStreamConsumer mockConsumer = EasyMock.createStrictMock(DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    assert iter.hasNext() : "unable to read event";
    DbusEvent event1 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event2 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event3 = iter.next();
    initMockExceptionStreamConsumer3EventFullLifecycle(mockConsumer, event1, event2, event3, keyCounts, exception);
    assert3EventFullLifecycleWithFailure(callback, event1, event2, event3);
    EasyMock.verify(mockConsumer);
    //    assert keyCounts.get(1L).get() == 1 : "invalid number of event(1) calls: " + keyCounts.get(1L).get();
    assert keyCounts.get(2L).get() == 1 : "invalid number of event(2) calls:" + keyCounts.get(2L).get();
    assert keyCounts.get(3L).get() == 1 : "invalid number of event(3) calls:" + keyCounts.get(3L).get();
}