Search in sources :

Example 41 with Hashtable

use of java.util.Hashtable in project databus by linkedin.

the class TestMultiConsumerCallback method test3IndependentStreamConsumersHappyPath.

@Test(groups = { "small", "functional" })
public void test3IndependentStreamConsumersHappyPath() {
    LOG.info("\n\nstarting test3IndependentStreamConsumersHappyPath()");
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 1, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 2, 2, (short) 3, keyCounts);
    eventsBuf.endEvents(100L);
    DatabusStreamConsumer mockConsumer1 = EasyMock.createStrictMock("consumer1", DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer1 = new SelectingDatabusCombinedConsumer(mockConsumer1);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    DatabusStreamConsumer mockConsumer2 = EasyMock.createStrictMock("consumer2", DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer2 = new SelectingDatabusCombinedConsumer(mockConsumer2);
    EasyMock.makeThreadSafe(mockConsumer2, true);
    DatabusStreamConsumer mockConsumer3 = EasyMock.createStrictMock("consumer3", DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer3 = new SelectingDatabusCombinedConsumer(mockConsumer3);
    EasyMock.makeThreadSafe(mockConsumer3, true);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg1 = new DatabusV2ConsumerRegistration(sdccMockConsumer1, sources, null);
    DatabusV2ConsumerRegistration consumerReg2 = new DatabusV2ConsumerRegistration(sdccMockConsumer2, sources, null);
    DatabusV2ConsumerRegistration consumerReg3 = new DatabusV2ConsumerRegistration(sdccMockConsumer3, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg1, consumerReg2, consumerReg3);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    assert iter.hasNext() : "unable to read event";
    DbusEvent event1 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event2 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event3 = iter.next();
    initMockStreamConsumer3EventFullLifecycle(mockConsumer1, event1, event2, event3, keyCounts);
    initMockStreamConsumer3EventFullLifecycle(mockConsumer2, event1, event2, event3, keyCounts);
    initMockStreamConsumer3EventFullLifecycle(mockConsumer3, event1, event2, event3, keyCounts);
    assert3EventFullLifecycle(callback, event1, event2, event3);
    EasyMock.verify(mockConsumer1);
    EasyMock.verify(mockConsumer2);
    EasyMock.verify(mockConsumer3);
    assert keyCounts.get(1L).get() == 3 : "invalid number of event(1) calls: " + keyCounts.get(1L).get();
    assert keyCounts.get(2L).get() == 3 : "invalid number of event(2) calls:" + keyCounts.get(2L).get();
    assert keyCounts.get(3L).get() == 3 : "invalid number of event(3) calls:" + keyCounts.get(3L).get();
}
Also used : DatabusStreamConsumer(com.linkedin.databus.client.pub.DatabusStreamConsumer) DbusEvent(com.linkedin.databus.core.DbusEvent) HashMap(java.util.HashMap) Hashtable(java.util.Hashtable) ArrayList(java.util.ArrayList) DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) IdNamePair(com.linkedin.databus.core.util.IdNamePair) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest) AfterTest(org.testng.annotations.AfterTest)

Example 42 with Hashtable

use of java.util.Hashtable in project databus by linkedin.

the class TestMultiConsumerCallback method test1ConsumerTimeout.

@Test
public void test1ConsumerTimeout() {
    LOG.info("\n\nstarting test1ConsumerTimeout()");
    //create dummy events
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 2, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 3, 1, (short) 2, keyCounts);
    eventsBuf.endEvents(100L);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    //skip over the first system event
    iter.next();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event1 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event2 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event3 = iter.next().createCopy();
    //make up some sources
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    //create the consumer mock up
    DatabusStreamConsumer mockConsumer1 = EasyMock.createStrictMock("consumer1", DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer1 = new SelectingDatabusCombinedConsumer(mockConsumer1);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer1, sources, null);
    EasyMock.expect(mockConsumer1.onStartConsumption()).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "startConsumption() called"), 150));
    EasyMock.expect(mockConsumer1.onStartDataEventSequence(null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartDataEventSequence() called"), 110));
    EasyMock.expect(mockConsumer1.onStartSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 40));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 50));
    EasyMock.expect(mockConsumer1.onDataEvent(event2, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(2) called"), 210));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 40));
    EasyMock.expect(mockConsumer1.onEndSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 50));
    EasyMock.replay(mockConsumer1);
    ConsumerCallbackStats consumerStatsCollector = new ConsumerCallbackStats(1, "test", "test", true, false, null);
    UnifiedClientStats unifiedStatsCollector = new UnifiedClientStats(1, "test", "test.unified");
    //Create and fire up callbacks
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 100, new StreamConsumerCallbackFactory(consumerStatsCollector, unifiedStatsCollector), consumerStatsCollector, unifiedStatsCollector, null, null);
    callback.setSourceMap(sourcesMap);
    ConsumerCallbackResult startConsumptionRes = callback.onStartConsumption();
    Assert.assertTrue(ConsumerCallbackResult.isFailure(startConsumptionRes), "startConsumption() failed");
    ConsumerCallbackResult startWindowRes = callback.onStartDataEventSequence(null);
    Assert.assertTrue(ConsumerCallbackResult.isFailure(startWindowRes), "startDataEventSequence() failed");
    ConsumerCallbackResult startSourceRes = callback.onStartSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startSourceRes), "startSources(source1) succeeded");
    ConsumerCallbackResult event1Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event1Res), "onDataEvent(1) succeeded");
    ConsumerCallbackResult event2Res = callback.onDataEvent(event2, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event2Res), "onDataEvent(2) queued up");
    ConsumerCallbackResult event3Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event3Res), "onDataEvent(1) queued up");
    ConsumerCallbackResult endSourceRes = callback.onEndSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isFailure(endSourceRes), "onEndSource fails because of timeout in onDataEvent(2)");
    EasyMock.reset(mockConsumer1);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    EasyMock.expect(mockConsumer1.onStartSource("source2", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 150)).times(0, 1);
    EasyMock.expect(mockConsumer1.onDataEvent(event3, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(3) called"), 40));
    EasyMock.expect(mockConsumer1.onEndSource("source2", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 60));
    EasyMock.replay(mockConsumer1);
    startSourceRes = callback.onStartSource("source2", null);
    Assert.assertTrue(ConsumerCallbackResult.isFailure(startSourceRes), "startSources(source2) fails");
    event1Res = callback.onDataEvent(event3, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event1Res), "onDataEvent(3) succeeded");
    endSourceRes = callback.onEndSource("source2", null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(endSourceRes), "onEndSource succeeds");
    long eventsErrProcessed = consumerStatsCollector.getNumErrorsProcessed();
    long totalEvents = consumerStatsCollector.getNumEventsReceived();
    long totalEventsProcessed = consumerStatsCollector.getNumEventsProcessed();
    System.out.println("eventsReceived = " + consumerStatsCollector.getNumEventsReceived() + " eventsProcessed=" + consumerStatsCollector.getNumEventsProcessed());
    System.out.println("eventsErrProcessed =" + consumerStatsCollector.getNumErrorsProcessed() + " eventsErrReceived=" + consumerStatsCollector.getNumErrorsReceived() + " totalEvents=" + consumerStatsCollector.getNumEventsReceived() + " totalEventsProcessed=" + totalEventsProcessed);
    //FIXME
    Assert.assertTrue(totalEvents >= totalEventsProcessed + eventsErrProcessed);
    Assert.assertTrue(eventsErrProcessed > 0);
    Assert.assertTrue(totalEventsProcessed < totalEvents);
//NOTE: We don't verify because all the canceled callbacks are not detected by EasyMock
//EasyMock.verify(mockConsumer1);
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) IdNamePair(com.linkedin.databus.core.util.IdNamePair) UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) DbusEvent(com.linkedin.databus.core.DbusEvent) DatabusStreamConsumer(com.linkedin.databus.client.pub.DatabusStreamConsumer) Hashtable(java.util.Hashtable) ConsumerCallbackStats(com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats) ConsumerCallbackResult(com.linkedin.databus.client.pub.ConsumerCallbackResult) DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest) AfterTest(org.testng.annotations.AfterTest)

Example 43 with Hashtable

use of java.util.Hashtable in project databus by linkedin.

the class TestMultiConsumerCallback method test2ConsumerTimeout.

@Test
public void test2ConsumerTimeout() {
    Logger log = Logger.getLogger("TestMultiConsumerCallback.test2ConsumerTimeout");
    //Logger.getRootLogger().setLevel(Level.INFO);
    log.info("\n\nstarting test2ConsumerTimeout()");
    log.info("create dummy events");
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 2, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 3, 1, (short) 2, keyCounts);
    eventsBuf.endEvents(100L);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    //skip over the first system event
    iter.next();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event1 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    DbusEvent event2 = iter.next().createCopy();
    Assert.assertTrue(iter.hasNext(), "unable to read event");
    log.info("make up some sources");
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    log.info("create the consumer mock up");
    DatabusStreamConsumer mockConsumer1 = EasyMock.createStrictMock("consumer1", DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer1 = new SelectingDatabusCombinedConsumer(mockConsumer1);
    EasyMock.makeThreadSafe(mockConsumer1, true);
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer1, sources, null);
    EasyMock.expect(mockConsumer1.onStartConsumption()).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "startConsumption() called"), 1));
    EasyMock.expect(mockConsumer1.onStartDataEventSequence(null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartDataEventSequence() called"), 1));
    EasyMock.expect(mockConsumer1.onStartSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 1));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 1));
    EasyMock.expect(mockConsumer1.onDataEvent(event2, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(2) called"), 1));
    EasyMock.expect(mockConsumer1.onDataEvent(event1, null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onDataEvet(1) called"), 1));
    EasyMock.expect(mockConsumer1.onEndSource("source1", null)).andAnswer(new SleepingAnswer<ConsumerCallbackResult>(new LoggedAnswer<ConsumerCallbackResult>(ConsumerCallbackResult.SUCCESS, LOG, Level.DEBUG, "onStartSource() called"), 1));
    EasyMock.replay(mockConsumer1);
    ConsumerCallbackStats consumerStatsCollector = new ConsumerCallbackStats(1, "test", "test", true, false, null);
    UnifiedClientStats unifiedStatsCollector = new UnifiedClientStats(1, "test", "test.unified");
    log.info("Create and fire up callbacks");
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    TimingOutMultiConsumerCallback callback = new TimingOutMultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 300, new StreamConsumerCallbackFactory(consumerStatsCollector, unifiedStatsCollector), consumerStatsCollector, unifiedStatsCollector, 3);
    callback.setSourceMap(sourcesMap);
    ConsumerCallbackResult startConsumptionRes = callback.onStartConsumption();
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startConsumptionRes), "startConsumption() succeeded: " + startConsumptionRes);
    ConsumerCallbackResult startWindowRes = callback.onStartDataEventSequence(null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startWindowRes), "startDataEventSequence() succeeded");
    ConsumerCallbackResult startSourceRes = callback.onStartSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(startSourceRes), "startSources(source1) succeeded");
    ConsumerCallbackResult event1Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event1Res), "onDataEvent(1) succeeded");
    ConsumerCallbackResult event2Res = callback.onDataEvent(event2, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event2Res), "onDataEvent(2) queued up");
    ConsumerCallbackResult event3Res = callback.onDataEvent(event1, null);
    Assert.assertTrue(ConsumerCallbackResult.isSuccess(event3Res), "onDataEvent(1) queued up");
    ConsumerCallbackResult endSourceRes = callback.onEndSource("source1", null);
    Assert.assertTrue(ConsumerCallbackResult.isFailure(endSourceRes), "onEndSource fails because of timeout in onDataEvent(2)");
    EasyMock.reset(mockConsumer1);
    log.info("test2ConsumerTimeout: end");
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Logger(org.apache.log4j.Logger) IdNamePair(com.linkedin.databus.core.util.IdNamePair) UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats) DbusEvent(com.linkedin.databus.core.DbusEvent) DatabusStreamConsumer(com.linkedin.databus.client.pub.DatabusStreamConsumer) Hashtable(java.util.Hashtable) ConsumerCallbackStats(com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats) ConsumerCallbackResult(com.linkedin.databus.client.pub.ConsumerCallbackResult) DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest) AfterTest(org.testng.annotations.AfterTest)

Example 44 with Hashtable

use of java.util.Hashtable in project databus by linkedin.

the class DummySuccessfulErrorCountingConsumer method testDispatcherRetriesExhausted.

@Test
public void testDispatcherRetriesExhausted() throws Exception {
    final Logger log = Logger.getLogger("TestDatabusHttpClient.testDispatcherRetriesExhausted");
    log.info("start");
    DatabusHttpClientImpl.Config clientConfig = new DatabusHttpClientImpl.Config();
    clientConfig.getConnectionDefaults().getPullerRetries().setMaxRetryNum(3);
    clientConfig.getConnectionDefaults().getDispatcherRetries().setMaxRetryNum(1);
    clientConfig.getContainer().getJmx().setRmiEnabled(false);
    int port = Utils.getAvailablePort(10100);
    clientConfig.getContainer().setHttpPort(0);
    final DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConfig);
    port = Utils.getAvailablePort(8898);
    @SuppressWarnings("unused") ServerInfo s1 = registerRelay(1, "relay1", new InetSocketAddress("localhost", port), "S1", client);
    //DummyStreamConsumer listener1 = new DummyStreamConsumer("consumer1");
    final DummySuccessfulErrorCountingConsumer listener1 = new DummySuccessfulErrorCountingConsumer("consumer1", true);
    client.registerDatabusStreamListener(listener1, null, "S1");
    Thread startThread = new Thread(new Runnable() {

        @Override
        public void run() {
            client.start();
        }
    }, "client start thread");
    startThread.start();
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            return client.getRelayConnections().size() == 1;
        }
    }, "client started", 1000, log);
    DatabusSourcesConnection dsc = client.getRelayConnections().get(0);
    RelayDispatcher rd = (RelayDispatcher) dsc.getRelayDispatcher();
    Assert.assertEquals(true, null != dsc);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    final String SOURCE1_SCHEMA_STR = "{\"name\":\"source1\",\"type\":\"record\",\"fields\":[{\"name\":\"s\",\"type\":\"string\"}]}";
    final String SOURCE2_SCHEMA_STR = "{\"name\":\"source2\",\"type\":\"record\",\"fields\":[{\"name\":\"s\",\"type\":\"string\"}]}";
    ;
    final String SOURCE3_SCHEMA_STR = "{\"name\":\"source3\",\"type\":\"record\",\"fields\":[{\"name\":\"s\",\"type\":\"string\"}]}";
    ;
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    int source1EventsNum = 2;
    int source2EventsNum = 2;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    // Send dummy e
    DbusEventBuffer eventsBuf = dsc.getDataEventsBuffer();
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L, null);
    rd.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    rd.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            return listener1._errorCount == 1;
        }
    }, "error callback", 1000, log);
    client.shutdown();
    log.info("done");
}
Also used : HashMap(java.util.HashMap) ServerInfo(com.linkedin.databus.client.pub.ServerInfo) InetSocketAddress(java.net.InetSocketAddress) ArrayList(java.util.ArrayList) Logger(org.apache.log4j.Logger) IdNamePair(com.linkedin.databus.core.util.IdNamePair) List(java.util.List) ArrayList(java.util.ArrayList) ConditionCheck(com.linkedin.databus2.test.ConditionCheck) Hashtable(java.util.Hashtable) Checkpoint(com.linkedin.databus.core.Checkpoint) DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RegisterResponseEntry(com.linkedin.databus2.core.container.request.RegisterResponseEntry) Test(org.testng.annotations.Test)

Example 45 with Hashtable

use of java.util.Hashtable in project databus by linkedin.

the class TestGenericDispatcher method testOneWindowHappyPath.

@Test(groups = { "small", "functional" })
public void testOneWindowHappyPath() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testOneWindowHappyPath");
    //log.setLevel(Level.DEBUG);
    log.info("start");
    int source1EventsNum = 2;
    int source2EventsNum = 2;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    final StateVerifyingStreamConsumer svsConsumer = new StateVerifyingStreamConsumer(null);
    //svsConsumer.getLog().setLevel(Level.DEBUG);
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(svsConsumer, keyCounts, srcidCounts);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    dispatcher.setSchemaIdCheck(false);
    Thread dispatcherThread = new Thread(dispatcher, "testOneWindowHappyPath-dispatcher");
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L, null);
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    Checkpoint cp = dispatcher.getDispatcherState().getLastSuccessfulCheckpoint();
    Assert.assertEquals(Checkpoint.FULLY_CONSUMED_WINDOW_OFFSET, cp.getWindowOffset());
    Assert.assertEquals(cp.getWindowScn(), cp.getPrevScn());
    for (long i = 1; i <= source1EventsNum + source2EventsNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
    }
    assertEquals("correct amount of callbacks for srcid 1", source1EventsNum, srcidCounts.get((short) 1).intValue());
    assertEquals("correct amount of callbacks for srcid 2", source2EventsNum, srcidCounts.get((short) 2).intValue());
    verifyNoLocks(log, eventsBuf);
    log.info("end\n");
}
Also used : DatabusV2ConsumerRegistration(com.linkedin.databus.client.consumer.DatabusV2ConsumerRegistration) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Logger(org.apache.log4j.Logger) IdNamePair(com.linkedin.databus.core.util.IdNamePair) List(java.util.List) ArrayList(java.util.ArrayList) StreamConsumerCallbackFactory(com.linkedin.databus.client.consumer.StreamConsumerCallbackFactory) AbstractDatabusStreamConsumer(com.linkedin.databus.client.consumer.AbstractDatabusStreamConsumer) DatabusStreamConsumer(com.linkedin.databus.client.pub.DatabusStreamConsumer) Hashtable(java.util.Hashtable) MultiConsumerCallback(com.linkedin.databus.client.consumer.MultiConsumerCallback) DatabusSubscription(com.linkedin.databus.core.data_model.DatabusSubscription) Checkpoint(com.linkedin.databus.core.Checkpoint) UncaughtExceptionTrackingThread(com.linkedin.databus.core.util.UncaughtExceptionTrackingThread) Checkpoint(com.linkedin.databus.core.Checkpoint) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) SelectingDatabusCombinedConsumer(com.linkedin.databus.client.consumer.SelectingDatabusCombinedConsumer) RegisterResponseEntry(com.linkedin.databus2.core.container.request.RegisterResponseEntry) Test(org.testng.annotations.Test)

Aggregations

Hashtable (java.util.Hashtable)1752 Test (org.junit.Test)374 ArrayList (java.util.ArrayList)355 AsynchClientTask (cbit.vcell.client.task.AsynchClientTask)219 ICompilationUnit (org.eclipse.jdt.core.ICompilationUnit)142 CompilationUnit (org.eclipse.jdt.core.dom.CompilationUnit)137 IPackageFragment (org.eclipse.jdt.core.IPackageFragment)136 HashMap (java.util.HashMap)104 IOException (java.io.IOException)91 Dictionary (java.util.Dictionary)91 Vector (java.util.Vector)88 File (java.io.File)86 Map (java.util.Map)84 Bundle (org.osgi.framework.Bundle)84 BundleContext (org.osgi.framework.BundleContext)78 Configuration (org.osgi.service.cm.Configuration)75 Enumeration (java.util.Enumeration)70 BundleDescription (org.eclipse.osgi.service.resolver.BundleDescription)70 State (org.eclipse.osgi.service.resolver.State)70 ServiceRegistration (org.osgi.framework.ServiceRegistration)65