Examples with DatabusSubscription com.linkedin.databus.core.data_model.DatabusSubscription used on opensource projects

Example 46 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class MockBootstrapConnection method createBootstrapPullThread.

private BootstrapPullThread createBootstrapPullThread(boolean failBsConnection, boolean throwBSConnException, boolean muteTransition, boolean readDataThrowException, boolean readDataException, String exceptionName, int freeReadSpace, int numBytesRead, boolean phaseCompleted, long startScn, long targetScn, String... sourceNames) throws Exception {
    List<String> sources = Arrays.asList(sourceNames);
    Properties clientProps = new Properties();
    clientProps.setProperty("client.container.httpPort", "0");
    clientProps.setProperty("client.container.jmx.rmiEnabled", "false");
    clientProps.setProperty("client.runtime.bootstrap.enabled", "true");
    clientProps.setProperty("client.runtime.bootstrap.service(1).name", "bs1");
    clientProps.setProperty("client.runtime.bootstrap.service(1).host", "localhost");
    clientProps.setProperty("client.runtime.bootstrap.service(1).port", "10001");
    clientProps.setProperty("client.runtime.bootstrap.service(1).sources", "source1");
    clientProps.setProperty("client.runtime.bootstrap.service(2).name", "bs2");
    clientProps.setProperty("client.runtime.bootstrap.service(2).host", "localhost");
    clientProps.setProperty("client.runtime.bootstrap.service(2).port", "10002");
    clientProps.setProperty("client.runtime.bootstrap.service(2).sources", "source1");
    clientProps.setProperty("client.runtime.bootstrap.service(3).name", "bs3");
    clientProps.setProperty("client.runtime.bootstrap.service(3).host", "localhost");
    clientProps.setProperty("client.runtime.bootstrap.service(3).port", "10003");
    clientProps.setProperty("client.runtime.bootstrap.service(3).sources", "source1");
    clientProps.setProperty("client.runtime.relay(1).name", "relay1");
    clientProps.setProperty("client.runtime.relay(1).port", "10001");
    clientProps.setProperty("client.runtime.relay(1).sources", "source1");
    clientProps.setProperty("client.runtime.relay(2).name", "relay2");
    clientProps.setProperty("client.runtime.relay(2).port", "10002");
    clientProps.setProperty("client.runtime.relay(2).sources", "source1");
    clientProps.setProperty("client.runtime.relay(3).name", "relay3");
    clientProps.setProperty("client.runtime.relay(3).port", "10003");
    clientProps.setProperty("client.runtime.relay(3).sources", "source1");
    clientProps.setProperty("client.connectionDefaults.eventBuffer.maxSize", "100000");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.maxRetryNum", "9");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.sleepIncFactor", "1.0");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.sleepIncDelta", "1");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.initSleep", "1");
    DatabusHttpClientImpl.Config clientConfBuilder = new DatabusHttpClientImpl.Config();
    ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>("client.", clientConfBuilder);
    configLoader.loadConfig(clientProps);
    DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build();
    DatabusSourcesConnection.StaticConfig srcConnConf = clientConf.getConnectionDefaults();
    DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConf);
    client.registerDatabusBootstrapListener(new LoggingConsumer(), null, "source1");
    Assert.assertNotNull(client, "client instantiation ok");
    DatabusHttpClientImpl.RuntimeConfig clientRtConf = clientConf.getRuntime().build();
    //we keep the index of the next server we expect to see
    AtomicInteger serverIdx = new AtomicInteger(-1);
    //generate the order in which we should see the servers
    List<ServerInfo> relayOrder = new ArrayList<ServerInfo>(clientRtConf.getRelays());
    if (LOG.isInfoEnabled()) {
        StringBuilder sb = new StringBuilder();
        for (ServerInfo serverInfo : relayOrder) {
            sb.append(serverInfo.getName());
            sb.append(" ");
        }
        LOG.info("Relay order:" + sb.toString());
    }
    List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
    sourcesResponse.add(new IdNamePair(1L, "source1"));
    Map<Long, List<RegisterResponseEntry>> registerResponse = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> regResponse = new ArrayList<RegisterResponseEntry>();
    regResponse.add(new RegisterResponseEntry(1L, (short) 1, SCHEMA$.toString()));
    registerResponse.put(1L, regResponse);
    ChunkedBodyReadableByteChannel channel = EasyMock.createMock(ChunkedBodyReadableByteChannel.class);
    if (!readDataException) {
        EasyMock.expect(channel.getMetadata("x-dbus-error-cause")).andReturn(null).anyTimes();
        EasyMock.expect(channel.getMetadata("x-dbus-req-id")).andReturn(null).anyTimes();
        EasyMock.expect(channel.getMetadata("x-dbus-error")).andReturn(null).anyTimes();
    } else {
        EasyMock.expect(channel.getMetadata("x-dbus-error-cause")).andReturn(exceptionName).anyTimes();
        EasyMock.expect(channel.getMetadata("x-dbus-req-id")).andReturn(exceptionName).anyTimes();
        EasyMock.expect(channel.getMetadata("x-dbus-error")).andReturn(exceptionName).anyTimes();
    }
    if (phaseCompleted)
        EasyMock.expect(channel.getMetadata("PhaseCompleted")).andReturn("true").anyTimes();
    else
        EasyMock.expect(channel.getMetadata("PhaseCompleted")).andReturn(null).anyTimes();
    EasyMock.replay(channel);
    DbusEventBuffer dbusBuffer = EasyMock.createMock(DbusEventBuffer.class);
    dbusBuffer.endEvents(false, -1, false, false, null);
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(dbusBuffer.injectEvent(EasyMock.<DbusEventInternalReadable>notNull())).andReturn(true).anyTimes();
    EasyMock.expect(dbusBuffer.getEventSerializationVersion()).andReturn(DbusEventFactory.DBUS_EVENT_V1).anyTimes();
    EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull(), org.easymock.EasyMock.<List<InternalDatabusEventsListener>>notNull(), org.easymock.EasyMock.<DbusEventsStatisticsCollector>isNull())).andReturn(numBytesRead).anyTimes();
    if (readDataThrowException) {
        EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull())).andThrow(new RuntimeException("dummy")).anyTimes();
    } else {
        EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull())).andReturn(numBytesRead).anyTimes();
    }
    EasyMock.expect(dbusBuffer.acquireIterator(EasyMock.<String>notNull())).andReturn(null).anyTimes();
    dbusBuffer.waitForFreeSpace((int) (10000 * 100.0 / clientConf.getPullerBufferUtilizationPct()));
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(dbusBuffer.getBufferFreeReadSpace()).andReturn(freeReadSpace).anyTimes();
    EasyMock.replay(dbusBuffer);
    //This guy succeeds on /sources but fails on /register
    MockBootstrapConnection mockSuccessConn = new MockBootstrapConnection(startScn, targetScn, channel, serverIdx, muteTransition);
    DatabusBootstrapConnectionFactory mockConnFactory = org.easymock.EasyMock.createMock("mockRelayFactory", DatabusBootstrapConnectionFactory.class);
    if (throwBSConnException) {
        EasyMock.expect(mockConnFactory.createConnection(EasyMock.<ServerInfo>notNull(), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andThrow(new RuntimeException("Mock Error")).anyTimes();
    } else if (failBsConnection) {
        EasyMock.expect(mockConnFactory.createConnection(EasyMock.<ServerInfo>notNull(), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(null).anyTimes();
    } else {
        EasyMock.expect(mockConnFactory.createConnection(EasyMock.<ServerInfo>notNull(), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockSuccessConn).anyTimes();
    }
    List<DatabusSubscription> sourcesSubList = DatabusSubscription.createSubscriptionList(sources);
    // Create ConnectionState
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sources);
    // Mock Bootstrap Puller
    RelayPullThread mockRelayPuller = EasyMock.createMock("rpt", RelayPullThread.class);
    mockRelayPuller.enqueueMessage(EasyMock.notNull());
    EasyMock.expectLastCall().anyTimes();
    // Mock Relay Dispatcher
    BootstrapDispatcher mockDispatcher = EasyMock.createMock("rd", BootstrapDispatcher.class);
    mockDispatcher.enqueueMessage(EasyMock.notNull());
    EasyMock.expectLastCall().anyTimes();
    //Set up mock for sources connection
    DatabusSourcesConnection sourcesConn2 = EasyMock.createMock(DatabusSourcesConnection.class);
    EasyMock.expect(sourcesConn2.getSourcesNames()).andReturn(Arrays.asList("source1")).anyTimes();
    EasyMock.expect(sourcesConn2.getSubscriptions()).andReturn(sourcesSubList).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionConfig()).andReturn(srcConnConf).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionStatus()).andReturn(new DatabusComponentStatus("dummy")).anyTimes();
    EasyMock.expect(sourcesConn2.getLocalRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getUnifiedClientStats()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapConnFactory()).andReturn(mockConnFactory).anyTimes();
    EasyMock.expect(sourcesConn2.loadPersistentCheckpoint()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getDataEventsBuffer()).andReturn(dbusBuffer).anyTimes();
    EasyMock.expect(sourcesConn2.isBootstrapEnabled()).andReturn(true).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapRegistrations()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapServices()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapEventsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getRelayPullThread()).andReturn(mockRelayPuller).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapDispatcher()).andReturn(mockDispatcher).anyTimes();
    EasyMock.makeThreadSafe(mockConnFactory, true);
    EasyMock.makeThreadSafe(mockDispatcher, true);
    EasyMock.makeThreadSafe(mockRelayPuller, true);
    EasyMock.makeThreadSafe(sourcesConn2, true);
    EasyMock.replay(mockConnFactory);
    EasyMock.replay(sourcesConn2);
    EasyMock.replay(mockDispatcher);
    EasyMock.replay(mockRelayPuller);
    BootstrapPullThread bsPuller = new BootstrapPullThread("RelayPuller", sourcesConn2, dbusBuffer, connStateFactory, clientRtConf.getBootstrap().getServicesSet(), new ArrayList<DbusKeyCompositeFilterConfig>(), clientConf.getPullerBufferUtilizationPct(), null, new DbusEventV2Factory(), null, null);
    mockSuccessConn.setCallback(bsPuller);
    return bsPuller;
}

Example 47 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class MockRemoteExceptionHandler method testShutdownRace.

@Test
public /**
   * DDSDBUS-1904: test a race condition between shutting down a connection and a reconnect to the
   * relay which may cause a Netty connection to be left open after the RelayPuller shutdown.
   * The idea is to trigger a shutdown exactly while we are trying to connect to a relay. We achieve
   * that by rigging MockRelayConnection to call RelayPuller.shutdown() right before responding
   * to requestSources();
   *  */
void testShutdownRace() throws Exception {
    final Logger log = Logger.getLogger("TestRelayPullThread.testShutdownRace");
    log.setLevel(Level.INFO);
    log.info("start");
    //elaborate test setup
    List<String> sources = Arrays.asList("source1");
    Properties clientProps = new Properties();
    clientProps.setProperty("client.container.httpPort", "0");
    clientProps.setProperty("client.runtime.bootstrap.enabled", "false");
    clientProps.setProperty("client.runtime.relay(1).name", "relay1");
    clientProps.setProperty("client.runtime.relay(1).port", "10001");
    clientProps.setProperty("client.runtime.relay(1).sources", "source1");
    clientProps.setProperty("client.connectionDefaults.eventBuffer.maxSize", "100000");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.maxRetryNum", "9");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.sleepIncFactor", "1.0");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.sleepIncDelta", "1");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.initSleep", "1");
    DatabusHttpClientImpl client = new DatabusHttpClientImpl("client.", clientProps);
    Assert.assertNotNull(client, "client instantiation ok");
    final DatabusHttpClientImpl.StaticConfig clientConf = client.getClientStaticConfig();
    final DatabusSourcesConnection.StaticConfig srcConnConf = clientConf.getConnectionDefaults();
    DatabusHttpClientImpl.RuntimeConfig clientRtConf = clientConf.getRuntime().build();
    DbusEventBuffer.StaticConfig bufferConf = clientConf.getConnectionDefaults().getEventBuffer();
    DbusEventBuffer relayBuffer = new DbusEventBuffer(bufferConf);
    DbusEventBuffer bootstrapBuffer = new DbusEventBuffer(bufferConf);
    //we keep the index of the next server we expect to see
    AtomicInteger serverIdx = new AtomicInteger(-1);
    Set<ServerInfo> relays = clientRtConf.getRelaysSet();
    //generate the order in which we should see the servers
    List<ServerInfo> relayOrder = new ArrayList<ServerInfo>(relays);
    if (LOG.isInfoEnabled()) {
        StringBuilder sb = new StringBuilder();
        for (ServerInfo serverInfo : relayOrder) {
            sb.append(serverInfo.getName());
            sb.append(" ");
        }
        LOG.info("Relay order:" + sb.toString());
    }
    List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
    sourcesResponse.add(new IdNamePair(1L, "source1"));
    RegisterResponseEntry rre1 = new RegisterResponseEntry(1L, (short) 1, SCHEMA$.toString());
    final HashMap<Long, List<RegisterResponseEntry>> registerResponse = new HashMap<Long, List<RegisterResponseEntry>>();
    registerResponse.put(1L, Arrays.asList(rre1));
    //This guy succeeds on both /sources and /register
    final MockRelayConnection relayConn1 = new MockRelayConnection(sourcesResponse, registerResponse, null, serverIdx);
    //This guy will succeed on /sources but will force a shutdown so that
    //the success message is ignored
    final MockRelayConnection relayConn2 = new MockRelayConnectionForTestShutdownRace(sourcesResponse, null, null, serverIdx, log);
    DatabusRelayConnectionFactory mockConnFactory = EasyMock.createMock("mockRelayFactory", DatabusRelayConnectionFactory.class);
    // expected scenario:  create relayConn1 -> /sources success -> /register success ->
    // /stream error -> create relayConn2 -> call /sources -> shut down puller -> return
    // success for /sources
    EasyMock.expect(mockConnFactory.createRelayConnection(serverNameMatcher(serverIdx, relayOrder), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(relayConn1);
    EasyMock.expect(mockConnFactory.createRelayConnection(serverNameMatcher(serverIdx, relayOrder), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(relayConn2);
    EasyMock.replay(mockConnFactory);
    List<DatabusSubscription> sourcesSubList = DatabusSubscription.createSubscriptionList(sources);
    //Dummy connection object as expected by the puller thread
    // Note that in this case, it is ok to pass Set<relays> as all the relays serve the same source "source1"
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sources);
    DatabusSourcesConnection sourcesConn = new DatabusSourcesConnection(srcConnConf, sourcesSubList, relays, null, null, null, relayBuffer, bootstrapBuffer, Executors.newCachedThreadPool(), null, null, null, null, null, null, null, mockConnFactory, null, null, null, null, new DbusEventV1Factory(), connStateFactory);
    final RelayPullThread relayPuller = new RelayPullThread("RelayPuller", sourcesConn, relayBuffer, connStateFactory, relays, new ArrayList<DbusKeyCompositeFilterConfig>(), !clientConf.getRuntime().getBootstrap().isEnabled(), clientConf.isReadLatestScnOnErrorEnabled(), clientConf.getPullerBufferUtilizationPct(), Integer.MAX_VALUE, ManagementFactory.getPlatformMBeanServer(), new DbusEventV1Factory(), null);
    //relayPuller.getLog().setLevel(Level.INFO);
    RemoteExceptionHandler mockRemoteExceptionHandler = new MockRemoteExceptionHandler(sourcesConn, relayBuffer, relayPuller);
    Field field = relayPuller.getClass().getDeclaredField("_remoteExceptionHandler");
    field.setAccessible(true);
    field.set(relayPuller, mockRemoteExceptionHandler);
    relayConn1.setCallback(relayPuller);
    relayConn2.setCallback(relayPuller);
    //Let the show begin
    final Thread relayPullerThread = new Thread(relayPuller, "testShutdownRace.RelayPuller");
    relayPullerThread.setDaemon(true);
    relayPullerThread.start();
    relayPuller.enqueueMessage(LifecycleMessage.createStartMessage());
    //wait for the puller to go the STREAM_REQUEST_SUCCESS state
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            return null != relayConn1.getLastStateMsg();
        }
    }, "wait for call from the puller to the relay connection", 500, log);
    //wait for puller thread to shutdown
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            log.debug(relayPuller.getMessageHistoryLog());
            return !relayPullerThread.isAlive();
        }
    }, "wait for puller to shutdown", 1000, log);
    EasyMock.verify(mockConnFactory);
    Assert.assertEquals(relayPuller.getLastOpenConnection(), null);
    log.info("done");
}

Example 48 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class DummySuccessfulErrorCountingConsumer method testUnregisterDatabusStreamListener.

@Test
public void testUnregisterDatabusStreamListener() throws Exception {
    DatabusHttpClientImpl.Config clientConfig = new DatabusHttpClientImpl.Config();
    clientConfig.getContainer().getJmx().setRmiEnabled(false);
    clientConfig.getContainer().setHttpPort(13403);
    DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConfig);
    registerRelay(1, "relay1", new InetSocketAddress("localhost", 8888), "S1,S2", client);
    registerRelay(2, "relay2", new InetSocketAddress("localhost", 7777), "S1,S3", client);
    registerRelay(3, "relay1.1", new InetSocketAddress("localhost", 8887), "S1,S2", client);
    registerRelay(4, "relay3", new InetSocketAddress("localhost", 6666), "S3,S4,S5", client);
    DummyStreamConsumer listener1 = new DummyStreamConsumer("consumer1");
    client.registerDatabusStreamListener(listener1, null, "S1");
    client.registerDatabusStreamListener(listener1, null, "S2");
    List<DatabusSubscription> ls1 = DatabusSubscription.createSubscriptionList(Arrays.asList("S1", "S2"));
    List<DatabusSubscription> ls2 = DatabusSubscription.createSubscriptionList(Arrays.asList("S1", "S3"));
    int consumersNum1 = safeListSize(client.getRelayGroupStreamConsumers().get(ls1)) + safeListSize(client.getRelayGroupStreamConsumers().get(ls2));
    assertEquals("expect two consumers in (S1,S2) or (S1,S3)", 2, consumersNum1);
    assertTrue("expect at least one consumer in (S1,S2)", safeListSize(client.getRelayGroupStreamConsumers().get(ls1)) >= 1);
    DummyStreamConsumer listener2 = new DummyStreamConsumer("consumer2");
    client.registerDatabusStreamListener(listener2, null, "S1");
    int consumersNum2 = safeListSize(client.getRelayGroupStreamConsumers().get(ls1)) + safeListSize(client.getRelayGroupStreamConsumers().get(ls2));
    assertEquals("expect three consumers in (S1,S2) or (S1,S3)", 3, consumersNum2);
    client.unregisterDatabusStreamListener(listener1);
    int consumersNum3 = safeListSize(client.getRelayGroupStreamConsumers().get(ls1)) + safeListSize(client.getRelayGroupStreamConsumers().get(ls2));
    //assertEquals("one consumer + 1-2 logging consumer in (S1,S2) or (S1,S3)", consumersNum2 - 2,
    //             consumersNum3);
    client.unregisterDatabusStreamListener(listener1);
    int consumersNum4 = safeListSize(client.getRelayGroupStreamConsumers().get(ls1)) + safeListSize(client.getRelayGroupStreamConsumers().get(ls2));
    assertEquals("expect one consumer in (S1,S2) or (S1,S3)", consumersNum3, consumersNum4);
    client.unregisterDatabusStreamListener(listener2);
}

Example 49 with DatabusSubscription

use of com.linkedin.databus.core.data_model.DatabusSubscription in project databus by linkedin.

the class TestDatabusSourcesConnection method testLoggerNameV2_2.

@Test
public void testLoggerNameV2_2() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException {
    // They are of same db, but may have different
    String source1 = "com.linkedin.events.db.dbPrefix1.tableName1";
    String source2 = "com.linkedin.events.db.dbPrefix2.tableName2";
    List<String> ls = new ArrayList<String>();
    ls.add(source1);
    ls.add(source2);
    List<DatabusSubscription> ds = DatabusSubscription.createSubscriptionList(ls);
    DatabusSourcesConnection dsc = DatabusSourcesConnection.createDatabusSourcesConnectionForTesting();
    String partName = DatabusSubscription.getPrettyNameForListOfSubscriptions(ds);
    Assert.assertEquals(partName, "dbPrefix1_tableName1_dbPrefix2_tableName2");
    Class<?>[] arg1 = new Class<?>[] { List.class, String.class };
    Method constructPrettyNameForLogging = DatabusSourcesConnection.class.getDeclaredMethod("constructPrettyNameForLogging", arg1);
    constructPrettyNameForLogging.setAccessible(true);
    Object prettyName = constructPrettyNameForLogging.invoke(dsc, ds, "test_1234");
    Assert.assertEquals(prettyName, "dbPrefix1_tableName1_dbPrefix2_tableName2_test_1234");
}