Example 36 with DbusEventBuffer
use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.
the class DummySuccessfulErrorCountingConsumer method testRelayFailoverPartialWindow1.
/**
* Client switches from R1 to R2 on server close and from R2 to R3 on timeout while in the middle of the windows.
*
* <pre>
* R1 EVB Boundaries --------------------------------------------------------------------------------
* ^ ^ ^ ^ ^ ^ ^ ^ ^ ^
* 0 10 20 30 40 50 60 70 80 90
*
* R2 EVB Boundaries --------------------------------------------------------------------------------
* ^ ^ ^ ^ ^
* 0 20 40 60 80
*
* R3 EVB Boundaries --------------------------------------------------------------------------------
* ^ ^ ^ ^
* 0 30 60 90
* </pre>
*
* Switch from R1 to R2 happens when windowScn = 10 and windowOffset = 8th event.
* Switch from R2 to R3 happens when windowScn = 40 and windowOffset = 3rd event.
*
* @throws Exception
*/
@Test
public void testRelayFailoverPartialWindow1() throws Exception {
final boolean debugOn = false;
final Logger log = Logger.getLogger("TestDatabusHttpClient.testRelayFailoverPartialWindow1");
log.setLevel(Level.INFO);
final int eventsNum = 200;
DbusEventInfo[] eventInfos = createSampleSchema1Events(eventsNum);
final long timeoutMult = debugOn ? 100000 : 1;
log.info("simulate relay buffers");
DbusEventBuffer[] relayBuffer = new DbusEventBuffer[RELAY_PORT.length];
List<List<Integer>> eventOfs = new ArrayList<List<Integer>>(3);
List<List<DbusEventKey>> eventKeys = new ArrayList<List<DbusEventKey>>(3);
for (int i = 0; i < RELAY_PORT.length; ++i) {
relayBuffer[i] = new DbusEventBuffer(_bufCfg);
relayBuffer[i].start(0);
WriteEventsResult wrRes = writeEventsToBuffer(relayBuffer[i], eventInfos, (i + 1) * 10);
List<Integer> ofs = wrRes.getOffsets();
eventOfs.add(ofs);
eventKeys.add(wrRes.getKeys());
}
List<DbusEventKey> key = eventKeys.get(0);
for (int i = 1; i < RELAY_PORT.length; ++i) {
assertEquals(" For Size index : " + i, key.size(), eventKeys.get(i).size());
assertEquals(" For index : " + i, key, eventKeys.get(i));
key = eventKeys.get(i);
}
int resp1EnfOfs = eventOfs.get(0).get(8);
log.info("figure out an event offset inside a window:" + resp1EnfOfs);
log.info("create client");
_stdClientCfgBuilder.getContainer().setReadTimeoutMs(DEFAULT_READ_TIMEOUT_MS);
final DatabusHttpClientImpl client = new DatabusHttpClientImpl(_stdClientCfgBuilder.build());
final TestConsumer consumer = new TestConsumer();
client.registerDatabusStreamListener(consumer, null, SOURCE1_NAME);
client.start();
try {
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return client._relayConnections.size() == 1;
}
}, "sources connection present", 100, log);
final DatabusSourcesConnection clientConn = client._relayConnections.get(0);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != clientConn.getRelayPullThread().getLastOpenConnection();
}
}, "relay connection present", 100, log);
final NettyHttpDatabusRelayConnection relayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
final NettyHttpDatabusRelayConnectionInspector relayConnInsp = new NettyHttpDatabusRelayConnectionInspector(relayConn);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != relayConnInsp.getChannel() && relayConnInsp.getChannel().isConnected();
}
}, "client connected", 200, log);
log.info("figure out the connection to the relay");
Channel clientChannel = relayConnInsp.getChannel();
InetSocketAddress relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
SocketAddress clientAddr = clientChannel.getLocalAddress();
int relayPort = relayAddr.getPort();
log.info("relay selected: " + relayPort);
SimpleTestServerConnection relay = null;
for (int i = 0; i < RELAY_PORT.length; ++i) {
if (relayPort == RELAY_PORT[i])
relay = _dummyServer[i];
}
assertTrue(null != relay);
final SocketAddress testClientAddr = clientAddr;
final SimpleTestServerConnection testRelay = relay;
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != testRelay.getChildChannel(testClientAddr);
}
}, "relay detects new connection", 1000, log);
Channel serverChannel = relay.getChildChannel(clientAddr);
assertTrue(null != serverChannel);
ChannelPipeline serverPipeline = serverChannel.getPipeline();
SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
log.info("process the /sources request");
NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
objCapture.clear();
log.info("send back the /sources response");
HttpResponse sourcesResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
sourcesResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
sourcesResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
return "1".equals(idListString);
}
}, "client processes /sources response", 100, log);
log.info("process the /register request");
NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
objCapture.clear();
log.info("send back the /register response");
RegisterResponseEntry entry = new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR);
String responseStr = NettyTestUtils.generateRegisterResponse(entry);
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
}
}, "client processes /register response", 100, log);
log.info("process /stream call and return a partial window");
NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*", 1000);
objCapture.clear();
log.info("send back the /stream response");
final DbusEventsStatisticsCollector stats = new DbusEventsStatisticsCollector(1, "test1", true, false, null);
Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
ChannelBuffer streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[0], cp, resp1EnfOfs, stats);
HttpResponse streamResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
streamResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
streamResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
log.info("make sure the client processes the /stream response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("LastWritten SCN:" + clientConn.getDataEventsBuffer().lastWrittenScn());
return clientConn.getDataEventsBuffer().lastWrittenScn() == 10;
}
}, "client receives /stream response", 1100, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("events num=" + consumer.getEventNum());
return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
}
}, "client processes /stream response", 110000, log);
assertEquals(-1, consumer.getRollbackScn());
int rollbackNum = 0;
assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
List<DbusEventKey> expKeys = eventKeys.get(0).subList(0, (int) stats.getTotalStats().getNumDataEvents());
List<Long> expSeqs = new ArrayList<Long>();
for (int i = 0; i < stats.getTotalStats().getNumDataEvents(); i++) expSeqs.add(10L);
long numEvents = stats.getTotalStats().getNumDataEvents();
assertEquals("Keys", expKeys, consumer.getKeys());
assertEquals("Sequences", expSeqs, consumer.getSequences());
log.info("now kill the relay and wait for a failover");
serverChannel.close();
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != relayConnInsp.getChannel() && !relayConnInsp.getChannel().isConnected();
}
}, "client disconnected", 200, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
}
}, "new netty connection", 200, log);
log.info("/////////// FAKING CONNECTION TO NEW RELAY //////////////");
final NettyHttpDatabusRelayConnection newRelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
final NettyHttpDatabusRelayConnectionInspector newRelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(newRelayConn);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != newRelayConnInsp.getChannel() && newRelayConnInsp.getChannel().isConnected();
}
}, "client connected to new relay", 200, log);
log.info("figure out the connection to the relay");
clientChannel = newRelayConnInsp.getChannel();
relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
clientAddr = clientChannel.getLocalAddress();
relayPort = relayAddr.getPort();
log.info("new relay selected: " + relayPort);
relay = null;
int relayIdx = 0;
for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
if (relayPort == RELAY_PORT[relayIdx])
relay = _dummyServer[relayIdx];
}
assertTrue(null != relay);
serverChannel = relay.getChildChannel(clientAddr);
assertTrue(null != serverChannel);
serverPipeline = serverChannel.getPipeline();
objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
log.info("process the /sources request");
NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
objCapture.clear();
log.info("send back the /sources response");
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
return "1".equals(idListString);
}
}, "client processes /sources response", 100, log);
log.info("process the /register request");
NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
objCapture.clear();
log.info("send back the /register response");
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.info("make sure the client processes the /register response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
}
}, "client processes /register response", 100, log);
log.info("process /stream call and return a partial window");
Matcher streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
String cpString = streamMatcher.group(1);
objCapture.clear();
int respStartOfs = eventOfs.get(1).get(1);
int respEndOfs = eventOfs.get(1).get(34);
cp = new Checkpoint(cpString);
//last window read was partial
assertTrue(cp.getWindowOffset() > 0);
streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[1], cp, respEndOfs - respStartOfs, stats);
NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
return clientConn.getDataEventsBuffer().lastWrittenScn() == 40;
}
}, "client receives /stream response", 1100, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("events num=" + consumer.getEventNum());
return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
}
}, "client processes /stream response", 11000, log);
assertEquals(20, consumer.getRollbackScn());
log.info("one more onStartDataEventSequence because of the rolback");
++rollbackNum;
assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
expKeys.addAll(eventKeys.get(1).subList(0, (int) (stats.getTotalStats().getNumDataEvents() - numEvents)));
for (int i = 0; i < stats.getTotalStats().getNumDataEvents() - numEvents; i++) expSeqs.add((i / 20) * 20 + 20L);
assertEquals("Keys", expKeys, consumer.getKeys());
assertEquals("Sequences", expSeqs, consumer.getSequences());
numEvents = stats.getTotalStats().getNumDataEvents();
///////////////////////////////////
//simulate a timeout on the server; the client would have sent a /stream call and there
//will be no response from the server, so eventually it should time out and switch servers
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("Channel :" + newRelayConnInsp.getChannel());
return (null != newRelayConnInsp.getChannel()) && (!newRelayConnInsp.getChannel().isConnected());
}
}, "waiting for a reconnect", (long) (DEFAULT_READ_TIMEOUT_MS * 1.5), log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
}
}, "new netty connection", 200, log);
final NettyHttpDatabusRelayConnection new2RelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
final NettyHttpDatabusRelayConnectionInspector new2RelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(new2RelayConn);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != new2RelayConnInsp.getChannel() && new2RelayConnInsp.getChannel().isConnected();
}
}, "client connected to third relay", 200, log);
log.info("figure out the connection to the relay");
clientChannel = new2RelayConnInsp.getChannel();
relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
clientAddr = clientChannel.getLocalAddress();
relayPort = relayAddr.getPort();
log.info("third relay selected: " + relayPort);
relay = null;
relayIdx = 0;
for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
if (relayPort == RELAY_PORT[relayIdx])
relay = _dummyServer[relayIdx];
}
assertTrue(null != relay);
serverChannel = relay.getChildChannel(clientAddr);
assertTrue(null != serverChannel);
serverPipeline = serverChannel.getPipeline();
objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
log.info("process the /sources request");
NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
objCapture.clear();
log.info("send back the /sources response");
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
return "1".equals(idListString);
}
}, "client processes /sources response", 100, log);
log.info("process the /register request");
NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
objCapture.clear();
log.info("SEND BACK THE /register RESPONSE");
//clientConn.getRelayDispatcher().getLog().setLevel(Level.DEBUG);
//RangeBasedReaderWriterLock.LOG.setLevel(Level.DEBUG);
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
}
}, "client processes /register response", timeoutMult * 100, log);
log.info("PROCESS the /stream CALL AND RETURN A PARTIAL WINDOW");
streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
cpString = streamMatcher.group(1);
objCapture.clear();
respStartOfs = eventOfs.get(2).get(1);
respEndOfs = eventOfs.get(2).get(84);
log.debug("Checkpoint String is :" + cpString);
cp = new Checkpoint(cpString);
log.info("last window read was partial. So the client would have reset the windowOffset");
assertTrue("Is WindowOffset Cleared", cp.getWindowOffset() == -1);
assertEquals("WindowSCN == PrevSCN. Ckpt :" + cp, cp.getWindowScn(), cp.getPrevScn());
streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[2], cp, respEndOfs - respStartOfs, stats);
NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
log.debug("NumEvents already seen :" + numEvents);
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn() + ", NumEvents :" + stats.getTotalStats().getNumDataEvents());
return clientConn.getDataEventsBuffer().lastWrittenScn() == 90;
}
}, "client receives /stream response, Sequences :" + consumer.getSequences(), timeoutMult * 1100, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("events num=" + consumer.getEventNum());
return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
}
}, "client processes /stream response", timeoutMult * 1100, log);
log.info("one more onStartDataEventSequence because of the rollback");
assertEquals(30, consumer.getRollbackScn());
++rollbackNum;
assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
expKeys.addAll(eventKeys.get(2).subList(0, (int) (stats.getTotalStats().getNumDataEvents() - numEvents)));
for (int i = 0; i < stats.getTotalStats().getNumDataEvents() - numEvents; i++) expSeqs.add((i / 30) * 30 + 30L);
assertEquals("Keys", expKeys, consumer.getKeys());
assertEquals("Sequences", expSeqs, consumer.getSequences());
numEvents = stats.getTotalStats().getNumDataEvents();
assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
} finally {
client.shutdown();
}
}
Also used :
NettyHttpDatabusRelayConnection(com.linkedin.databus.client.netty.NettyHttpDatabusRelayConnection)
NettyHttpDatabusRelayConnectionInspector(com.linkedin.databus.client.netty.NettyHttpDatabusRelayConnectionInspector)
DefaultHttpChunk(org.jboss.netty.handler.codec.http.DefaultHttpChunk)
Matcher(java.util.regex.Matcher)
InetSocketAddress(java.net.InetSocketAddress)
ArrayList(java.util.ArrayList)
DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector)
Logger(org.apache.log4j.Logger)
ChannelBuffer(org.jboss.netty.buffer.ChannelBuffer)
DbusEventInfo(com.linkedin.databus.core.DbusEventInfo)
List(java.util.List)
ArrayList(java.util.ArrayList)
SocketAddress(java.net.SocketAddress)
InetSocketAddress(java.net.InetSocketAddress)
ConditionCheck(com.linkedin.databus2.test.ConditionCheck)
Channel(org.jboss.netty.channel.Channel)
SimpleTestServerConnection(com.linkedin.databus2.test.container.SimpleTestServerConnection)
DefaultHttpResponse(org.jboss.netty.handler.codec.http.DefaultHttpResponse)
HttpResponse(org.jboss.netty.handler.codec.http.HttpResponse)
Checkpoint(com.linkedin.databus.core.Checkpoint)
ChannelPipeline(org.jboss.netty.channel.ChannelPipeline)
DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer)
AtomicInteger(java.util.concurrent.atomic.AtomicInteger)
Checkpoint(com.linkedin.databus.core.Checkpoint)
SimpleObjectCaptureHandler(com.linkedin.databus2.test.container.SimpleObjectCaptureHandler)
DefaultHttpResponse(org.jboss.netty.handler.codec.http.DefaultHttpResponse)
RegisterResponseEntry(com.linkedin.databus2.core.container.request.RegisterResponseEntry)
DbusEventKey(com.linkedin.databus.core.DbusEventKey)
DefaultHttpChunk(org.jboss.netty.handler.codec.http.DefaultHttpChunk)
HttpChunk(org.jboss.netty.handler.codec.http.HttpChunk)
Test(org.testng.annotations.Test)
Example 37 with DbusEventBuffer
use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.
the class MockRemoteExceptionHandler method testShutdownRace.
@Test
public /**
* DDSDBUS-1904: test a race condition between shutting down a connection and a reconnect to the
* relay which may cause a Netty connection to be left open after the RelayPuller shutdown.
* The idea is to trigger a shutdown exactly while we are trying to connect to a relay. We achieve
* that by rigging MockRelayConnection to call RelayPuller.shutdown() right before responding
* to requestSources();
* */
void testShutdownRace() throws Exception {
final Logger log = Logger.getLogger("TestRelayPullThread.testShutdownRace");
log.setLevel(Level.INFO);
log.info("start");
//elaborate test setup
List<String> sources = Arrays.asList("source1");
Properties clientProps = new Properties();
clientProps.setProperty("client.container.httpPort", "0");
clientProps.setProperty("client.runtime.bootstrap.enabled", "false");
clientProps.setProperty("client.runtime.relay(1).name", "relay1");
clientProps.setProperty("client.runtime.relay(1).port", "10001");
clientProps.setProperty("client.runtime.relay(1).sources", "source1");
clientProps.setProperty("client.connectionDefaults.eventBuffer.maxSize", "100000");
clientProps.setProperty("client.connectionDefaults.pullerRetries.maxRetryNum", "9");
clientProps.setProperty("client.connectionDefaults.pullerRetries.sleepIncFactor", "1.0");
clientProps.setProperty("client.connectionDefaults.pullerRetries.sleepIncDelta", "1");
clientProps.setProperty("client.connectionDefaults.pullerRetries.initSleep", "1");
DatabusHttpClientImpl client = new DatabusHttpClientImpl("client.", clientProps);
Assert.assertNotNull(client, "client instantiation ok");
final DatabusHttpClientImpl.StaticConfig clientConf = client.getClientStaticConfig();
final DatabusSourcesConnection.StaticConfig srcConnConf = clientConf.getConnectionDefaults();
DatabusHttpClientImpl.RuntimeConfig clientRtConf = clientConf.getRuntime().build();
DbusEventBuffer.StaticConfig bufferConf = clientConf.getConnectionDefaults().getEventBuffer();
DbusEventBuffer relayBuffer = new DbusEventBuffer(bufferConf);
DbusEventBuffer bootstrapBuffer = new DbusEventBuffer(bufferConf);
//we keep the index of the next server we expect to see
AtomicInteger serverIdx = new AtomicInteger(-1);
Set<ServerInfo> relays = clientRtConf.getRelaysSet();
//generate the order in which we should see the servers
List<ServerInfo> relayOrder = new ArrayList<ServerInfo>(relays);
if (LOG.isInfoEnabled()) {
StringBuilder sb = new StringBuilder();
for (ServerInfo serverInfo : relayOrder) {
sb.append(serverInfo.getName());
sb.append(" ");
}
LOG.info("Relay order:" + sb.toString());
}
List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
sourcesResponse.add(new IdNamePair(1L, "source1"));
RegisterResponseEntry rre1 = new RegisterResponseEntry(1L, (short) 1, SCHEMA$.toString());
final HashMap<Long, List<RegisterResponseEntry>> registerResponse = new HashMap<Long, List<RegisterResponseEntry>>();
registerResponse.put(1L, Arrays.asList(rre1));
//This guy succeeds on both /sources and /register
final MockRelayConnection relayConn1 = new MockRelayConnection(sourcesResponse, registerResponse, null, serverIdx);
//This guy will succeed on /sources but will force a shutdown so that
//the success message is ignored
final MockRelayConnection relayConn2 = new MockRelayConnectionForTestShutdownRace(sourcesResponse, null, null, serverIdx, log);
DatabusRelayConnectionFactory mockConnFactory = EasyMock.createMock("mockRelayFactory", DatabusRelayConnectionFactory.class);
// expected scenario: create relayConn1 -> /sources success -> /register success ->
// /stream error -> create relayConn2 -> call /sources -> shut down puller -> return
// success for /sources
EasyMock.expect(mockConnFactory.createRelayConnection(serverNameMatcher(serverIdx, relayOrder), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(relayConn1);
EasyMock.expect(mockConnFactory.createRelayConnection(serverNameMatcher(serverIdx, relayOrder), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(relayConn2);
EasyMock.replay(mockConnFactory);
List<DatabusSubscription> sourcesSubList = DatabusSubscription.createSubscriptionList(sources);
//Dummy connection object as expected by the puller thread
// Note that in this case, it is ok to pass Set<relays> as all the relays serve the same source "source1"
ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sources);
DatabusSourcesConnection sourcesConn = new DatabusSourcesConnection(srcConnConf, sourcesSubList, relays, null, null, null, relayBuffer, bootstrapBuffer, Executors.newCachedThreadPool(), null, null, null, null, null, null, null, mockConnFactory, null, null, null, null, new DbusEventV1Factory(), connStateFactory);
final RelayPullThread relayPuller = new RelayPullThread("RelayPuller", sourcesConn, relayBuffer, connStateFactory, relays, new ArrayList<DbusKeyCompositeFilterConfig>(), !clientConf.getRuntime().getBootstrap().isEnabled(), clientConf.isReadLatestScnOnErrorEnabled(), clientConf.getPullerBufferUtilizationPct(), Integer.MAX_VALUE, ManagementFactory.getPlatformMBeanServer(), new DbusEventV1Factory(), null);
//relayPuller.getLog().setLevel(Level.INFO);
RemoteExceptionHandler mockRemoteExceptionHandler = new MockRemoteExceptionHandler(sourcesConn, relayBuffer, relayPuller);
Field field = relayPuller.getClass().getDeclaredField("_remoteExceptionHandler");
field.setAccessible(true);
field.set(relayPuller, mockRemoteExceptionHandler);
relayConn1.setCallback(relayPuller);
relayConn2.setCallback(relayPuller);
//Let the show begin
final Thread relayPullerThread = new Thread(relayPuller, "testShutdownRace.RelayPuller");
relayPullerThread.setDaemon(true);
relayPullerThread.start();
relayPuller.enqueueMessage(LifecycleMessage.createStartMessage());
//wait for the puller to go the STREAM_REQUEST_SUCCESS state
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != relayConn1.getLastStateMsg();
}
}, "wait for call from the puller to the relay connection", 500, log);
//wait for puller thread to shutdown
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug(relayPuller.getMessageHistoryLog());
return !relayPullerThread.isAlive();
}
}, "wait for puller to shutdown", 1000, log);
EasyMock.verify(mockConnFactory);
Assert.assertEquals(relayPuller.getLastOpenConnection(), null);
log.info("done");
}
Also used :
HashMap(java.util.HashMap)
ServerInfo(com.linkedin.databus.client.pub.ServerInfo)
ArrayList(java.util.ArrayList)
Logger(org.apache.log4j.Logger)
Properties(java.util.Properties)
Field(java.lang.reflect.Field)
IdNamePair(com.linkedin.databus.core.util.IdNamePair)
List(java.util.List)
ArrayList(java.util.ArrayList)
RemoteExceptionHandler(com.linkedin.databus.client.netty.RemoteExceptionHandler)
ConditionCheck(com.linkedin.databus2.test.ConditionCheck)
DbusKeyCompositeFilterConfig(com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig)
DatabusSubscription(com.linkedin.databus.core.data_model.DatabusSubscription)
DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer)
AtomicInteger(java.util.concurrent.atomic.AtomicInteger)
RegisterResponseEntry(com.linkedin.databus2.core.container.request.RegisterResponseEntry)
DbusEventV1Factory(com.linkedin.databus.core.DbusEventV1Factory)
Test(org.testng.annotations.Test)
Example 38 with DbusEventBuffer
use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.
the class DummySuccessfulErrorCountingConsumer method testInStreamError.
/**
* Tests the logic of the client to disconnect from a relay while still processing
* data from the relay. We want to make sure that ChunkedBodyReadableByteChannel
* and thus the StreamHttpResponseProcessor do net get blocked with unprocessed data.
* One way to trigger such an error is to insert some invalid data inside the
* response and then some more data ( > 4MB which is the current buffer size). */
@Test
public void testInStreamError() throws Exception {
final Logger log = Logger.getLogger("TestDatabusHttpClient.testInStreamError");
//log.setLevel(Level.DEBUG);
final int eventsNum = 20;
DbusEventInfo[] eventInfos = createSampleSchema1Events(eventsNum);
//simulate relay buffers
DbusEventBuffer relayBuffer = new DbusEventBuffer(_bufCfg);
relayBuffer.start(0);
writeEventsToBuffer(relayBuffer, eventInfos, 4);
//prepare stream response
Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
final DbusEventsStatisticsCollector stats = new DbusEventsStatisticsCollector(1, "test1", true, false, null);
ChannelBuffer streamResPrefix = NettyTestUtils.streamToChannelBuffer(relayBuffer, cp, 20000, stats);
final StringBuilder alotOfDeadbeef = new StringBuilder();
for (int i = 0; i < 1024 * 1024; ++i) {
alotOfDeadbeef.append("DEADBEEF ");
}
ChannelBuffer streamResSuf = ChannelBuffers.wrappedBuffer(alotOfDeadbeef.toString().getBytes("UTF-8"));
final ChannelBuffer streamRes = ChannelBuffers.wrappedBuffer(streamResPrefix, streamResSuf);
//create client
_stdClientCfgBuilder.getContainer().setReadTimeoutMs(DEFAULT_READ_TIMEOUT_MS);
final DatabusHttpClientImpl client = new DatabusHttpClientImpl(_stdClientCfgBuilder.build());
final TestConsumer consumer = new TestConsumer();
client.registerDatabusStreamListener(consumer, null, SOURCE1_NAME);
client.start();
try {
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return client._relayConnections.size() == 1;
}
}, "sources connection present", 100, log);
final DatabusSourcesConnection clientConn = client._relayConnections.get(0);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != clientConn.getRelayPullThread().getLastOpenConnection();
}
}, "relay connection present", 100, log);
final NettyHttpDatabusRelayConnection relayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
final NettyHttpDatabusRelayConnectionInspector relayConnInsp = new NettyHttpDatabusRelayConnectionInspector(relayConn);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != relayConnInsp.getChannel() && relayConnInsp.getChannel().isConnected();
}
}, "client connected", 200, log);
//figure out the connection to the relay
Channel clientChannel = relayConnInsp.getChannel();
InetSocketAddress relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
SocketAddress clientAddr = clientChannel.getLocalAddress();
int relayPort = relayAddr.getPort();
log.info("relay selected: " + relayPort);
SimpleTestServerConnection relay = null;
for (int i = 0; i < RELAY_PORT.length; ++i) {
if (relayPort == RELAY_PORT[i])
relay = _dummyServer[i];
}
assertTrue(null != relay);
final SocketAddress testClientAddr = clientAddr;
final SimpleTestServerConnection testRelay = relay;
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != testRelay.getChildChannel(testClientAddr);
}
}, "relay detects new connection", 1000, log);
Channel serverChannel = relay.getChildChannel(clientAddr);
assertTrue(null != serverChannel);
ChannelPipeline serverPipeline = serverChannel.getPipeline();
SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
//process the /sources request
NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
objCapture.clear();
//send back the /sources response
HttpResponse sourcesResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
sourcesResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
sourcesResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
//make sure the client processes the response correctly
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
return "1".equals(idListString);
}
}, "client processes /sources response", 100, log);
log.debug("process the /register request");
NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
objCapture.clear();
log.debug("send back the /register response");
RegisterResponseEntry entry = new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR);
String responseStr = NettyTestUtils.generateRegisterResponse(entry);
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.debug("make sure the client processes the response /register correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
}
}, "client processes /register response", 100, log);
log.debug("process /stream call and return a response with garbled suffix");
NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*", 1000);
objCapture.clear();
final HttpResponse streamResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
streamResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
streamResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
//send the response asynchronously in case the client blocks
final Thread streamRespThread = new Thread(new Runnable() {
@Override
public void run() {
NettyTestUtils.sendServerResponses(testRelay, testClientAddr, streamResp, new DefaultHttpChunk(streamRes), 60000);
}
}, "send /stream resp");
streamRespThread.setDaemon(true);
streamRespThread.start();
log.debug("make sure the client disconnects and recovers from the /stream response");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != relayConnInsp.getChannel() && !relayConnInsp.getChannel().isConnected();
}
}, "client disconnected", 30000, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
}
}, "new netty connection", 30000, log);
//make sure the relay send thread is dead
streamRespThread.join(100);
Assert.assertTrue(!streamRespThread.isAlive());
log.debug("PHASE 2: make sure the client has fully recovered and able to connect to another relay");
final NettyHttpDatabusRelayConnection newRelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
final NettyHttpDatabusRelayConnectionInspector newRelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(newRelayConn);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != newRelayConnInsp.getChannel() && newRelayConnInsp.getChannel().isConnected();
}
}, "client connected to new relay", 200, log);
//figure out the connection to the relay
clientChannel = newRelayConnInsp.getChannel();
relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
clientAddr = clientChannel.getLocalAddress();
relayPort = relayAddr.getPort();
log.info("new relay selected: " + relayPort);
relay = null;
int relayIdx = 0;
for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
if (relayPort == RELAY_PORT[relayIdx])
relay = _dummyServer[relayIdx];
}
assertTrue(null != relay);
serverChannel = relay.getChildChannel(clientAddr);
assertTrue(null != serverChannel);
serverPipeline = serverChannel.getPipeline();
objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
//process the /sources request
NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
objCapture.clear();
//send back the /sources response
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
//make sure the client processes the response correctly
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
return "1".equals(idListString);
}
}, "client processes /sources response", 100, log);
//process the /register request
NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
objCapture.clear();
//send back the /register response
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
//make sure the client processes the response correctly
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
}
}, "client processes /register response", 100, log);
//process /stream call and return a partial window
Matcher streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
String cpString = streamMatcher.group(1);
log.debug("/stream checkpoint: " + cpString);
objCapture.clear();
NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamResPrefix));
//make sure the client processes the response correctly
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
return clientConn.getDataEventsBuffer().lastWrittenScn() == 20;
}
}, "client receives /stream response", 1100, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("events num=" + consumer.getEventNum());
return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
}
}, "client processes /stream response", 11000, log);
} finally {
client.shutdown();
}
}
Also used :
NettyHttpDatabusRelayConnection(com.linkedin.databus.client.netty.NettyHttpDatabusRelayConnection)
NettyHttpDatabusRelayConnectionInspector(com.linkedin.databus.client.netty.NettyHttpDatabusRelayConnectionInspector)
DefaultHttpChunk(org.jboss.netty.handler.codec.http.DefaultHttpChunk)
Matcher(java.util.regex.Matcher)
InetSocketAddress(java.net.InetSocketAddress)
DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector)
Logger(org.apache.log4j.Logger)
ChannelBuffer(org.jboss.netty.buffer.ChannelBuffer)
DbusEventInfo(com.linkedin.databus.core.DbusEventInfo)
SocketAddress(java.net.SocketAddress)
InetSocketAddress(java.net.InetSocketAddress)
ConditionCheck(com.linkedin.databus2.test.ConditionCheck)
Channel(org.jboss.netty.channel.Channel)
SimpleTestServerConnection(com.linkedin.databus2.test.container.SimpleTestServerConnection)
DefaultHttpResponse(org.jboss.netty.handler.codec.http.DefaultHttpResponse)
HttpResponse(org.jboss.netty.handler.codec.http.HttpResponse)
Checkpoint(com.linkedin.databus.core.Checkpoint)
ChannelPipeline(org.jboss.netty.channel.ChannelPipeline)
DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer)
Checkpoint(com.linkedin.databus.core.Checkpoint)
SimpleObjectCaptureHandler(com.linkedin.databus2.test.container.SimpleObjectCaptureHandler)
DefaultHttpResponse(org.jboss.netty.handler.codec.http.DefaultHttpResponse)
RegisterResponseEntry(com.linkedin.databus2.core.container.request.RegisterResponseEntry)
DefaultHttpChunk(org.jboss.netty.handler.codec.http.DefaultHttpChunk)
HttpChunk(org.jboss.netty.handler.codec.http.HttpChunk)
Test(org.testng.annotations.Test)
Example 39 with DbusEventBuffer
use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.
the class BootstrapPullThread method doReadBootstrapEvents.
protected void doReadBootstrapEvents(ConnectionState curState) {
boolean success = true;
boolean debugEnabled = _log.isDebugEnabled();
boolean enqueueMessage = true;
try {
Checkpoint cp = curState.getCheckpoint();
DbusEventBuffer eventBuffer = curState.getDataEventsBuffer();
if (debugEnabled)
_log.debug("Sending bootstrap events to buffer");
//eventBuffer.startEvents();
DbusEventInternalReadable cpEvent = getEventFactory().createCheckpointEvent(cp);
byte[] cpEventBytes = new byte[cpEvent.size()];
if (debugEnabled) {
_log.debug("checkpoint event size: " + cpEventBytes.length);
_log.debug("checkpoint event:" + cpEvent.toString());
}
cpEvent.getRawBytes().get(cpEventBytes);
ByteArrayInputStream cpIs = new ByteArrayInputStream(cpEventBytes);
ReadableByteChannel cpRbc = Channels.newChannel(cpIs);
UnifiedClientStats unifiedClientStats = _sourcesConn.getUnifiedClientStats();
sendHeartbeat(unifiedClientStats);
int ecnt = eventBuffer.readEvents(cpRbc);
success = (ecnt > 0);
if (!success) {
_log.error("Unable to write bootstrap phase marker");
} else {
ChunkedBodyReadableByteChannel readChannel = curState.getReadChannel();
String remoteErrorName = RemoteExceptionHandler.getExceptionName(readChannel);
Throwable remoteError = _remoteExceptionHandler.getException(readChannel);
if (null != remoteError && remoteError instanceof BootstrapDatabaseTooOldException) {
_log.error("Bootstrap database is too old!");
_remoteExceptionHandler.handleException(remoteError);
curState.switchToStreamResponseError();
} else if (null != remoteErrorName) {
//remote processing error
_log.error("read events error: " + RemoteExceptionHandler.getExceptionMessage(readChannel));
curState.switchToStreamResponseError();
} else {
sendHeartbeat(unifiedClientStats);
int eventsNum = eventBuffer.readEvents(readChannel, curState.getListeners(), _sourcesConn.getBootstrapEventsStatsCollector());
if (eventsNum == 0 && _remoteExceptionHandler.getPendingEventSize(readChannel) > eventBuffer.getMaxReadBufferCapacity()) {
String err = "ReadBuffer max capacity(" + eventBuffer.getMaxReadBufferCapacity() + ") is less than event size(" + _remoteExceptionHandler.getPendingEventSize(readChannel) + "). Increase databus.client.connectionDefaults.bstEventBuffer.maxEventSize and restart.";
_log.fatal(err);
enqueueMessage(LifecycleMessage.createSuspendOnErroMessage(new PendingEventTooLargeException(err)));
return;
} else {
resetServerRetries();
if (debugEnabled)
_log.debug("Sending events to buffer");
numEventsInCurrentState += eventsNum;
_log.info("Bootstrap events read so far: " + numEventsInCurrentState);
String status = readChannel.getMetadata("PhaseCompleted");
final BootstrapCheckpointHandler ckptHandler = curState.getBstCheckpointHandler();
if (status != null) {
// set status in checkpoint to indicate that we are done with the current source
if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_CATCHUP) {
ckptHandler.finalizeCatchupPhase(cp);
} else if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT) {
ckptHandler.finalizeSnapshotPhase(cp);
} else {
throw new RuntimeException("Invalid bootstrap phase: " + cp.getConsumptionMode());
}
_log.info("Bootstrap events read :" + numEventsInCurrentState + " during phase:" + cp.getConsumptionMode() + " [" + cp.getBootstrapSnapshotSourceIndex() + "," + cp.getBootstrapCatchupSourceIndex() + "]");
numEventsInCurrentState = 0;
} else {
// question: how is snapshotOffset maintained in ckpt
if (eventsNum > 0) {
cp.bootstrapCheckPoint();
}
}
curState.switchToStreamResponseDone();
}
}
}
} catch (InterruptedException ie) {
_log.error("interupted", ie);
success = false;
} catch (InvalidEventException e) {
_log.error("error reading events from server: " + e.getMessage(), e);
success = false;
} catch (RuntimeException e) {
_log.error("runtime error reading events from server: " + e.getMessage(), e);
success = false;
}
if (toTearConnAfterHandlingResponse()) {
tearConnectionAndEnqueuePickServer();
enqueueMessage = false;
} else if (!success) {
curState.switchToPickServer();
}
if (enqueueMessage)
enqueueMessage(curState);
}
Also used :
ReadableByteChannel(java.nio.channels.ReadableByteChannel)
UnifiedClientStats(com.linkedin.databus.client.pub.mbean.UnifiedClientStats)
PendingEventTooLargeException(com.linkedin.databus.core.PendingEventTooLargeException)
DbusEventInternalReadable(com.linkedin.databus.core.DbusEventInternalReadable)
Checkpoint(com.linkedin.databus.core.Checkpoint)
BootstrapCheckpointHandler(com.linkedin.databus.core.BootstrapCheckpointHandler)
DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer)
BootstrapDatabaseTooOldException(com.linkedin.databus2.core.container.request.BootstrapDatabaseTooOldException)
Checkpoint(com.linkedin.databus.core.Checkpoint)
ByteArrayInputStream(java.io.ByteArrayInputStream)
InvalidEventException(com.linkedin.databus.core.InvalidEventException)
Example 40 with DbusEventBuffer
use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.
the class TestMultiConsumerCallback method test3GroupedStreamConsumersHappyPath.
@Test(groups = { "small", "functional" })
public void test3GroupedStreamConsumersHappyPath() {
LOG.info("\n\nstarting test3GroupedStreamConsumersHappyPath()");
Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
eventsBuf.start(0);
eventsBuf.startEvents();
initBufferWithEvents(eventsBuf, 1, 1, (short) 1, keyCounts);
initBufferWithEvents(eventsBuf, 2, 2, (short) 3, keyCounts);
eventsBuf.endEvents(100L);
DatabusStreamConsumer mockConsumer1 = EasyMock.createStrictMock("consumer1", DatabusStreamConsumer.class);
EasyMock.makeThreadSafe(mockConsumer1, true);
DatabusStreamConsumer mockConsumer2 = EasyMock.createStrictMock("consumer2", DatabusStreamConsumer.class);
EasyMock.makeThreadSafe(mockConsumer2, true);
DatabusStreamConsumer mockConsumer3 = EasyMock.createStrictMock("consumer3", DatabusStreamConsumer.class);
EasyMock.makeThreadSafe(mockConsumer3, true);
DatabusCombinedConsumer sdccMockConsumer1 = new SelectingDatabusCombinedConsumer(mockConsumer1);
DatabusCombinedConsumer sdccMockConsumer2 = new SelectingDatabusCombinedConsumer(mockConsumer2);
DatabusCombinedConsumer sdccMockConsumer3 = new SelectingDatabusCombinedConsumer(mockConsumer3);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg1 = new DatabusV2ConsumerRegistration(Arrays.asList(sdccMockConsumer1, sdccMockConsumer2, sdccMockConsumer3), sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg1);
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
callback.setSourceMap(sourcesMap);
DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
assert iter.hasNext() : "unable to read event";
DbusEvent event1 = iter.next();
assert iter.hasNext() : "unable to read event";
DbusEvent event2 = iter.next();
assert iter.hasNext() : "unable to read event";
DbusEvent event3 = iter.next();
initMockStreamConsumer3OptEventFullLifecycle(mockConsumer1, event1, event2, event3, keyCounts);
initMockStreamConsumer3OptEventFullLifecycle(mockConsumer2, event1, event2, event3, keyCounts);
initMockStreamConsumer3OptEventFullLifecycle(mockConsumer3, event1, event2, event3, keyCounts);
assert3EventFullLifecycle(callback, event1, event2, event3);
EasyMock.verify(mockConsumer1);
EasyMock.verify(mockConsumer2);
EasyMock.verify(mockConsumer3);
assert (keyCounts.get(1L).get() + keyCounts.get(2L).get() + keyCounts.get(3L).get()) == 3 : "invalid number of calls: " + keyCounts.get(1L).get() + "," + keyCounts.get(2L).get() + "," + keyCounts.get(3L).get();
}
Also used :
DatabusStreamConsumer(com.linkedin.databus.client.pub.DatabusStreamConsumer)
DbusEvent(com.linkedin.databus.core.DbusEvent)
HashMap(java.util.HashMap)
Hashtable(java.util.Hashtable)
ArrayList(java.util.ArrayList)
DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer)
AtomicInteger(java.util.concurrent.atomic.AtomicInteger)
IdNamePair(com.linkedin.databus.core.util.IdNamePair)
DatabusCombinedConsumer(com.linkedin.databus.client.pub.DatabusCombinedConsumer)
Test(org.testng.annotations.Test)
BeforeTest(org.testng.annotations.BeforeTest)
AfterTest(org.testng.annotations.AfterTest)
Aggregations
DbusEventBuffer (com.linkedin.databus.core.DbusEventBuffer)46
Test (org.testng.annotations.Test)29
ArrayList (java.util.ArrayList)22
HashMap (java.util.HashMap)19
AtomicInteger (java.util.concurrent.atomic.AtomicInteger)19
Logger (org.apache.log4j.Logger)18
IdNamePair (com.linkedin.databus.core.util.IdNamePair)17
Checkpoint (com.linkedin.databus.core.Checkpoint)14
RegisterResponseEntry (com.linkedin.databus2.core.container.request.RegisterResponseEntry)12
ConditionCheck (com.linkedin.databus2.test.ConditionCheck)12
DbusEvent (com.linkedin.databus.core.DbusEvent)11
DbusEventsStatisticsCollector (com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector)11
Hashtable (java.util.Hashtable)11
DatabusStreamConsumer (com.linkedin.databus.client.pub.DatabusStreamConsumer)10
ServerInfo (com.linkedin.databus.client.pub.ServerInfo)10
DatabusSubscription (com.linkedin.databus.core.data_model.DatabusSubscription)9
AfterTest (org.testng.annotations.AfterTest)9
BeforeTest (org.testng.annotations.BeforeTest)9
InetSocketAddress (java.net.InetSocketAddress)7
NettyHttpDatabusRelayConnection (com.linkedin.databus.client.netty.NettyHttpDatabusRelayConnection)6
