Example 16 with SimpleTestServerConnection
use of com.linkedin.databus2.test.container.SimpleTestServerConnection in project databus by linkedin.
the class DummySuccessfulErrorCountingConsumer method testRelayFailoverPartialWindow1.
/**
* Client switches from R1 to R2 on server close and from R2 to R3 on timeout while in the middle of the windows.
*
* <pre>
* R1 EVB Boundaries --------------------------------------------------------------------------------
* ^ ^ ^ ^ ^ ^ ^ ^ ^ ^
* 0 10 20 30 40 50 60 70 80 90
*
* R2 EVB Boundaries --------------------------------------------------------------------------------
* ^ ^ ^ ^ ^
* 0 20 40 60 80
*
* R3 EVB Boundaries --------------------------------------------------------------------------------
* ^ ^ ^ ^
* 0 30 60 90
* </pre>
*
* Switch from R1 to R2 happens when windowScn = 10 and windowOffset = 8th event.
* Switch from R2 to R3 happens when windowScn = 40 and windowOffset = 3rd event.
*
* @throws Exception
*/
@Test
public void testRelayFailoverPartialWindow1() throws Exception {
final boolean debugOn = false;
final Logger log = Logger.getLogger("TestDatabusHttpClient.testRelayFailoverPartialWindow1");
log.setLevel(Level.INFO);
final int eventsNum = 200;
DbusEventInfo[] eventInfos = createSampleSchema1Events(eventsNum);
final long timeoutMult = debugOn ? 100000 : 1;
log.info("simulate relay buffers");
DbusEventBuffer[] relayBuffer = new DbusEventBuffer[RELAY_PORT.length];
List<List<Integer>> eventOfs = new ArrayList<List<Integer>>(3);
List<List<DbusEventKey>> eventKeys = new ArrayList<List<DbusEventKey>>(3);
for (int i = 0; i < RELAY_PORT.length; ++i) {
relayBuffer[i] = new DbusEventBuffer(_bufCfg);
relayBuffer[i].start(0);
WriteEventsResult wrRes = writeEventsToBuffer(relayBuffer[i], eventInfos, (i + 1) * 10);
List<Integer> ofs = wrRes.getOffsets();
eventOfs.add(ofs);
eventKeys.add(wrRes.getKeys());
}
List<DbusEventKey> key = eventKeys.get(0);
for (int i = 1; i < RELAY_PORT.length; ++i) {
assertEquals(" For Size index : " + i, key.size(), eventKeys.get(i).size());
assertEquals(" For index : " + i, key, eventKeys.get(i));
key = eventKeys.get(i);
}
int resp1EnfOfs = eventOfs.get(0).get(8);
log.info("figure out an event offset inside a window:" + resp1EnfOfs);
log.info("create client");
_stdClientCfgBuilder.getContainer().setReadTimeoutMs(DEFAULT_READ_TIMEOUT_MS);
final DatabusHttpClientImpl client = new DatabusHttpClientImpl(_stdClientCfgBuilder.build());
final TestConsumer consumer = new TestConsumer();
client.registerDatabusStreamListener(consumer, null, SOURCE1_NAME);
client.start();
try {
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return client._relayConnections.size() == 1;
}
}, "sources connection present", 100, log);
final DatabusSourcesConnection clientConn = client._relayConnections.get(0);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != clientConn.getRelayPullThread().getLastOpenConnection();
}
}, "relay connection present", 100, log);
final NettyHttpDatabusRelayConnection relayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
final NettyHttpDatabusRelayConnectionInspector relayConnInsp = new NettyHttpDatabusRelayConnectionInspector(relayConn);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != relayConnInsp.getChannel() && relayConnInsp.getChannel().isConnected();
}
}, "client connected", 200, log);
log.info("figure out the connection to the relay");
Channel clientChannel = relayConnInsp.getChannel();
InetSocketAddress relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
SocketAddress clientAddr = clientChannel.getLocalAddress();
int relayPort = relayAddr.getPort();
log.info("relay selected: " + relayPort);
SimpleTestServerConnection relay = null;
for (int i = 0; i < RELAY_PORT.length; ++i) {
if (relayPort == RELAY_PORT[i])
relay = _dummyServer[i];
}
assertTrue(null != relay);
final SocketAddress testClientAddr = clientAddr;
final SimpleTestServerConnection testRelay = relay;
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != testRelay.getChildChannel(testClientAddr);
}
}, "relay detects new connection", 1000, log);
Channel serverChannel = relay.getChildChannel(clientAddr);
assertTrue(null != serverChannel);
ChannelPipeline serverPipeline = serverChannel.getPipeline();
SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
log.info("process the /sources request");
NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
objCapture.clear();
log.info("send back the /sources response");
HttpResponse sourcesResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
sourcesResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
sourcesResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
return "1".equals(idListString);
}
}, "client processes /sources response", 100, log);
log.info("process the /register request");
NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
objCapture.clear();
log.info("send back the /register response");
RegisterResponseEntry entry = new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR);
String responseStr = NettyTestUtils.generateRegisterResponse(entry);
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
}
}, "client processes /register response", 100, log);
log.info("process /stream call and return a partial window");
NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*", 1000);
objCapture.clear();
log.info("send back the /stream response");
final DbusEventsStatisticsCollector stats = new DbusEventsStatisticsCollector(1, "test1", true, false, null);
Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
ChannelBuffer streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[0], cp, resp1EnfOfs, stats);
HttpResponse streamResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
streamResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
streamResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
log.info("make sure the client processes the /stream response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("LastWritten SCN:" + clientConn.getDataEventsBuffer().lastWrittenScn());
return clientConn.getDataEventsBuffer().lastWrittenScn() == 10;
}
}, "client receives /stream response", 1100, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("events num=" + consumer.getEventNum());
return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
}
}, "client processes /stream response", 110000, log);
assertEquals(-1, consumer.getRollbackScn());
int rollbackNum = 0;
assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
List<DbusEventKey> expKeys = eventKeys.get(0).subList(0, (int) stats.getTotalStats().getNumDataEvents());
List<Long> expSeqs = new ArrayList<Long>();
for (int i = 0; i < stats.getTotalStats().getNumDataEvents(); i++) expSeqs.add(10L);
long numEvents = stats.getTotalStats().getNumDataEvents();
assertEquals("Keys", expKeys, consumer.getKeys());
assertEquals("Sequences", expSeqs, consumer.getSequences());
log.info("now kill the relay and wait for a failover");
serverChannel.close();
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != relayConnInsp.getChannel() && !relayConnInsp.getChannel().isConnected();
}
}, "client disconnected", 200, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
}
}, "new netty connection", 200, log);
log.info("/////////// FAKING CONNECTION TO NEW RELAY //////////////");
final NettyHttpDatabusRelayConnection newRelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
final NettyHttpDatabusRelayConnectionInspector newRelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(newRelayConn);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != newRelayConnInsp.getChannel() && newRelayConnInsp.getChannel().isConnected();
}
}, "client connected to new relay", 200, log);
log.info("figure out the connection to the relay");
clientChannel = newRelayConnInsp.getChannel();
relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
clientAddr = clientChannel.getLocalAddress();
relayPort = relayAddr.getPort();
log.info("new relay selected: " + relayPort);
relay = null;
int relayIdx = 0;
for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
if (relayPort == RELAY_PORT[relayIdx])
relay = _dummyServer[relayIdx];
}
assertTrue(null != relay);
serverChannel = relay.getChildChannel(clientAddr);
assertTrue(null != serverChannel);
serverPipeline = serverChannel.getPipeline();
objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
log.info("process the /sources request");
NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
objCapture.clear();
log.info("send back the /sources response");
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
return "1".equals(idListString);
}
}, "client processes /sources response", 100, log);
log.info("process the /register request");
NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
objCapture.clear();
log.info("send back the /register response");
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.info("make sure the client processes the /register response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
}
}, "client processes /register response", 100, log);
log.info("process /stream call and return a partial window");
Matcher streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
String cpString = streamMatcher.group(1);
objCapture.clear();
int respStartOfs = eventOfs.get(1).get(1);
int respEndOfs = eventOfs.get(1).get(34);
cp = new Checkpoint(cpString);
//last window read was partial
assertTrue(cp.getWindowOffset() > 0);
streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[1], cp, respEndOfs - respStartOfs, stats);
NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
return clientConn.getDataEventsBuffer().lastWrittenScn() == 40;
}
}, "client receives /stream response", 1100, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("events num=" + consumer.getEventNum());
return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
}
}, "client processes /stream response", 11000, log);
assertEquals(20, consumer.getRollbackScn());
log.info("one more onStartDataEventSequence because of the rolback");
++rollbackNum;
assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
expKeys.addAll(eventKeys.get(1).subList(0, (int) (stats.getTotalStats().getNumDataEvents() - numEvents)));
for (int i = 0; i < stats.getTotalStats().getNumDataEvents() - numEvents; i++) expSeqs.add((i / 20) * 20 + 20L);
assertEquals("Keys", expKeys, consumer.getKeys());
assertEquals("Sequences", expSeqs, consumer.getSequences());
numEvents = stats.getTotalStats().getNumDataEvents();
///////////////////////////////////
//simulate a timeout on the server; the client would have sent a /stream call and there
//will be no response from the server, so eventually it should time out and switch servers
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("Channel :" + newRelayConnInsp.getChannel());
return (null != newRelayConnInsp.getChannel()) && (!newRelayConnInsp.getChannel().isConnected());
}
}, "waiting for a reconnect", (long) (DEFAULT_READ_TIMEOUT_MS * 1.5), log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
}
}, "new netty connection", 200, log);
final NettyHttpDatabusRelayConnection new2RelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
final NettyHttpDatabusRelayConnectionInspector new2RelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(new2RelayConn);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != new2RelayConnInsp.getChannel() && new2RelayConnInsp.getChannel().isConnected();
}
}, "client connected to third relay", 200, log);
log.info("figure out the connection to the relay");
clientChannel = new2RelayConnInsp.getChannel();
relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
clientAddr = clientChannel.getLocalAddress();
relayPort = relayAddr.getPort();
log.info("third relay selected: " + relayPort);
relay = null;
relayIdx = 0;
for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
if (relayPort == RELAY_PORT[relayIdx])
relay = _dummyServer[relayIdx];
}
assertTrue(null != relay);
serverChannel = relay.getChildChannel(clientAddr);
assertTrue(null != serverChannel);
serverPipeline = serverChannel.getPipeline();
objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
log.info("process the /sources request");
NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
objCapture.clear();
log.info("send back the /sources response");
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
return "1".equals(idListString);
}
}, "client processes /sources response", 100, log);
log.info("process the /register request");
NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
objCapture.clear();
log.info("SEND BACK THE /register RESPONSE");
//clientConn.getRelayDispatcher().getLog().setLevel(Level.DEBUG);
//RangeBasedReaderWriterLock.LOG.setLevel(Level.DEBUG);
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
}
}, "client processes /register response", timeoutMult * 100, log);
log.info("PROCESS the /stream CALL AND RETURN A PARTIAL WINDOW");
streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
cpString = streamMatcher.group(1);
objCapture.clear();
respStartOfs = eventOfs.get(2).get(1);
respEndOfs = eventOfs.get(2).get(84);
log.debug("Checkpoint String is :" + cpString);
cp = new Checkpoint(cpString);
log.info("last window read was partial. So the client would have reset the windowOffset");
assertTrue("Is WindowOffset Cleared", cp.getWindowOffset() == -1);
assertEquals("WindowSCN == PrevSCN. Ckpt :" + cp, cp.getWindowScn(), cp.getPrevScn());
streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[2], cp, respEndOfs - respStartOfs, stats);
NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
log.debug("NumEvents already seen :" + numEvents);
log.info("make sure the client processes the response correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn() + ", NumEvents :" + stats.getTotalStats().getNumDataEvents());
return clientConn.getDataEventsBuffer().lastWrittenScn() == 90;
}
}, "client receives /stream response, Sequences :" + consumer.getSequences(), timeoutMult * 1100, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("events num=" + consumer.getEventNum());
return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
}
}, "client processes /stream response", timeoutMult * 1100, log);
log.info("one more onStartDataEventSequence because of the rollback");
assertEquals(30, consumer.getRollbackScn());
++rollbackNum;
assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
expKeys.addAll(eventKeys.get(2).subList(0, (int) (stats.getTotalStats().getNumDataEvents() - numEvents)));
for (int i = 0; i < stats.getTotalStats().getNumDataEvents() - numEvents; i++) expSeqs.add((i / 30) * 30 + 30L);
assertEquals("Keys", expKeys, consumer.getKeys());
assertEquals("Sequences", expSeqs, consumer.getSequences());
numEvents = stats.getTotalStats().getNumDataEvents();
assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
} finally {
client.shutdown();
}
}
Also used :
NettyHttpDatabusRelayConnection(com.linkedin.databus.client.netty.NettyHttpDatabusRelayConnection)
NettyHttpDatabusRelayConnectionInspector(com.linkedin.databus.client.netty.NettyHttpDatabusRelayConnectionInspector)
DefaultHttpChunk(org.jboss.netty.handler.codec.http.DefaultHttpChunk)
Matcher(java.util.regex.Matcher)
InetSocketAddress(java.net.InetSocketAddress)
ArrayList(java.util.ArrayList)
DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector)
Logger(org.apache.log4j.Logger)
ChannelBuffer(org.jboss.netty.buffer.ChannelBuffer)
DbusEventInfo(com.linkedin.databus.core.DbusEventInfo)
List(java.util.List)
ArrayList(java.util.ArrayList)
SocketAddress(java.net.SocketAddress)
InetSocketAddress(java.net.InetSocketAddress)
ConditionCheck(com.linkedin.databus2.test.ConditionCheck)
Channel(org.jboss.netty.channel.Channel)
SimpleTestServerConnection(com.linkedin.databus2.test.container.SimpleTestServerConnection)
DefaultHttpResponse(org.jboss.netty.handler.codec.http.DefaultHttpResponse)
HttpResponse(org.jboss.netty.handler.codec.http.HttpResponse)
Checkpoint(com.linkedin.databus.core.Checkpoint)
ChannelPipeline(org.jboss.netty.channel.ChannelPipeline)
DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer)
AtomicInteger(java.util.concurrent.atomic.AtomicInteger)
Checkpoint(com.linkedin.databus.core.Checkpoint)
SimpleObjectCaptureHandler(com.linkedin.databus2.test.container.SimpleObjectCaptureHandler)
DefaultHttpResponse(org.jboss.netty.handler.codec.http.DefaultHttpResponse)
RegisterResponseEntry(com.linkedin.databus2.core.container.request.RegisterResponseEntry)
DbusEventKey(com.linkedin.databus.core.DbusEventKey)
DefaultHttpChunk(org.jboss.netty.handler.codec.http.DefaultHttpChunk)
HttpChunk(org.jboss.netty.handler.codec.http.HttpChunk)
Test(org.testng.annotations.Test)
Example 17 with SimpleTestServerConnection
use of com.linkedin.databus2.test.container.SimpleTestServerConnection in project databus by linkedin.
the class DummySuccessfulErrorCountingConsumer method captureAndReplyRegisterRequest.
private void captureAndReplyRegisterRequest(SimpleObjectCaptureHandler objCapture, SimpleTestServerConnection relay, SocketAddress clientAddr, final DatabusSourcesConnection clientConn, Logger log) throws JsonGenerationException, JsonMappingException, IOException {
log.debug("process the /register request");
NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
objCapture.clear();
log.debug("send back the /register response");
RegisterResponseEntry entry = new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR);
String responseStr = NettyTestUtils.generateRegisterResponse(entry);
HttpResponse registerResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, registerResp, body);
log.debug("make sure the client processes the response /register correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
}
}, "client processes /register response", 100, log);
}
Also used :
ConditionCheck(com.linkedin.databus2.test.ConditionCheck)
DefaultHttpChunk(org.jboss.netty.handler.codec.http.DefaultHttpChunk)
DefaultHttpResponse(org.jboss.netty.handler.codec.http.DefaultHttpResponse)
RegisterResponseEntry(com.linkedin.databus2.core.container.request.RegisterResponseEntry)
DefaultHttpResponse(org.jboss.netty.handler.codec.http.DefaultHttpResponse)
HttpResponse(org.jboss.netty.handler.codec.http.HttpResponse)
DefaultHttpChunk(org.jboss.netty.handler.codec.http.DefaultHttpChunk)
HttpChunk(org.jboss.netty.handler.codec.http.HttpChunk)
Example 18 with SimpleTestServerConnection
use of com.linkedin.databus2.test.container.SimpleTestServerConnection in project databus by linkedin.
the class DummySuccessfulErrorCountingConsumer method setUpClass.
@BeforeClass
public void setUpClass() throws InvalidConfigException {
//set up logging
TestUtil.setupLoggingWithTimestampedFile(true, "/tmp/TestDatabusHttpClient_", ".log", Level.INFO);
InternalLoggerFactory.setDefaultFactory(new Log4JLoggerFactory());
//initialize relays
for (int relayN = 0; relayN < RELAY_PORT.length; ++relayN) {
_dummyServer[relayN] = new SimpleTestServerConnection(_eventFactory.getByteOrder(), SimpleTestServerConnection.ServerType.NIO);
_dummyServer[relayN].setPipelineFactory(new ChannelPipelineFactory() {
@Override
public ChannelPipeline getPipeline() throws Exception {
return Channels.pipeline(new LoggingHandler(InternalLogLevel.DEBUG), new HttpServerCodec(), new LoggingHandler(InternalLogLevel.DEBUG), new SimpleObjectCaptureHandler());
}
});
_dummyServer[relayN].start(RELAY_PORT[relayN]);
}
//create standard client config
DatabusHttpClientImpl.Config clientCfgBuilder = new DatabusHttpClientImpl.Config();
clientCfgBuilder.getContainer().setHttpPort(0);
clientCfgBuilder.getContainer().getJmx().setRmiEnabled(false);
clientCfgBuilder.getContainer().setReadTimeoutMs(10000000);
clientCfgBuilder.getConnectionDefaults().getPullerRetries().setInitSleep(10);
clientCfgBuilder.getRuntime().getBootstrap().setEnabled(false);
clientCfgBuilder.getCheckpointPersistence().setClearBeforeUse(true);
for (int i = 0; i < RELAY_PORT.length; ++i) {
clientCfgBuilder.getRuntime().getRelay(Integer.toString(i)).setHost("localhost");
clientCfgBuilder.getRuntime().getRelay(Integer.toString(i)).setPort(RELAY_PORT[i]);
clientCfgBuilder.getRuntime().getRelay(Integer.toString(i)).setSources(SOURCE1_NAME);
}
_stdClientCfgBuilder = clientCfgBuilder;
_stdClientCfg = clientCfgBuilder.build();
//create standard relay buffer config
DbusEventBuffer.Config bufCfgBuilder = new DbusEventBuffer.Config();
bufCfgBuilder.setAllocationPolicy(AllocationPolicy.HEAP_MEMORY.toString());
bufCfgBuilder.setMaxSize(100000);
bufCfgBuilder.setScnIndexSize(128);
bufCfgBuilder.setAverageEventSize(1);
_bufCfg = bufCfgBuilder.build();
}
Also used :
LoggingHandler(org.jboss.netty.handler.logging.LoggingHandler)
SimpleTestServerConnection(com.linkedin.databus2.test.container.SimpleTestServerConnection)
Checkpoint(com.linkedin.databus.core.Checkpoint)
InvalidConfigException(com.linkedin.databus.core.util.InvalidConfigException)
JsonGenerationException(org.codehaus.jackson.JsonGenerationException)
UnsupportedEncodingException(java.io.UnsupportedEncodingException)
JsonMappingException(org.codehaus.jackson.map.JsonMappingException)
IOException(java.io.IOException)
DatabusClientException(com.linkedin.databus.client.pub.DatabusClientException)
ChannelPipeline(org.jboss.netty.channel.ChannelPipeline)
DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer)
SimpleObjectCaptureHandler(com.linkedin.databus2.test.container.SimpleObjectCaptureHandler)
HttpServerCodec(org.jboss.netty.handler.codec.http.HttpServerCodec)
Log4JLoggerFactory(org.jboss.netty.logging.Log4JLoggerFactory)
ChannelPipelineFactory(org.jboss.netty.channel.ChannelPipelineFactory)
BeforeClass(org.testng.annotations.BeforeClass)
Example 19 with SimpleTestServerConnection
use of com.linkedin.databus2.test.container.SimpleTestServerConnection in project databus by linkedin.
the class DummySuccessfulErrorCountingConsumer method testInStreamError.
/**
* Tests the logic of the client to disconnect from a relay while still processing
* data from the relay. We want to make sure that ChunkedBodyReadableByteChannel
* and thus the StreamHttpResponseProcessor do net get blocked with unprocessed data.
* One way to trigger such an error is to insert some invalid data inside the
* response and then some more data ( > 4MB which is the current buffer size). */
@Test
public void testInStreamError() throws Exception {
final Logger log = Logger.getLogger("TestDatabusHttpClient.testInStreamError");
//log.setLevel(Level.DEBUG);
final int eventsNum = 20;
DbusEventInfo[] eventInfos = createSampleSchema1Events(eventsNum);
//simulate relay buffers
DbusEventBuffer relayBuffer = new DbusEventBuffer(_bufCfg);
relayBuffer.start(0);
writeEventsToBuffer(relayBuffer, eventInfos, 4);
//prepare stream response
Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
final DbusEventsStatisticsCollector stats = new DbusEventsStatisticsCollector(1, "test1", true, false, null);
ChannelBuffer streamResPrefix = NettyTestUtils.streamToChannelBuffer(relayBuffer, cp, 20000, stats);
final StringBuilder alotOfDeadbeef = new StringBuilder();
for (int i = 0; i < 1024 * 1024; ++i) {
alotOfDeadbeef.append("DEADBEEF ");
}
ChannelBuffer streamResSuf = ChannelBuffers.wrappedBuffer(alotOfDeadbeef.toString().getBytes("UTF-8"));
final ChannelBuffer streamRes = ChannelBuffers.wrappedBuffer(streamResPrefix, streamResSuf);
//create client
_stdClientCfgBuilder.getContainer().setReadTimeoutMs(DEFAULT_READ_TIMEOUT_MS);
final DatabusHttpClientImpl client = new DatabusHttpClientImpl(_stdClientCfgBuilder.build());
final TestConsumer consumer = new TestConsumer();
client.registerDatabusStreamListener(consumer, null, SOURCE1_NAME);
client.start();
try {
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return client._relayConnections.size() == 1;
}
}, "sources connection present", 100, log);
final DatabusSourcesConnection clientConn = client._relayConnections.get(0);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != clientConn.getRelayPullThread().getLastOpenConnection();
}
}, "relay connection present", 100, log);
final NettyHttpDatabusRelayConnection relayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
final NettyHttpDatabusRelayConnectionInspector relayConnInsp = new NettyHttpDatabusRelayConnectionInspector(relayConn);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != relayConnInsp.getChannel() && relayConnInsp.getChannel().isConnected();
}
}, "client connected", 200, log);
//figure out the connection to the relay
Channel clientChannel = relayConnInsp.getChannel();
InetSocketAddress relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
SocketAddress clientAddr = clientChannel.getLocalAddress();
int relayPort = relayAddr.getPort();
log.info("relay selected: " + relayPort);
SimpleTestServerConnection relay = null;
for (int i = 0; i < RELAY_PORT.length; ++i) {
if (relayPort == RELAY_PORT[i])
relay = _dummyServer[i];
}
assertTrue(null != relay);
final SocketAddress testClientAddr = clientAddr;
final SimpleTestServerConnection testRelay = relay;
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != testRelay.getChildChannel(testClientAddr);
}
}, "relay detects new connection", 1000, log);
Channel serverChannel = relay.getChildChannel(clientAddr);
assertTrue(null != serverChannel);
ChannelPipeline serverPipeline = serverChannel.getPipeline();
SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
//process the /sources request
NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
objCapture.clear();
//send back the /sources response
HttpResponse sourcesResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
sourcesResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
sourcesResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
//make sure the client processes the response correctly
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
return "1".equals(idListString);
}
}, "client processes /sources response", 100, log);
log.debug("process the /register request");
NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
objCapture.clear();
log.debug("send back the /register response");
RegisterResponseEntry entry = new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR);
String responseStr = NettyTestUtils.generateRegisterResponse(entry);
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
log.debug("make sure the client processes the response /register correctly");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
}
}, "client processes /register response", 100, log);
log.debug("process /stream call and return a response with garbled suffix");
NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*", 1000);
objCapture.clear();
final HttpResponse streamResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
streamResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
streamResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
//send the response asynchronously in case the client blocks
final Thread streamRespThread = new Thread(new Runnable() {
@Override
public void run() {
NettyTestUtils.sendServerResponses(testRelay, testClientAddr, streamResp, new DefaultHttpChunk(streamRes), 60000);
}
}, "send /stream resp");
streamRespThread.setDaemon(true);
streamRespThread.start();
log.debug("make sure the client disconnects and recovers from the /stream response");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != relayConnInsp.getChannel() && !relayConnInsp.getChannel().isConnected();
}
}, "client disconnected", 30000, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
}
}, "new netty connection", 30000, log);
//make sure the relay send thread is dead
streamRespThread.join(100);
Assert.assertTrue(!streamRespThread.isAlive());
log.debug("PHASE 2: make sure the client has fully recovered and able to connect to another relay");
final NettyHttpDatabusRelayConnection newRelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
final NettyHttpDatabusRelayConnectionInspector newRelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(newRelayConn);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return null != newRelayConnInsp.getChannel() && newRelayConnInsp.getChannel().isConnected();
}
}, "client connected to new relay", 200, log);
//figure out the connection to the relay
clientChannel = newRelayConnInsp.getChannel();
relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
clientAddr = clientChannel.getLocalAddress();
relayPort = relayAddr.getPort();
log.info("new relay selected: " + relayPort);
relay = null;
int relayIdx = 0;
for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
if (relayPort == RELAY_PORT[relayIdx])
relay = _dummyServer[relayIdx];
}
assertTrue(null != relay);
serverChannel = relay.getChildChannel(clientAddr);
assertTrue(null != serverChannel);
serverPipeline = serverChannel.getPipeline();
objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
//process the /sources request
NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
objCapture.clear();
//send back the /sources response
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
//make sure the client processes the response correctly
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
return "1".equals(idListString);
}
}, "client processes /sources response", 100, log);
//process the /register request
NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
objCapture.clear();
//send back the /register response
body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
//make sure the client processes the response correctly
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
}
}, "client processes /register response", 100, log);
//process /stream call and return a partial window
Matcher streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
String cpString = streamMatcher.group(1);
log.debug("/stream checkpoint: " + cpString);
objCapture.clear();
NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamResPrefix));
//make sure the client processes the response correctly
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
return clientConn.getDataEventsBuffer().lastWrittenScn() == 20;
}
}, "client receives /stream response", 1100, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("events num=" + consumer.getEventNum());
return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
}
}, "client processes /stream response", 11000, log);
} finally {
client.shutdown();
}
}
Also used :
NettyHttpDatabusRelayConnection(com.linkedin.databus.client.netty.NettyHttpDatabusRelayConnection)
NettyHttpDatabusRelayConnectionInspector(com.linkedin.databus.client.netty.NettyHttpDatabusRelayConnectionInspector)
DefaultHttpChunk(org.jboss.netty.handler.codec.http.DefaultHttpChunk)
Matcher(java.util.regex.Matcher)
InetSocketAddress(java.net.InetSocketAddress)
DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector)
Logger(org.apache.log4j.Logger)
ChannelBuffer(org.jboss.netty.buffer.ChannelBuffer)
DbusEventInfo(com.linkedin.databus.core.DbusEventInfo)
SocketAddress(java.net.SocketAddress)
InetSocketAddress(java.net.InetSocketAddress)
ConditionCheck(com.linkedin.databus2.test.ConditionCheck)
Channel(org.jboss.netty.channel.Channel)
SimpleTestServerConnection(com.linkedin.databus2.test.container.SimpleTestServerConnection)
DefaultHttpResponse(org.jboss.netty.handler.codec.http.DefaultHttpResponse)
HttpResponse(org.jboss.netty.handler.codec.http.HttpResponse)
Checkpoint(com.linkedin.databus.core.Checkpoint)
ChannelPipeline(org.jboss.netty.channel.ChannelPipeline)
DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer)
Checkpoint(com.linkedin.databus.core.Checkpoint)
SimpleObjectCaptureHandler(com.linkedin.databus2.test.container.SimpleObjectCaptureHandler)
DefaultHttpResponse(org.jboss.netty.handler.codec.http.DefaultHttpResponse)
RegisterResponseEntry(com.linkedin.databus2.core.container.request.RegisterResponseEntry)
DefaultHttpChunk(org.jboss.netty.handler.codec.http.DefaultHttpChunk)
HttpChunk(org.jboss.netty.handler.codec.http.HttpChunk)
Test(org.testng.annotations.Test)
Aggregations
SimpleTestServerConnection (com.linkedin.databus2.test.container.SimpleTestServerConnection)17
ChannelPipeline (org.jboss.netty.channel.ChannelPipeline)17
SimpleObjectCaptureHandler (com.linkedin.databus2.test.container.SimpleObjectCaptureHandler)11
ChannelBuffer (org.jboss.netty.buffer.ChannelBuffer)11
Test (org.testng.annotations.Test)11
DbusEventBuffer (com.linkedin.databus.core.DbusEventBuffer)10
ConditionCheck (com.linkedin.databus2.test.ConditionCheck)9
DefaultHttpChunk (org.jboss.netty.handler.codec.http.DefaultHttpChunk)8
DefaultHttpResponse (org.jboss.netty.handler.codec.http.DefaultHttpResponse)8
HttpChunk (org.jboss.netty.handler.codec.http.HttpChunk)8
HttpResponse (org.jboss.netty.handler.codec.http.HttpResponse)8
Checkpoint (com.linkedin.databus.core.Checkpoint)7
RegisterResponseEntry (com.linkedin.databus2.core.container.request.RegisterResponseEntry)7
NettyHttpDatabusRelayConnection (com.linkedin.databus.client.netty.NettyHttpDatabusRelayConnection)6
NettyHttpDatabusRelayConnectionInspector (com.linkedin.databus.client.netty.NettyHttpDatabusRelayConnectionInspector)6
DbusEventInfo (com.linkedin.databus.core.DbusEventInfo)6
DbusEventsStatisticsCollector (com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector)6
InvalidConfigException (com.linkedin.databus.core.util.InvalidConfigException)6
InetSocketAddress (java.net.InetSocketAddress)6
SocketAddress (java.net.SocketAddress)6
