Examples with HttpChunk org.jboss.netty.handler.codec.http.HttpChunk used on opensource projects

Example 26 with HttpChunk

use of org.jboss.netty.handler.codec.http.HttpChunk in project databus by linkedin.

the class TestResponseProcessors method testStartSCNHappyPathChunked.

@Test
public void testStartSCNHappyPathChunked() throws Exception {
    TestAbstractQueue queue = new TestAbstractQueue();
    TestConnectionStateMessage stateMsg = new TestConnectionStateMessage();
    DefaultHttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    TestRemoteExceptionHandler remoteExHandler = new TestRemoteExceptionHandler();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    BootstrapStartScnHttpResponseProcessor processor = new BootstrapStartScnHttpResponseProcessor(null, queue, stateMsg, cp, remoteExHandler, null);
    ChannelBuffer buf = getScnResponse();
    HttpChunk httpChunk = new DefaultHttpChunk(buf);
    HttpChunkTrailer httpChunkTrailer = new DefaultHttpChunkTrailer();
    processor.startResponse(httpResponse);
    processor.addChunk(httpChunk);
    processor.addTrailer(httpChunkTrailer);
    processor.finishResponse();
    Assert.assertEquals("Error Handled", false, processor._errorHandled);
    Assert.assertEquals("Processor Response State", AbstractHttpResponseProcessorDecorator.ResponseStatus.CHUNKS_FINISHED, processor._responseStatus);
    Assert.assertEquals("Actor Queue Size", 1, queue.getMessages().size());
    Assert.assertEquals("Expected ConnectionStateMessage", "TestConnectionStateMessage", queue.getMessages().get(0).getClass().getSimpleName());
    TestConnectionStateMessage gotMsg = (TestConnectionStateMessage) (queue.getMessages().get(0));
    Assert.assertEquals("Expected ConnectionStateMessage State", TestConnectionStateMessage.State.STARTSCN_RESPONSE_SUCCESS, gotMsg._state);
    Assert.assertEquals("StartSCN Response Id Check", new Long(5678912), cp.getBootstrapStartScn());
}

Example 27 with HttpChunk

use of org.jboss.netty.handler.codec.http.HttpChunk in project databus by linkedin.

the class HttpRequestLoggingHandler method writeRequested.

@Override
public void writeRequested(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
    if (e.getMessage() instanceof HttpRequest) {
        HttpRequest req = (HttpRequest) e.getMessage();
        startHttpRequest(true, req);
    } else if (e.getMessage() instanceof HttpResponse) {
        HttpResponse resp = (HttpResponse) e.getMessage();
        startHttpResponse(false, resp);
    } else if (e.getMessage() instanceof HttpChunk) {
        HttpChunk chunk = (HttpChunk) e.getMessage();
        processHttpChunk(false, chunk);
    }
    super.writeRequested(ctx, e);
}

Example 28 with HttpChunk

use of org.jboss.netty.handler.codec.http.HttpChunk in project databus by linkedin.

the class HttpRequestLoggingHandler method messageReceived.

@Override
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
    if (e.getMessage() instanceof HttpRequest) {
        // HttpRequest is for a relay from client/bootstrap, hence it is an inbound request
        // ( outbound = false )
        HttpRequest req = (HttpRequest) e.getMessage();
        startHttpRequest(false, req);
    } else if (e.getMessage() instanceof HttpResponse) {
        // HttpResponse is from a relay to a client/bootstrap, hence it is an outbound request
        // ( outbound = true )
        HttpResponse resp = (HttpResponse) e.getMessage();
        startHttpResponse(true, resp);
    } else if (e.getMessage() instanceof HttpChunk) {
        HttpChunk chunk = (HttpChunk) e.getMessage();
        processHttpChunk(true, chunk);
        if (State.OUTBOUND_RESPONSE_END == _state) {
            endHttpResponse(true);
        }
    }
    super.messageReceived(ctx, e);
}

Example 29 with HttpChunk

use of org.jboss.netty.handler.codec.http.HttpChunk in project databus by linkedin.

the class DummySuccessfulErrorCountingConsumer method testRelayFailoverPartialWindow2.

@Test
public /**
 * R1 EVB Boundaries ------------------------------------------------------------------------------
 *                   ^                        ^                          ^                    ^
 *                   0                        30                         60                   90
 * R2 EVB Boundaries --------------------------------------------------------------------------------
 *                   ^               ^                   ^               ^            ^
 *                   0               20                  40              60           80
 *
 * R3 EVB Boundaries --------------------------------------------------------------------------------
 *                   ^      ^        ^         ^         ^       ^       ^      ^     ^        ^
 *                   0      10       20        30        40      50      60     70    80       90
 *
 *  Client switches from R1 to R2 on server close and from R2 to R3 on timeout while in the middle of the windows.
 *
 *  Switch from R1 -> R2 happens when windowScn = 30 and WindowOffset = 8th event
 *  Switch from R2 -> R3 happens when windowScn = 40 and windowOffset = 5th event
 *
 * @throws Exception
 */
void testRelayFailoverPartialWindow2() throws Exception {
    final Logger log = Logger.getLogger("TestDatabusHttpClient.testRelayFailoverPartialWindow2");
    // log.setLevel(Level.DEBUG);
    final int eventsNum = 200;
    DbusEventInfo[] eventInfos = createSampleSchema1Events(eventsNum);
    // simulate relay buffers
    DbusEventBuffer[] relayBuffer = new DbusEventBuffer[RELAY_PORT.length];
    List<List<Integer>> eventOfs = new ArrayList<List<Integer>>(3);
    List<List<DbusEventKey>> eventKeys = new ArrayList<List<DbusEventKey>>(3);
    for (int i = 0; i < RELAY_PORT.length; ++i) {
        relayBuffer[i] = new DbusEventBuffer(_bufCfg);
        relayBuffer[i].start(0);
        WriteEventsResult wrRes = writeEventsToBuffer(relayBuffer[i], eventInfos, (RELAY_PORT.length - i) * 10);
        List<Integer> ofs = wrRes.getOffsets();
        eventOfs.add(ofs);
        eventKeys.add(wrRes.getKeys());
    }
    List<DbusEventKey> key = eventKeys.get(0);
    for (int i = 1; i < RELAY_PORT.length; ++i) {
        assertEquals(key, eventKeys.get(i));
        key = eventKeys.get(i);
    }
    // figure out an event offset inside a window
    int resp1EnfOfs = eventOfs.get(0).get(8);
    // create client
    _stdClientCfgBuilder.getContainer().setReadTimeoutMs(DEFAULT_READ_TIMEOUT_MS);
    final DatabusHttpClientImpl client = new DatabusHttpClientImpl(_stdClientCfgBuilder.build());
    final TestConsumer consumer = new TestConsumer();
    client.registerDatabusStreamListener(consumer, null, SOURCE1_NAME);
    client.start();
    try {
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return client._relayConnections.size() == 1;
            }
        }, "sources connection present", 100, log);
        final DatabusSourcesConnection clientConn = client._relayConnections.get(0);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != clientConn.getRelayPullThread().getLastOpenConnection();
            }
        }, "relay connection present", 100, log);
        final NettyHttpDatabusRelayConnection relayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector relayConnInsp = new NettyHttpDatabusRelayConnectionInspector(relayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != relayConnInsp.getChannel() && relayConnInsp.getChannel().isConnected();
            }
        }, "client connected", 200, log);
        // figure out the connection to the relay
        Channel clientChannel = relayConnInsp.getChannel();
        InetSocketAddress relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        SocketAddress clientAddr = clientChannel.getLocalAddress();
        int relayPort = relayAddr.getPort();
        log.info("relay selected: " + relayPort);
        SimpleTestServerConnection relay = null;
        for (int i = 0; i < RELAY_PORT.length; ++i) {
            if (relayPort == RELAY_PORT[i])
                relay = _dummyServer[i];
        }
        assertTrue(null != relay);
        final SocketAddress testClientAddr = clientAddr;
        final SimpleTestServerConnection testRelay = relay;
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != testRelay.getChildChannel(testClientAddr);
            }
        }, "relay detects new connection", 1000, log);
        Channel serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        ChannelPipeline serverPipeline = serverChannel.getPipeline();
        SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        // process the /sources request
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        // send back the /sources response
        HttpResponse sourcesResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        sourcesResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        sourcesResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
        HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        // make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        // process the /register request
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        // send back the /register response
        RegisterResponseEntry entry = new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR);
        String responseStr = NettyTestUtils.generateRegisterResponse(entry);
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        // make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", 100, log);
        // process /stream call and return a partial window
        NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*", 1000);
        objCapture.clear();
        // send back the /stream response
        final DbusEventsStatisticsCollector stats = new DbusEventsStatisticsCollector(1, "test1", true, false, null);
        Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
        ChannelBuffer streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[0], cp, resp1EnfOfs, stats);
        HttpResponse streamResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        streamResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        streamResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
        NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
        // make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("LastWritten SCN:" + clientConn.getDataEventsBuffer().lastWrittenScn());
                return clientConn.getDataEventsBuffer().lastWrittenScn() == 30;
            }
        }, "client receives /stream response", 1100, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.error("events num=" + consumer.getEventNum());
                return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
        }, "client processes /stream response", 110000, log);
        assertEquals(-1, consumer.getRollbackScn());
        int rollbackNum = 0;
        assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
        List<DbusEventKey> expKeys = eventKeys.get(0).subList(0, (int) stats.getTotalStats().getNumDataEvents());
        List<Long> expSeqs = new ArrayList<Long>();
        for (int i = 0; i < stats.getTotalStats().getNumDataEvents(); i++) expSeqs.add(30L);
        long numEvents = stats.getTotalStats().getNumDataEvents();
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        numEvents = stats.getTotalStats().getNumDataEvents();
        // now kill the relay and wait for a failover
        serverChannel.close();
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != relayConnInsp.getChannel() && !relayConnInsp.getChannel().isConnected();
            }
        }, "client disconnected", 200, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
        }, "new netty connection", 200, log);
        // ///////// FAKING CONNECTION TO NEW RELAY
        final NettyHttpDatabusRelayConnection newRelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector newRelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(newRelayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != newRelayConnInsp.getChannel() && newRelayConnInsp.getChannel().isConnected();
            }
        }, "client connected to new relay", 200, log);
        // figure out the connection to the relay
        clientChannel = newRelayConnInsp.getChannel();
        relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        clientAddr = clientChannel.getLocalAddress();
        relayPort = relayAddr.getPort();
        log.info("new relay selected: " + relayPort);
        relay = null;
        int relayIdx = 0;
        for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
            if (relayPort == RELAY_PORT[relayIdx])
                relay = _dummyServer[relayIdx];
        }
        assertTrue(null != relay);
        serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        serverPipeline = serverChannel.getPipeline();
        objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        // process the /sources request
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        // send back the /sources response
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        // make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        // process the /register request
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        // send back the /register response
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        // make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", 100, log);
        // process /stream call and return a partial window
        Matcher streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
        String cpString = streamMatcher.group(1);
        objCapture.clear();
        int respStartOfs = eventOfs.get(1).get(1);
        int respEndOfs = eventOfs.get(1).get(26);
        cp = new Checkpoint(cpString);
        // last window read was partial
        assertTrue(cp.getWindowOffset() > 0);
        streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[1], cp, respEndOfs - respStartOfs, stats);
        NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
        // make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
                return clientConn.getDataEventsBuffer().lastWrittenScn() == 40;
            }
        }, "client receives /stream response", 1100, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("events num=" + consumer.getEventNum());
                return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
        }, "client processes /stream response", 11000, log);
        assertEquals(20, consumer.getRollbackScn());
        // add one more onStartDataEventsSequence after the rollback()
        ++rollbackNum;
        assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
        expKeys.addAll(eventKeys.get(1).subList(0, (int) (stats.getTotalStats().getNumDataEvents() - numEvents)));
        for (int i = 0; i < (stats.getTotalStats().getNumDataEvents() - numEvents); i++) expSeqs.add((i / 20) * 20 + 20L);
        log.info("Expected NumKeys :" + expKeys.size() + ", Got NumKeys :" + consumer.getKeys().size());
        for (int i = 0; i < expKeys.size(); i++) {
            if (!consumer.getKeys().contains(expKeys.get(i)))
                log.error(i + " Key :" + expKeys.get(i) + " missing !!");
            else
                log.info(i + " Key :" + expKeys.get(i) + " present !!");
        }
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        numEvents = stats.getTotalStats().getNumDataEvents();
        assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
        // /////////////////////////////////
        // simulate a timeout on the server; the client would have sent a /stream call and there
        // will be no response from the server, so eventually it should time out and switch servers
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("Channel :" + newRelayConnInsp.getChannel());
                return (null != newRelayConnInsp.getChannel()) && (!newRelayConnInsp.getChannel().isConnected());
            }
        }, "waiting for a reconnect", (long) (DEFAULT_READ_TIMEOUT_MS * 1.5), log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
        }, "new netty connection", 200, log);
        final NettyHttpDatabusRelayConnection new2RelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector new2RelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(new2RelayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != new2RelayConnInsp.getChannel() && new2RelayConnInsp.getChannel().isConnected();
            }
        }, "client connected to third relay", 200, log);
        // figure out the connection to the relay
        clientChannel = new2RelayConnInsp.getChannel();
        relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        clientAddr = clientChannel.getLocalAddress();
        relayPort = relayAddr.getPort();
        log.info("third relay selected: " + relayPort);
        relay = null;
        relayIdx = 0;
        for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
            if (relayPort == RELAY_PORT[relayIdx])
                relay = _dummyServer[relayIdx];
        }
        assertTrue(null != relay);
        serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        serverPipeline = serverChannel.getPipeline();
        objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        // process the /sources request
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        // send back the /sources response
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        // make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        // process the /register request
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        // send back the /register response
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        // make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /sources response", 100, log);
        // process /stream call and return a partial window
        streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
        cpString = streamMatcher.group(1);
        objCapture.clear();
        respStartOfs = eventOfs.get(2).get(20);
        respEndOfs = eventOfs.get(2).get(89);
        log.debug("Checkpoint String is :" + cpString);
        cp = new Checkpoint(cpString);
        // last window read was partial. So the client would have reset the windowOffset
        assertTrue("Is WindowOffset Cleared", cp.getWindowOffset() == -1);
        assertEquals("WindowSCN == PrevSCN. Ckpt :" + cp, cp.getWindowScn(), cp.getPrevScn());
        streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[2], cp, respEndOfs - respStartOfs, stats);
        NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
        // make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
                return clientConn.getDataEventsBuffer().lastWrittenScn() == 90;
            }
        }, "client receives /stream response, Sequence :" + consumer.getSequences(), 10100, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("events num=" + consumer.getEventNum());
                return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
        }, "client processes /stream response", 110000, log);
        assertEquals(30, consumer.getRollbackScn());
        // add one more onStartDataEventsSequence because of the rollback
        ++rollbackNum;
        assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
        expKeys.addAll(eventKeys.get(2).subList(20, (int) (stats.getTotalStats().getNumDataEvents() - numEvents + 20)));
        for (int i = 0; i < (stats.getTotalStats().getNumDataEvents() - numEvents); i++) expSeqs.add((i / 10) * 10 + 30L);
        numEvents = stats.getTotalStats().getNumDataEvents();
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences Size ", expSeqs.size(), consumer.getSequences().size());
        for (int i = 0; i < expSeqs.size(); i++) {
            System.out.println(i + " Exp : " + expSeqs.get(i) + ", Got :" + consumer.getSequences().get(i));
            if (expSeqs.get(i) != consumer.getSequences().get(i))
                throw new RuntimeException("Failed at " + i);
        }
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        numEvents = stats.getTotalStats().getNumDataEvents();
        assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
        // Expect none of the keys streamed to be missed.
        expKeys.clear();
        cp = clientConn.getRelayPullThread().getConnectionState().getCheckpoint();
        int offset = (int) ((cp.getWindowOffset() < 0) ? 0 : cp.getWindowOffset());
        LOG.info("Client ckpt is :" + cp);
        expKeys.addAll(eventKeys.get(2).subList(0, 80 + offset));
        for (int i = 0; i < expKeys.size(); i++) {
            if (!consumer.getKeys().contains(expKeys.get(i))) {
                log.error(i + " Key :" + expKeys.get(i) + " missing !!");
                throw new RuntimeException("Failed at " + i);
            } else
                log.info(i + " Key :" + expKeys.get(i) + " present !!");
        }
    } finally {
        client.shutdown();
    }
}

Example 30 with HttpChunk

use of org.jboss.netty.handler.codec.http.HttpChunk in project databus by linkedin.

the class DummySuccessfulErrorCountingConsumer method testInStreamTimeOut2.

/**
 * Tests the logic of the client to handle Timeout that comes while processing stream request.
 *  the script:
 *     setup client and connect to one of the servers
 *     wait for /sources and register call and replay
 *     save the 'future' of the write operation for the /stream call. Replace this future down the stream with the fake one,
 *       so the notification of write completion will never come
 *     make server send only headers info first
 *     make server send data, but intercept the message before it reaches the client. At this moment fire WriteTimeout
 *        exception from a separate thread.
 *     Make sure PullerThread doesn't get two error messages (and as a result tries to setup up two new connections)
 */
@Test
public void testInStreamTimeOut2() throws Exception {
    final Logger log = Logger.getLogger("TestDatabusHttpClient.testInStreamTimeout2");
    MockServerChannelHandler.LOG.setLevel(Level.DEBUG);
    // log.setLevel(Level.);
    final int eventsNum = 20;
    DbusEventInfo[] eventInfos = createSampleSchema1Events(eventsNum);
    // simulate relay buffers
    DbusEventBuffer relayBuffer = new DbusEventBuffer(_bufCfg);
    relayBuffer.start(0);
    writeEventsToBuffer(relayBuffer, eventInfos, 4);
    // prepare stream response
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
    final DbusEventsStatisticsCollector stats = new DbusEventsStatisticsCollector(1, "test1", true, false, null);
    // create ChunnelBuffer and fill it with events from relayBuffer
    ChannelBuffer streamResPrefix = NettyTestUtils.streamToChannelBuffer(relayBuffer, cp, 20000, stats);
    // create client
    _stdClientCfgBuilder.getContainer().setReadTimeoutMs(DEFAULT_READ_TIMEOUT_MS);
    final DatabusHttpClientImpl client = new DatabusHttpClientImpl(_stdClientCfgBuilder.build());
    final TestConsumer consumer = new TestConsumer();
    client.registerDatabusStreamListener(consumer, null, SOURCE1_NAME);
    // connect to a relay created in SetupClass (one out of three)
    client.start();
    // wait until a connection made
    try {
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return client._relayConnections.size() == 1;
            }
        }, "sources connection present", 100, log);
        // get the connection
        final DatabusSourcesConnection clientConn = client._relayConnections.get(0);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != clientConn.getRelayPullThread().getLastOpenConnection();
            }
        }, "relay connection present", 100, log);
        // figure out connection details
        final NettyHttpDatabusRelayConnection relayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector relayConnInsp = new NettyHttpDatabusRelayConnectionInspector(relayConn);
        // wait until client is connected
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != relayConnInsp.getChannel() && relayConnInsp.getChannel().isConnected();
            }
        }, "client connected", 200, log);
        // figure out which port we got connected to on the server side
        Channel clientChannel = relayConnInsp.getChannel();
        InetSocketAddress relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        int relayPort = relayAddr.getPort();
        log.info("relay selected: " + relayPort);
        // add our handler to the client's pipeline which will generate the timeout
        MockServerChannelHandler mock = new MockServerChannelHandler();
        clientChannel.getPipeline().addBefore("inflater", "mockServer", mock);
        Map<String, ChannelHandler> map = clientChannel.getPipeline().toMap();
        boolean handlerFound = false;
        for (Map.Entry<String, ChannelHandler> m : map.entrySet()) {
            if (LOG.isDebugEnabled())
                LOG.debug(m.getKey() + "=>" + m.getValue());
            if (m.getKey().equals("mockServer"))
                handlerFound = true;
        }
        Assert.assertTrue(handlerFound, "handler added");
        SimpleTestServerConnection relay = null;
        // Find the relay's object
        for (int i = 0; i < RELAY_PORT.length; ++i) {
            if (relayPort == RELAY_PORT[i])
                relay = _dummyServer[i];
        }
        assertTrue(null != relay);
        SocketAddress clientAddr = clientChannel.getLocalAddress();
        final SocketAddress testClientAddr = clientAddr;
        final SimpleTestServerConnection testRelay = relay;
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != testRelay.getChildChannel(testClientAddr);
            }
        }, "relay detects new connection", 1000, log);
        Channel serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        ChannelPipeline serverPipeline = serverChannel.getPipeline();
        SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        // process the /sources request
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        // send back the /sources response
        HttpResponse sourcesResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        sourcesResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        sourcesResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
        HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        // make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        log.info("process the /register request");
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        String msgHistory = clientConn.getRelayPullThread().getMessageHistoryLog();
        log.info("MSG HISTORY before: " + msgHistory);
        // make sure our handler will save the 'future' of the next write operation - 'stream'
        mock.enableSaveTheFuture(true);
        log.info("send back the /register response");
        RegisterResponseEntry entry = new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR);
        String responseStr = NettyTestUtils.generateRegisterResponse(entry);
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.info("make sure the client processes the response /register correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", 100, log);
        mock.disableWriteComplete(true);
        log.info("process /stream call and return a response");
        NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*", 1000);
        objCapture.clear();
        log.info("***1");
        // disable save future as it should be saved by now
        mock.enableSaveTheFuture(false);
        log.info("***2");
        final HttpResponse streamResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        streamResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        streamResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
        log.info("***3");
        // timeout for local netty calls (in test only)
        int timeout = 1000;
        // send header info
        relay.sendServerResponse(clientAddr, sourcesResp, timeout);
        TestUtil.sleep(1000);
        log.info("***4");
        // when write data arrives from the server - we want to simulate/throw WriteTimeoutException
        mock.enableThrowWTOException(true);
        // send data
        relay.sendServerResponse(clientAddr, new DefaultHttpChunk(streamResPrefix), timeout);
        relay.sendServerResponse(clientAddr, HttpChunk.LAST_CHUNK, timeout);
        log.info("***5");
        // make sure close channel event and future failure are propagated
        TestUtil.sleep(3000);
        // get the history and validate it
        String expectedHistory = "[START, PICK_SERVER, REQUEST_SOURCES, SOURCES_RESPONSE_SUCCESS, REQUEST_REGISTER, REGISTER_RESPONSE_SUCCESS, REQUEST_STREAM, STREAM_REQUEST_SUCCESS, STREAM_RESPONSE_DONE, REQUEST_STREAM, STREAM_REQUEST_ERROR, PICK_SERVER, REQUEST_SOURCES]".trim();
        msgHistory = clientConn.getRelayPullThread().getMessageHistoryLog().trim();
        log.info("***6");
        LOG.info("MSG HISTORY: " + msgHistory);
        Assert.assertEquals(msgHistory, expectedHistory, "Puller thread message history doesn't match");
        log.info("***7");
    } catch (Exception e2) {
        log.info("Got exception" + e2);
    } finally {
        client.shutdown();
    }
}