Examples with ConditionCheck com.linkedin.databus2.test.ConditionCheck used on opensource projects

Example 26 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class SimpleClientPipelineFactoryWithSleep method testClientSimpleRequestResponse.

@Test
public void testClientSimpleRequestResponse() {
    DbusEventFactory eventFactory = new DbusEventV1Factory();
    SimpleTestServerConnection srvConn = new SimpleTestServerConnection(eventFactory.getByteOrder());
    srvConn.setPipelineFactory(new SimpleServerPipelineFactory());
    boolean serverStarted = srvConn.startSynchronously(101, CONNECT_TIMEOUT_MS);
    Assert.assertTrue(serverStarted, "server started");
    final SimpleTestClientConnection clientConn = new SimpleTestClientConnection(eventFactory.getByteOrder());
    clientConn.setPipelineFactory(new SimpleClientPipelineFactoryWithSleep(200));
    boolean clientConnected = clientConn.startSynchronously(101, CONNECT_TIMEOUT_MS);
    Assert.assertTrue(clientConnected, "client connected");
    //hook in to key places in the server pipeline
    ChannelPipeline lastSrvConnPipeline = srvConn.getLastConnChannel().getPipeline();
    SimpleTestMessageReader srvMsgReader = (SimpleTestMessageReader) lastSrvConnPipeline.get(SimpleTestMessageReader.class.getSimpleName());
    ExceptionListenerTestHandler srvExceptionListener = (ExceptionListenerTestHandler) lastSrvConnPipeline.get(ExceptionListenerTestHandler.class.getSimpleName());
    //hook in to key places in the client pipeline
    ChannelPipeline clientPipeline = clientConn.getChannel().getPipeline();
    final ExceptionListenerTestHandler clientExceptionListener = (ExceptionListenerTestHandler) clientPipeline.get(ExceptionListenerTestHandler.class.getSimpleName());
    //System.err.println("Current thread: " + Thread.currentThread());
    //send a request in a separate thread because the client will intentionally block to simulate
    //a timeout
    final ChannelBuffer msg = ChannelBuffers.wrappedBuffer("hello".getBytes(Charset.defaultCharset()));
    Thread sendThread1 = new Thread(new Runnable() {

        @Override
        public void run() {
            //System.err.println(Thread.currentThread().toString() + ": sending message");
            clientConn.getChannel().write(msg);
        }
    }, "send msg thread");
    sendThread1.start();
    //System.err.println(Thread.currentThread().toString() + ": waiting for 10");
    try {
        Thread.sleep(50);
    } catch (InterruptedException ie) {
    }
    ;
    //System.err.println(Thread.currentThread().toString() + ": done Waiting for 10");
    //make sure the server has not received the message
    Assert.assertNull(srvExceptionListener.getLastException(), "no server errors");
    Assert.assertNull(clientExceptionListener.getLastException(), "no errors yet");
    Assert.assertTrue(!"hello".equals(srvMsgReader.getMsg()), "message not read yet");
    //wait for the write timeout
    try {
        Thread.sleep(300);
    } catch (InterruptedException ie) {
    }
    ;
    //System.err.println("Done Waiting for 300");
    //the client should have timed out and closed the connection
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            return null != clientExceptionListener.getLastException();
        }
    }, "client error", 1000, null);
    Assert.assertTrue(null != clientExceptionListener.getLastException(), "client error");
    Assert.assertTrue(clientExceptionListener.getLastException() instanceof ClosedChannelException || clientExceptionListener.getLastException() instanceof WriteTimeoutException, "client error test");
    Assert.assertTrue(!lastSrvConnPipeline.getChannel().isConnected(), "client has disconnected");
    Assert.assertTrue(!clientPipeline.getChannel().isConnected(), "closed connection to server");
}

Example 27 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class DummyRemoteExceptionHandler method runRegisterV4.

private void runRegisterV4(final String subtestName, final String protocolVersionHeader, final String responseStr, final TestResponseProcessors.TestConnectionStateMessage.State expectedRegisterState) throws IOException, ScnNotFoundException, OffsetNotFoundException //throws JsonGenerationException, JsonMappingException, IOException, ScnNotFoundException, OffsetNotFoundException
{
    final Logger log = Logger.getLogger("TestNettyHttpDatabusRelayConnection." + subtestName);
    //log.setLevel(Level.DEBUG);
    TestingConnectionCallback callback = TestingConnectionCallback.createAndStart(subtestName);
    DummyRemoteExceptionHandler remoteExceptionHandler = new DummyRemoteExceptionHandler();
    final NettyHttpDatabusRelayConnection conn = (NettyHttpDatabusRelayConnection) CONN_FACTORY.createRelayConnection(RELAY_SERVER_INFO, callback, remoteExceptionHandler);
    // verify the version - current DBUS_EVENT_V1
    Assert.assertEquals(MAX_EVENT_VERSION, conn.getMaxEventVersion());
    try {
        // connect to server and send /sources
        TestResponseProcessors.TestConnectionStateMessage msg = new TestResponseProcessors.TestConnectionStateMessage();
        conn.requestSources(msg);
        waitForServerConnection(conn, log);
        // introspect connection to server
        Channel channel = conn._channel;
        final SocketAddress clientAddr = channel.getLocalAddress();
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != _dummyServer.getChildChannel(clientAddr);
            }
        }, "client connection established", 1000, log);
        Channel serverChannel = _dummyServer.getChildChannel(clientAddr);
        ChannelPipeline serverPipeline = serverChannel.getPipeline();
        SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        // verify server gets the /sources request
        HttpResponse sourcesResp = runHappyPathSources(log, callback, remoteExceptionHandler, clientAddr, objCapture);
        // send /register and check result
        doRegisterV4(log, callback, remoteExceptionHandler, conn, msg, clientAddr, objCapture, sourcesResp, protocolVersionHeader, responseStr, expectedRegisterState);
        callback.clearLastMsg();
        objCapture.clear();
    } finally {
        conn.close();
        callback.shutdown();
    }
}

Example 28 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class DummyRemoteExceptionHandler method runServerRegisterDisconnectIteration.

private void runServerRegisterDisconnectIteration(final Logger log, TestingConnectionCallback callback, DummyRemoteExceptionHandler remoteExceptionHandler, final NettyHttpDatabusRelayConnection conn) throws JsonGenerationException, JsonMappingException, IOException, ScnNotFoundException, OffsetNotFoundException {
    //connect to server and send /sources
    TestResponseProcessors.TestConnectionStateMessage msg = new TestResponseProcessors.TestConnectionStateMessage();
    conn.requestSources(msg);
    waitForServerConnection(conn, log);
    //introspect connection to server
    Channel channel = conn._channel;
    SocketAddress clientAddr = channel.getLocalAddress();
    final SocketAddress finalClientAddr = clientAddr;
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            return _dummyServer.getChildChannel(finalClientAddr) != null;
        }
    }, "client connected", 100, log);
    Channel serverChannel = _dummyServer.getChildChannel(clientAddr);
    ChannelPipeline serverPipeline = serverChannel.getPipeline();
    SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
    //verify server gets the /source request
    runHappyPathSources(log, callback, remoteExceptionHandler, clientAddr, objCapture);
    callback.clearLastMsg();
    objCapture.clear();
    conn.requestRegister("1", msg);
    //verify server gets the /register request
    HttpRequest msgReq = captureRequest(objCapture);
    Assert.assertTrue(msgReq.getUri().startsWith("/register"));
    serverChannel.close();
    waitForCallback(callback, TestResponseProcessors.TestConnectionStateMessage.State.REGISTER_RESPONSE_ERROR, log);
    Assert.assertNull(remoteExceptionHandler.getLastException());
    Assert.assertEquals(1, callback.getAllMsgs().size());
    callback.clearLastMsg();
    objCapture.clear();
}

Example 29 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class TestNettyHttpDatabusBootstrapConnection method sendStartScnHappyPath.

private void sendStartScnHappyPath(final NettyHttpDatabusBootstrapConnection conn, Checkpoint cp, DummyDatabusBootstrapConnectionStateMessage bstCallback, String sourceNames, long startScn, Logger log) throws IOException, JsonGenerationException, JsonMappingException {
    //send startSCN()
    conn.requestStartScn(cp, bstCallback, sourceNames);
    TestUtil.assertWithBackoff(new ConditionCheck() {

        @Override
        public boolean check() {
            return null != conn._channel && conn._channel.isConnected();
        }
    }, "wait for client to connect", 1000, log);
    //introspect connection to server
    Channel channel = conn._channel;
    SocketAddress clientAddr = channel.getLocalAddress();
    Channel serverChannel = _dummyServer.getChildChannel(clientAddr);
    ChannelPipeline serverPipeline = serverChannel.getPipeline();
    SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
    Assert.assertTrue(objCapture.waitForMessage(1000, 0));
    Object msgObj = objCapture.getMessages().get(0);
    Assert.assertTrue(msgObj instanceof HttpRequest);
    //verify we got a /startSCN call
    HttpRequest msgReq = (HttpRequest) msgObj;
    Assert.assertTrue(msgReq.getUri().startsWith("/startSCN"));
    //send back some response
    HttpResponse sourcesResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    sourcesResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
    sourcesResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
    ObjectMapper objMapper = new ObjectMapper();
    HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(objMapper.writeValueAsBytes(String.valueOf(startScn))));
    NettyTestUtils.sendServerResponses(_dummyServer, clientAddr, sourcesResp, body);
}

Example 30 with ConditionCheck

use of com.linkedin.databus2.test.ConditionCheck in project databus by linkedin.

the class TestGoldenGateEventProducer method testGGParserStats.

/**
   * test collection of parser stats, especially lag between parsed and added files
   * @throws Exception
   */
@Test
public void testGGParserStats() throws Exception {
    short[] sourceIds = new short[] { 505, 506 };
    String[] sourceNames = new String[] { "source1", "source2" };
    // setup trail Files directory
    File ggTrailDir = FileUtils.createTempDir("testGGParserStats");
    // configure physical source
    String uri = "gg://" + ggTrailDir.getAbsolutePath() + ":x3";
    PhysicalSourceStaticConfig pssc = buildSimplePssc(sourceIds, sourceNames, uri);
    LOG.info("Uri=" + uri);
    // create schema
    Schema s = Schema.parse(sourceAvroSchema);
    VersionedSchema vs = new VersionedSchema(new VersionedSchemaId("source1", (short) 3), s, null);
    // mock for schema registry
    SchemaRegistryService srs = EasyMock.createMock(SchemaRegistryService.class);
    EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName("source1")).andReturn(vs).anyTimes();
    EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName("source2")).andReturn(vs).anyTimes();
    EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName(null)).andReturn(vs);
    // mock for MaxSCNReadWriter
    MaxSCNReaderWriter mscn = EasyMock.createMock(MaxSCNReaderWriter.class);
    EasyMock.expect(mscn.getMaxScn()).andReturn((long) -2).atLeastOnce();
    mscn.saveMaxScn(EasyMock.anyLong());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.replay(mscn);
    EasyMock.replay(srs);
    int totalTransWritten = 0;
    int totalFilesWritten = 0;
    // buffer
    DbusEventBufferAppendable mb = createBufMult(pssc);
    // start GG producer
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, srs, mb, null, mscn);
    // create first 2 files
    addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x301"), 100, 4);
    addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x302"), 200, 4);
    totalTransWritten = 8;
    totalFilesWritten = 2;
    // get hold of parser stats object
    final GGParserStatistics ggParserStats = gg.getParserStats();
    // all should be 0
    Assert.assertEquals(0, ggParserStats.getNumFilesParsed());
    Assert.assertEquals(0, ggParserStats.getNumFilesAdded());
    Assert.assertEquals(0, ggParserStats.getFilesLag());
    Assert.assertEquals(0, ggParserStats.getTimeLag());
    Assert.assertEquals(0, ggParserStats.getBytesLag());
    try {
        LOG.info("starting event producer");
        // -2 here does nothing. actual setting happens thru the mock of
        gg.start(-2);
        // MaxSCNReadWriter
        // let it parse first files
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return ggParserStats.getNumFilesParsed() == 2 && (8 * _transactionPatternSize == ggParserStats.getNumBytesTotalParsed());
            }
        }, "First two files parsed", 2000, LOG);
        // stats in the interim
        Assert.assertEquals(2, ggParserStats.getNumFilesParsed());
        Assert.assertEquals(2, ggParserStats.getNumFilesAdded());
        Assert.assertEquals(0, ggParserStats.getFilesLag());
        Assert.assertEquals(0, ggParserStats.getTimeLag());
        Assert.assertEquals(0, ggParserStats.getBytesLag());
        Assert.assertEquals(totalTransWritten * _transactionPatternSize, ggParserStats.getNumBytesTotalParsed());
        gg.pause();
        // the file will get parsed but not processed
        addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x303"), 300, 4);
        totalTransWritten += 4;
        totalFilesWritten++;
        // to get more then a ms lag time
        TestUtil.sleep(2000);
        addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x304"), 400, 4);
        totalTransWritten += 4;
        totalFilesWritten++;
        // to guarantee we picked up stats update (stats are updated
        TestUtil.sleep(6000);
        // every 5 seconds)
        // now we should be 2 files behind. parser thread gets paused AFTER it start
        // processing the file
        // so the actuall value will be 1 file behind
        // 303(already started being parsed), only 304 is behind
        int lagFiles = 1;
        // 1 file, 4 transactions each
        long lagBytes = 1 * 4 * _transactionPatternSize;
        /*
       * Assert.assertEquals(totalFilesWritten-1, ggParserStats.getNumFilesParsed());
       * Assert.assertEquals(totalFilesWritten, ggParserStats.getNumFilesAdded());
       * Assert.assertEquals(lagFiles, ggParserStats.getFilesLag()); // because 303 got
       * parsed
       *
       * // we added 4 files and parsed 3 , so the diff should be 1 file size (4
       * trasactions in 1 file) Assert.assertEquals(lagBytes,
       * ggParserStats.getBytesLag()); Assert.assertTrue(ggParserStats.getTimeLag()>0);
       */
        gg.unpause();
        TestUtil.sleep(5000);
        // now we should catchup
        Assert.assertEquals(4, ggParserStats.getNumFilesParsed());
        Assert.assertEquals(4, ggParserStats.getNumFilesAdded());
        Assert.assertEquals(0, ggParserStats.getFilesLag());
        Assert.assertEquals(0, ggParserStats.getTimeLag());
        Assert.assertEquals(0, ggParserStats.getBytesLag());
        // append to a file
        LOG.info("pausing again");
        gg.pause();
        addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x304"), 410, 4);
        totalTransWritten += 4;
        TestUtil.sleep(1000);
        addToTrailFile(new File(ggTrailDir.getAbsolutePath() + "/x304"), 420, 4);
        totalTransWritten += 4;
        TestUtil.sleep(2000);
        gg.unpause();
        TestUtil.sleep(5500);
        // should be still up
        Assert.assertEquals(4, ggParserStats.getNumFilesParsed());
        Assert.assertEquals(4, ggParserStats.getNumFilesAdded());
        Assert.assertEquals(0, ggParserStats.getFilesLag());
        Assert.assertEquals(0, ggParserStats.getTimeLag());
        Assert.assertEquals(0, ggParserStats.getBytesLag());
        // assert the stats
        int totalFilesSize = totalTransWritten * _transactionPatternSize;
        Assert.assertEquals((totalFilesSize / totalFilesWritten), ggParserStats.getAvgFileSize());
        Assert.assertEquals(true, ggParserStats.getAvgParseTransactionTimeNs() > 0);
        Assert.assertEquals("part1", ggParserStats.getPhysicalSourceName());
        Assert.assertEquals(totalFilesSize / totalTransWritten, ggParserStats.getAvgTransactionSize());
        Assert.assertEquals(423, ggParserStats.getMaxScn());
        // 2
        Assert.assertEquals(totalTransWritten * 2, ggParserStats.getNumTotalEvents());
        // events
        // per
        // transaction
        Assert.assertEquals(totalTransWritten, ggParserStats.getNumTransactionsTotal());
        Assert.assertEquals(totalTransWritten, ggParserStats.getNumTransactionsWithEvents());
        Assert.assertEquals(0, ggParserStats.getNumTransactionsWithoutEvents());
        Assert.assertEquals(true, ggParserStats.getTimeSinceLastAccessMs() > 0);
        Assert.assertEquals(totalTransWritten * _transactionPatternSize, ggParserStats.getNumBytesTotalParsed());
        Assert.assertEquals("NumSCNRegressions", 0, ggParserStats.getNumSCNRegressions());
        Assert.assertEquals("LastSCNRegressed", -1, ggParserStats.getLastRegressedScn());
    } finally {
        gg.shutdown();
    }
    return;
}