Examples with DbusEventBuffer com.linkedin.databus.core.DbusEventBuffer used on opensource projects

Example 26 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class DummyRemoteExceptionHandler method testServerPartialDisconnect.

@Test
public void testServerPartialDisconnect() throws IOException, ScnNotFoundException, OffsetNotFoundException {
    final Logger log = Logger.getLogger("TestNettyHttpDatabusRelayConnection.testServerPartialDisconnect");
    DbusEventBuffer buf = createSimpleBuffer();
    TestingConnectionCallback callback = TestingConnectionCallback.createAndStart("testHappyPath");
    DummyRemoteExceptionHandler remoteExceptionHandler = new DummyRemoteExceptionHandler();
    final NettyHttpDatabusRelayConnection conn = (NettyHttpDatabusRelayConnection) CONN_FACTORY.createRelayConnection(RELAY_SERVER_INFO, callback, remoteExceptionHandler);
    try {
        runServerPartialStreamIteration(log, buf, callback, remoteExceptionHandler, conn);
    } finally {
        conn.close();
        callback.shutdown();
    }
}

Example 27 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class DummyRemoteExceptionHandler method testServerRandomScenario.

/**
   * <p>Random sequence of test iterations
   */
@Test
public void testServerRandomScenario() throws IOException, ScnNotFoundException, OffsetNotFoundException {
    final Logger log = Logger.getLogger("TestNettyHttpDatabusRelayConnection.testServerRandomScenario");
    log.info("in");
    DbusEventBuffer buf = createSimpleBuffer();
    TestingConnectionCallback callback = TestingConnectionCallback.createAndStart("testServerRandomScenario");
    DummyRemoteExceptionHandler remoteExceptionHandler = new DummyRemoteExceptionHandler();
    final NettyHttpDatabusRelayConnection conn = (NettyHttpDatabusRelayConnection) CONN_FACTORY.createRelayConnection(RELAY_SERVER_INFO, callback, remoteExceptionHandler);
    conn.getHandler().getLog().setLevel(_logLevel);
    try {
        final int iterNum = 15;
        Random rng = new Random();
        for (int i = 1; i <= iterNum; ++i) {
            int testId = rng.nextInt(10);
            switch(testId) {
                case 0:
                    {
                        log.info("======> step " + i + ": runHappyPathIteration");
                        runHappyPathIteration(log, buf, callback, remoteExceptionHandler, conn);
                        break;
                    }
                case 1:
                    {
                        log.info("======> step " + i + ": runServerSourcesDisconnectIteration");
                        runServerSourcesDisconnectIteration(log, callback, remoteExceptionHandler, conn);
                        conn.getHandler().reset();
                        break;
                    }
                case 2:
                    {
                        log.info("======> step " + i + ": runServerSourcesReadTimeoutIteration");
                        runServerSourcesReadTimeoutIteration(log, callback, remoteExceptionHandler, conn);
                        conn.getHandler().reset();
                        break;
                    }
                case 3:
                    {
                        log.info("======> step " + i + ": runServerRegisterDisconnectIteration");
                        runServerRegisterDisconnectIteration(log, callback, remoteExceptionHandler, conn);
                        conn.getHandler().reset();
                        break;
                    }
                case 4:
                    {
                        log.info("======> step " + i + ": runServerRegisterReqDisconnectIteration");
                        runServerRegisterReqDisconnectIteration(log, callback, remoteExceptionHandler, conn);
                        conn.getHandler().reset();
                        break;
                    }
                case 5:
                    {
                        log.info("======> step " + i + ": runServerRegisterReadTimeoutIteration");
                        runServerRegisterReadTimeoutIteration(log, callback, remoteExceptionHandler, conn);
                        conn.getHandler().reset();
                        break;
                    }
                case 6:
                    {
                        log.info("======> step " + i + ": runServerStreamDisconnectIteration");
                        runServerStreamDisconnectIteration(log, callback, remoteExceptionHandler, conn);
                        conn.getHandler().reset();
                        break;
                    }
                case 7:
                    {
                        log.info("======> step " + i + ": runServerStreamReqDisconnectIteration");
                        runServerStreamReqDisconnectIteration(log, callback, remoteExceptionHandler, conn);
                        conn.getHandler().reset();
                        break;
                    }
                case 8:
                    {
                        log.info("======> step " + i + ": runServerStreamReadTimeoutIteration");
                        runServerStreamReadTimeoutIteration(log, callback, remoteExceptionHandler, conn);
                        conn.getHandler().reset();
                        break;
                    }
                case 9:
                    {
                        log.info("======> step " + i + ": runServerPartialStreamTimeoutIteration");
                        runServerPartialStreamTimeoutIteration(log, buf, callback, remoteExceptionHandler, conn);
                        conn.getHandler().reset();
                        break;
                    }
                default:
                    {
                        Assert.fail("step " + i + ": unknown test id: " + testId);
                    }
                    log.info("======> step " + i + ": complete.");
                    if (0 != testId) {
                        //if it was an iteration with an error, sleep a bit until we make sure the client
                        //channel has been disconnected
                        TestUtil.assertWithBackoff(new ConditionCheck() {

                            @Override
                            public boolean check() {
                                return !conn._channel.isConnected();
                            }
                        }, "wait for child channel closure ", 1024, log);
                    }
            }
        }
    } finally {
        conn.close();
        callback.shutdown();
        log.info("out");
    }
}

Example 28 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class DummyRemoteExceptionHandler method testHttpRequestLoggingHandlerRequest.

@Test
public void testHttpRequestLoggingHandlerRequest() throws Exception {
    final Logger log = Logger.getLogger("TestNettyHttpDatabusRelayConnection.testHttpRequestLoggingHandlerRequest");
    DbusEventBuffer buf = createSimpleBuffer();
    TestingConnectionCallback callback = TestingConnectionCallback.createAndStart("testHttpRequestLoggingHandlerRequest");
    DummyRemoteExceptionHandler remoteExceptionHandler = new DummyRemoteExceptionHandler();
    final NettyHttpDatabusRelayConnection conn = (NettyHttpDatabusRelayConnection) CONN_FACTORY.createRelayConnection(RELAY_SERVER_INFO, callback, remoteExceptionHandler);
    try {
        runHappyPathIteration(log, buf, callback, remoteExceptionHandler, conn);
    } finally {
        conn.close();
        callback.shutdown();
    }
}

Example 29 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class DatabusEventRandomProducer method run.

@Override
public void run() {
    _stopGeneration.set(false);
    _suspendGeneration.set(false);
    long startTime = System.nanoTime();
    long startTime0 = startTime;
    long realStartTime = System.currentTimeMillis() * 1000000L;
    _numEventsGenerated = 0;
    int eventBatchSize = 10;
    long currTime = startTime;
    long sleepTimes = 0;
    long sleepTime = 0;
    long firstScn = _startScn;
    long maxScn;
    long numEventsGeneratedAfterResume = 0;
    // get the free space at the beginning as the buffer size
    long eventBufferSize = 0;
    long sizeDataEventsBeforeResume = 0;
    long currSizeDataEvents = _statsCollector.getTotalStats().getSizeDataEvents() * _statsCollector.getTotalStats().getNumDataEvents();
    maxScn = firstScn;
    /* fix firstSCN setting and figure minimum free buffers size available */
    long minSpace = Long.MAX_VALUE;
    for (DbusEventBuffer buf : _dbusEventBuffer.bufIterable()) {
        if (buf.getMinScn() < 0)
            buf.start(firstScn - 1);
        long tmpMinSpace = buf.getBufferFreeSpace();
        if (tmpMinSpace < minSpace)
            minSpace = tmpMinSpace;
    }
    if (_dbusEventBuffer.bufsNum() > 0)
        eventBufferSize = minSpace;
    try {
        _rateMonitor = new RateMonitor("RandomProducer");
        _rateMonitor.start();
        while (!_stopGeneration.get()) {
            LOG.info("Resume. currTime (ms) = " + currTime / MILLISECONDS_IN_NANOS + ", startTime (ms) = " + startTime / MILLISECONDS_IN_NANOS + ", elapseTime = " + (currTime - startTime) / MILLISECONDS_IN_NANOS + ", _duration=" + _duration + ", numEventsToGenerate = " + _numEventsToGenerate.get() + ", StartScn = " + firstScn + ", _minEventsPerWindow=" + _minEventsPerWindow + ", _maxEventsPerWindow =" + _maxEventsPerWindow + ", _keyMin =" + _keyMin + ", _keyMax =" + _keyMax + ", _minLength=" + _minLength + ", _maxLength=" + _maxLength + ", numEvents = " + numEventsGeneratedAfterResume + ", eventBufferSize = " + eventBufferSize + ", sources = " + _sources.size());
            while (!_stopGeneration.get() && !_suspendGeneration.get() && (currTime - startTime < _duration * MILLISECONDS_IN_NANOS) && (numEventsGeneratedAfterResume < _numEventsToGenerate.get()) && (currSizeDataEvents - sizeDataEventsBeforeResume < _percentOfBufferToGenerate.get() / 100.0 * eventBufferSize)) {
                eventBatchSize = _minEventsPerWindow + RngUtils.randomPositiveShort(_realRng) % (_maxEventsPerWindow - _minEventsPerWindow);
                long before = System.nanoTime();
                _rateMonitor.resume();
                maxScn = produceNRandomEvents(firstScn, realStartTime + (currTime - startTime0), eventBatchSize, _sources, _keyMin.get(), _keyMax.get(), _minLength, _maxLength, _schemas);
                assert (maxScn >= firstScn);
                _rateMonitor.ticks(eventBatchSize + 1);
                _rateMonitor.suspend();
                currTime = System.nanoTime();
                currSizeDataEvents = _statsCollector.getTotalStats().getSizeDataEvents() * _statsCollector.getTotalStats().getNumDataEvents();
                firstScn = maxScn + 1;
                _numEventsGenerated += eventBatchSize;
                numEventsGeneratedAfterResume += eventBatchSize;
                _totalGenerationTime += (currTime - before);
                long nextTime = (long) (startTime + numEventsGeneratedAfterResume * _tickInNanos);
                if (nextTime > currTime) {
                    try {
                        ++sleepTimes;
                        sleepTime += (nextTime - currTime);
                        long milliseconds = (nextTime - currTime) / MILLISECONDS_IN_NANOS;
                        int nanoseconds = (int) ((nextTime - currTime) - (milliseconds * MILLISECONDS_IN_NANOS));
                        Thread.sleep(milliseconds, nanoseconds);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                        return;
                    }
                    currTime = System.nanoTime();
                }
            }
            LOG.info("Suspended. currTime (ms) = " + currTime / MILLISECONDS_IN_NANOS + ", startTime (ms) = " + startTime / MILLISECONDS_IN_NANOS + ", elapseTime = " + (currTime - startTime) / MILLISECONDS_IN_NANOS + ", numEvents = " + numEventsGeneratedAfterResume + ", eventBufferSize = " + eventBufferSize + ", currEventSize = " + currSizeDataEvents + ", startEventSize = " + sizeDataEventsBeforeResume + ", EventSizeDelta = " + (currSizeDataEvents - sizeDataEventsBeforeResume));
            LOG.info(getRateOfProduction());
            //Suspend till resumed
            synchronized (this) {
                if (!_stopGeneration.get()) {
                    boolean doWait = false;
                    // Checking again for suspend inside synchronized block to make sure we didn't miss resume outside the synchronized block.
                    if (!_suspendGeneration.get()) {
                        if (currTime - startTime >= _duration * MILLISECONDS_IN_NANOS || (numEventsGeneratedAfterResume >= _numEventsToGenerate.get()) || (currSizeDataEvents - sizeDataEventsBeforeResume >= _percentOfBufferToGenerate.get() / 100.0 * eventBufferSize)) {
                            // Completed this round. Suspending myself till someone calls resume or stop
                            doWait = true;
                            _suspendGeneration.set(true);
                        } else {
                            // case when someone called suspend and resumed immediately before I reached here
                            doWait = false;
                        }
                    } else {
                        // User requested suspension
                        doWait = true;
                    }
                    while (doWait && (!_stopGeneration.get()) && (_suspendGeneration.get())) {
                        try {
                            this.wait();
                        } catch (InterruptedException ie) {
                            LOG.info("Got Interrupted during suspension: " + ie.getMessage());
                        }
                    }
                    // reset startTime to wall clock
                    startTime = System.nanoTime();
                    currTime = startTime;
                    // reset the event size and number of events
                    numEventsGeneratedAfterResume = 0;
                    sizeDataEventsBeforeResume = currSizeDataEvents;
                }
            }
        }
        _stopGeneration.set(true);
    } catch (RuntimeException e) {
        LOG.error("event generation error:" + e.getMessage(), e);
    } catch (KeyTypeNotImplementedException e) {
        LOG.error("event generation error:" + e.getMessage(), e);
    } finally {
        LOG.info("Produced " + _numEventsGenerated + " events in " + (currTime - startTime) + " nanoseconds.");
        LOG.info("Slept a total of " + sleepTimes + " times for a duration of " + sleepTime + " nanoseconds.");
        LOG.info("Busy time = " + (currTime - startTime - sleepTime) + " nanoseconds.");
        _rateMonitor.stop();
        _generationStopped.countDown();
    }
}

Example 30 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class DummyRemoteExceptionHandler method testServerPartialTimeout.

@Test
public void testServerPartialTimeout() throws IOException, ScnNotFoundException, OffsetNotFoundException {
    final Logger log = Logger.getLogger("TestNettyHttpDatabusRelayConnection.testServerPartialResponse");
    DbusEventBuffer buf = createSimpleBuffer();
    TestingConnectionCallback callback = TestingConnectionCallback.createAndStart("testHappyPath");
    DummyRemoteExceptionHandler remoteExceptionHandler = new DummyRemoteExceptionHandler();
    final NettyHttpDatabusRelayConnection conn = (NettyHttpDatabusRelayConnection) CONN_FACTORY.createRelayConnection(RELAY_SERVER_INFO, callback, remoteExceptionHandler);
    try {
        runServerPartialStreamTimeoutIteration(log, buf, callback, remoteExceptionHandler, conn);
    } finally {
        conn.close();
        callback.shutdown();
    }
}