Examples with DbusEventBuffer com.linkedin.databus.core.DbusEventBuffer used on opensource projects

Example 1 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class TestDatabusRelayMain method testPendingEventSize.

@Test
public /**
 * When the relay has no events, we should not get the x-dbus-pending-event-size even if we present a small buffer.
 * When the relay has events, we should see the header on a small buffer but see an event when the buffer
 * is large enough, and should not see the header in the large buffer case.
 */
void testPendingEventSize() throws Exception {
    DatabusRelayMain relay = null;
    try {
        final short srcId = 104;
        final String srcName = "foo";
        PhysicalSourceConfig pConfig = new PhysicalSourceConfig();
        pConfig.setId(srcId);
        pConfig.setName(srcName);
        pConfig.setUri("mock");
        short lid = (short) (srcId + 1);
        LogicalSourceConfig lConf = new LogicalSourceConfig();
        lConf.setId(lid);
        lConf.setName(srcName);
        // this is table name in the oracle source world
        lConf.setUri(srcName);
        lConf.setPartitionFunction("constant:1");
        pConfig.addSource(lConf);
        int relayPort = Utils.getAvailablePort(11994);
        final int relayId = 666;
        HttpRelay.Config httpRelayConfig = new HttpRelay.Config();
        ServerContainer.Config containerConfig = DatabusRelayTestUtil.createContainerConfig(relayId, relayPort);
        DbusEventBuffer.Config bufferConfig = DatabusRelayTestUtil.createBufferConfig(10000, 250, 100);
        httpRelayConfig.setContainer(containerConfig);
        httpRelayConfig.setEventBuffer(bufferConfig);
        httpRelayConfig.setStartDbPuller("true");
        PhysicalSourceStaticConfig[] pStaticConfigs = new PhysicalSourceStaticConfig[1];
        for (LogicalSourceConfig lsc : pConfig.getSources()) {
            httpRelayConfig.setSourceName("" + lsc.getId(), lsc.getName());
        }
        pStaticConfigs[0] = pConfig.build();
        relay = new DatabusRelayMain(httpRelayConfig.build(), pStaticConfigs);
        relay.start();
        // Insert one event into the relay.
        LogicalSource lsrc = new LogicalSource((int) lid, srcName);
        DbusEventBuffer buf = relay.getEventBuffer().getDbusEventBuffer(lsrc);
        byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
        final long prevScn = 99;
        final long eventScn = 101;
        buf.start(prevScn);
        buf.startEvents();
        Assert.assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, lid, schema, new byte[100], false, null));
        buf.endEvents(eventScn, null);
        HttpResponseHandler handler = new HttpResponseHandler();
        // On a good buffer length we should not see the extra header.
        testClient(relayPort, 1000, 100L, handler);
        Assert.assertNull(handler._pendingEventHeader, "Received pending event header on full buffer");
        // We should see the extra header when we get 0 events and the next event is too big to fit in
        testClient(relayPort, 10, 100L, handler);
        Assert.assertNotNull(handler._pendingEventHeader);
        Assert.assertEquals(Integer.valueOf(handler._pendingEventHeader).intValue(), 161);
        // But if there are no events, then we should not see the header even if buffer is very small
        handler._pendingEventHeader = null;
        testClient(relayPort, 10, 1005L, handler);
        Assert.assertNull(handler._pendingEventHeader, "Received pending event header on full buffer");
    } finally {
        relay.shutdownUninteruptibly();
    }
}

Example 2 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class TestDatabusRelayEvents method testV2Events.

/**
 * Stuffs an event buffer with both a v1 and a v2 event, then reads the buffer two ways:
 * first accepting only v1 events (verifying conversion of the v2 event to v1); then accepting
 * both v1 and v2 events.
 *
 * Note that the version of the _EOP_ events must match the version of the event factory,
 * regardless of the versions of any preceding "real" events.  (This matches DbusEventBuffer
 * behavior; see the serializeLongKeyEndOfPeriodMarker() call in endEvents() for details.)
 */
@Test
public void testV2Events() throws KeyTypeNotImplementedException, InvalidEventException, IOException, DatabusException {
    final Logger log = Logger.getLogger("TestDatabusRelayEvents.testV2Events");
    log.setLevel(Level.DEBUG);
    String[] srcs = { "com.linkedin.events.example.fake.FakeSchema" };
    String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
    short srcId = 2;
    short pId = 1;
    int relayPort = Utils.getAvailablePort(11993);
    // create relay
    final DatabusRelayMain relay1 = createRelay(relayPort, pId, srcs);
    DatabusRelayTestUtil.RelayRunner r1 = null;
    ClientRunner cr = null;
    try {
        // EventProducer[] producers = relay1.getProducers();
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("Relay created");
        DbusEventBufferMult bufMult = relay1.getEventBuffer();
        PhysicalPartition pPartition = new PhysicalPartition((int) pId, pSourceName);
        DbusEventBuffer buf = (DbusEventBuffer) bufMult.getDbusEventBufferAppendable(pPartition);
        log.info("create some events");
        long windowScn = 100L;
        ByteBuffer serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(), pId, DbusEventFactory.DBUS_EVENT_V2);
        ReadableByteChannel channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
        int readEvents = buf.readEvents(channel);
        log.info("successfully read in " + readEvents + " events ");
        channel.close();
        windowScn = 101L;
        serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(), pId, DbusEventFactory.DBUS_EVENT_V1);
        channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
        readEvents = buf.readEvents(channel);
        log.info("successfully read in " + readEvents + " events ");
        channel.close();
        log.info("starting relay on port " + relayPort);
        r1.start();
        // TestUtil.sleep(10*1000);
        // wait until relay comes up
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return relay1.isRunningStatus();
            }
        }, "Relay hasn't come up completely ", 30000, LOG);
        log.info("now create client");
        String srcSubscriptionString = TestUtil.join(srcs, ",");
        String serverName = "localhost:" + relayPort;
        final EventsCountingConsumer countingConsumer = new EventsCountingConsumer();
        int id = (RngUtils.randomPositiveInt() % 10000) + 1;
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V1, 0);
        cr = new ClientRunner(clientConn);
        log.info("starting client");
        cr.start();
        // wait till client gets the event
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                int events = countingConsumer.getNumDataEvents();
                LOG.info("client got " + events + " events");
                return events == 2;
            }
        }, "Consumer didn't get 2 events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(countingConsumer.getNumDataEvents(), 2);
        Assert.assertEquals(countingConsumer.getNumWindows(), 2);
        Assert.assertEquals(countingConsumer.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1), 2);
        log.info("shutdown first client");
        clientConn.stop();
        cr.shutdown();
        TestUtil.sleep(1000);
        cr = null;
        log.info("start another client who understands V2");
        final EventsCountingConsumer countingConsumer1 = new EventsCountingConsumer();
        clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer1, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V2, 0);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("wait till client gets the event");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                int events = countingConsumer1.getNumDataEvents();
                LOG.debug("client got " + events + " events");
                return events == 2;
            }
        }, "Consumer didn't get 2 events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(countingConsumer1.getNumDataEvents(), 2);
        Assert.assertEquals(countingConsumer1.getNumWindows(), 2);
        Assert.assertEquals(countingConsumer1.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1), 1);
        Assert.assertEquals(countingConsumer1.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V2), 1);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 3 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class TestGoldenGateEventProducer method createBufMult.

/**
 * Creates a DbusBufMult
 */
private DbusEventBufferAppendable createBufMult(PhysicalSourceStaticConfig pssc) throws InvalidConfigException {
    DbusEventBuffer.StaticConfig config = null;
    if (config == null) {
        try {
            DbusEventBuffer.Config cfgBuilder = new DbusEventBuffer.Config();
            cfgBuilder.setMaxSize(10 * 1024 * 1024);
            cfgBuilder.setScnIndexSize(2 * 1024 * 1024);
            cfgBuilder.setAllocationPolicy("MMAPPED_MEMORY");
            config = cfgBuilder.build();
        } catch (InvalidConfigException e) {
            fail("invalid configuration", e);
        }
    }
    PhysicalSourceStaticConfig[] pConfigs = new PhysicalSourceStaticConfig[1];
    pConfigs[0] = pssc;
    DbusEventBufferMult eventBufferMult = new DbusEventBufferMult(pConfigs, config, new DbusEventV2Factory());
    for (DbusEventBuffer b : eventBufferMult.bufIterable()) {
        b.start(1);
    }
    DbusEventBufferAppendable buf = eventBufferMult.getDbusEventBufferAppendable(505);
    return buf;
}

Example 4 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class TestGoldenGateEventProducer method testAddEventToBuffer.

@Test
public void testAddEventToBuffer() throws InvalidConfigException, UnsupportedKeyException, DatabusException {
    // No rate control
    long rate = 0;
    PhysicalSourceStaticConfig pssc = buildPssc(rate, 0L);
    long scn = 10;
    DbusEventBuffer mb = (DbusEventBuffer) createBufMult(pssc);
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
    int sourceId = 505;
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
    Object key = new String("name");
    Schema.Type keyType = Schema.Type.RECORD;
    ColumnsState.KeyPair kp = new ColumnsState.KeyPair(key, keyType);
    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(1);
    keyPairs.add(kp);
    Schema s = Schema.parse(avroSchema);
    GenericRecord gr = new GenericData.Record(s);
    gr.put("name", "phani");
    DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    db.add(dbi);
    TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
    dbUpdates.add(dbUpdate);
    long timestamp = System.nanoTime();
    gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
    Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);
    DbusEventIterator iter = mb.acquireIterator("test");
    int count = 0;
    long eventTs = 0;
    while (iter.hasNext()) {
        DbusEvent e = iter.next();
        if (count == 1) {
            // first event prev control event
            eventTs = e.timestampInNanos();
        }
        count++;
    }
    Assert.assertEquals("Event timestamp in Ns", timestamp, eventTs);
    Assert.assertEquals("Got events ", 3, count);
    return;
}

Example 5 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class RelayEventProducer method createDatabusSourcesConnection.

public static DatabusSourcesConnection createDatabusSourcesConnection(String producerName, int id, String serverName, String subscriptionString, DatabusCombinedConsumer consumer, long internalBufferMaxSize, int largestEventSize, long consumerTimeoutMs, long pollIntervalMs, long connTimeoutMs, int consumerParallelism, boolean blockingBuffer, DatabusClientNettyThreadPools nettyThreadPools, int noEventsTimeoutSec, int maxEventVersion, int initReadBufferSize) throws InvalidConfigException {
    // the assumption here is that the list of subscriptions will become the
    // list of sources hosted by the relay
    Set<ServerInfo> relayServices = createServerInfo(serverName, subscriptionString);
    // null bootstrapService
    Set<ServerInfo> bootstrapServices = null;
    // create subscription objects based on what is required by subscription
    String[] subscriptionList = subscriptionString.split(",");
    List<DatabusSubscription> subsList = DatabusSubscription.createSubscriptionList(Arrays.asList(subscriptionList));
    List<String> sourcesStrList = DatabusSubscription.getStrList(subsList);
    LOG.info("The sourcesList is " + sourcesStrList);
    // create registration objects with consumers
    List<DatabusV2ConsumerRegistration> relayConsumers = createDatabusV2ConsumerRegistration(consumer, sourcesStrList);
    List<DatabusV2ConsumerRegistration> bstConsumers = null;
    // setup sources connection config
    DatabusSourcesConnection.Config confBuilder = new DatabusSourcesConnection.Config();
    confBuilder.setId(id);
    // consume whatever is in relay
    confBuilder.setConsumeCurrent(true);
    // this is set to false as the behaviour is to read the latest SCN when SCN is not found, the buffer isn't cleared
    // as such , so a possibility of gaps in events arises. What we want ideally is to clear existing buffer and then consume from latest SCN
    confBuilder.setReadLatestScnOnError(false);
    // 10s max consumer timeout
    confBuilder.setConsumerTimeBudgetMs(consumerTimeoutMs);
    // poll interval in ms and infinite retry;
    confBuilder.getPullerRetries().setMaxRetryNum(-1);
    confBuilder.getPullerRetries().setInitSleep(pollIntervalMs);
    confBuilder.setConsumerParallelism(consumerParallelism);
    confBuilder.getDispatcherRetries().setMaxRetryNum(1);
    // internal buffer conf
    DbusEventBuffer.Config bufferConf = new DbusEventBuffer.Config();
    bufferConf.setMaxSize(internalBufferMaxSize);
    bufferConf.setAllocationPolicy("DIRECT_MEMORY");
    if (initReadBufferSize > 0) {
        bufferConf.setAverageEventSize(initReadBufferSize);
    }
    bufferConf.setMaxEventSize(largestEventSize);
    // client buffer's scn index- not used
    bufferConf.setScnIndexSize(64 * 1024);
    String queuePolicy = blockingBuffer ? "BLOCK_ON_WRITE" : "OVERWRITE_ON_WRITE";
    bufferConf.setQueuePolicy(queuePolicy);
    // get appropriate checkpointThresholdPct
    double newCkptPct = confBuilder.computeSafeCheckpointThresholdPct(bufferConf);
    if (newCkptPct < 5.0 || newCkptPct > 95.0) {
        LOG.warn("Not setting required checkpointThresholdPct : " + newCkptPct + "to  accommodate largestEventSize= " + largestEventSize + " in buffer of size " + bufferConf.getMaxSize());
        if (newCkptPct <= 0.0) {
            // unlikely to happen: if it does retain default
            newCkptPct = confBuilder.getCheckpointThresholdPct();
        }
        if (newCkptPct < 5.0) {
            newCkptPct = 5.0;
        } else if (newCkptPct > 95.0) {
            newCkptPct = 95.0;
        }
    }
    LOG.info("Setting checkpointThresholdPct:" + newCkptPct);
    confBuilder.setCheckpointThresholdPct(newCkptPct);
    confBuilder.setEventBuffer(bufferConf);
    confBuilder.setNoEventsConnectionResetTimeSec(noEventsTimeoutSec);
    DatabusSourcesConnection.StaticConfig connConfig = confBuilder.build();
    // internal buffers of databus client library
    DbusEventBuffer buffer = new DbusEventBuffer(connConfig.getEventBuffer());
    buffer.start(0);
    DbusEventBuffer bootstrapBuffer = null;
    // Create threadpools and netty managers
    // read - write timeout in ms
    long readTimeoutMs = connTimeoutMs;
    long writeTimeoutMs = connTimeoutMs;
    long bstReadTimeoutMs = connTimeoutMs;
    int protocolVersion = 2;
    // connection factory
    NettyHttpConnectionFactory defaultConnFactory = new NettyHttpConnectionFactory(nettyThreadPools.getBossExecutorService(), nettyThreadPools.getIoExecutorService(), null, nettyThreadPools.getTimer(), writeTimeoutMs, readTimeoutMs, bstReadTimeoutMs, protocolVersion, maxEventVersion, nettyThreadPools.getChannelGroup());
    // Create Thread pool for consumer threads
    int maxThreadsNum = 1;
    int keepAliveMs = 1000;
    ThreadPoolExecutor defaultExecutorService = new OrderedMemoryAwareThreadPoolExecutor(maxThreadsNum, 0, 0, keepAliveMs, TimeUnit.MILLISECONDS);
    ConsumerCallbackStats relayConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.cons", producerName + ".inbound.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
    ConsumerCallbackStats bootstrapConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.bs.cons", producerName + ".inbound.bs.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
    UnifiedClientStats unifiedClientStats = new UnifiedClientStats(id, producerName + ".inbound.unified.cons", producerName + ".inbound.unified.cons", true, false, UnifiedClientStats.DEFAULT_DEADNESS_THRESHOLD_MS, null, ManagementFactory.getPlatformMBeanServer());
    DatabusRelayConnectionFactory relayConnFactory = defaultConnFactory;
    DatabusBootstrapConnectionFactory bootstrapConnFactory = defaultConnFactory;
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sourcesStrList);
    DatabusSourcesConnection conn = new DatabusSourcesConnection(connConfig, subsList, relayServices, bootstrapServices, relayConsumers, bstConsumers, buffer, bootstrapBuffer, defaultExecutorService, // getContainerStatsCollector(),
    null, // getInboundEventStatisticsCollector(),
    null, // getBootstrapEventsStatsCollector(),
    null, // relay callback stats
    relayConsumerStats, // bootstrap callback stats
    bootstrapConsumerStats, // combined relay/bootstrap callback stats
    unifiedClientStats, // getCheckpointPersistenceProvider(),
    null, relayConnFactory, bootstrapConnFactory, // getHttpStatsCollector(),
    null, // RegistrationId
    null, null, new DbusEventV2Factory(), // TODO Get the ref to factory from HttpRelay.
    connStateFactory);
    return conn;
}