Examples with DbusEventBuffer com.linkedin.databus.core.DbusEventBuffer used on opensource projects

Example 6 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class TestGoldenGateEventProducer method createBufMult.

/**
   * Creates a DbusBufMult
   */
private DbusEventBufferAppendable createBufMult(PhysicalSourceStaticConfig pssc) throws InvalidConfigException {
    DbusEventBuffer.StaticConfig config = null;
    if (config == null) {
        try {
            DbusEventBuffer.Config cfgBuilder = new DbusEventBuffer.Config();
            cfgBuilder.setMaxSize(10 * 1024 * 1024);
            cfgBuilder.setScnIndexSize(2 * 1024 * 1024);
            cfgBuilder.setAllocationPolicy("MMAPPED_MEMORY");
            config = cfgBuilder.build();
        } catch (InvalidConfigException e) {
            fail("invalid configuration", e);
        }
    }
    PhysicalSourceStaticConfig[] pConfigs = new PhysicalSourceStaticConfig[1];
    pConfigs[0] = pssc;
    DbusEventBufferMult eventBufferMult = new DbusEventBufferMult(pConfigs, config, new DbusEventV2Factory());
    for (DbusEventBuffer b : eventBufferMult.bufIterable()) {
        b.start(1);
    }
    DbusEventBufferAppendable buf = eventBufferMult.getDbusEventBufferAppendable(505);
    return buf;
}

Example 7 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class TestDatabusRelayEvents method testV2Events.

/**
   * Stuffs an event buffer with both a v1 and a v2 event, then reads the buffer two ways:
   * first accepting only v1 events (verifying conversion of the v2 event to v1); then accepting
   * both v1 and v2 events.
   *
   * Note that the version of the _EOP_ events must match the version of the event factory,
   * regardless of the versions of any preceding "real" events.  (This matches DbusEventBuffer
   * behavior; see the serializeLongKeyEndOfPeriodMarker() call in endEvents() for details.)
   */
@Test
public void testV2Events() throws KeyTypeNotImplementedException, InvalidEventException, IOException, DatabusException {
    final Logger log = Logger.getLogger("TestDatabusRelayEvents.testV2Events");
    log.setLevel(Level.DEBUG);
    String[] srcs = { "com.linkedin.events.example.fake.FakeSchema" };
    String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
    short srcId = 2;
    short pId = 1;
    int relayPort = Utils.getAvailablePort(11993);
    // create relay
    final DatabusRelayMain relay1 = createRelay(relayPort, pId, srcs);
    DatabusRelayTestUtil.RelayRunner r1 = null;
    ClientRunner cr = null;
    try {
        //EventProducer[] producers = relay1.getProducers();
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("Relay created");
        DbusEventBufferMult bufMult = relay1.getEventBuffer();
        PhysicalPartition pPartition = new PhysicalPartition((int) pId, pSourceName);
        DbusEventBuffer buf = (DbusEventBuffer) bufMult.getDbusEventBufferAppendable(pPartition);
        log.info("create some events");
        long windowScn = 100L;
        ByteBuffer serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(), pId, DbusEventFactory.DBUS_EVENT_V2);
        ReadableByteChannel channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
        int readEvents = buf.readEvents(channel);
        log.info("successfully read in " + readEvents + " events ");
        channel.close();
        windowScn = 101L;
        serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(), pId, DbusEventFactory.DBUS_EVENT_V1);
        channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
        readEvents = buf.readEvents(channel);
        log.info("successfully read in " + readEvents + " events ");
        channel.close();
        log.info("starting relay on port " + relayPort);
        r1.start();
        //TestUtil.sleep(10*1000);
        // wait until relay comes up
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return relay1.isRunningStatus();
            }
        }, "Relay hasn't come up completely ", 30000, LOG);
        log.info("now create client");
        String srcSubscriptionString = TestUtil.join(srcs, ",");
        String serverName = "localhost:" + relayPort;
        final EventsCountingConsumer countingConsumer = new EventsCountingConsumer();
        int id = (RngUtils.randomPositiveInt() % 10000) + 1;
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V1, 0);
        cr = new ClientRunner(clientConn);
        log.info("starting client");
        cr.start();
        // wait till client gets the event
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                int events = countingConsumer.getNumDataEvents();
                LOG.info("client got " + events + " events");
                return events == 2;
            }
        }, "Consumer didn't get 2 events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(countingConsumer.getNumDataEvents(), 2);
        Assert.assertEquals(countingConsumer.getNumWindows(), 2);
        Assert.assertEquals(countingConsumer.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1), 2);
        log.info("shutdown first client");
        clientConn.stop();
        cr.shutdown();
        TestUtil.sleep(1000);
        cr = null;
        log.info("start another client who understands V2");
        final EventsCountingConsumer countingConsumer1 = new EventsCountingConsumer();
        clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer1, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V2, 0);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("wait till client gets the event");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                int events = countingConsumer1.getNumDataEvents();
                LOG.debug("client got " + events + " events");
                return events == 2;
            }
        }, "Consumer didn't get 2 events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(countingConsumer1.getNumDataEvents(), 2);
        Assert.assertEquals(countingConsumer1.getNumWindows(), 2);
        Assert.assertEquals(countingConsumer1.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1), 1);
        Assert.assertEquals(countingConsumer1.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V2), 1);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 8 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class TestDatabusRelayMain method testPendingEventSize.

@Test
public /**
   * When the relay has no events, we should not get the x-dbus-pending-event-size even if we present a small buffer.
   * When the relay has events, we should see the header on a small buffer but see an event when the buffer
   * is large enough, and should not see the header in the large buffer case.
   */
void testPendingEventSize() throws Exception {
    DatabusRelayMain relay = null;
    try {
        final short srcId = 104;
        final String srcName = "foo";
        PhysicalSourceConfig pConfig = new PhysicalSourceConfig();
        pConfig.setId(srcId);
        pConfig.setName(srcName);
        pConfig.setUri("mock");
        short lid = (short) (srcId + 1);
        LogicalSourceConfig lConf = new LogicalSourceConfig();
        lConf.setId(lid);
        lConf.setName(srcName);
        // this is table name in the oracle source world
        lConf.setUri(srcName);
        lConf.setPartitionFunction("constant:1");
        pConfig.addSource(lConf);
        int relayPort = Utils.getAvailablePort(11994);
        final int relayId = 666;
        HttpRelay.Config httpRelayConfig = new HttpRelay.Config();
        ServerContainer.Config containerConfig = DatabusRelayTestUtil.createContainerConfig(relayId, relayPort);
        DbusEventBuffer.Config bufferConfig = DatabusRelayTestUtil.createBufferConfig(10000, 250, 100);
        httpRelayConfig.setContainer(containerConfig);
        httpRelayConfig.setEventBuffer(bufferConfig);
        httpRelayConfig.setStartDbPuller("true");
        PhysicalSourceStaticConfig[] pStaticConfigs = new PhysicalSourceStaticConfig[1];
        for (LogicalSourceConfig lsc : pConfig.getSources()) {
            httpRelayConfig.setSourceName("" + lsc.getId(), lsc.getName());
        }
        pStaticConfigs[0] = pConfig.build();
        relay = new DatabusRelayMain(httpRelayConfig.build(), pStaticConfigs);
        relay.start();
        // Insert one event into the relay.
        LogicalSource lsrc = new LogicalSource((int) lid, srcName);
        DbusEventBuffer buf = relay.getEventBuffer().getDbusEventBuffer(lsrc);
        byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
        final long prevScn = 99;
        final long eventScn = 101;
        buf.start(prevScn);
        buf.startEvents();
        Assert.assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, lid, schema, new byte[100], false, null));
        buf.endEvents(eventScn, null);
        HttpResponseHandler handler = new HttpResponseHandler();
        // On a good buffer length we should not see the extra header.
        testClient(relayPort, 1000, 100L, handler);
        Assert.assertNull(handler._pendingEventHeader, "Received pending event header on full buffer");
        // We should see the extra header when we get 0 events and the next event is too big to fit in
        testClient(relayPort, 10, 100L, handler);
        Assert.assertNotNull(handler._pendingEventHeader);
        Assert.assertEquals(Integer.valueOf(handler._pendingEventHeader).intValue(), 161);
        // But if there are no events, then we should not see the header even if buffer is very small
        handler._pendingEventHeader = null;
        testClient(relayPort, 10, 1005L, handler);
        Assert.assertNull(handler._pendingEventHeader, "Received pending event header on full buffer");
    } finally {
        relay.shutdownUninteruptibly();
    }
}

Example 9 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class DatabusHttpClientImpl method initializeRelayConnections.

private synchronized void initializeRelayConnections() {
    for (List<DatabusSubscription> subsList : _relayGroups.keySet()) {
        List<String> sourcesStrList = DatabusSubscription.getStrList(subsList);
        List<DatabusV2ConsumerRegistration> relayConsumers = getRelayGroupStreamConsumers().get(subsList);
        //nothing to do
        if (null == relayConsumers || 0 == relayConsumers.size())
            continue;
        try {
            DatabusSourcesConnection.StaticConfig connConfig = getClientStaticConfig().getConnection(sourcesStrList);
            if (null == connConfig) {
                connConfig = getClientStaticConfig().getConnectionDefaults();
            }
            // make sure we have the right policy.
            if (!connConfig.getEventBuffer().isEnableScnIndex() && connConfig.getEventBuffer().getQueuePolicy() != DbusEventBuffer.QueuePolicy.BLOCK_ON_WRITE) {
                throw new InvalidConfigException("If SCN index is disabled, queue policy must be BLOCK_ON_WRITE");
            }
            CheckpointPersistenceProvider cpPersistenceProvder = getCheckpointPersistenceProvider();
            if (null != cpPersistenceProvder && getClientStaticConfig().getCheckpointPersistence().isClearBeforeUse()) {
                cpPersistenceProvder.removeCheckpoint(sourcesStrList);
            }
            ServerInfo server0 = _relayGroups.get(subsList).iterator().next();
            ArrayList<DatabusV2ConsumerRegistration> bstConsumersRegs = new ArrayList<DatabusV2ConsumerRegistration>();
            for (List<DatabusSubscription> bstSubSourcesList : getRelayGroupBootstrapConsumers().keySet()) {
                List<DatabusV2ConsumerRegistration> bstRegsistrations = getRelayGroupBootstrapConsumers().get(bstSubSourcesList);
                for (DatabusV2ConsumerRegistration bstConsumerReg : bstRegsistrations) {
                    if (server0.supportsSources(bstConsumerReg.getSources())) {
                        bstConsumersRegs.add(bstConsumerReg);
                    }
                }
            }
            DbusEventBuffer eventBuffer = connConfig.getEventBuffer().getOrCreateEventBuffer(_eventFactory);
            eventBuffer.setDropOldEvents(true);
            eventBuffer.start(0);
            DbusEventBuffer bootstrapBuffer = null;
            // create bootstrap only if it is enabled
            if (_clientStaticConfig.getRuntime().getBootstrap().isEnabled()) {
                bootstrapBuffer = new DbusEventBuffer(connConfig.getEventBuffer());
                bootstrapBuffer.setDropOldEvents(false);
                bootstrapBuffer.start(0);
            }
            LOG.info("The sourcesList is " + sourcesStrList);
            LOG.info("The relayGroupStreamConsumers is " + getRelayGroupStreamConsumers().get(subsList));
            Set<ServerInfo> relays = _relayGroups.get(subsList);
            Set<ServerInfo> bootstrapServices = _bootstrapGroups.get(subsList);
            String statsCollectorName = generateSubsStatsName(sourcesStrList);
            int ownerId = getContainerStaticConfig().getId();
            _bootstrapEventsStats.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".inbound.bs", true, false, getMbeanServer()));
            _inBoundStatsCollectors.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".inbound", true, false, getMbeanServer()));
            _outBoundStatsCollectors.addStatsCollector(statsCollectorName, new DbusEventsStatisticsCollector(ownerId, statsCollectorName + ".outbound", true, false, getMbeanServer()));
            _consumerStatsCollectors.addStatsCollector(statsCollectorName, new ConsumerCallbackStats(ownerId, statsCollectorName + ".inbound.cons", statsCollectorName + ".inbound.cons", true, false, null, getMbeanServer()));
            _bsConsumerStatsCollectors.addStatsCollector(statsCollectorName, new ConsumerCallbackStats(ownerId, statsCollectorName + ".inbound.bs.cons", statsCollectorName + ".inbound.bs.cons", true, false, null, getMbeanServer()));
            _unifiedClientStatsCollectors.addStatsCollector(statsCollectorName, new UnifiedClientStats(ownerId, statsCollectorName + ".inbound.unified.cons", statsCollectorName + ".inbound.unified.cons", true, false, _clientStaticConfig.getPullerThreadDeadnessThresholdMs(), null, getMbeanServer()));
            ConnectionStateFactory connStateFactory = new ConnectionStateFactory(DatabusSubscription.getStrList(subsList));
            DatabusSourcesConnection newConn = new DatabusSourcesConnection(connConfig, subsList, relays, bootstrapServices, relayConsumers, //_relayGroupBootstrapConsumers.get(sourcesList),
            bstConsumersRegs, eventBuffer, bootstrapBuffer, getDefaultExecutorService(), getContainerStatsCollector(), _inBoundStatsCollectors.getStatsCollector(statsCollectorName), _bootstrapEventsStats.getStatsCollector(statsCollectorName), _consumerStatsCollectors.getStatsCollector(statsCollectorName), _bsConsumerStatsCollectors.getStatsCollector(statsCollectorName), _unifiedClientStatsCollectors.getStatsCollector(statsCollectorName), getCheckpointPersistenceProvider(), getRelayConnFactory(), getBootstrapConnFactory(), getHttpStatsCollector(), null, this, _eventFactory, connStateFactory);
            newConn.start();
            _relayConnections.add(newConn);
        } catch (Exception e) {
            LOG.error("connection initialization issue for source(s):" + subsList + "; please check your configuration", e);
        }
    }
    if (0 == _relayConnections.size()) {
        LOG.warn("No connections specified");
    }
}

Example 10 with DbusEventBuffer

use of com.linkedin.databus.core.DbusEventBuffer in project databus by linkedin.

the class DispatcherState method resetIterators.

public void resetIterators() {
    if (null != _lastSuccessfulIterator) {
        setLastSuccessfulIterator(null);
        _lastSuccessfulScn = null;
        _lastSuccessfulCheckpoint = null;
    }
    if (null != _eventsIterator) {
        DbusEventBuffer eventBuffer = _eventsIterator.getEventBuffer();
        String iteratorName = _eventsIterator.getIdentifier();
        _eventsIterator.close();
        _eventsIterator = eventBuffer.acquireIterator(iteratorName);
        if (LOG.isDebugEnabled()) {
            String msg = "Reset event iterator to: " + _eventsIterator;
            DbusLogAccumulator.addLog(msg, LOG);
        }
        resetSourceInfo();
    }
}