Examples with DbusEventBufferAppendable com.linkedin.databus.core.DbusEventBufferAppendable used on opensource projects

Example 1 with DbusEventBufferAppendable

use of com.linkedin.databus.core.DbusEventBufferAppendable in project databus by linkedin.

the class DatabusRelayMain method addOneProducer.

/** overrides HTTP relay method */
@Override
public void addOneProducer(PhysicalSourceStaticConfig pConfig) throws DatabusException, EventCreationException, UnsupportedKeyException, SQLException, InvalidConfigException {
    // Register a command to allow start/stop/status of the relay
    List<EventProducer> plist = new ArrayList<EventProducer>();
    PhysicalPartition pPartition = pConfig.getPhysicalPartition();
    MaxSCNReaderWriter maxScnReaderWriters = _maxScnReaderWriters.getOrCreateHandler(pPartition);
    LOG.info("Starting server container with maxScnReaderWriter:" + maxScnReaderWriters);
    // Get the event buffer
    DbusEventBufferAppendable dbusEventBuffer = getEventBuffer().getDbusEventBufferAppendable(pPartition);
    // Get the schema registry service
    SchemaRegistryService schemaRegistryService = getSchemaRegistryService();
    // Get a stats collector per physical source
    addPhysicalPartitionCollectors(pPartition);
    String statsCollectorName = pPartition.toSimpleString();
    /*
		 * _inBoundStatsCollectors.addStatsCollector(statsCollectorName, new
		 * DbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
		 * statsCollectorName+".inbound", true, false, getMbeanServer()));
		 *
		 * _outBoundStatsCollectors.addStatsCollector(statsCollectorName, new
		 * DbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
		 * statsCollectorName+".outbound", true, false, getMbeanServer()));
		 */
    // Create the event producer
    String uri = pConfig.getUri();
    if (uri == null)
        throw new DatabusException("Uri is required to start the relay");
    uri = uri.trim();
    EventProducer producer = null;
    if (uri.startsWith("jdbc:")) {
        SourceType sourceType = pConfig.getReplBitSetter().getSourceType();
        if (SourceType.TOKEN.equals(sourceType))
            throw new DatabusException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");
        // if a buffer for this partiton exists - we are overwri
        producer = new OracleEventProducerFactory().buildEventProducer(pConfig, schemaRegistryService, dbusEventBuffer, getMbeanServer(), _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else if (uri.startsWith("mock")) {
        // Get all relevant pConfig attributes
        //TODO add real instantiation
        EventProducerServiceProvider mockProvider = _producersRegistry.getEventProducerServiceProvider("mock");
        if (null == mockProvider) {
            throw new DatabusRuntimeException("relay event producer not available: " + "mock");
        }
        producer = mockProvider.createProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else if (uri.startsWith("gg:")) {
        producer = new GoldenGateEventProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else if (uri.startsWith("mysql:")) {
        LOG.info("Adding OpenReplicatorEventProducer for uri :" + uri);
        final String serviceName = "or";
        EventProducerServiceProvider orProvider = _producersRegistry.getEventProducerServiceProvider(serviceName);
        if (null == orProvider) {
            throw new DatabusRuntimeException("relay event producer not available: " + serviceName);
        }
        producer = orProvider.createProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else {
        // Get all relevant pConfig attributes and initialize the nettyThreadPool objects
        RelayEventProducer.DatabusClientNettyThreadPools nettyThreadPools = new RelayEventProducer.DatabusClientNettyThreadPools(0, getNetworkTimeoutTimer(), getBossExecutorService(), getIoExecutorService(), getHttpChannelGroup());
        producer = new RelayEventProducer(pConfig, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters, nettyThreadPools);
    }
    // if a buffer for this partiton exists - we are overwriting it.
    _producers.put(pPartition, producer);
    plist.add(producer);
    // append 'monitoring event producer'
    if (producer instanceof OracleEventProducer) {
        MonitoringEventProducer monitoringProducer = new MonitoringEventProducer("dbMonitor." + pPartition.toSimpleString(), pConfig.getName(), pConfig.getUri(), ((OracleEventProducer) producer).getMonitoredSourceInfos(), getMbeanServer());
        _monitoringProducers.put(pPartition, monitoringProducer);
        plist.add(monitoringProducer);
    }
    if (_csEventRequestProcessor == null)
        _csEventRequestProcessor = new ControlSourceEventsRequestProcessor(null, this, plist);
    else
        _csEventRequestProcessor.addEventProducers(plist);
    RequestProcessorRegistry processorRegistry = getProcessorRegistry();
    processorRegistry.reregister(ControlSourceEventsRequestProcessor.COMMAND_NAME, _csEventRequestProcessor);
}

Example 2 with DbusEventBufferAppendable

use of com.linkedin.databus.core.DbusEventBufferAppendable in project databus by linkedin.

the class TestGoldenGateEventProducer method createBufMult.

/**
   * Creates a DbusBufMult
   */
private DbusEventBufferAppendable createBufMult(PhysicalSourceStaticConfig pssc) throws InvalidConfigException {
    DbusEventBuffer.StaticConfig config = null;
    if (config == null) {
        try {
            DbusEventBuffer.Config cfgBuilder = new DbusEventBuffer.Config();
            cfgBuilder.setMaxSize(10 * 1024 * 1024);
            cfgBuilder.setScnIndexSize(2 * 1024 * 1024);
            cfgBuilder.setAllocationPolicy("MMAPPED_MEMORY");
            config = cfgBuilder.build();
        } catch (InvalidConfigException e) {
            fail("invalid configuration", e);
        }
    }
    PhysicalSourceStaticConfig[] pConfigs = new PhysicalSourceStaticConfig[1];
    pConfigs[0] = pssc;
    DbusEventBufferMult eventBufferMult = new DbusEventBufferMult(pConfigs, config, new DbusEventV2Factory());
    for (DbusEventBuffer b : eventBufferMult.bufIterable()) {
        b.start(1);
    }
    DbusEventBufferAppendable buf = eventBufferMult.getDbusEventBufferAppendable(505);
    return buf;
}

Example 3 with DbusEventBufferAppendable

use of com.linkedin.databus.core.DbusEventBufferAppendable in project databus by linkedin.

the class TestDatabusRelayEvents method testEventConversion.

@Test
public /**
   * append event V2 to the buffer and stream it to the client
   * which only accepts events V1. Make sure it got converted
   */
void testEventConversion() throws InterruptedException, IOException, DatabusException {
    final Logger log = Logger.getLogger("TestDatabusRelayEvents.testEventConversion");
    log.setLevel(Level.INFO);
    DatabusRelayTestUtil.RelayRunner r1 = null;
    ClientRunner cr = null;
    try {
        String[] srcs = { "com.linkedin.events.example.fake.FakeSchema" };
        int pId = 1;
        int srcId = 2;
        int relayPort = Utils.getAvailablePort(11994);
        ;
        final DatabusRelayMain relay1 = createRelay(relayPort, pId, srcs);
        Assert.assertNotNull(relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("Relay created");
        DbusEventBufferMult bufMult = relay1.getEventBuffer();
        String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
        PhysicalPartition pPartition = new PhysicalPartition(pId, pSourceName);
        DbusEventBufferAppendable buf = bufMult.getDbusEventBufferAppendable(pPartition);
        DbusEventKey key = new DbusEventKey(123L);
        byte[] schemaId = relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray();
        byte[] payload = RngUtils.randomString(100).getBytes(Charset.defaultCharset());
        DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, 100L, (short) pId, (short) pId, 897L, (short) srcId, schemaId, payload, false, true);
        eventInfo.setEventSerializationVersion(DbusEventFactory.DBUS_EVENT_V2);
        buf.startEvents();
        buf.appendEvent(key, eventInfo, null);
        buf.endEvents(100L, null);
        r1.start();
        log.info("Relay started");
        // wait until relay comes up
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return relay1.isRunningStatus();
            }
        }, "Relay hasn't come up completely ", 7000, LOG);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcs, ",");
        String serverName = "localhost:" + relayPort;
        final EventsCountingConsumer countingConsumer = new EventsCountingConsumer();
        int id = (RngUtils.randomPositiveInt() % 10000) + 1;
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V1, 0);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("Consumer started");
        // wait till client gets the event
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return countingConsumer.getNumDataEvents() == 1;
            }
        }, "Consumer didn't get any events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(1, countingConsumer.getNumDataEvents());
        Assert.assertEquals(1, countingConsumer.getNumWindows());
        Assert.assertEquals(1, countingConsumer.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1));
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 4 with DbusEventBufferAppendable

use of com.linkedin.databus.core.DbusEventBufferAppendable in project databus by linkedin.

the class TestGoldenGateEventProducer method testTransactionsWithDuplicateSCN.

/**
   *
   * Test to cover merging of transactions with same SCNs
   */
@Test
public void testTransactionsWithDuplicateSCN() throws Exception {
    short[] sourceIds = new short[] { 505, 506 };
    String[] sourceNames = new String[] { "source1", "source2" };
    PhysicalSourceStaticConfig pssc = buildSimplePssc(sourceIds, sourceNames, "gg:///tmp:xxx");
    DbusEventBufferAppendable mb = createBufMult(pssc);
    // start producer
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    Object handleXmlCallbackObject = getHandleXmlCallbackInnerInstance(gg);
    Method method = getOnTransactionEndMethod();
    // generates the updates
    List<String> keys = new ArrayList<String>();
    keys.add("key1");
    // SCN = 10 - Case where both transactions having same SCNs : same set of sources and
    // same keys
    long scn = 10;
    long timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC;
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates1 = generateUpdates(sourceIds, keys, scn);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
    timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates2 = // Same SCN as before
    generateUpdates(sourceIds, keys, scn);
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp + 1, scn) });
    // Expect no events as events are not yet appended to buffer.
    testStats(gg, new EventStatsValues(sourceIds[0], 0, 0, 0, 0, 0), new EventStatsValues(sourceIds[1], 0, 0, 0, 0, 0), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 0, 0, 0, 0, 0));
    // SCN = 11 - Case where both transactions having same SCNs : same set of sources but
    // different keys
    {
        scn = 11;
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 1;
        keys.clear();
        keys.add("key2");
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        // Testing for SCN = 10 case
        testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 1, 0, 10), new EventStatsValues(sourceIds[1], 5, 0, 1, 0, 10), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 2, 0, 10));
    }
    // SCN = 12 - Case where both transactions having same SCNs but different set of
    // sources
    {
        scn = 12;
        keys.clear();
        keys.add("key2");
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 2;
        dbUpdates1 = generateUpdates(new short[] { sourceIds[1] }, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN
        dbUpdates2 = generateUpdates(new short[] { sourceIds[0] }, keys, scn);
        // as before
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        // Testing for SCN = 11 case
        testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 3, 0, 11), new EventStatsValues(sourceIds[1], 5, 0, 3, 0, 11), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 6, 0, 11));
    }
    // SCN = 13 - Case where more than 2 transactions having same SCNs and keys. The keys
    // will be merged.
    {
        scn = 13;
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 3;
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        // Testing for SCN = 12 case
        testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 4, 0, 12), new EventStatsValues(sourceIds[1], 5, 0, 4, 0, 12), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 8, 0, 12));
    }
    // SCN = 14 - Case where more than 2 transactions having same SCNs but different keys.
    {
        scn = 14;
        timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 3;
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        keys.clear();
        keys.add("key5");
        dbUpdates1 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
        // Same SCN as before
        dbUpdates2 = generateUpdates(sourceIds, keys, scn);
        method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates2, new TransactionInfo(0, 0, timestamp, scn) });
        // Testing for SCN = 13 case
        testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 5, 0, 13), new EventStatsValues(sourceIds[1], 5, 0, 5, 0, 13), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 10, 0, 13));
    }
    // THis is an extra-call but the corresponding events will not be added to EVB.
    // This is needed to flush the events in the above call to EVB
    scn = 15;
    timestamp = System.currentTimeMillis() * DbusConstants.NUM_NSECS_IN_MSEC + 4;
    method.invoke(handleXmlCallbackObject, new Object[] { dbUpdates1, new TransactionInfo(0, 0, timestamp, scn) });
    // Testing for SCN = 12 case
    testStats(gg, new EventStatsValues(sourceIds[0], 5, 0, 7, 0, 14), new EventStatsValues(sourceIds[1], 5, 0, 7, 0, 14), new EventStatsValues(GoldenGateEventProducer.GLOBAL_SOURCE_ID, 5, 0, 14, 0, 14));
}

Example 5 with DbusEventBufferAppendable

use of com.linkedin.databus.core.DbusEventBufferAppendable in project databus by linkedin.

the class DatabusEventRandomProducer method produceNRandomEvents.

public long produceNRandomEvents(long startScn, long currentTime, int numberOfEvents, List<IdNamePair> sources, long keyMin, long keyMax, int minLength, int maxLength, List<Schema> schemas) throws KeyTypeNotImplementedException {
    // random number between startScn and startScn + 100;
    long endScn = startScn + 1 + (RngUtils.randomPositiveLong() % 100L);
    long scnDiff = endScn - startScn;
    long maxScn = startScn;
    int numSources = sources.size();
    int eventsPerSource = numberOfEvents / numSources + 1;
    if (eventsPerSource <= 0)
        eventsPerSource = 1;
    _dbusEventBuffer.startAllEvents();
    assert endScn > startScn;
    if (LOG.isDebugEnabled()) {
        LOG.debug("endScn = " + endScn + " startScn = " + startScn);
    }
    byte[] defaultSchemaId = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
    // trace 1% samples
    boolean enableTracing = (RngUtils.randomPositiveLong() % 100L <= 1);
    for (int i = 0; i < numberOfEvents; ++i) {
        // random key between 0 and 100M
        DbusEventKey key = new DbusEventKey(RngUtils.randomPositiveLong(keyMin, keyMax));
        long scn = startScn + (i / scnDiff);
        //short srcId = sources.get((Integer) (RngUtils.randomPositiveShort() % sources.size())).getId().shortValue();
        short srcId = sources.get(i / eventsPerSource).getId().shortValue();
        byte[] schemaId = (_schemaIds != null) ? _schemaIds.get((long) srcId) : defaultSchemaId;
        genEventsPerSource.put((int) srcId, genEventsPerSource.get((int) srcId) + 1);
        String value = null;
        int rnd = RngUtils.randomPositiveShort();
        int length = minLength + rnd % (maxLength - minLength);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Creating random record with SCN =" + scn + " and length =" + length);
        }
        // 1K of row data
        value = RngUtils.randomString(length);
        //short lPartitionId = (short) (key.getLongKey() % Short.MAX_VALUE);
        short lPartitionId = LogicalSourceConfig.DEFAULT_LOGICAL_SOURCE_PARTITION;
        short pPartitionId = _dbusEventBuffer.getPhysicalPartition(srcId).getId().shortValue();
        DbusEventBufferAppendable buf = _dbusEventBuffer.getDbusEventBufferAppendable(srcId);
        boolean appended = buf.appendEvent(key, pPartitionId, lPartitionId, currentTime, srcId, schemaId, value.getBytes(Charset.defaultCharset()), enableTracing, _statsCollector);
        assert appended == true;
        maxScn = Math.max(scn, maxScn);
        if (LOG.isDebugEnabled() && (i % 7 == 0))
            LOG.debug("Produce: srcId=" + srcId + ";pSid=" + _dbusEventBuffer.getPhysicalPartition(srcId) + ";scn=" + scn + "; buf(hc)=" + buf.hashCode());
    }
    // for debug purposes - collect stats how many events for each srcId were generated
    StringBuilder s = new StringBuilder(100);
    for (Entry<Integer, Integer> e : genEventsPerSource.entrySet()) {
        s.append(e.getKey() + ":" + e.getValue() + ",");
    }
    LOG.info("generated sources/events = " + s.toString());
    _dbusEventBuffer.endAllEvents(maxScn, getCurrentNanoTime(), _statsCollector);
    if (LOG.isDebugEnabled())
        LOG.debug("Produce:window:" + maxScn);
    assert maxScn >= startScn;
    return maxScn;
}