Examples with DbusEventInfo com.linkedin.databus.core.DbusEventInfo used on opensource projects

Example 1 with DbusEventInfo

use of com.linkedin.databus.core.DbusEventInfo in project databus by linkedin.

the class GoldenGateEventProducer method addEventToBuffer.

/**
   *
   * @param dbUpdates  The dbUpdates present in the current transaction
   * @param ti The meta information about the transaction. (See TransactionInfo class for more details).
   * @throws DatabusException
   * @throws UnsupportedKeyException
   */
protected void addEventToBuffer(List<TransactionState.PerSourceTransactionalUpdate> dbUpdates, TransactionInfo ti) throws DatabusException, UnsupportedKeyException {
    if (dbUpdates.size() == 0)
        throw new DatabusException("Cannot handle empty dbUpdates");
    long scn = ti.getScn();
    long timestamp = ti.getTransactionTimeStampNs();
    EventSourceStatistics globalStats = getSource(GLOBAL_SOURCE_ID).getStatisticsBean();
    /**
     * We skip the start scn of the relay, we have already added a EOP for this SCN in the buffer.
     * Why is this not a problem ?
     * There are two cases:
     * 1. When we use the earliest/latest scn if there is no maxScn (We don't really have a start point). So it's really OK to miss the first event.
     * 2. If it's the maxSCN, then event was already seen by the relay.
     */
    if (scn == _startPrevScn.get()) {
        _log.info("Skipping this transaction, EOP already send for this event");
        return;
    }
    getEventBuffer().startEvents();
    int eventsInTransactionCount = 0;
    List<EventReaderSummary> summaries = new ArrayList<EventReaderSummary>();
    for (int i = 0; i < dbUpdates.size(); ++i) {
        GenericRecord record = null;
        TransactionState.PerSourceTransactionalUpdate perSourceUpdate = dbUpdates.get(i);
        short sourceId = (short) perSourceUpdate.getSourceId();
        // prepare stats collection per source
        EventSourceStatistics perSourceStats = getSource(sourceId).getStatisticsBean();
        Iterator<DbUpdateState.DBUpdateImage> dbUpdateIterator = perSourceUpdate.getDbUpdatesSet().iterator();
        int eventsInDbUpdate = 0;
        long dbUpdatesEventsSize = 0;
        long startDbUpdatesMs = System.currentTimeMillis();
        while (//TODO verify if there is any case where we need to rollback.
        dbUpdateIterator.hasNext()) {
            DbUpdateState.DBUpdateImage dbUpdate = dbUpdateIterator.next();
            //Construct the Databus Event key, determine the key type and construct the key
            Object keyObj = obtainKey(dbUpdate);
            DbusEventKey eventKey = new DbusEventKey(keyObj);
            //Get the logicalparition id
            PartitionFunction partitionFunction = _partitionFunctionHashMap.get((int) sourceId);
            short lPartitionId = partitionFunction.getPartition(eventKey);
            record = dbUpdate.getGenericRecord();
            //Write the event to the buffer
            if (record == null)
                throw new DatabusException("Cannot write event to buffer because record = " + record);
            if (record.getSchema() == null)
                throw new DatabusException("The record does not have a schema (null schema)");
            try {
                //Collect stats on number of dbUpdates for one source
                eventsInDbUpdate++;
                //Count of all the events in the current transaction
                eventsInTransactionCount++;
                // Serialize the row
                ByteArrayOutputStream bos = new ByteArrayOutputStream();
                Encoder encoder = new BinaryEncoder(bos);
                GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(record.getSchema());
                writer.write(record, encoder);
                byte[] serializedValue = bos.toByteArray();
                //Get the md5 for the schema
                SchemaId schemaId = SchemaId.createWithMd5(dbUpdate.getSchema());
                //Determine the operation type and convert to dbus opcode
                DbusOpcode opCode;
                if (dbUpdate.getOpType() == DbUpdateState.DBUpdateImage.OpType.INSERT || dbUpdate.getOpType() == DbUpdateState.DBUpdateImage.OpType.UPDATE) {
                    opCode = DbusOpcode.UPSERT;
                    if (_log.isDebugEnabled())
                        _log.debug("The event with scn " + scn + " is INSERT/UPDATE");
                } else if (dbUpdate.getOpType() == DbUpdateState.DBUpdateImage.OpType.DELETE) {
                    opCode = DbusOpcode.DELETE;
                    if (_log.isDebugEnabled())
                        _log.debug("The event with scn " + scn + " is DELETE");
                } else {
                    throw new DatabusException("Unknown opcode from dbUpdate for event with scn:" + scn);
                }
                //Construct the dbusEvent info
                DbusEventInfo dbusEventInfo = new DbusEventInfo(opCode, scn, (short) _pConfig.getId(), lPartitionId, timestamp, sourceId, schemaId.getByteArray(), serializedValue, false, false);
                dbusEventInfo.setReplicated(dbUpdate.isReplicated());
                perSourceStats.addEventCycle(1, ti.getTransactionTimeRead(), serializedValue.length, scn);
                globalStats.addEventCycle(1, ti.getTransactionTimeRead(), serializedValue.length, scn);
                long tsEnd = System.currentTimeMillis();
                perSourceStats.addTimeOfLastDBAccess(tsEnd);
                globalStats.addTimeOfLastDBAccess(tsEnd);
                //Append to the event buffer
                getEventBuffer().appendEvent(eventKey, dbusEventInfo, _statsCollector);
                _rc.incrementEventCount();
                dbUpdatesEventsSize += serializedValue.length;
            } catch (IOException io) {
                perSourceStats.addError();
                globalStats.addEmptyEventCycle();
                _log.error("Cannot create byte stream payload: " + dbUpdates.get(i).getSourceId());
            }
        }
        long endDbUpdatesMs = System.currentTimeMillis();
        long dbUpdatesElapsedTimeMs = endDbUpdatesMs - startDbUpdatesMs;
        // Log Event Summary at logical source level
        EventReaderSummary summary = new EventReaderSummary(sourceId, _monitoredSources.get(sourceId).getSourceName(), scn, eventsInDbUpdate, dbUpdatesEventsSize, -1L, /* Not supported */
        dbUpdatesElapsedTimeMs, timestamp, timestamp, -1L);
        if (_eventsLog.isInfoEnabled()) {
            _eventsLog.info(summary.toString());
        }
        summaries.add(summary);
        if (_log.isDebugEnabled())
            _log.debug("There are " + eventsInDbUpdate + " events seen in the current dbUpdate");
    }
    // Log Event Summary at Physical source level
    ReadEventCycleSummary summary = new ReadEventCycleSummary(_pConfig.getName(), summaries, scn, -1);
    if (_eventsLog.isInfoEnabled()) {
        _eventsLog.info(summary.toString());
    }
    _log.info("Writing " + eventsInTransactionCount + " events from transaction with scn: " + scn);
    if (scn <= 0)
        throw new DatabusException("Unable to write events to buffer because of negative/zero scn: " + scn);
    getEventBuffer().endEvents(scn, _statsCollector);
    _scn.set(scn);
    if (getMaxScnReaderWriter() != null) {
        try {
            getMaxScnReaderWriter().saveMaxScn(_scn.get());
        } catch (DatabusException e) {
            _log.error("Cannot save scn = " + _scn + " for physical source = " + getName(), e);
        }
    }
}

Example 2 with DbusEventInfo

use of com.linkedin.databus.core.DbusEventInfo in project databus by linkedin.

the class TestDatabusRelayEvents method addEvent.

/**
   *  adds create a bytebuffer with a serialized event and EOW event using readEvents interface
   * @param scn - scn of the event
   * @param srcId - src id of the event
   * @param pId - partition id of the event
   * @param ver - serialization version of the event (V1 or V2)
   * @return - buffer with serialized events
   * @throws KeyTypeNotImplementedException
   */
private ByteBuffer addEvent(long scn, short srcId, byte[] schemaId, short pId, byte ver) throws KeyTypeNotImplementedException {
    DbusEventKey key = new DbusEventKey(123L);
    byte[] payload = RngUtils.randomString(100).getBytes(Charset.defaultCharset());
    DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, scn, pId, pId, 897L, srcId, schemaId, payload, false, true);
    eventInfo.setEventSerializationVersion(ver);
    // serialize it into a buffer
    int newEventSize = DbusEventFactory.computeEventLength(key, eventInfo);
    // now create an end of window event
    DbusEventInfo eventInfoEOP = new // must be the same as physicalPartition
    DbusEventInfo(// must be the same as physicalPartition
    null, // must be the same as physicalPartition
    scn, // must be the same as physicalPartition
    pId, // must be the same as physicalPartition
    pId, 900L, DbusEventInternalWritable.EOPMarkerSrcId, DbusEventInternalWritable.emptyMd5, DbusEventInternalWritable.EOPMarkerValue, //enable tracing
    false, // autocommit
    true);
    eventInfoEOP.setEventSerializationVersion(_eventFactory.getVersion());
    // serialize it into buffer
    int newEventSizeEOP = DbusEventFactory.computeEventLength(DbusEventInternalWritable.EOPMarkerKey, eventInfoEOP);
    int totalSize = newEventSize + newEventSizeEOP;
    ByteBuffer serializationBuffer = ByteBuffer.allocate(totalSize);
    serializationBuffer.order(ByteOrder.BIG_ENDIAN);
    // event itself
    int size = DbusEventFactory.serializeEvent(key, serializationBuffer, eventInfo);
    // EOP
    int size1 = _eventFactory.serializeLongKeyEndOfPeriodMarker(serializationBuffer, eventInfoEOP);
    assert totalSize == (size + size1);
    serializationBuffer.flip();
    return serializationBuffer;
}

Example 3 with DbusEventInfo

use of com.linkedin.databus.core.DbusEventInfo in project databus by linkedin.

the class TestDatabusRelayEvents method testEventConversion.

@Test
public /**
   * append event V2 to the buffer and stream it to the client
   * which only accepts events V1. Make sure it got converted
   */
void testEventConversion() throws InterruptedException, IOException, DatabusException {
    final Logger log = Logger.getLogger("TestDatabusRelayEvents.testEventConversion");
    log.setLevel(Level.INFO);
    DatabusRelayTestUtil.RelayRunner r1 = null;
    ClientRunner cr = null;
    try {
        String[] srcs = { "com.linkedin.events.example.fake.FakeSchema" };
        int pId = 1;
        int srcId = 2;
        int relayPort = Utils.getAvailablePort(11994);
        ;
        final DatabusRelayMain relay1 = createRelay(relayPort, pId, srcs);
        Assert.assertNotNull(relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("Relay created");
        DbusEventBufferMult bufMult = relay1.getEventBuffer();
        String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
        PhysicalPartition pPartition = new PhysicalPartition(pId, pSourceName);
        DbusEventBufferAppendable buf = bufMult.getDbusEventBufferAppendable(pPartition);
        DbusEventKey key = new DbusEventKey(123L);
        byte[] schemaId = relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray();
        byte[] payload = RngUtils.randomString(100).getBytes(Charset.defaultCharset());
        DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, 100L, (short) pId, (short) pId, 897L, (short) srcId, schemaId, payload, false, true);
        eventInfo.setEventSerializationVersion(DbusEventFactory.DBUS_EVENT_V2);
        buf.startEvents();
        buf.appendEvent(key, eventInfo, null);
        buf.endEvents(100L, null);
        r1.start();
        log.info("Relay started");
        // wait until relay comes up
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return relay1.isRunningStatus();
            }
        }, "Relay hasn't come up completely ", 7000, LOG);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcs, ",");
        String serverName = "localhost:" + relayPort;
        final EventsCountingConsumer countingConsumer = new EventsCountingConsumer();
        int id = (RngUtils.randomPositiveInt() % 10000) + 1;
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V1, 0);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("Consumer started");
        // wait till client gets the event
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return countingConsumer.getNumDataEvents() == 1;
            }
        }, "Consumer didn't get any events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(1, countingConsumer.getNumDataEvents());
        Assert.assertEquals(1, countingConsumer.getNumWindows());
        Assert.assertEquals(1, countingConsumer.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1));
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 4 with DbusEventInfo

use of com.linkedin.databus.core.DbusEventInfo in project databus by linkedin.

the class BootstrapEventBuffer method appendEvent.

public boolean appendEvent(DbusEventKey key, DbusEventKey seederChunkKey, short pPartitionId, short lPartitionId, long timeStamp, short srcId, byte[] schemaId, byte[] value, boolean enableTracing, boolean isReplicated, DbusEventsStatisticsCollector statsCollector) {
    DbusEventInfo eventInfo = new DbusEventInfo(null, 0L, pPartitionId, lPartitionId, timeStamp, srcId, schemaId, value, enableTracing, false);
    eventInfo.setReplicated(isReplicated);
    return appendEvent(key, seederChunkKey, eventInfo, statsCollector);
}

Example 5 with DbusEventInfo

use of com.linkedin.databus.core.DbusEventInfo in project databus by linkedin.

the class TestUnifiedClientStats method createEvent.

private DbusEvent createEvent(long timestampNs) {
    DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, // SCN
    6004L, // physical partition ID
    (short) 1, // logical partition ID
    (short) 1, timestampNs, // srcId
    (short) 1, // payloadSchemaMd5
    SOURCE1_SCHEMAID, // payload
    SOURCE1_PAYLOAD, // enableTracing
    false, // autocommit
    true);
    DbusEventKey key = new DbusEventKey("myKey".getBytes(Charset.forName("UTF-8")));
    ByteBuffer buf = ByteBuffer.allocate(1000).order(ByteOrder.BIG_ENDIAN);
    try {
        DbusEventFactory.serializeEvent(key, buf, eventInfo);
    } catch (KeyTypeNotImplementedException ex) {
        fail("string key type not supported by DbusEventV2Factory?!? " + ex.getLocalizedMessage());
    }
    return _eventFactory.createReadOnlyDbusEventFromBuffer(buf, 0);
}