Examples with DbusEvent com.linkedin.databus.core.DbusEvent used on opensource projects

Example 1 with DbusEvent

use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.

the class TestGoldenGateEventProducer method testAddEventToBuffer.

@Test
public void testAddEventToBuffer() throws InvalidConfigException, UnsupportedKeyException, DatabusException {
    // No rate control
    long rate = 0;
    PhysicalSourceStaticConfig pssc = buildPssc(rate, 0L);
    long scn = 10;
    DbusEventBuffer mb = (DbusEventBuffer) createBufMult(pssc);
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
    int sourceId = 505;
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
    Object key = new String("name");
    Schema.Type keyType = Schema.Type.RECORD;
    ColumnsState.KeyPair kp = new ColumnsState.KeyPair(key, keyType);
    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(1);
    keyPairs.add(kp);
    Schema s = Schema.parse(avroSchema);
    GenericRecord gr = new GenericData.Record(s);
    gr.put("name", "phani");
    DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    db.add(dbi);
    TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
    dbUpdates.add(dbUpdate);
    long timestamp = System.nanoTime();
    gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
    Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);
    DbusEventIterator iter = mb.acquireIterator("test");
    int count = 0;
    long eventTs = 0;
    while (iter.hasNext()) {
        DbusEvent e = iter.next();
        if (count == 1) {
            // first event prev control event
            eventTs = e.timestampInNanos();
        }
        count++;
    }
    Assert.assertEquals("Event timestamp in Ns", timestamp, eventTs);
    Assert.assertEquals("Got events ", 3, count);
    return;
}

Example 2 with DbusEvent

use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.

the class DbusEventAppender method run.

@Override
public void run() {
    //append events into buffer serially with varying  window sizes;
    long lastScn = -1;
    _count = 0;
    int dataEventCount = 0;
    int bootstrapCheckpoints = _bootstrapCheckpointPerWindow;
    int maxCount = (int) (_fraction * _events.size());
    for (DbusEvent ev : _events) {
        if (dataEventCount >= maxCount) {
            break;
        }
        long evScn = ev.sequence();
        if (lastScn != evScn) {
            //new window;
            if (lastScn == -1) {
                // Test DDSDBUS-1109 by skipping the start() call. The scn Index should be set for streamEvents() to work correctly
                if (_invokeStartOnBuffer) {
                    _buffer.start(evScn - 1);
                }
                _buffer.startEvents();
                if (_bootstrapCheckpoint != null) {
                    //add the initial checkpoint event to dispatcher's buffer to simulate bootstrap
                    addBootstrapCheckpointEventToBuffer(evScn - 1, dataEventCount, 1);
                }
            } else {
                ++_count;
                if (_callInternalListeners) {
                    if (0 == bootstrapCheckpoints) {
                        _buffer.endEvents(lastScn, _stats);
                    } else {
                        addBootstrapCheckpointEventToBuffer(lastScn, dataEventCount, 1);
                    }
                    --bootstrapCheckpoints;
                } else {
                    if (0 == bootstrapCheckpoints) {
                        _buffer.endEvents(true, lastScn, false, false, _stats);
                    } else {
                        //simulate bootstrap calls (snapshot)
                        addBootstrapCheckpointEventToBuffer(lastScn, dataEventCount, 1);
                    }
                    --bootstrapCheckpoints;
                }
                if (!_bufferReflector.validateBuffer()) {
                    throw new RuntimeException("Buffer validation 1 failed");
                }
                if (bootstrapCheckpoints < 0) {
                    _buffer.startEvents();
                    bootstrapCheckpoints = _bootstrapCheckpointPerWindow;
                }
            }
            if (!_bufferReflector.validateBuffer()) {
                throw new RuntimeException("Buffer validation 2 failed");
            }
            lastScn = evScn;
        }
        dataEventCount = addEventToBuffer(ev, dataEventCount);
        ++_count;
    }
    if ((lastScn != -1) && (maxCount == _events.size())) {
        ++_count;
        _buffer.endEvents(lastScn, _stats);
    }
    _dataEvents = dataEventCount;
}

Example 3 with DbusEvent

use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.

the class BootstrapDBCleanerQueryExecutor method getLastEventinLog.

private BootstrapDBRow getLastEventinLog(BootstrapLogInfo logInfo, DbusEventFactory eventFactory) throws SQLException {
    Statement stmt = null;
    ResultSet rs = null;
    long id = -1;
    long scn = -1;
    String key = null;
    DbusEvent event = null;
    try {
        String tableName = logInfo.getLogTable();
        StringBuilder sql = new StringBuilder();
        sql.append("select id, srckey, scn, val from ");
        sql.append(tableName);
        sql.append(" where id = ( select max(id) from ");
        sql.append(tableName);
        sql.append(" )");
        stmt = _conn.createStatement();
        rs = stmt.executeQuery(sql.toString());
        if (rs.next()) {
            int i = 1;
            id = rs.getLong(i++);
            String srcKey = rs.getString(i++);
            scn = rs.getLong(i++);
            ByteBuffer tmpBuffer = ByteBuffer.wrap(rs.getBytes(i));
            LOG.info("BUFFER SIZE:" + tmpBuffer.limit());
            event = eventFactory.createReadOnlyDbusEventFromBuffer(tmpBuffer, tmpBuffer.position());
            LOG.info("Last Row for log (" + logInfo + ") - ID :" + id + ", srcKey :" + srcKey + ", SCN :" + scn + ", Event :" + event.toString());
        } else {
            LOG.error("No ResultSet for query :" + sql.toString());
        }
    } finally {
        DBHelper.close(rs, stmt, null);
    }
    return new BootstrapDBRow(id, key, scn, event);
}

Example 4 with DbusEvent

use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.

the class RelayPullThread method doReadDataEvents.

protected void doReadDataEvents(ConnectionState curState) {
    boolean debugEnabled = _log.isDebugEnabled();
    boolean enqueueMessage = true;
    try {
        ChunkedBodyReadableByteChannel readChannel = curState.getReadChannel();
        Checkpoint cp = curState.getCheckpoint();
        curState.setRelayFellOff(false);
        String remoteErrorName = RemoteExceptionHandler.getExceptionName(readChannel);
        Throwable knownRemoteError = _remoteExceptionHandler.getException(readChannel);
        if (null != knownRemoteError && knownRemoteError instanceof ScnNotFoundException) {
            if (toTearConnAfterHandlingResponse()) {
                tearConnectionAndEnqueuePickServer();
                enqueueMessage = false;
            } else {
                curState.setRelayFellOff(true);
                if (_retriesOnFallOff.getRemainingRetriesNum() > 0) {
                    _log.error("Got SCNNotFoundException. Retry (" + _retriesOnFallOff.getRetriesNum() + ") out of " + _retriesOnFallOff.getConfig().getMaxRetryNum());
                    curState.switchToPickServer();
                } else {
                    enqueueMessage = onRelayFellOff(curState, cp, knownRemoteError);
                }
            }
        } else if (null != remoteErrorName) {
            if (toTearConnAfterHandlingResponse()) {
                tearConnectionAndEnqueuePickServer();
                enqueueMessage = false;
            } else {
                //remote processing error
                _log.error("read events error: " + RemoteExceptionHandler.getExceptionMessage(readChannel));
                curState.switchToStreamResponseError();
            }
        } else {
            /*DispatcherState dispatchState = curState.getDispatcherState();
          dispatchState.switchToDispatchEvents();
          _dispatcherThread.addNewStateBlocking(dispatchState);*/
            if (debugEnabled)
                _log.debug("Sending events to buffer");
            DbusEventsStatisticsCollector connCollector = _sourcesConn.getInboundEventsStatsCollector();
            if (curState.isSCNRegress()) {
                _log.info("SCN Regress requested !! Sending a SCN Regress Message to dispatcher. Curr Ckpt :" + curState.getCheckpoint());
                DbusEvent regressEvent = getEventFactory().createSCNRegressEvent(new SCNRegressMessage(curState.getCheckpoint()));
                writeEventToRelayDispatcher(curState, regressEvent, "SCN Regress Event from ckpt :" + curState.getCheckpoint());
                curState.setSCNRegress(false);
            }
            UnifiedClientStats unifiedClientStats = _sourcesConn.getUnifiedClientStats();
            if (unifiedClientStats != null) {
                // failsafe:  we're definitely not bootstrapping here
                unifiedClientStats.setBootstrappingState(false);
                sendHeartbeat(unifiedClientStats);
            }
            int eventsNum = curState.getDataEventsBuffer().readEvents(readChannel, curState.getListeners(), connCollector);
            boolean resetConnection = false;
            if (eventsNum > 0) {
                _timeSinceEventsSec = System.currentTimeMillis();
                cp.checkPoint();
            } else {
                // check how long it has been since we got some events
                if (_remoteExceptionHandler.getPendingEventSize(readChannel) > curState.getDataEventsBuffer().getMaxReadBufferCapacity()) {
                    // The relay had a pending event that we can never accommodate. This is fatal error.
                    String err = "ReadBuffer max capacity(" + curState.getDataEventsBuffer().getMaxReadBufferCapacity() + ") is less than event size(" + _remoteExceptionHandler.getPendingEventSize(readChannel) + "). Increase databus.client.connectionDefaults.eventBuffer.maxEventSize and restart.";
                    _log.fatal(err);
                    enqueueMessage(LifecycleMessage.createSuspendOnErroMessage(new PendingEventTooLargeException(err)));
                    return;
                } else {
                    if (_noEventsConnectionResetTimeSec > 0) {
                        // unless the feature is disabled (<=0)
                        resetConnection = (System.currentTimeMillis() - _timeSinceEventsSec) / 1000 > _noEventsConnectionResetTimeSec;
                        if (resetConnection) {
                            _timeSinceEventsSec = System.currentTimeMillis();
                            _log.warn("about to reset connection to relay " + curState.getServerInetAddress() + ", because there were no events for " + _noEventsConnectionResetTimeSec + "secs");
                        }
                    }
                }
            }
            if (debugEnabled)
                _log.debug("Events read: " + eventsNum);
            // if it has been too long since we got non-empty responses - the relay may be stuck, try to reconnect
            if (toTearConnAfterHandlingResponse() || resetConnection) {
                tearConnectionAndEnqueuePickServer();
                enqueueMessage = false;
            } else {
                curState.switchToStreamResponseDone();
                resetServerRetries();
            }
        }
        if (enqueueMessage)
            enqueueMessage(curState);
    } catch (InterruptedException ie) {
        _log.warn("interrupted", ie);
        curState.switchToStreamResponseError();
        enqueueMessage(curState);
    } catch (InvalidEventException e) {
        _log.error("error reading events from server:" + e, e);
        curState.switchToStreamResponseError();
        enqueueMessage(curState);
    } catch (RuntimeException e) {
        _log.error("runtime error reading events from server: " + e, e);
        curState.switchToStreamResponseError();
        enqueueMessage(curState);
    }
}

Example 5 with DbusEvent

use of com.linkedin.databus.core.DbusEvent in project databus by linkedin.

the class TestInternalMetadata method testGetMetadata_UnhappyPath_EventHasNoMetadata.

/**
   * Verifies that getMetadata() returns null if event has no metadata.
   */
@Test
public void testGetMetadata_UnhappyPath_EventHasNoMetadata() throws Exception {
    LOG.info("starting testGetMetadata_UnhappyPath_EventHasNoMetadata()");
    // build the event without any metadata
    DbusEvent event = createEvent(null);
    // create a metadata schema set, just because we like to
    VersionedSchemaSet metadataSchemaSet = new VersionedSchemaSet();
    metadataSchemaSet.add(SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE, METADATA_SCHEMA_VERSION, new SchemaId(METADATA_SCHEMA_CHECKSUM), CORRECT_METADATA_SCHEMA, // preserve original string
    true);
    // now create the decoder and attempt to use it to extract and decode the event's metadata
    DbusEventAvroDecoder eventDecoder = createDecoder(metadataSchemaSet);
    try {
        GenericRecord reuse = null;
        GenericRecord decodedMetadata = eventDecoder.getMetadata(event, reuse);
        Assert.assertNull(decodedMetadata, "getMetadata() should have returned null;");
    } catch (Exception ex) {
        Assert.fail("getMetadata() should not have thrown exception: " + ex);
    }
    LOG.info("leaving testGetMetadata_UnhappyPath_EventHasNoMetadata()");
}