Examples with DbusEventInternalReadable com.linkedin.databus.core.DbusEventInternalReadable used on opensource projects

Example 1 with DbusEventInternalReadable

use of com.linkedin.databus.core.DbusEventInternalReadable in project databus by linkedin.

the class DbusEventBufferConsumer method run.

@Override
public void run() {
    reset();
    try {
        int totalEvents = 0;
        long allTotalEvents = 0;
        DbusEventIterator iDbusEvent = _buffer.acquireIterator("Test_DbusEventBufferConsumer");
        do {
            if (!iDbusEvent.hasNext()) {
                if (!_invalidEvent) {
                    if (_deletionInterval > 0) {
                        iDbusEvent.remove();
                    }
                    try {
                        iDbusEvent.await();
                    } catch (InterruptedException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                        return;
                    }
                } else {
                    //try and consume as many events as possible
                    if (allTotalEvents >= eventsReadTillInvalidEvent()) {
                        LOG.info("total events read until invalid event=" + allTotalEvents + "; terminating");
                        stop();
                    }
                }
            }
            while (iDbusEvent.hasNext()) {
                DbusEventInternalReadable e = iDbusEvent.next();
                ++allTotalEvents;
                if (!e.isCheckpointMessage() && !e.isControlMessage() && !e.isEndOfPeriodMarker()) {
                    //needs to be idempotent; so - ensure that duplicates are dropped;
                    if (!_seenKeys.contains(e.key())) {
                        //deep copy
                        _out.add(e.createCopy());
                        _seenKeys.add(e.key());
                        ++totalEvents;
                    }
                }
                if ((_deletionInterval > 0) && allTotalEvents % _deletionInterval == 0) {
                    iDbusEvent.remove();
                }
            }
        } while (totalEvents < _maxEvents && !_stop);
        iDbusEvent.remove();
    } catch (RuntimeException e) {
        _exceptionThrown = e;
        LOG.error("consumer exception:" + e.getMessage(), e);
    } catch (Error e) {
        _exceptionThrown = e;
        LOG.error("consumer error:" + e.getMessage(), e);
    }
}

Example 2 with DbusEventInternalReadable

use of com.linkedin.databus.core.DbusEventInternalReadable in project databus by linkedin.

the class BootstrapEventWriter method onCheckpointEvent.

@Override
public void onCheckpointEvent(Checkpoint currentCheckpoint, DbusEventsStatisticsCollector curStatsCollector) {
    // refresh LOG level
    _debug = LOG.isDebugEnabled();
    // store values in the internal structure
    currentCheckpoint.bootstrapCheckPoint();
    // write ckpt back to client
    DbusEventInternalReadable checkpointEvent = _eventFactory.createCheckpointEvent(currentCheckpoint);
    checkpointEvent.writeTo(_writeChannel, _encoding);
// LOG.info("Sending snapshot checkpoint to client: " + currentCheckpoint.toString());
}

Example 3 with DbusEventInternalReadable

use of com.linkedin.databus.core.DbusEventInternalReadable in project databus by linkedin.

the class BootstrapTableReaderV2 method execute.

public void execute() throws SQLException {
    ResultSet rs = null;
    boolean hasMore = true;
    long curId = -1;
    try {
        _log.info("Executing query : " + _queryString);
        ByteBuffer buffer = ByteBuffer.allocateDirect(MAX_EVENT_SIZE);
        int count = 0;
        DbusEventInternalReadable event = _eventFactory.createReadOnlyDbusEventFromBuffer(buffer, 0);
        _eventHandler.onStart(_queryString);
        while (hasMore) {
            _log.debug("currentId=" + curId);
            _query.setLong(1, curId);
            rs = _query.executeQuery();
            hasMore = false;
            while (rs.next()) {
                hasMore = true;
                buffer.clear();
                buffer.put(rs.getBytes("val"));
                curId = rs.getLong("id");
                event = event.reset(buffer, 0);
                GenericRecord record = _decoder.getGenericRecord(event);
                if (checkFilters(event, record)) {
                    _eventHandler.onRecord(event, record);
                }
                count++;
            }
            rs.close();
        }
        _eventHandler.onEnd(count);
    } finally {
        DBHelper.close(rs, _query, _jdbcConn);
    }
}

Example 4 with DbusEventInternalReadable

use of com.linkedin.databus.core.DbusEventInternalReadable in project databus by linkedin.

the class BootstrapPullThread method processBootstrapComplete.

/**
   * Update and persist checkpoint at the end of bootstrap phase so that
   * the online phase can continue from it.
   * @param cp
   * @throws IOException
   */
protected void processBootstrapComplete(Checkpoint cp, ConnectionState curState) throws IOException, DatabusException {
    logBootstrapPhase(DbusClientMode.BOOTSTRAP_CATCHUP, cp.getBootstrapSnapshotSourceIndex(), cp.getBootstrapCatchupSourceIndex());
    _log.info("Bootstrap got completed !! Checkpoint is :" + cp.toString());
    /*
     * DDS-989
     * WindowSCN need not match the bootstrapTargetSCN always when we are catching up multiple sources.
     * So set the windowSCN to be that of targetSCN as we are consistent as of targetSCN
     */
    cp.setWindowScn(cp.getBootstrapTargetScn());
    cp.setPrevScn(cp.getBootstrapTargetScn());
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    // clear Bootstrap scns for future bootstraps
    cp.resetBootstrap();
    DbusEventBuffer eventBuffer = curState.getDataEventsBuffer();
    try {
        DbusEventInternalReadable cpEvent = getEventFactory().createCheckpointEvent(cp);
        boolean success = eventBuffer.injectEvent(cpEvent);
        if (!success) {
            _log.error("Unable to write bootstrap phase marker");
        } else {
            //TODO need real partition for partitioned bootstrap
            DbusEventInternalReadable eopEvent = curState.createEopEvent(cp, getEventFactory());
            success = eventBuffer.injectEvent(eopEvent);
            if (!success) {
                _log.error("Unable to write bootstrap EOP marker");
            }
        }
    } catch (InvalidEventException iee) {
        _log.error("Unable to write bootstrap phase marker", iee);
    }
    unlockV3Bootstrap();
}

Example 5 with DbusEventInternalReadable

use of com.linkedin.databus.core.DbusEventInternalReadable in project databus by linkedin.

the class MockBootstrapConnection method testBootstrapPendingEvent.

// Make sure that we suspend on error when we get the x-dbus-pending-size header with a size that is
// larger than our dbusevent size.
@Test
public void testBootstrapPendingEvent() throws Exception {
    List<String> sources = Arrays.asList("source1");
    Properties clientProps = new Properties();
    clientProps.setProperty("client.container.httpPort", "0");
    clientProps.setProperty("client.container.jmx.rmiEnabled", "false");
    clientProps.setProperty("client.runtime.bootstrap.enabled", "true");
    clientProps.setProperty("client.runtime.bootstrap.service(1).name", "bs1");
    clientProps.setProperty("client.runtime.bootstrap.service(1).host", "localhost");
    clientProps.setProperty("client.runtime.bootstrap.service(1).port", "10001");
    clientProps.setProperty("client.runtime.bootstrap.service(1).sources", "source1");
    clientProps.setProperty("client.runtime.relay(1).name", "relay1");
    clientProps.setProperty("client.runtime.relay(1).port", "10001");
    clientProps.setProperty("client.runtime.relay(1).sources", "source1");
    clientProps.setProperty("client.connectionDefaults.eventBuffer.maxSize", "100000");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.maxRetryNum", "3");
    DatabusHttpClientImpl.Config clientConfBuilder = new DatabusHttpClientImpl.Config();
    ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>("client.", clientConfBuilder);
    configLoader.loadConfig(clientProps);
    DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build();
    DatabusSourcesConnection.StaticConfig srcConnConf = clientConf.getConnectionDefaults();
    DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConf);
    client.registerDatabusBootstrapListener(new LoggingConsumer(), null, "source1");
    Assert.assertNotNull(client, "client instantiation failed");
    DatabusHttpClientImpl.RuntimeConfig clientRtConf = clientConf.getRuntime().build();
    //we keep the index of the next server we expect to see
    AtomicInteger serverIdx = new AtomicInteger(-1);
    List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
    sourcesResponse.add(new IdNamePair(1L, "source1"));
    Map<Long, List<RegisterResponseEntry>> registerResponse = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> regResponse = new ArrayList<RegisterResponseEntry>();
    regResponse.add(new RegisterResponseEntry(1L, (short) 1, SCHEMA$.toString()));
    registerResponse.put(1L, regResponse);
    ChunkedBodyReadableByteChannel channel = EasyMock.createMock(ChunkedBodyReadableByteChannel.class);
    // getting the pending-event-size header is called twice, once for checking and once for logging.
    EasyMock.expect(channel.getMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE)).andReturn("1000000").times(2);
    EasyMock.expect(channel.getMetadata("x-dbus-error-cause")).andReturn(null).times(2);
    EasyMock.expect(channel.getMetadata("x-dbus-error")).andReturn(null).times(2);
    EasyMock.replay(channel);
    DbusEventBuffer dbusBuffer = EasyMock.createMock(DbusEventBuffer.class);
    dbusBuffer.endEvents(false, -1, false, false, null);
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(dbusBuffer.injectEvent(EasyMock.<DbusEventInternalReadable>notNull())).andReturn(true).anyTimes();
    EasyMock.expect(dbusBuffer.getEventSerializationVersion()).andReturn(DbusEventFactory.DBUS_EVENT_V1).anyTimes();
    EasyMock.expect(dbusBuffer.getMaxReadBufferCapacity()).andReturn(600).times(2);
    EasyMock.expect(dbusBuffer.getBufferFreeReadSpace()).andReturn(600000).times(2);
    EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull())).andReturn(1).times(1);
    EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull(), EasyMock.<List<InternalDatabusEventsListener>>notNull(), EasyMock.<DbusEventsStatisticsCollector>isNull())).andReturn(0).times(1);
    EasyMock.replay(dbusBuffer);
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sources);
    //This guy succeeds on /sources but fails on /register
    MockBootstrapConnection mockSuccessConn = new MockBootstrapConnection(10, 10, channel, serverIdx, false);
    DatabusBootstrapConnectionFactory mockConnFactory = org.easymock.EasyMock.createMock("mockRelayFactory", DatabusBootstrapConnectionFactory.class);
    //each server should be tried MAX_RETRIES time until all retries are exhausted
    EasyMock.expect(mockConnFactory.createConnection(EasyMock.<ServerInfo>notNull(), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockSuccessConn).anyTimes();
    List<DatabusSubscription> sourcesSubList = DatabusSubscription.createSubscriptionList(sources);
    DatabusSourcesConnection sourcesConn2 = EasyMock.createMock(DatabusSourcesConnection.class);
    EasyMock.expect(sourcesConn2.getSourcesNames()).andReturn(Arrays.asList("source1")).anyTimes();
    EasyMock.expect(sourcesConn2.getSubscriptions()).andReturn(sourcesSubList).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionConfig()).andReturn(srcConnConf).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionStatus()).andReturn(new DatabusComponentStatus("dummy")).anyTimes();
    EasyMock.expect(sourcesConn2.getLocalRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getUnifiedClientStats()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapConnFactory()).andReturn(mockConnFactory).anyTimes();
    EasyMock.expect(sourcesConn2.loadPersistentCheckpoint()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getDataEventsBuffer()).andReturn(dbusBuffer).anyTimes();
    EasyMock.expect(sourcesConn2.isBootstrapEnabled()).andReturn(true).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapRegistrations()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapServices()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapEventsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.makeThreadSafe(mockConnFactory, true);
    EasyMock.makeThreadSafe(sourcesConn2, true);
    EasyMock.replay(mockConnFactory);
    EasyMock.replay(sourcesConn2);
    BootstrapPullThread bsPuller = new BootstrapPullThread("RelayPuller", sourcesConn2, dbusBuffer, connStateFactory, clientRtConf.getBootstrap().getServicesSet(), new ArrayList<DbusKeyCompositeFilterConfig>(), clientConf.getPullerBufferUtilizationPct(), ManagementFactory.getPlatformMBeanServer(), new DbusEventV2Factory(), null, null);
    mockSuccessConn.setCallback(bsPuller);
    bsPuller.getComponentStatus().start();
    Checkpoint cp = _ckptHandlerSource1.createInitialBootstrapCheckpoint(null, 0L);
    //TODO remove
    //cp.setSnapshotSource("source1");
    //cp.setCatchupSource("source1");
    //cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    ConnectionState connState = bsPuller.getConnectionState();
    connState.switchToBootstrap(cp);
    testTransitionCase(bsPuller, StateId.BOOTSTRAP, StateId.REQUEST_START_SCN, cp);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.REQUEST_START_SCN, StateId.START_SCN_RESPONSE_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    Map<Long, List<RegisterResponseEntry>> entries = new HashMap<Long, List<RegisterResponseEntry>>();
    entries.put(1L, new ArrayList<RegisterResponseEntry>());
    connState.setSourcesSchemas(entries);
    connState.setCurrentBSServerInfo(bsPuller.getCurentServer());
    testTransitionCase(bsPuller, StateId.START_SCN_RESPONSE_SUCCESS, StateId.REQUEST_STREAM, null);
    bsPuller.getMessageQueue().clear();
    connState.getSourcesNameMap().put("source1", new IdNamePair(1L, "source1"));
    connState.getSourceIdMap().put(1L, new IdNamePair(1L, "source1"));
    testTransitionCase(bsPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_REQUEST_SUCCESS, "SUSPEND_ON_ERROR", null);
    EasyMock.verify(channel);
    EasyMock.verify(sourcesConn2);
    EasyMock.verify(dbusBuffer);
    EasyMock.verify(channel);
    EasyMock.verify(mockConnFactory);
}