Examples with DbusEventV2Factory com.linkedin.databus.core.DbusEventV2Factory used on opensource projects

Example 1 with DbusEventV2Factory

use of com.linkedin.databus.core.DbusEventV2Factory in project databus by linkedin.

the class TestDbusEventBufferNG method testSameBufferTraceOn.

@Test
public void testSameBufferTraceOn() throws IOException, InvalidConfigException {
    File buf1Trace = null;
    File buf2Trace = null;
    File buf3Trace = null;
    try {
        File tmpFile = File.createTempFile("TestDbusEventBufferNG-buffer", ".trace");
        //make sure the file was created, i.e. we have write permissions
        Assert.assertTrue(tmpFile.exists());
        //we just need the name
        Assert.assertTrue(tmpFile.delete());
        DbusEventBuffer.Config cfgBuilder = new DbusEventBuffer.Config();
        cfgBuilder.setMaxSize(10000);
        cfgBuilder.setScnIndexSize(1024);
        cfgBuilder.setAverageEventSize(1024);
        cfgBuilder.setAllocationPolicy(AllocationPolicy.HEAP_MEMORY.toString());
        cfgBuilder.getTrace().setOption(RelayEventTraceOption.Option.file.toString());
        cfgBuilder.getTrace().setFilename(tmpFile.getAbsolutePath());
        cfgBuilder.getTrace().setNeedFileSuffix(true);
        //test that the first gets created
        PhysicalPartition pp = new PhysicalPartition(1, "TestPart");
        DbusEventFactory eventFactory = new DbusEventV2Factory();
        DbusEventBuffer buf1 = new DbusEventBuffer(cfgBuilder.build(), pp, eventFactory);
        buf1Trace = new File(tmpFile.getAbsolutePath() + "." + pp.getName() + "_" + pp.getId());
        Assert.assertTrue(buf1Trace.exists());
        buf1.start(1);
        for (int i = 1; i <= 10; ++i) {
            buf1.startEvents();
            buf1.appendEvent(new DbusEventKey(i), (short) 0, (short) 0, 0L, (short) 0, new byte[16], new byte[10], false, null);
            buf1.endEvents(10 * i);
        }
        long trace1len = buf1Trace.length();
        Assert.assertTrue(trace1len > 0);
        //create the second buffer and check its trace is different
        DbusEventBuffer buf2 = new DbusEventBuffer(cfgBuilder.build(), pp, eventFactory);
        buf2Trace = new File(tmpFile.getAbsolutePath() + "." + pp.getName() + "_" + pp.getId() + ".1");
        Assert.assertTrue(buf2Trace.exists());
        buf2.start(1);
        for (int i = 1; i <= 5; ++i) {
            buf2.startEvents();
            buf2.appendEvent(new DbusEventKey(i), (short) 0, (short) 0, 0L, (short) 0, new byte[16], new byte[10], false, null);
            buf2.endEvents(10 * i);
        }
        long trace2len = buf2Trace.length();
        Assert.assertTrue(trace2len > 0);
        Assert.assertTrue(buf1Trace.length() == trace1len);
        //create a third buffer and check its trace is different
        DbusEventBuffer buf3 = new DbusEventBuffer(cfgBuilder.build(), pp, eventFactory);
        buf3Trace = new File(tmpFile.getAbsolutePath() + "." + pp.getName() + "_" + pp.getId() + ".2");
        Assert.assertTrue(buf3Trace.exists());
        buf3.start(1);
        for (int i = 1; i <= 6; ++i) {
            buf3.startEvents();
            buf3.appendEvent(new DbusEventKey(i), (short) 0, (short) 0, 0L, (short) 0, new byte[16], new byte[10], false, null);
            buf3.endEvents(10 * i);
        }
        long trace3len = buf3Trace.length();
        Assert.assertTrue(trace3len > 0);
        Assert.assertTrue(buf1Trace.length() == trace1len);
        Assert.assertTrue(buf2Trace.length() == trace2len);
    } finally {
        Assert.assertTrue(null == buf1Trace || buf1Trace.delete());
        Assert.assertTrue(null == buf2Trace || buf2Trace.delete());
        Assert.assertTrue(null == buf3Trace || buf3Trace.delete());
    }
}

Example 2 with DbusEventV2Factory

use of com.linkedin.databus.core.DbusEventV2Factory in project databus by linkedin.

the class RelayEventProducer method createDatabusSourcesConnection.

public static DatabusSourcesConnection createDatabusSourcesConnection(String producerName, int id, String serverName, String subscriptionString, DatabusCombinedConsumer consumer, long internalBufferMaxSize, int largestEventSize, long consumerTimeoutMs, long pollIntervalMs, long connTimeoutMs, int consumerParallelism, boolean blockingBuffer, DatabusClientNettyThreadPools nettyThreadPools, int noEventsTimeoutSec, int maxEventVersion, int initReadBufferSize) throws InvalidConfigException {
    // the assumption here is that the list of subscriptions will become the
    // list of sources hosted by the relay
    Set<ServerInfo> relayServices = createServerInfo(serverName, subscriptionString);
    // null bootstrapService
    Set<ServerInfo> bootstrapServices = null;
    // create subscription objects based on what is required by subscription
    String[] subscriptionList = subscriptionString.split(",");
    List<DatabusSubscription> subsList = DatabusSubscription.createSubscriptionList(Arrays.asList(subscriptionList));
    List<String> sourcesStrList = DatabusSubscription.getStrList(subsList);
    LOG.info("The sourcesList is " + sourcesStrList);
    // create registration objects with consumers
    List<DatabusV2ConsumerRegistration> relayConsumers = createDatabusV2ConsumerRegistration(consumer, sourcesStrList);
    List<DatabusV2ConsumerRegistration> bstConsumers = null;
    // setup sources connection config
    DatabusSourcesConnection.Config confBuilder = new DatabusSourcesConnection.Config();
    confBuilder.setId(id);
    // consume whatever is in relay
    confBuilder.setConsumeCurrent(true);
    //this is set to false as the behaviour is to read the latest SCN when SCN is not found, the buffer isn't cleared
    //as such , so a possibility of gaps in events arises. What we want ideally is to clear existing buffer and then consume from latest SCN
    confBuilder.setReadLatestScnOnError(false);
    // 10s max consumer timeout
    confBuilder.setConsumerTimeBudgetMs(consumerTimeoutMs);
    // poll interval in ms and infinite retry;
    confBuilder.getPullerRetries().setMaxRetryNum(-1);
    confBuilder.getPullerRetries().setInitSleep(pollIntervalMs);
    confBuilder.setConsumerParallelism(consumerParallelism);
    confBuilder.getDispatcherRetries().setMaxRetryNum(1);
    // internal buffer conf
    DbusEventBuffer.Config bufferConf = new DbusEventBuffer.Config();
    bufferConf.setMaxSize(internalBufferMaxSize);
    bufferConf.setAllocationPolicy("DIRECT_MEMORY");
    if (initReadBufferSize > 0) {
        bufferConf.setAverageEventSize(initReadBufferSize);
    }
    bufferConf.setMaxEventSize(largestEventSize);
    //client buffer's scn index- not used
    bufferConf.setScnIndexSize(64 * 1024);
    String queuePolicy = blockingBuffer ? "BLOCK_ON_WRITE" : "OVERWRITE_ON_WRITE";
    bufferConf.setQueuePolicy(queuePolicy);
    //get appropriate checkpointThresholdPct
    double newCkptPct = confBuilder.computeSafeCheckpointThresholdPct(bufferConf);
    if (newCkptPct < 5.0 || newCkptPct > 95.0) {
        LOG.warn("Not setting required checkpointThresholdPct : " + newCkptPct + "to  accommodate largestEventSize= " + largestEventSize + " in buffer of size " + bufferConf.getMaxSize());
        if (newCkptPct <= 0.0) {
            //unlikely to happen: if it does retain default
            newCkptPct = confBuilder.getCheckpointThresholdPct();
        }
        if (newCkptPct < 5.0) {
            newCkptPct = 5.0;
        } else if (newCkptPct > 95.0) {
            newCkptPct = 95.0;
        }
    }
    LOG.info("Setting checkpointThresholdPct:" + newCkptPct);
    confBuilder.setCheckpointThresholdPct(newCkptPct);
    confBuilder.setEventBuffer(bufferConf);
    confBuilder.setNoEventsConnectionResetTimeSec(noEventsTimeoutSec);
    DatabusSourcesConnection.StaticConfig connConfig = confBuilder.build();
    // internal buffers of databus client library
    DbusEventBuffer buffer = new DbusEventBuffer(connConfig.getEventBuffer());
    buffer.start(0);
    DbusEventBuffer bootstrapBuffer = null;
    // Create threadpools and netty managers
    // read - write timeout in ms
    long readTimeoutMs = connTimeoutMs;
    long writeTimeoutMs = connTimeoutMs;
    long bstReadTimeoutMs = connTimeoutMs;
    int protocolVersion = 2;
    // connection factory
    NettyHttpConnectionFactory defaultConnFactory = new NettyHttpConnectionFactory(nettyThreadPools.getBossExecutorService(), nettyThreadPools.getIoExecutorService(), null, nettyThreadPools.getTimer(), writeTimeoutMs, readTimeoutMs, bstReadTimeoutMs, protocolVersion, maxEventVersion, nettyThreadPools.getChannelGroup());
    // Create Thread pool for consumer threads
    int maxThreadsNum = 1;
    int keepAliveMs = 1000;
    ThreadPoolExecutor defaultExecutorService = new OrderedMemoryAwareThreadPoolExecutor(maxThreadsNum, 0, 0, keepAliveMs, TimeUnit.MILLISECONDS);
    ConsumerCallbackStats relayConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.cons", producerName + ".inbound.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
    ConsumerCallbackStats bootstrapConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.bs.cons", producerName + ".inbound.bs.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
    UnifiedClientStats unifiedClientStats = new UnifiedClientStats(id, producerName + ".inbound.unified.cons", producerName + ".inbound.unified.cons", true, false, UnifiedClientStats.DEFAULT_DEADNESS_THRESHOLD_MS, null, ManagementFactory.getPlatformMBeanServer());
    DatabusRelayConnectionFactory relayConnFactory = defaultConnFactory;
    DatabusBootstrapConnectionFactory bootstrapConnFactory = defaultConnFactory;
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sourcesStrList);
    DatabusSourcesConnection conn = new DatabusSourcesConnection(connConfig, subsList, relayServices, bootstrapServices, relayConsumers, bstConsumers, buffer, bootstrapBuffer, defaultExecutorService, // getContainerStatsCollector(),
    null, // getInboundEventStatisticsCollector(),
    null, // getBootstrapEventsStatsCollector(),
    null, // relay callback stats
    relayConsumerStats, // bootstrap callback stats
    bootstrapConsumerStats, // combined relay/bootstrap callback stats
    unifiedClientStats, // getCheckpointPersistenceProvider(),
    null, relayConnFactory, bootstrapConnFactory, // getHttpStatsCollector(),
    null, // RegistrationId
    null, null, new DbusEventV2Factory(), // TODO Get the ref to factory from HttpRelay.
    connStateFactory);
    return conn;
}

Example 3 with DbusEventV2Factory

use of com.linkedin.databus.core.DbusEventV2Factory in project databus by linkedin.

the class TestGoldenGateEventProducer method createBufMult.

/**
   * Creates a DbusBufMult
   */
private DbusEventBufferAppendable createBufMult(PhysicalSourceStaticConfig pssc) throws InvalidConfigException {
    DbusEventBuffer.StaticConfig config = null;
    if (config == null) {
        try {
            DbusEventBuffer.Config cfgBuilder = new DbusEventBuffer.Config();
            cfgBuilder.setMaxSize(10 * 1024 * 1024);
            cfgBuilder.setScnIndexSize(2 * 1024 * 1024);
            cfgBuilder.setAllocationPolicy("MMAPPED_MEMORY");
            config = cfgBuilder.build();
        } catch (InvalidConfigException e) {
            fail("invalid configuration", e);
        }
    }
    PhysicalSourceStaticConfig[] pConfigs = new PhysicalSourceStaticConfig[1];
    pConfigs[0] = pssc;
    DbusEventBufferMult eventBufferMult = new DbusEventBufferMult(pConfigs, config, new DbusEventV2Factory());
    for (DbusEventBuffer b : eventBufferMult.bufIterable()) {
        b.start(1);
    }
    DbusEventBufferAppendable buf = eventBufferMult.getDbusEventBufferAppendable(505);
    return buf;
}

Example 4 with DbusEventV2Factory

use of com.linkedin.databus.core.DbusEventV2Factory in project databus by linkedin.

the class DbusEventFactoryForTesting method createReadOnlyDbusEventFromBuffer.

/**
   * Creates a read-only DbusEvent out of an already initialized (serialized) buffer,
   * except with the specified SCN.  The event's header CRC is updated appropriately.
   *
   * @param buf        buffer containing the serialized event
   * @param position   byte-offset of the serialized event in the buffer
   * @param seq        sequence number (SCN) of the new event
   * @param version    desired version of the new event
   * @return  a read-only DbusEvent
   */
public static DbusEventInternalReadable createReadOnlyDbusEventFromBuffer(ByteBuffer buf, int position, long seq, byte version) {
    DbusEventFactory eventFactory;
    if (version == DbusEventFactory.DBUS_EVENT_V1) {
        eventFactory = new DbusEventV1Factory();
    } else if (version == DbusEventFactory.DBUS_EVENT_V2) {
        eventFactory = new DbusEventV2Factory();
    } else {
        throw new UnsupportedDbusEventVersionRuntimeException(version);
    }
    DbusEventInternalWritable event = eventFactory.createWritableDbusEventFromBuffer(buf, position);
    event.setSequence(seq);
    event.applyCrc();
    return event;
}

Example 5 with DbusEventV2Factory

use of com.linkedin.databus.core.DbusEventV2Factory in project databus by linkedin.

the class MockBootstrapConnection method testBootstrapPendingEvent.

// Make sure that we suspend on error when we get the x-dbus-pending-size header with a size that is
// larger than our dbusevent size.
@Test
public void testBootstrapPendingEvent() throws Exception {
    List<String> sources = Arrays.asList("source1");
    Properties clientProps = new Properties();
    clientProps.setProperty("client.container.httpPort", "0");
    clientProps.setProperty("client.container.jmx.rmiEnabled", "false");
    clientProps.setProperty("client.runtime.bootstrap.enabled", "true");
    clientProps.setProperty("client.runtime.bootstrap.service(1).name", "bs1");
    clientProps.setProperty("client.runtime.bootstrap.service(1).host", "localhost");
    clientProps.setProperty("client.runtime.bootstrap.service(1).port", "10001");
    clientProps.setProperty("client.runtime.bootstrap.service(1).sources", "source1");
    clientProps.setProperty("client.runtime.relay(1).name", "relay1");
    clientProps.setProperty("client.runtime.relay(1).port", "10001");
    clientProps.setProperty("client.runtime.relay(1).sources", "source1");
    clientProps.setProperty("client.connectionDefaults.eventBuffer.maxSize", "100000");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.maxRetryNum", "3");
    DatabusHttpClientImpl.Config clientConfBuilder = new DatabusHttpClientImpl.Config();
    ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>("client.", clientConfBuilder);
    configLoader.loadConfig(clientProps);
    DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build();
    DatabusSourcesConnection.StaticConfig srcConnConf = clientConf.getConnectionDefaults();
    DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConf);
    client.registerDatabusBootstrapListener(new LoggingConsumer(), null, "source1");
    Assert.assertNotNull(client, "client instantiation failed");
    DatabusHttpClientImpl.RuntimeConfig clientRtConf = clientConf.getRuntime().build();
    //we keep the index of the next server we expect to see
    AtomicInteger serverIdx = new AtomicInteger(-1);
    List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
    sourcesResponse.add(new IdNamePair(1L, "source1"));
    Map<Long, List<RegisterResponseEntry>> registerResponse = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> regResponse = new ArrayList<RegisterResponseEntry>();
    regResponse.add(new RegisterResponseEntry(1L, (short) 1, SCHEMA$.toString()));
    registerResponse.put(1L, regResponse);
    ChunkedBodyReadableByteChannel channel = EasyMock.createMock(ChunkedBodyReadableByteChannel.class);
    // getting the pending-event-size header is called twice, once for checking and once for logging.
    EasyMock.expect(channel.getMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE)).andReturn("1000000").times(2);
    EasyMock.expect(channel.getMetadata("x-dbus-error-cause")).andReturn(null).times(2);
    EasyMock.expect(channel.getMetadata("x-dbus-error")).andReturn(null).times(2);
    EasyMock.replay(channel);
    DbusEventBuffer dbusBuffer = EasyMock.createMock(DbusEventBuffer.class);
    dbusBuffer.endEvents(false, -1, false, false, null);
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(dbusBuffer.injectEvent(EasyMock.<DbusEventInternalReadable>notNull())).andReturn(true).anyTimes();
    EasyMock.expect(dbusBuffer.getEventSerializationVersion()).andReturn(DbusEventFactory.DBUS_EVENT_V1).anyTimes();
    EasyMock.expect(dbusBuffer.getMaxReadBufferCapacity()).andReturn(600).times(2);
    EasyMock.expect(dbusBuffer.getBufferFreeReadSpace()).andReturn(600000).times(2);
    EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull())).andReturn(1).times(1);
    EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull(), EasyMock.<List<InternalDatabusEventsListener>>notNull(), EasyMock.<DbusEventsStatisticsCollector>isNull())).andReturn(0).times(1);
    EasyMock.replay(dbusBuffer);
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sources);
    //This guy succeeds on /sources but fails on /register
    MockBootstrapConnection mockSuccessConn = new MockBootstrapConnection(10, 10, channel, serverIdx, false);
    DatabusBootstrapConnectionFactory mockConnFactory = org.easymock.EasyMock.createMock("mockRelayFactory", DatabusBootstrapConnectionFactory.class);
    //each server should be tried MAX_RETRIES time until all retries are exhausted
    EasyMock.expect(mockConnFactory.createConnection(EasyMock.<ServerInfo>notNull(), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockSuccessConn).anyTimes();
    List<DatabusSubscription> sourcesSubList = DatabusSubscription.createSubscriptionList(sources);
    DatabusSourcesConnection sourcesConn2 = EasyMock.createMock(DatabusSourcesConnection.class);
    EasyMock.expect(sourcesConn2.getSourcesNames()).andReturn(Arrays.asList("source1")).anyTimes();
    EasyMock.expect(sourcesConn2.getSubscriptions()).andReturn(sourcesSubList).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionConfig()).andReturn(srcConnConf).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionStatus()).andReturn(new DatabusComponentStatus("dummy")).anyTimes();
    EasyMock.expect(sourcesConn2.getLocalRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getUnifiedClientStats()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapConnFactory()).andReturn(mockConnFactory).anyTimes();
    EasyMock.expect(sourcesConn2.loadPersistentCheckpoint()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getDataEventsBuffer()).andReturn(dbusBuffer).anyTimes();
    EasyMock.expect(sourcesConn2.isBootstrapEnabled()).andReturn(true).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapRegistrations()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapServices()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapEventsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.makeThreadSafe(mockConnFactory, true);
    EasyMock.makeThreadSafe(sourcesConn2, true);
    EasyMock.replay(mockConnFactory);
    EasyMock.replay(sourcesConn2);
    BootstrapPullThread bsPuller = new BootstrapPullThread("RelayPuller", sourcesConn2, dbusBuffer, connStateFactory, clientRtConf.getBootstrap().getServicesSet(), new ArrayList<DbusKeyCompositeFilterConfig>(), clientConf.getPullerBufferUtilizationPct(), ManagementFactory.getPlatformMBeanServer(), new DbusEventV2Factory(), null, null);
    mockSuccessConn.setCallback(bsPuller);
    bsPuller.getComponentStatus().start();
    Checkpoint cp = _ckptHandlerSource1.createInitialBootstrapCheckpoint(null, 0L);
    //TODO remove
    //cp.setSnapshotSource("source1");
    //cp.setCatchupSource("source1");
    //cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    ConnectionState connState = bsPuller.getConnectionState();
    connState.switchToBootstrap(cp);
    testTransitionCase(bsPuller, StateId.BOOTSTRAP, StateId.REQUEST_START_SCN, cp);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.REQUEST_START_SCN, StateId.START_SCN_RESPONSE_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    Map<Long, List<RegisterResponseEntry>> entries = new HashMap<Long, List<RegisterResponseEntry>>();
    entries.put(1L, new ArrayList<RegisterResponseEntry>());
    connState.setSourcesSchemas(entries);
    connState.setCurrentBSServerInfo(bsPuller.getCurentServer());
    testTransitionCase(bsPuller, StateId.START_SCN_RESPONSE_SUCCESS, StateId.REQUEST_STREAM, null);
    bsPuller.getMessageQueue().clear();
    connState.getSourcesNameMap().put("source1", new IdNamePair(1L, "source1"));
    connState.getSourceIdMap().put(1L, new IdNamePair(1L, "source1"));
    testTransitionCase(bsPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_REQUEST_SUCCESS, "SUSPEND_ON_ERROR", null);
    EasyMock.verify(channel);
    EasyMock.verify(sourcesConn2);
    EasyMock.verify(dbusBuffer);
    EasyMock.verify(channel);
    EasyMock.verify(mockConnFactory);
}