Examples with PhysicalPartition com.linkedin.databus.core.data_model.PhysicalPartition used on opensource projects

Example 1 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class DatabusRequestExecutionHandler method messageReceived.

@Override
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
    if (e.getMessage() instanceof HttpRequest) {
        _httpRequest = (HttpRequest) e.getMessage();
        ctx.sendUpstream(e);
    }
    if (e.getMessage() instanceof DatabusRequest) {
        _dbusRequest = (DatabusRequest) e.getMessage();
        // If there is a physical partition stashed away, then restore it into the request now.
        if (ctx.getAttachment() != null && ctx.getAttachment() instanceof PhysicalPartition) {
            _dbusRequest.setCursorPartition((PhysicalPartition) (ctx.getAttachment()));
        }
        /*NettyStats nettyStats = _configManager.getNettyStats();
      boolean nettyStatsEnabled = nettyStats.isEnabled();
      CallCompletion callCompletion = nettyStatsEnabled ?
          nettyStats.getRequestHandler_writeResponse().startCall() :
          null;
      CallCompletion processRequestCompletion = null;*/
        try {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Creating response for command [" + _dbusRequest.getId() + "] " + _dbusRequest.getName());
            }
            // Decide whether to close the connection or not.
            boolean keepAlive = isKeepAlive(_httpRequest);
            HttpResponse response = generateEmptyResponse();
            if (LOG.isDebugEnabled()) {
                // We are debugging -- let's add some more info to the response
                response.addHeader(DatabusHttpHeaders.DATABUS_REQUEST_ID_HEADER, Long.toString(_dbusRequest.getId()));
            }
            // Write the response.
            ChunkedBodyWritableByteChannel responseChannel = null;
            try {
                responseChannel = new ChunkedBodyWritableByteChannel(e.getChannel(), response);
                _dbusRequest.setResponseContent(responseChannel);
                if (LOG.isDebugEnabled()) {
                    LOG.debug("About to run command [" + _dbusRequest.getId() + "] " + _dbusRequest.getName());
                }
                // FIXME   DDS-305: Rework the netty stats collector to use event-based stats aggregation
                /*if (nettyStatsEnabled)
          {
            processRequestCompletion = nettyStats.getRequestHandler_processRequest().startCall();
          }*/
                Future<DatabusRequest> responseFuture = _processorRegistry.run(_dbusRequest);
                ServerContainer.RuntimeConfig config = _dbusRequest.getConfig();
                int timeoutMs = config.getRequestProcessingBudgetMs();
                boolean done = responseFuture.isDone();
                while (!done) {
                    try {
                        responseFuture.get(timeoutMs, TimeUnit.MILLISECONDS);
                        done = true;
                        ctx.setAttachment(_dbusRequest.getCursorPartition());
                    } catch (InterruptedException ie) {
                        done = responseFuture.isDone();
                    } catch (Exception ex) {
                        done = true;
                        _dbusRequest.setError(ex);
                        // On any error, clear any context saved. We will start afresh in a new request.
                        ctx.setAttachment(null);
                    // FIXME   DDS-305: Rework the netty stats collector to use event-based stats aggregation
                    /*if (null != processRequestCompletion)
              {
                processRequestCompletion.endCallWithError(ex);
                processRequestCompletion = null;
              }*/
                    }
                }
            } finally {
                if (null != responseChannel) {
                    if (LOG.isDebugEnabled()) {
                        // Add some more debugging info
                        long curTimeMs = System.currentTimeMillis();
                        responseChannel.addMetadata(DatabusHttpHeaders.DATABUS_REQUEST_LATENCY_HEADER, Long.toString(curTimeMs - _dbusRequest.getCreateTimestampMs()));
                    }
                    responseChannel.close();
                }
                if (null != _dbusRequest.getResponseThrowable()) {
                    ContainerStatisticsCollector statsCollector = _serverContainer.getContainerStatsCollector();
                    if (null != statsCollector) {
                        statsCollector.registerContainerError(_dbusRequest.getResponseThrowable());
                    }
                }
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("Done runing command [" + _dbusRequest.getId() + "] " + _dbusRequest.getName());
            }
            // Close the non-keep-alive or hard-failed connection after the write operation is done.
            if (!keepAlive || null == responseChannel) {
                e.getChannel().close();
            }
        // FIXME   DDS-305: Rework the netty stats collector to use event-based stats aggregation
        /*if (null != callCompletion)
        {
          callCompletion.endCall();
        }*/
        } catch (RuntimeException ex) {
            LOG.error("HttpRequestHandler.writeResponse error", ex);
            // FIXME   DDS-305: Rework the netty stats collector to use event-based stats aggregation
            /*if (null != callCompletion)
        {
          callCompletion.endCallWithError(ex);
        }*/
            ContainerStatisticsCollector statsCollector = _serverContainer.getContainerStatsCollector();
            if (null != statsCollector)
                statsCollector.registerContainerError(ex);
        }
    } else {
        // Pass on everything else
        ctx.sendUpstream(e);
    }
}

Example 2 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class TestDatabusRelayEvents method testV2Events.

/**
 * Stuffs an event buffer with both a v1 and a v2 event, then reads the buffer two ways:
 * first accepting only v1 events (verifying conversion of the v2 event to v1); then accepting
 * both v1 and v2 events.
 *
 * Note that the version of the _EOP_ events must match the version of the event factory,
 * regardless of the versions of any preceding "real" events.  (This matches DbusEventBuffer
 * behavior; see the serializeLongKeyEndOfPeriodMarker() call in endEvents() for details.)
 */
@Test
public void testV2Events() throws KeyTypeNotImplementedException, InvalidEventException, IOException, DatabusException {
    final Logger log = Logger.getLogger("TestDatabusRelayEvents.testV2Events");
    log.setLevel(Level.DEBUG);
    String[] srcs = { "com.linkedin.events.example.fake.FakeSchema" };
    String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
    short srcId = 2;
    short pId = 1;
    int relayPort = Utils.getAvailablePort(11993);
    // create relay
    final DatabusRelayMain relay1 = createRelay(relayPort, pId, srcs);
    DatabusRelayTestUtil.RelayRunner r1 = null;
    ClientRunner cr = null;
    try {
        // EventProducer[] producers = relay1.getProducers();
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("Relay created");
        DbusEventBufferMult bufMult = relay1.getEventBuffer();
        PhysicalPartition pPartition = new PhysicalPartition((int) pId, pSourceName);
        DbusEventBuffer buf = (DbusEventBuffer) bufMult.getDbusEventBufferAppendable(pPartition);
        log.info("create some events");
        long windowScn = 100L;
        ByteBuffer serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(), pId, DbusEventFactory.DBUS_EVENT_V2);
        ReadableByteChannel channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
        int readEvents = buf.readEvents(channel);
        log.info("successfully read in " + readEvents + " events ");
        channel.close();
        windowScn = 101L;
        serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(), pId, DbusEventFactory.DBUS_EVENT_V1);
        channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
        readEvents = buf.readEvents(channel);
        log.info("successfully read in " + readEvents + " events ");
        channel.close();
        log.info("starting relay on port " + relayPort);
        r1.start();
        // TestUtil.sleep(10*1000);
        // wait until relay comes up
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return relay1.isRunningStatus();
            }
        }, "Relay hasn't come up completely ", 30000, LOG);
        log.info("now create client");
        String srcSubscriptionString = TestUtil.join(srcs, ",");
        String serverName = "localhost:" + relayPort;
        final EventsCountingConsumer countingConsumer = new EventsCountingConsumer();
        int id = (RngUtils.randomPositiveInt() % 10000) + 1;
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V1, 0);
        cr = new ClientRunner(clientConn);
        log.info("starting client");
        cr.start();
        // wait till client gets the event
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                int events = countingConsumer.getNumDataEvents();
                LOG.info("client got " + events + " events");
                return events == 2;
            }
        }, "Consumer didn't get 2 events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(countingConsumer.getNumDataEvents(), 2);
        Assert.assertEquals(countingConsumer.getNumWindows(), 2);
        Assert.assertEquals(countingConsumer.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1), 2);
        log.info("shutdown first client");
        clientConn.stop();
        cr.shutdown();
        TestUtil.sleep(1000);
        cr = null;
        log.info("start another client who understands V2");
        final EventsCountingConsumer countingConsumer1 = new EventsCountingConsumer();
        clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer1, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V2, 0);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("wait till client gets the event");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                int events = countingConsumer1.getNumDataEvents();
                LOG.debug("client got " + events + " events");
                return events == 2;
            }
        }, "Consumer didn't get 2 events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(countingConsumer1.getNumDataEvents(), 2);
        Assert.assertEquals(countingConsumer1.getNumWindows(), 2);
        Assert.assertEquals(countingConsumer1.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1), 1);
        Assert.assertEquals(countingConsumer1.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V2), 1);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 3 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class TestDatabusRelayEvents method testEventConversion.

@Test
public /**
 * append event V2 to the buffer and stream it to the client
 * which only accepts events V1. Make sure it got converted
 */
void testEventConversion() throws InterruptedException, IOException, DatabusException {
    final Logger log = Logger.getLogger("TestDatabusRelayEvents.testEventConversion");
    log.setLevel(Level.INFO);
    DatabusRelayTestUtil.RelayRunner r1 = null;
    ClientRunner cr = null;
    try {
        String[] srcs = { "com.linkedin.events.example.fake.FakeSchema" };
        int pId = 1;
        int srcId = 2;
        int relayPort = Utils.getAvailablePort(11994);
        ;
        final DatabusRelayMain relay1 = createRelay(relayPort, pId, srcs);
        Assert.assertNotNull(relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("Relay created");
        DbusEventBufferMult bufMult = relay1.getEventBuffer();
        String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
        PhysicalPartition pPartition = new PhysicalPartition(pId, pSourceName);
        DbusEventBufferAppendable buf = bufMult.getDbusEventBufferAppendable(pPartition);
        DbusEventKey key = new DbusEventKey(123L);
        byte[] schemaId = relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray();
        byte[] payload = RngUtils.randomString(100).getBytes(Charset.defaultCharset());
        DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, 100L, (short) pId, (short) pId, 897L, (short) srcId, schemaId, payload, false, true);
        eventInfo.setEventSerializationVersion(DbusEventFactory.DBUS_EVENT_V2);
        buf.startEvents();
        buf.appendEvent(key, eventInfo, null);
        buf.endEvents(100L, null);
        r1.start();
        log.info("Relay started");
        // wait until relay comes up
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return relay1.isRunningStatus();
            }
        }, "Relay hasn't come up completely ", 7000, LOG);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcs, ",");
        String serverName = "localhost:" + relayPort;
        final EventsCountingConsumer countingConsumer = new EventsCountingConsumer();
        int id = (RngUtils.randomPositiveInt() % 10000) + 1;
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V1, 0);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("Consumer started");
        // wait till client gets the event
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return countingConsumer.getNumDataEvents() == 1;
            }
        }, "Consumer didn't get any events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(1, countingConsumer.getNumDataEvents());
        Assert.assertEquals(1, countingConsumer.getNumWindows());
        Assert.assertEquals(1, countingConsumer.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1));
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 4 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class DatabusRelayMain method addOneProducer.

/**
 * overrides HTTP relay method
 */
@Override
public void addOneProducer(PhysicalSourceStaticConfig pConfig) throws DatabusException, EventCreationException, UnsupportedKeyException, SQLException, InvalidConfigException {
    // Register a command to allow start/stop/status of the relay
    List<EventProducer> plist = new ArrayList<EventProducer>();
    PhysicalPartition pPartition = pConfig.getPhysicalPartition();
    MaxSCNReaderWriter maxScnReaderWriters = _maxScnReaderWriters.getOrCreateHandler(pPartition);
    LOG.info("Starting server container with maxScnReaderWriter:" + maxScnReaderWriters);
    // Get the event buffer
    DbusEventBufferAppendable dbusEventBuffer = getEventBuffer().getDbusEventBufferAppendable(pPartition);
    // Get the schema registry service
    SchemaRegistryService schemaRegistryService = getSchemaRegistryService();
    // Get a stats collector per physical source
    addPhysicalPartitionCollectors(pPartition);
    String statsCollectorName = pPartition.toSimpleString();
    /*
		 * _inBoundStatsCollectors.addStatsCollector(statsCollectorName, new
		 * DbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
		 * statsCollectorName+".inbound", true, false, getMbeanServer()));
		 *
		 * _outBoundStatsCollectors.addStatsCollector(statsCollectorName, new
		 * DbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
		 * statsCollectorName+".outbound", true, false, getMbeanServer()));
		 */
    // Create the event producer
    String uri = pConfig.getUri();
    if (uri == null)
        throw new DatabusException("Uri is required to start the relay");
    uri = uri.trim();
    EventProducer producer = null;
    if (uri.startsWith("jdbc:")) {
        SourceType sourceType = pConfig.getReplBitSetter().getSourceType();
        if (SourceType.TOKEN.equals(sourceType))
            throw new DatabusException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");
        // if a buffer for this partiton exists - we are overwri
        producer = new OracleEventProducerFactory().buildEventProducer(pConfig, schemaRegistryService, dbusEventBuffer, getMbeanServer(), _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else if (uri.startsWith("mock")) {
        // Get all relevant pConfig attributes
        // TODO add real instantiation
        EventProducerServiceProvider mockProvider = _producersRegistry.getEventProducerServiceProvider("mock");
        if (null == mockProvider) {
            throw new DatabusRuntimeException("relay event producer not available: " + "mock");
        }
        producer = mockProvider.createProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else if (uri.startsWith("gg:")) {
        producer = new GoldenGateEventProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else if (uri.startsWith("mysql:")) {
        LOG.info("Adding OpenReplicatorEventProducer for uri :" + uri);
        final String serviceName = "or";
        EventProducerServiceProvider orProvider = _producersRegistry.getEventProducerServiceProvider(serviceName);
        if (null == orProvider) {
            throw new DatabusRuntimeException("relay event producer not available: " + serviceName);
        }
        producer = orProvider.createProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else {
        // Get all relevant pConfig attributes and initialize the nettyThreadPool objects
        RelayEventProducer.DatabusClientNettyThreadPools nettyThreadPools = new RelayEventProducer.DatabusClientNettyThreadPools(0, getNetworkTimeoutTimer(), getBossExecutorService(), getIoExecutorService(), getHttpChannelGroup());
        producer = new RelayEventProducer(pConfig, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters, nettyThreadPools);
    }
    // if a buffer for this partiton exists - we are overwriting it.
    _producers.put(pPartition, producer);
    plist.add(producer);
    // append 'monitoring event producer'
    if (producer instanceof OracleEventProducer) {
        MonitoringEventProducer monitoringProducer = new MonitoringEventProducer("dbMonitor." + pPartition.toSimpleString(), pConfig.getName(), pConfig.getUri(), ((OracleEventProducer) producer).getMonitoredSourceInfos(), getMbeanServer());
        _monitoringProducers.put(pPartition, monitoringProducer);
        plist.add(monitoringProducer);
    }
    if (_csEventRequestProcessor == null)
        _csEventRequestProcessor = new ControlSourceEventsRequestProcessor(null, this, plist);
    else
        _csEventRequestProcessor.addEventProducers(plist);
    RequestProcessorRegistry processorRegistry = getProcessorRegistry();
    processorRegistry.reregister(ControlSourceEventsRequestProcessor.COMMAND_NAME, _csEventRequestProcessor);
}

Example 5 with PhysicalPartition

use of com.linkedin.databus.core.data_model.PhysicalPartition in project databus by linkedin.

the class DatabusRelayMain method doShutdown.

@Override
protected void doShutdown() {
    LOG.warn("Shutting down Relay!");
    for (Entry<PhysicalPartition, EventProducer> entry : _producers.entrySet()) {
        PhysicalPartition pPartition = entry.getKey();
        EventProducer producer = entry.getValue();
        if (null != producer && (producer.isRunning() || producer.isPaused())) {
            producer.shutdown();
            try {
                producer.waitForShutdown();
            } catch (InterruptedException ie) {
            }
            LOG.info("EventProducer is shutdown!");
        }
        MonitoringEventProducer monitoringProducer = _monitoringProducers.get(pPartition);
        if (monitoringProducer != null) {
            if (monitoringProducer.isRunning()) {
                monitoringProducer.shutdown();
            }
            while (monitoringProducer.isRunning() || monitoringProducer.isPaused()) {
                try {
                    monitoringProducer.waitForShutdown();
                } catch (InterruptedException ie) {
                }
            }
            monitoringProducer.unregisterMBeans();
        }
    }
    super.doShutdown();
}