Examples with PhysicalSourceStaticConfig com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig used on opensource projects

Example 11 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class DatabusRelayMain method addOneProducer.

/** overrides HTTP relay method */
@Override
public void addOneProducer(PhysicalSourceStaticConfig pConfig) throws DatabusException, EventCreationException, UnsupportedKeyException, SQLException, InvalidConfigException {
    // Register a command to allow start/stop/status of the relay
    List<EventProducer> plist = new ArrayList<EventProducer>();
    PhysicalPartition pPartition = pConfig.getPhysicalPartition();
    MaxSCNReaderWriter maxScnReaderWriters = _maxScnReaderWriters.getOrCreateHandler(pPartition);
    LOG.info("Starting server container with maxScnReaderWriter:" + maxScnReaderWriters);
    // Get the event buffer
    DbusEventBufferAppendable dbusEventBuffer = getEventBuffer().getDbusEventBufferAppendable(pPartition);
    // Get the schema registry service
    SchemaRegistryService schemaRegistryService = getSchemaRegistryService();
    // Get a stats collector per physical source
    addPhysicalPartitionCollectors(pPartition);
    String statsCollectorName = pPartition.toSimpleString();
    /*
		 * _inBoundStatsCollectors.addStatsCollector(statsCollectorName, new
		 * DbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
		 * statsCollectorName+".inbound", true, false, getMbeanServer()));
		 *
		 * _outBoundStatsCollectors.addStatsCollector(statsCollectorName, new
		 * DbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
		 * statsCollectorName+".outbound", true, false, getMbeanServer()));
		 */
    // Create the event producer
    String uri = pConfig.getUri();
    if (uri == null)
        throw new DatabusException("Uri is required to start the relay");
    uri = uri.trim();
    EventProducer producer = null;
    if (uri.startsWith("jdbc:")) {
        SourceType sourceType = pConfig.getReplBitSetter().getSourceType();
        if (SourceType.TOKEN.equals(sourceType))
            throw new DatabusException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");
        // if a buffer for this partiton exists - we are overwri
        producer = new OracleEventProducerFactory().buildEventProducer(pConfig, schemaRegistryService, dbusEventBuffer, getMbeanServer(), _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else if (uri.startsWith("mock")) {
        // Get all relevant pConfig attributes
        //TODO add real instantiation
        EventProducerServiceProvider mockProvider = _producersRegistry.getEventProducerServiceProvider("mock");
        if (null == mockProvider) {
            throw new DatabusRuntimeException("relay event producer not available: " + "mock");
        }
        producer = mockProvider.createProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else if (uri.startsWith("gg:")) {
        producer = new GoldenGateEventProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else if (uri.startsWith("mysql:")) {
        LOG.info("Adding OpenReplicatorEventProducer for uri :" + uri);
        final String serviceName = "or";
        EventProducerServiceProvider orProvider = _producersRegistry.getEventProducerServiceProvider(serviceName);
        if (null == orProvider) {
            throw new DatabusRuntimeException("relay event producer not available: " + serviceName);
        }
        producer = orProvider.createProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
    } else {
        // Get all relevant pConfig attributes and initialize the nettyThreadPool objects
        RelayEventProducer.DatabusClientNettyThreadPools nettyThreadPools = new RelayEventProducer.DatabusClientNettyThreadPools(0, getNetworkTimeoutTimer(), getBossExecutorService(), getIoExecutorService(), getHttpChannelGroup());
        producer = new RelayEventProducer(pConfig, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters, nettyThreadPools);
    }
    // if a buffer for this partiton exists - we are overwriting it.
    _producers.put(pPartition, producer);
    plist.add(producer);
    // append 'monitoring event producer'
    if (producer instanceof OracleEventProducer) {
        MonitoringEventProducer monitoringProducer = new MonitoringEventProducer("dbMonitor." + pPartition.toSimpleString(), pConfig.getName(), pConfig.getUri(), ((OracleEventProducer) producer).getMonitoredSourceInfos(), getMbeanServer());
        _monitoringProducers.put(pPartition, monitoringProducer);
        plist.add(monitoringProducer);
    }
    if (_csEventRequestProcessor == null)
        _csEventRequestProcessor = new ControlSourceEventsRequestProcessor(null, this, plist);
    else
        _csEventRequestProcessor.addEventProducers(plist);
    RequestProcessorRegistry processorRegistry = getProcessorRegistry();
    processorRegistry.reregister(ControlSourceEventsRequestProcessor.COMMAND_NAME, _csEventRequestProcessor);
}

Example 12 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class GoldenGateEventProducer method buildGGMonitoredSourceInfo.

public GGMonitoredSourceInfo buildGGMonitoredSourceInfo(LogicalSourceStaticConfig sourceConfig, PhysicalSourceStaticConfig pConfig) throws DatabusException, InvalidConfigException {
    // udpate partition mapping
    PartitionFunction partitionFunction = GGEventGenerationFactory.buildPartitionFunction(sourceConfig);
    _partitionFunctionHashMap.put((int) sourceConfig.getId(), partitionFunction);
    EventSourceStatistics statisticsBean = new EventSourceStatistics(sourceConfig.getName());
    GGMonitoredSourceInfo sourceInfo = new GGMonitoredSourceInfo(sourceConfig.getId(), sourceConfig.getName(), statisticsBean);
    registerMbeans(sourceInfo);
    return sourceInfo;
}

Example 13 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class DatabusRelayMain method removeOneProducer.

/** overrides HTTP relay method */
@Override
public void removeOneProducer(PhysicalSourceStaticConfig pConfig) {
    PhysicalPartition pPartition = pConfig.getPhysicalPartition();
    List<EventProducer> plist = new ArrayList<EventProducer>();
    if (_producers != null && _producers.containsKey(pPartition))
        plist.add(_producers.remove(pPartition));
    if (_monitoringProducers != null && _monitoringProducers.containsKey(pPartition))
        plist.add(_monitoringProducers.remove(pPartition));
    if (plist.size() > 0 && _csEventRequestProcessor != null)
        _csEventRequestProcessor.removeEventProducers(plist);
}

Example 14 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class TestGoldenGateEventProducer method testAddEventToBuffer.

@Test
public void testAddEventToBuffer() throws InvalidConfigException, UnsupportedKeyException, DatabusException {
    // No rate control
    long rate = 0;
    PhysicalSourceStaticConfig pssc = buildPssc(rate, 0L);
    long scn = 10;
    DbusEventBuffer mb = (DbusEventBuffer) createBufMult(pssc);
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
    int sourceId = 505;
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
    Object key = new String("name");
    Schema.Type keyType = Schema.Type.RECORD;
    ColumnsState.KeyPair kp = new ColumnsState.KeyPair(key, keyType);
    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(1);
    keyPairs.add(kp);
    Schema s = Schema.parse(avroSchema);
    GenericRecord gr = new GenericData.Record(s);
    gr.put("name", "phani");
    DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    db.add(dbi);
    TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
    dbUpdates.add(dbUpdate);
    long timestamp = System.nanoTime();
    gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
    Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);
    DbusEventIterator iter = mb.acquireIterator("test");
    int count = 0;
    long eventTs = 0;
    while (iter.hasNext()) {
        DbusEvent e = iter.next();
        if (count == 1) {
            // first event prev control event
            eventTs = e.timestampInNanos();
        }
        count++;
    }
    Assert.assertEquals("Event timestamp in Ns", timestamp, eventTs);
    Assert.assertEquals("Got events ", 3, count);
    return;
}

Example 15 with PhysicalSourceStaticConfig

use of com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig in project databus by linkedin.

the class TestGoldenGateEventProducer method createBufMult.

/**
   * Creates a DbusBufMult
   */
private DbusEventBufferAppendable createBufMult(PhysicalSourceStaticConfig pssc) throws InvalidConfigException {
    DbusEventBuffer.StaticConfig config = null;
    if (config == null) {
        try {
            DbusEventBuffer.Config cfgBuilder = new DbusEventBuffer.Config();
            cfgBuilder.setMaxSize(10 * 1024 * 1024);
            cfgBuilder.setScnIndexSize(2 * 1024 * 1024);
            cfgBuilder.setAllocationPolicy("MMAPPED_MEMORY");
            config = cfgBuilder.build();
        } catch (InvalidConfigException e) {
            fail("invalid configuration", e);
        }
    }
    PhysicalSourceStaticConfig[] pConfigs = new PhysicalSourceStaticConfig[1];
    pConfigs[0] = pssc;
    DbusEventBufferMult eventBufferMult = new DbusEventBufferMult(pConfigs, config, new DbusEventV2Factory());
    for (DbusEventBuffer b : eventBufferMult.bufIterable()) {
        b.start(1);
    }
    DbusEventBufferAppendable buf = eventBufferMult.getDbusEventBufferAppendable(505);
    return buf;
}