use of com.linkedin.databus2.relay.OracleEventProducerFactory in project databus by linkedin.
the class BootstrapAvroFileSeederMain method init.
public static void init(String[] args) throws Exception {
parseArgs(args);
File sourcesJson = new File(_sSourcesConfigFile);
ObjectMapper mapper = new ObjectMapper();
PhysicalSourceConfig physicalSourceConfig = mapper.readValue(sourcesJson, PhysicalSourceConfig.class);
physicalSourceConfig.checkForNulls();
Config config = new Config();
ConfigLoader<StaticConfig> configLoader = new ConfigLoader<StaticConfig>("databus.seed.", config);
_sStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
// Make sure the URI from the configuration file identifies an Oracle JDBC source.
String uri = physicalSourceConfig.getUri();
if (!uri.startsWith("jdbc:oracle")) {
throw new InvalidConfigException("Invalid source URI (" + physicalSourceConfig.getUri() + "). Only jdbc:oracle: URIs are supported.");
}
OracleEventProducerFactory factory = new BootstrapSeederOracleEventProducerFactory(_sStaticConfig.getController().getPKeyNameMap());
// Parse each one of the logical sources
_sources = new ArrayList<OracleTriggerMonitoredSourceInfo>();
FileSystemSchemaRegistryService schemaRegistryService = FileSystemSchemaRegistryService.build(_sStaticConfig.getSchemaRegistry().getFileSystem());
for (LogicalSourceConfig sourceConfig : physicalSourceConfig.getSources()) {
OracleTriggerMonitoredSourceInfo source = factory.buildOracleMonitoredSourceInfo(sourceConfig.build(), physicalSourceConfig.build(), schemaRegistryService);
_sources.add(source);
}
_sSeeder = new BootstrapDBSeeder(_sStaticConfig.getBootstrap(), _sources);
_sBootstrapBuffer = new BootstrapEventBuffer(_sStaticConfig.getController().getCommitInterval() * 2);
_sWriterThread = new BootstrapSeederWriterThread(_sBootstrapBuffer, _sSeeder);
_sReader = new BootstrapAvroFileEventReader(_sStaticConfig.getController(), _sources, _sSeeder.getLastRows(), _sBootstrapBuffer);
}
use of com.linkedin.databus2.relay.OracleEventProducerFactory in project databus by linkedin.
the class BootstrapSeederMain method init.
public static void init(String[] args) throws Exception {
parseArgs(args);
// Load the source configuration JSON file
// File sourcesJson = new File("integration-test/config/sources-member2.json");
File sourcesJson = new File(_sSourcesConfigFile);
ObjectMapper mapper = new ObjectMapper();
PhysicalSourceConfig physicalSourceConfig = mapper.readValue(sourcesJson, PhysicalSourceConfig.class);
physicalSourceConfig.checkForNulls();
Config config = new Config();
ConfigLoader<StaticConfig> configLoader = new ConfigLoader<StaticConfig>("databus.seed.", config);
_sStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);
// Make sure the URI from the configuration file identifies an Oracle JDBC source.
String uri = physicalSourceConfig.getUri();
if (!uri.startsWith("jdbc:oracle")) {
throw new InvalidConfigException("Invalid source URI (" + physicalSourceConfig.getUri() + "). Only jdbc:oracle: URIs are supported.");
}
String sourceTypeStr = physicalSourceConfig.getReplBitSetter().getSourceType();
if (SourceType.TOKEN.toString().equalsIgnoreCase(sourceTypeStr))
throw new InvalidConfigException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");
// Create the OracleDataSource used to get DB connection(s)
try {
Class oracleDataSourceClass = OracleJarUtils.loadClass("oracle.jdbc.pool.OracleDataSource");
Object ods = oracleDataSourceClass.newInstance();
Method setURLMethod = oracleDataSourceClass.getMethod("setURL", String.class);
setURLMethod.invoke(ods, uri);
_sDataStore = (DataSource) ods;
} catch (Exception e) {
String errMsg = "Error creating a data source object ";
LOG.error(errMsg, e);
throw e;
}
// TODO: Need a better way than relaying on RelayFactory for generating MonitoredSourceInfo
OracleEventProducerFactory factory = new BootstrapSeederOracleEventProducerFactory(_sStaticConfig.getController().getPKeyNameMap());
// Parse each one of the logical sources
_sources = new ArrayList<OracleTriggerMonitoredSourceInfo>();
FileSystemSchemaRegistryService schemaRegistryService = FileSystemSchemaRegistryService.build(_sStaticConfig.getSchemaRegistry().getFileSystem());
Set<String> seenUris = new HashSet<String>();
for (LogicalSourceConfig sourceConfig : physicalSourceConfig.getSources()) {
String srcUri = sourceConfig.getUri();
if (seenUris.contains(srcUri)) {
String msg = "Uri (" + srcUri + ") is used for more than one sources. Currently Bootstrap Seeder cannot support seeding sources with the same URI together. Please have them run seperately !!";
LOG.fatal(msg);
throw new InvalidConfigException(msg);
}
seenUris.add(srcUri);
OracleTriggerMonitoredSourceInfo source = factory.buildOracleMonitoredSourceInfo(sourceConfig.build(), physicalSourceConfig.build(), schemaRegistryService);
_sources.add(source);
}
_sSeeder = new BootstrapDBSeeder(_sStaticConfig.getBootstrap(), _sources);
_sBootstrapBuffer = new BootstrapEventBuffer(_sStaticConfig.getController().getCommitInterval() * 2);
_sWriterThread = new BootstrapSeederWriterThread(_sBootstrapBuffer, _sSeeder);
_sReader = new BootstrapSrcDBEventReader(_sDataStore, _sBootstrapBuffer, _sStaticConfig.getController(), _sources, _sSeeder.getLastRows(), _sSeeder.getLastKeys(), 0);
}
use of com.linkedin.databus2.relay.OracleEventProducerFactory in project databus by linkedin.
the class DatabusRelayMain method addOneProducer.
/**
* overrides HTTP relay method
*/
@Override
public void addOneProducer(PhysicalSourceStaticConfig pConfig) throws DatabusException, EventCreationException, UnsupportedKeyException, SQLException, InvalidConfigException {
// Register a command to allow start/stop/status of the relay
List<EventProducer> plist = new ArrayList<EventProducer>();
PhysicalPartition pPartition = pConfig.getPhysicalPartition();
MaxSCNReaderWriter maxScnReaderWriters = _maxScnReaderWriters.getOrCreateHandler(pPartition);
LOG.info("Starting server container with maxScnReaderWriter:" + maxScnReaderWriters);
// Get the event buffer
DbusEventBufferAppendable dbusEventBuffer = getEventBuffer().getDbusEventBufferAppendable(pPartition);
// Get the schema registry service
SchemaRegistryService schemaRegistryService = getSchemaRegistryService();
// Get a stats collector per physical source
addPhysicalPartitionCollectors(pPartition);
String statsCollectorName = pPartition.toSimpleString();
/*
* _inBoundStatsCollectors.addStatsCollector(statsCollectorName, new
* DbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
* statsCollectorName+".inbound", true, false, getMbeanServer()));
*
* _outBoundStatsCollectors.addStatsCollector(statsCollectorName, new
* DbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
* statsCollectorName+".outbound", true, false, getMbeanServer()));
*/
// Create the event producer
String uri = pConfig.getUri();
if (uri == null)
throw new DatabusException("Uri is required to start the relay");
uri = uri.trim();
EventProducer producer = null;
if (uri.startsWith("jdbc:")) {
SourceType sourceType = pConfig.getReplBitSetter().getSourceType();
if (SourceType.TOKEN.equals(sourceType))
throw new DatabusException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");
// if a buffer for this partiton exists - we are overwri
producer = new OracleEventProducerFactory().buildEventProducer(pConfig, schemaRegistryService, dbusEventBuffer, getMbeanServer(), _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
} else if (uri.startsWith("mock")) {
// Get all relevant pConfig attributes
// TODO add real instantiation
EventProducerServiceProvider mockProvider = _producersRegistry.getEventProducerServiceProvider("mock");
if (null == mockProvider) {
throw new DatabusRuntimeException("relay event producer not available: " + "mock");
}
producer = mockProvider.createProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
} else if (uri.startsWith("gg:")) {
producer = new GoldenGateEventProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
} else if (uri.startsWith("mysql:")) {
LOG.info("Adding OpenReplicatorEventProducer for uri :" + uri);
final String serviceName = "or";
EventProducerServiceProvider orProvider = _producersRegistry.getEventProducerServiceProvider(serviceName);
if (null == orProvider) {
throw new DatabusRuntimeException("relay event producer not available: " + serviceName);
}
producer = orProvider.createProducer(pConfig, schemaRegistryService, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters);
} else {
// Get all relevant pConfig attributes and initialize the nettyThreadPool objects
RelayEventProducer.DatabusClientNettyThreadPools nettyThreadPools = new RelayEventProducer.DatabusClientNettyThreadPools(0, getNetworkTimeoutTimer(), getBossExecutorService(), getIoExecutorService(), getHttpChannelGroup());
producer = new RelayEventProducer(pConfig, dbusEventBuffer, _inBoundStatsCollectors.getStatsCollector(statsCollectorName), maxScnReaderWriters, nettyThreadPools);
}
// if a buffer for this partiton exists - we are overwriting it.
_producers.put(pPartition, producer);
plist.add(producer);
// append 'monitoring event producer'
if (producer instanceof OracleEventProducer) {
MonitoringEventProducer monitoringProducer = new MonitoringEventProducer("dbMonitor." + pPartition.toSimpleString(), pConfig.getName(), pConfig.getUri(), ((OracleEventProducer) producer).getMonitoredSourceInfos(), getMbeanServer());
_monitoringProducers.put(pPartition, monitoringProducer);
plist.add(monitoringProducer);
}
if (_csEventRequestProcessor == null)
_csEventRequestProcessor = new ControlSourceEventsRequestProcessor(null, this, plist);
else
_csEventRequestProcessor.addEventProducers(plist);
RequestProcessorRegistry processorRegistry = getProcessorRegistry();
processorRegistry.reregister(ControlSourceEventsRequestProcessor.COMMAND_NAME, _csEventRequestProcessor);
}
Aggregations