use of com.linkedin.databus2.core.container.request.ProcessorRegistrationConflictException in project databus by linkedin.
the class IntegratedDummyDatabusConsumer method initConn.
public void initConn(List<String> sources) throws IOException, InvalidConfigException, DatabusClientException, DatabusException {
StringBuilder sourcesString = new StringBuilder();
boolean firstSrc = true;
for (String source : sources) {
if (!firstSrc)
sourcesString.append(",");
firstSrc = false;
sourcesString.append(source);
}
_fileBasedCallback.init();
ArrayList<DatabusBootstrapConsumer> bootstrapCallbacks = new ArrayList<DatabusBootstrapConsumer>();
bootstrapCallbacks.add(this);
ArrayList<DatabusStreamConsumer> streamCallbacks = new ArrayList<DatabusStreamConsumer>();
streamCallbacks.add(this);
DatabusHttpClientImpl.Config clientConfigBuilder = new DatabusHttpClientImpl.Config();
clientConfigBuilder.getContainer().getJmx().setJmxServicePort(5555);
clientConfigBuilder.getContainer().setId(545454);
clientConfigBuilder.getContainer().setHttpPort(8082);
clientConfigBuilder.getCheckpointPersistence().setType(ProviderType.FILE_SYSTEM.toString());
clientConfigBuilder.getCheckpointPersistence().getFileSystem().setRootDirectory("./integratedconsumer-checkpoints");
clientConfigBuilder.getCheckpointPersistence().setClearBeforeUse(true);
clientConfigBuilder.getRuntime().getBootstrap().setEnabled(true);
DatabusSourcesConnection.Config srcDefaultConfig = new DatabusSourcesConnection.Config();
srcDefaultConfig.setFreeBufferThreshold((int) (_maxEventBufferSize * 0.05));
srcDefaultConfig.setCheckpointThresholdPct(80);
// 60 sec before retries (unless disabled)
srcDefaultConfig.setConsumerTimeBudgetMs(_useConsumerTimeout ? 60000 : 0);
// max of 3 retries
srcDefaultConfig.getDispatcherRetries().setMaxRetryNum(3);
clientConfigBuilder.setConnectionDefaults(srcDefaultConfig);
DbusEventBuffer.Config eventBufferConfig = clientConfigBuilder.getConnectionDefaults().getEventBuffer();
eventBufferConfig.setMaxSize(_maxEventBufferSize);
eventBufferConfig.setAverageEventSize(_maxReadBufferSize);
// TODO: the following shall be used once we can set eventbuffer for bootstrap through the config builder (DDSDBUS-82)
// For now, bootstrap buffer will use the same config as relay buffer.
// clientConfigBuilder.getConnectionDefaults().
DatabusHttpClientImpl.StaticConfig clientConfig = clientConfigBuilder.build();
ServerInfoBuilder relayBuilder = clientConfig.getRuntime().getRelay("1");
relayBuilder.setName("DefaultRelay");
relayBuilder.setHost("localhost");
relayBuilder.setPort(9000);
relayBuilder.setSources(sourcesString.toString());
ServerInfoBuilder bootstrapBuilder = clientConfig.getRuntime().getBootstrap().getService("2");
bootstrapBuilder.setName("DefaultBootstrapServices");
bootstrapBuilder.setHost("localhost");
bootstrapBuilder.setPort(6060);
bootstrapBuilder.setSources(sourcesString.toString());
_dbusClient = new DatabusHttpClientImpl(clientConfig);
_dbusClient.registerDatabusStreamListener(this, sources, null);
_dbusClient.registerDatabusBootstrapListener(this, sources, null);
// add pause processor
try {
_dbusClient.getProcessorRegistry().register(ConsumerPauseRequestProcessor.COMMAND_NAME, new ConsumerPauseRequestProcessor(null, this));
} catch (ProcessorRegistrationConflictException e) {
LOG.error("Failed to register " + ConsumerPauseRequestProcessor.COMMAND_NAME);
}
}
use of com.linkedin.databus2.core.container.request.ProcessorRegistrationConflictException in project databus by linkedin.
the class CommandsRegistry method registerCommand.
/**
* Registers a new command
* @param name the HTTP name of the command (null for no HTTP interface)
* @param opcode the TCP opcode (null for no TCP inteface)
* @param binaryParserFactory the factory for binary parsers for this command to be used
* by the TCP interface (can be null if opcode is null)
* @param execHandlerFactory the factory for ChannelHandlers to be used for executing
* commands of the specified type. The factory will be used for
* both HTTP and TCP commands if those interfaces are enabled.
* Must not be null.
* @throws ProcessorRegistrationConflictException
*/
public void registerCommand(String name, Byte opcode, BinaryCommandParserFactory binaryParserFactory, RequestExecutionHandlerFactory execHandlerFactory) throws ProcessorRegistrationConflictException {
if (null != opcode) {
if (_binaryParsers.containsKey(opcode))
throw new ProcessorRegistrationConflictException(name);
_binaryParsers.put(opcode, binaryParserFactory);
_tcpExecHandlers.put(opcode, execHandlerFactory);
}
if (null != name) {
_httpExecHandlers.put(name, execHandlerFactory);
}
}
use of com.linkedin.databus2.core.container.request.ProcessorRegistrationConflictException in project databus by linkedin.
the class DatabusRelayMain method initProducers.
public void initProducers() throws InvalidConfigException, DatabusException, EventCreationException, UnsupportedKeyException, SQLException, ProcessorRegistrationConflictException {
LOG.info("initializing producers");
for (PhysicalSourceStaticConfig pConfig : _pConfigs) {
addOneProducer(pConfig);
}
this.setDbPullerStart(_relayStaticConfig.getStartDbPuller());
LOG.info("done initializing producers");
}
use of com.linkedin.databus2.core.container.request.ProcessorRegistrationConflictException in project databus by linkedin.
the class ClusterFileLoggingClient method mainFunction.
public void mainFunction(String[] args) throws Exception {
String[] leftOverArgs = processLocalArgs(args);
Properties startupProps = DatabusHttpClientImpl.processCommandLineArgs(leftOverArgs);
DatabusHttpClientImpl.Config clientConfigBuilder = new DatabusHttpClientImpl.Config();
clientConfigBuilder.getContainer().setIdFromName(MODULE + ".localhost");
if (_enableBootStrap) {
clientConfigBuilder.getRuntime().getBootstrap().setEnabled(true);
}
ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>("databus.client.", clientConfigBuilder);
String[] sources = getSources();
StringBuilder sourcesString = new StringBuilder();
boolean firstSrc = true;
for (String source : sources) {
if (!firstSrc)
sourcesString.append(",");
firstSrc = false;
sourcesString.append(source);
}
if (_httpPort != null) {
startupProps.put("databus.client.container.httpPort", _httpPort);
}
if (_jmxServicePort != null) {
startupProps.put("databus.client.container.jmx.jmxServicePort", _jmxServicePort);
}
if (_checkpointFileRootDir != null) {
startupProps.put("databus.client.checkpointPersistence.fileSystem.rootDirectory", _checkpointFileRootDir);
}
DatabusHttpClientImpl.StaticConfig clientConfig = configLoader.loadConfig(startupProps);
// set up relay
ServerInfoBuilder relayBuilder = clientConfig.getRuntime().getRelay("1");
relayBuilder.setName("DefaultRelay");
if (_relayHost != null) {
relayBuilder.setHost(_relayHost);
}
if (_relayPort != null) {
relayBuilder.setPort(Integer.parseInt(_relayPort));
}
relayBuilder.setSources(sourcesString.toString());
// set up bootstrap
if (_enableBootStrap) {
ServerInfoBuilder bootstrapBuilder = clientConfig.getRuntime().getBootstrap().getService("2");
bootstrapBuilder.setName("DefaultBootstrapServices");
if (_bootstrapHost != null) {
bootstrapBuilder.setHost(_bootstrapHost);
}
if (_bootstrapPort != null) {
bootstrapBuilder.setPort(Integer.parseInt(_bootstrapPort));
}
bootstrapBuilder.setSources(sourcesString.toString());
}
// set up listeners
DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConfig);
List<DatabusRegistration> regs = new ArrayList<DatabusRegistration>();
for (String cluster : _clusters) {
DatabusRegistration reg = client.registerCluster(cluster, createConsumerFactory(cluster, _valueDumpFile, _eventDumpFile), createServerSideFactory(cluster), createPartitionListener(cluster), sources);
regs.add(reg);
}
// add pause processor
try {
client.getProcessorRegistry().register(ContainerOperationProcessor.COMMAND_NAME, new ContainerOperationProcessor(null, client));
} catch (ProcessorRegistrationConflictException e) {
LOG.error("Failed to register " + ConsumerPauseRequestProcessor.COMMAND_NAME);
}
DatabusClientShutdownThread shutdownThread = new DatabusClientShutdownThread(client);
Runtime.getRuntime().addShutdownHook(shutdownThread);
client.startAndBlock();
}
use of com.linkedin.databus2.core.container.request.ProcessorRegistrationConflictException in project databus by linkedin.
the class SimpleFileLoggingConsumer method mainFunction.
public void mainFunction(String[] args) throws Exception {
String[] leftOverArgs = processLocalArgs(args);
Properties startupProps = DatabusHttpClientImpl.processCommandLineArgs(leftOverArgs);
DatabusHttpClientImpl.Config clientConfigBuilder = new DatabusHttpClientImpl.Config();
clientConfigBuilder.getContainer().setIdFromName(MODULE + ".localhost");
if (_enableBootStrap) {
clientConfigBuilder.getRuntime().getBootstrap().setEnabled(true);
}
ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>("databus.client.", clientConfigBuilder);
String[] sources = addSources();
StringBuilder sourcesString = new StringBuilder();
boolean firstSrc = true;
for (String source : sources) {
if (!firstSrc)
sourcesString.append(",");
firstSrc = false;
sourcesString.append(source);
}
if (_httpPort != null) {
startupProps.put("databus.client.container.httpPort", _httpPort);
}
if (_jmxServicePort != null) {
startupProps.put("databus.client.container.jmx.jmxServicePort", _jmxServicePort);
}
if (_checkpointFileRootDir != null) {
startupProps.put("databus.client.checkpointPersistence.fileSystem.rootDirectory", _checkpointFileRootDir);
}
DatabusHttpClientImpl.StaticConfig clientConfig = configLoader.loadConfig(startupProps);
// set up relay
ServerInfoBuilder relayBuilder = clientConfig.getRuntime().getRelay("1");
relayBuilder.setName("DefaultRelay");
if (_relayHost != null) {
relayBuilder.setHost(_relayHost);
}
if (_relayPort != null) {
relayBuilder.setPort(Integer.parseInt(_relayPort));
}
relayBuilder.setSources(sourcesString.toString());
// set up bootstrap
if (_enableBootStrap) {
ServerInfoBuilder bootstrapBuilder = clientConfig.getRuntime().getBootstrap().getService("2");
bootstrapBuilder.setName("DefaultBootstrapServices");
if (_bootstrapHost != null) {
bootstrapBuilder.setHost(_bootstrapHost);
}
if (_bootstrapPort != null) {
bootstrapBuilder.setPort(Integer.parseInt(_bootstrapPort));
}
bootstrapBuilder.setSources(sourcesString.toString());
}
// handler server side filtering, can either pass a config file or set from command line
DbusKeyCompositeFilterConfig filterConfig = createServerSideFilterConfig(_filterConfFile, startupProps);
// set up listeners
DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConfig);
// dump decoded payload values and raw (undecoded) events
DatabusFileLoggingConsumer consumer = createTypedConsumer(_valueDumpFile, _eventDumpFile);
if (_eventPattern != null) {
consumer.setEventPattern(_eventPattern);
}
DatabusRegistration reg = client.register(consumer, sources);
if (!(reg instanceof DatabusV2RegistrationImpl)) {
throw new RuntimeException("Unexpected type for registration Object !!");
}
if (null != filterConfig)
reg.withServerSideFilter(filterConfig);
// add pause processor
try {
client.getProcessorRegistry().register(ConsumerPauseRequestProcessor.COMMAND_NAME, new ConsumerPauseRequestProcessor(null, consumer));
client.getProcessorRegistry().register(ContainerOperationProcessor.COMMAND_NAME, new ContainerOperationProcessor(null, client));
} catch (ProcessorRegistrationConflictException e) {
LOG.error("Failed to register " + ConsumerPauseRequestProcessor.COMMAND_NAME);
}
DatabusClientShutdownThread shutdownThread = new DatabusClientShutdownThread(client);
Runtime.getRuntime().addShutdownHook(shutdownThread);
// this should automatically start the registration
client.startAndBlock();
}
Aggregations