Examples with IdNamePair com.linkedin.databus.core.util.IdNamePair used on opensource projects

Example 36 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class SchemaMetaDataManager method updateAndGetNewSrcId.

public short updateAndGetNewSrcId(String dbName, String srcName) throws DatabusException {
    _maxSrcId++;
    dbName = dbName.toLowerCase(Locale.ENGLISH);
    if (null != _idNameRegistry.getSource(srcName))
        throw new DatabusException("Source Name (" + srcName + ") already available in the schema registry !!");
    TreeSet<String> s = null;
    if (_physicalToLogicalSrcMap.containsKey(dbName)) {
        s = _physicalToLogicalSrcMap.get(dbName);
    } else {
        s = new TreeSet<String>();
        _physicalToLogicalSrcMap.put(dbName, s);
    }
    s.add(srcName);
    IdNamePair pair = new IdNamePair(Long.valueOf(_maxSrcId), srcName);
    LogicalSource src = new LogicalSource(pair);
    List<LogicalSource> newSources = new ArrayList<LogicalSource>();
    newSources.add(src);
    _idNameRegistry.add(newSources);
    return _maxSrcId;
}

Example 37 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class HttpRelay method main.

public static void main(String[] args) throws Exception {
    Cli cli = new Cli();
    cli.processCommandLineArgs(args);
    cli.parseRelayConfig();
    StaticConfig staticConfig = cli.getRelayConfigBuilder().build();
    HttpRelay relay = new HttpRelay(staticConfig, cli.getPhysicalSourceStaticConfigs());
    RequestProcessorRegistry processorRegistry = relay.getProcessorRegistry();
    //Changes to add schemaId to event; DDSDBUS-3421
    //The long term fix is to remove DatabusEventRandomProducer in favour of RelayEventGenerator
    //The medium term fix is to send SchemaRegistry to DatabusEventRandomProducer, but move RandomProducer to databus-relay-impl (from databus-core-impl)
    //Reason: SchemaHelper classes required to parse/generate schemaId from schemaRegistry requires databus-schemas-core which depends on databus-core-impl
    SchemaRegistryService sr = relay.getSchemaRegistryService();
    HashMap<Long, byte[]> schemaIds = new HashMap<Long, byte[]>(staticConfig.getSourceIds().size());
    for (IdNamePair pair : staticConfig.getSourceIds()) {
        LOG.info("Http Relay Schema Reg:" + pair.getName() + " id=" + pair.getId());
        String schemaStr = sr.fetchLatestSchemaBySourceName(pair.getName());
        if (schemaStr != null) {
            Schema s = Schema.parse(schemaStr);
            byte[] sid = SchemaHelper.getSchemaId(s.toString());
            LOG.info("Found schema! Adding schemaId for sourceName=" + pair.getName() + " id=" + pair.getId() + " schemaId=" + sid);
            schemaIds.put(pair.getId(), sid);
        } else {
            byte[] defaultSid = "abcde".getBytes(Charset.defaultCharset());
            LOG.info("Didn't find schema! Adding default schemaId for sourceName=" + pair.getName() + "id=" + pair.getId() + " schemaId=" + defaultSid);
            schemaIds.put(pair.getId(), defaultSid);
        }
    }
    DatabusEventProducer randomEventProducer = new DatabusEventRandomProducer(relay.getEventBuffer(), 10, 100, 1000, staticConfig.getSourceIds(), schemaIds);
    // specify stats collector for this producer
    ((DatabusEventRandomProducer) randomEventProducer).setStatsCollector(relay.getInboundEventStatisticsCollector());
    processorRegistry.register(EchoRequestProcessor.COMMAND_NAME, new EchoRequestProcessor(null));
    processorRegistry.register(SleepRequestProcessor.COMMAND_NAME, new SleepRequestProcessor(null));
    processorRegistry.register(GenerateDataEventsRequestProcessor.COMMAND_NAME, new GenerateDataEventsRequestProcessor(null, relay, randomEventProducer));
    processorRegistry.register(LoadDataEventsRequestProcessor.COMMAND_NAME, new LoadDataEventsRequestProcessor(relay.getDefaultExecutorService(), relay));
    LOG.info("source = " + relay.getSourcesIdNameRegistry().getAllSources());
    try {
        relay.registerShutdownHook();
        relay.startAndBlock();
    } catch (Exception e) {
        LOG.error("Error starting the relay", e);
    }
    LOG.info("Exiting relay");
}

Example 38 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class SourcesRequestProcessor method process.

@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException {
    int protoVersion = request.getOptionalIntParam(VERSION_PARAM_NAME, 1);
    ObjectMapper mapper = new ObjectMapper();
    StringWriter out = new StringWriter(10240);
    Collection<LogicalSource> sources = _relay.getSourcesIdNameRegistry().getAllSources();
    if (1 == protoVersion) {
        ArrayList<IdNamePair> sourcePairs = new ArrayList<IdNamePair>(sources.size());
        for (LogicalSource source : sources) sourcePairs.add(new IdNamePair(source.getId().longValue(), source.getName()));
        mapper.writeValue(out, sourcePairs);
    } else if (2 == protoVersion)
        mapper.writeValue(out, sources);
    else
        throw new InvalidRequestParamValueException(COMMAND_NAME, VERSION_PARAM_NAME, Integer.toString(protoVersion));
    byte[] resultBytes = out.toString().getBytes(Charset.defaultCharset());
    request.getResponseContent().write(ByteBuffer.wrap(resultBytes));
    HttpStatisticsCollector relayStatsCollector = _relay.getHttpStatisticsCollector();
    if (null != relayStatsCollector) {
        HttpStatisticsCollector connStatsCollector = (HttpStatisticsCollector) request.getParams().get(relayStatsCollector.getName());
        if (null != connStatsCollector) {
            connStatsCollector.registerSourcesCall();
        } else {
            relayStatsCollector.registerSourcesCall();
        }
    }
    return request;
}

Example 39 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class TestRelayCommandsLocal method testSourcesTrackingCommand.

@Test
public void testSourcesTrackingCommand() throws Exception {
    LOG.debug("\n\nstarting testSourcesTrackingCommand()\n");
    HttpRequest httpRequest = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/sources");
    httpRequest.setHeader(DatabusHttpHeaders.DBUS_CLIENT_HOST_HDR, "localhost");
    httpRequest.setHeader(DatabusHttpHeaders.DBUS_CLIENT_SERVICE_HDR, "unittestclient");
    SimpleTestHttpClient httpClient = SimpleTestHttpClient.createLocal(TimeoutPolicy.ALL_TIMEOUTS);
    SimpleHttpResponseHandler respHandler = httpClient.sendRequest(_serverAddress, httpRequest);
    assertTrue("failed to get a response", respHandler.awaitResponseUninterruptedly(1, TimeUnit.SECONDS));
    ByteArrayInputStream in = new ByteArrayInputStream(respHandler.getReceivedBytes());
    ObjectMapper objMapper = new ObjectMapper();
    List<IdNamePair> res = objMapper.readValue(in, new TypeReference<List<IdNamePair>>() {
    });
    assertNotNull("no result", res);
    if (LOG.isDebugEnabled()) {
        LOG.debug("/sources response:" + new String(respHandler.getReceivedBytes()));
    }
    HashSet<IdNamePair> origSet = new HashSet<IdNamePair>(_staticConfig.getSourceIds());
    HashSet<IdNamePair> resSet = new HashSet<IdNamePair>(res);
    Assert.assertEquals(origSet, resSet);
}

Example 40 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class TestRelayCommandsLocal method testSources1Command.

@Test
public void testSources1Command() throws Exception {
    LOG.debug("\n\nstarting testSources1Command()\n");
    HttpRequest httpRequest = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/sources");
    SimpleTestHttpClient httpClient = SimpleTestHttpClient.createLocal(TimeoutPolicy.ALL_TIMEOUTS);
    SimpleHttpResponseHandler respHandler = httpClient.sendRequest(_serverAddress, httpRequest);
    assertTrue("failed to get a response", respHandler.awaitResponseUninterruptedly(1, TimeUnit.SECONDS));
    ByteArrayInputStream in = new ByteArrayInputStream(respHandler.getReceivedBytes());
    ObjectMapper objMapper = new ObjectMapper();
    List<IdNamePair> res = objMapper.readValue(in, new TypeReference<List<IdNamePair>>() {
    });
    assertNotNull("no result", res);
    if (LOG.isDebugEnabled()) {
        LOG.debug("/sources response:" + new String(respHandler.getReceivedBytes()));
    }
    HashSet<IdNamePair> origSet = new HashSet<IdNamePair>(_staticConfig.getSourceIds());
    HashSet<IdNamePair> resSet = new HashSet<IdNamePair>(res);
    Assert.assertEquals(origSet, resSet);
}