Examples with IdNamePair com.linkedin.databus.core.util.IdNamePair used on opensource projects

Example 41 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class RelayPullThread method buildSubsList.

/**
   * Build the string of subs to be sent in the register call.
   * */
private String buildSubsList(List<DatabusSubscription> subs, Map<String, IdNamePair> sourceNameMap) {
    StringBuilder sb = new StringBuilder(128);
    sb.append("[");
    boolean first = true;
    for (DatabusSubscription sub : subs) {
        if (!first)
            sb.append(',');
        DatabusSubscription realSub = sub;
        LogicalSource ls = sub.getLogicalSource();
        if (!ls.idKnown() && !ls.isWildcard()) {
            IdNamePair sourceEntry = sourceNameMap.get(ls.getName());
            if (null == sourceEntry) {
                //this should never happen
                throw new RuntimeException("FATAL! unable to find logical source " + ls.getName() + " in " + sourceNameMap);
            }
            realSub = new DatabusSubscription(sub.getPhysicalSource(), sub.getPhysicalPartition(), new LogicalSourceId(new LogicalSource(sourceEntry.getId().intValue(), ls.getName()), sub.getLogicalPartition().getId()));
        }
        sb.append(realSub.toJsonString());
        first = false;
    }
    sb.append("]");
    return sb.toString();
}

Example 42 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class GenerateDataEventsRequestProcessor method process.

@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException {
    String action = request.getParams().getProperty(DatabusRequest.PATH_PARAM_NAME, "");
    if (action.equals("check")) {
        boolean genRunning = _producer.checkRunning();
        StringBuilder resBuilder = new StringBuilder(1024);
        Formatter fmt = new Formatter(resBuilder);
        fmt.format("{\"genDataEventsRunning\":\"%b\"}", genRunning);
        request.getResponseContent().write(ByteBuffer.wrap(resBuilder.toString().getBytes(Charset.defaultCharset())));
    } else if (action.equals("stop")) {
        _producer.stopGeneration();
        request.getResponseContent().write(ByteBuffer.wrap("{\"genDataEventsRunning\":\"send-stop\"}".getBytes(Charset.defaultCharset())));
    } else if (action.equals("suspend")) {
        _producer.suspendGeneration();
        request.getResponseContent().write(ByteBuffer.wrap("{\"genDataEventsRunning\":\"send-suspend\"}".getBytes(Charset.defaultCharset())));
    } else if (action.equals("resume")) {
        long numEventToGenerate = request.getOptionalLongParam(NUM_EVENTS_TO_GENERATE, Long.MAX_VALUE);
        long keyMin = request.getOptionalLongParam(KEY_MIN_PARAM, 0L);
        long keyMax = request.getOptionalLongParam(KEY_MAX_PARAM, Long.MAX_VALUE);
        int percentOfBufferToGenerate = request.getOptionalIntParam(PERCENT_BUFFER_TO_GENERATE, Integer.MAX_VALUE);
        _producer.resumeGeneration(numEventToGenerate, percentOfBufferToGenerate, keyMin, keyMax);
        request.getResponseContent().write(ByteBuffer.wrap("{\"genDataEventsRunning\":\"send-resume\"}".getBytes(Charset.defaultCharset())));
    } else if (action.equals("start")) {
        long fromScn = request.getRequiredLongParam(SCN_PARAM);
        long durationMs = request.getRequiredLongParam(DURATION_MS);
        int eventsPerSec = request.getRequiredIntParam(EVENTS_PER_SEC_PARAM);
        long numEventToGenerate = request.getOptionalLongParam(NUM_EVENTS_TO_GENERATE, Long.MAX_VALUE);
        int percentOfBufferToGenerate = request.getOptionalIntParam(PERCENT_BUFFER_TO_GENERATE, Integer.MAX_VALUE);
        long keyMin = request.getOptionalLongParam(KEY_MIN_PARAM, 0L);
        long keyMax = request.getOptionalLongParam(KEY_MAX_PARAM, Long.MAX_VALUE);
        String sourcesListStr = request.getRequiredStringParam(SOURCES_NAME_PARAM);
        String[] sourcesStrArray = sourcesListStr.split(",");
        List<IdNamePair> sourcesIdList = new ArrayList<IdNamePair>(sourcesStrArray.length);
        for (String sourceIdStr : sourcesStrArray) {
            try {
                Integer id = Integer.valueOf(sourceIdStr);
                LogicalSource source = _relay.getSourcesIdNameRegistry().getSource(id);
                if (null != source)
                    sourcesIdList.add(source.asIdNamePair());
                else
                    LOG.error("unable to find source id: " + id);
            } catch (NumberFormatException nfe) {
                throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_NAME_PARAM, sourceIdStr);
            }
        }
        //We have to use the global stats collector because the generation can go beyond the lifespan
        //of the connection
        boolean tryStart = _producer.startGeneration(fromScn, eventsPerSec, durationMs, numEventToGenerate, percentOfBufferToGenerate, keyMin, keyMax, sourcesIdList, _relayStatsCollector);
        StringBuilder resBuilder = new StringBuilder(1024);
        Formatter fmt = new Formatter(resBuilder);
        fmt.format("{\"genDataEventsStarted\":\"%b\"}", tryStart);
        request.getResponseContent().write(ByteBuffer.wrap(resBuilder.toString().getBytes(Charset.defaultCharset())));
    } else {
        throw new InvalidRequestParamValueException(COMMAND_NAME, "request path", action);
    }
    return request;
}

Example 43 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class SchemaMetaDataManager method populateSrcNameIdMap.

private void populateSrcNameIdMap() throws IOException {
    int ln = 0;
    File file = new File(_idNameMapFile);
    BufferedReader reader = null;
    try {
        reader = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
        String line = null;
        List<IdNamePair> pairCollection = new ArrayList<IdNamePair>();
        Set<String> srcNames = new HashSet<String>();
        Set<Short> srcIds = new HashSet<Short>();
        while ((line = reader.readLine()) != null) {
            line = line.trim();
            ln++;
            // omit  comments
            if (line.startsWith("#"))
                continue;
            // if line has comments elsewhere, omit comtents to the right of it.
            String[] parts = line.split("#");
            line = parts[0];
            String[] toks = line.split(":");
            if (toks.length != 2)
                throw new RuntimeException("SrcIdToName map file (" + file + ") is corrupted at line number :" + ln);
            short srcId = new Short(toks[0]);
            String srcName = toks[1];
            _maxSrcId = (short) Math.max(_maxSrcId, srcId);
            if (srcIds.contains(srcId)) {
                throw new RuntimeException("Duplicate SrcId (" + srcId + ") present in the file :" + _idNameMapFile);
            } else if (srcNames.contains(srcName)) {
                throw new RuntimeException("Duplicate Source name (" + srcName + ") present in the file :" + _idNameMapFile);
            }
            srcIds.add(srcId);
            srcNames.add(srcName);
            pairCollection.add(new IdNamePair(Long.valueOf(srcId), srcName));
        }
        _idNameRegistry.updateFromIdNamePairs(pairCollection);
    } finally {
        if (null != reader)
            reader.close();
    }
}

Example 44 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class TestRelayCommandsLocal method doTestSources2Command.

private void doTestSources2Command() throws Exception {
    HttpRequest httpRequest = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/sources");
    SimpleTestHttpClient httpClient = SimpleTestHttpClient.createLocal(TimeoutPolicy.ALL_TIMEOUTS);
    SimpleHttpResponseHandler respHandler = httpClient.sendRequest(_serverAddress, httpRequest);
    assertTrue("failed to get a response", respHandler.awaitResponseUninterruptedly(1, TimeUnit.SECONDS));
    ByteArrayInputStream in = new ByteArrayInputStream(respHandler.getReceivedBytes());
    ObjectMapper objMapper = new ObjectMapper();
    List<IdNamePair> res = objMapper.readValue(in, new TypeReference<List<IdNamePair>>() {
    });
    assertNotNull("no result", res);
    if (LOG.isDebugEnabled()) {
        LOG.debug("/sources response:" + new String(respHandler.getReceivedBytes()));
    }
    HashSet<IdNamePair> origSet = new HashSet<IdNamePair>(_staticConfig.getSourceIds());
    HashSet<IdNamePair> resSet = new HashSet<IdNamePair>(res);
    Assert.assertEquals(origSet, resSet);
}

Example 45 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class TestGenericDispatcher method testTwoWindowEventCallbackFailure.

@Test(groups = { "small", "functional" })
public void testTwoWindowEventCallbackFailure() {
    final Logger log = Logger.getLogger("TestGenericDispatcher.testTwoWindowEventCallbackFailure");
    log.info("start");
    int source1EventsNum = 2;
    int source2EventsNum = 2;
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();
    final TestGenericDispatcherEventBuffer eventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short) 1, keyCounts, srcidCounts);
    eventsBuf.endEvents(100L);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short) 2, keyCounts, srcidCounts);
    eventsBuf.endEvents(200L);
    DatabusStreamConsumer mockConsumer = new EventCountingConsumer(new StateVerifyingStreamConsumer(new DataEventFailingStreamConsumer((short) 2)), keyCounts, srcidCounts);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newSingleThreadExecutor(), 1000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
    RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), eventsBuf, callback, null, null, null, null, null);
    Thread dispatcherThread = new Thread(dispatcher);
    //dispatcherThread.setDaemon(true);
    dispatcherThread.start();
    HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
    List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
    l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
    l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
    l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
    schemaMap.put(1L, l1);
    schemaMap.put(2L, l2);
    schemaMap.put(3L, l3);
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
    dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
    try {
        Thread.sleep(2000);
    } catch (InterruptedException ie) {
    }
    dispatcher.shutdown();
    for (long i = 1; i <= source1EventsNum; ++i) {
        assertEquals("correct amount of callbacks for key " + i, 1, keyCounts.get(i).intValue());
    }
    for (long i = source2EventsNum + 1; i <= source1EventsNum + source2EventsNum; ++i) {
        assert keyCounts.get(1L + source1EventsNum).intValue() > 1 : "correct amount of callbacks for key " + i + ":" + keyCounts.get(i).intValue();
    }
    verifyNoLocks(null, eventsBuf);
    log.info("end\n");
}