Examples with IdNamePair com.linkedin.databus.core.util.IdNamePair used on opensource projects

Example 6 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class TestRelayCommandsLocal method testSources1Command.

@Test
public void testSources1Command() throws Exception {
    LOG.debug("\n\nstarting testSources1Command()\n");
    HttpRequest httpRequest = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/sources");
    SimpleTestHttpClient httpClient = SimpleTestHttpClient.createLocal(TimeoutPolicy.ALL_TIMEOUTS);
    SimpleHttpResponseHandler respHandler = httpClient.sendRequest(_serverAddress, httpRequest);
    assertTrue("failed to get a response", respHandler.awaitResponseUninterruptedly(1, TimeUnit.SECONDS));
    ByteArrayInputStream in = new ByteArrayInputStream(respHandler.getReceivedBytes());
    ObjectMapper objMapper = new ObjectMapper();
    List<IdNamePair> res = objMapper.readValue(in, new TypeReference<List<IdNamePair>>() {
    });
    assertNotNull("no result", res);
    if (LOG.isDebugEnabled()) {
        LOG.debug("/sources response:" + new String(respHandler.getReceivedBytes()));
    }
    HashSet<IdNamePair> origSet = new HashSet<IdNamePair>(_staticConfig.getSourceIds());
    HashSet<IdNamePair> resSet = new HashSet<IdNamePair>(res);
    Assert.assertEquals(origSet, resSet);
}

Example 7 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class GenericDispatcher method doCheckStartSource.

protected boolean doCheckStartSource(DispatcherState curState, Long eventSrcId, SchemaId schemaId) {
    boolean success = true;
    if (eventSrcId >= 0) {
        IdNamePair source = curState.getSources().get(eventSrcId);
        if (null == source) {
            _log.error("Unable to find source: srcid=" + eventSrcId);
            success = false;
        } else {
            VersionedSchema verSchema = curState.getSchemaSet().getLatestVersionByName(source.getName());
            VersionedSchema exactSchema = _schemaIdCheck ? curState.getSchemaSet().getById(schemaId) : null;
            if (null == verSchema) {
                _log.error("Unable to find schema: srcid=" + source.getId() + " name=" + source.getName());
                success = false;
            } else if (_schemaIdCheck && null == exactSchema) {
                _log.error("Unable to find schema: srcid=" + source.getId() + " name=" + source.getName() + " schemaId=" + schemaId);
                success = false;
            } else if (verSchema.getSchema() != curState.getCurrentSourceSchema()) {
                curState.switchToStartStreamSource(source, verSchema.getSchema());
                success = doStartStreamSource(curState);
            }
        }
    }
    return success;
}

Example 8 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class RelayPullThread method doSourcesResponseSuccess.

protected void doSourcesResponseSuccess(ConnectionState curState) {
    mergeRelayCallsStats();
    Map<String, IdNamePair> sourceNameMap = curState.getSourcesNameMap();
    StringBuilder sb = new StringBuilder();
    boolean firstSource = true;
    boolean error = false;
    List<IdNamePair> sourcesList = new ArrayList<IdNamePair>(_sourcesConn.getSourcesNames().size());
    if (curState.getRelayConnection().getProtocolVersion() < 3) {
        // we pass a list of subscriptions.
        for (String sourceName : _sourcesConn.getSourcesNames()) {
            IdNamePair source = sourceNameMap.get(sourceName);
            if (null == source) {
                _log.error("Source not found on server: " + sourceName);
                error = true;
                break;
            } else {
                if (!firstSource)
                    sb.append(',');
                sb.append(source.getId().toString());
                firstSource = false;
                sourcesList.add(source);
            }
        }
    }
    String sourcesIdList = sb.toString();
    String subsString = error ? "ERROR" : buildSubsList(curState.getSubscriptions(), sourceNameMap);
    if (_log.isDebugEnabled()) {
        _log.debug("Source ids: " + sourcesIdList);
        _log.debug("Subs : " + subsString);
    }
    _sourcesConn.getRelayDispatcher().enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(curState.getSources(), sourcesIdList));
    if (toTearConnAfterHandlingResponse()) {
        tearConnectionAndEnqueuePickServer();
    } else {
        if (error) {
            curState.switchToPickServer();
        } else {
            curState.switchToRequestSourcesSchemas(sourcesIdList, subsString);
            if (_sourcesConn.isBootstrapEnabled()) {
                _sourcesConn.getBootstrapPuller().enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(curState.getSources(), curState.getSourcesIdListString()));
            }
            String hostHdr = curState.getHostName(), svcHdr = curState.getSvcName();
            _log.info("Connected to relay " + hostHdr + " with a service identifier " + svcHdr);
        }
        enqueueMessage(curState);
    }
}

Example 9 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class RelayPullThread method doRequestStream.

protected void doRequestStream(ConnectionState curState) {
    boolean debugEnabled = _log.isDebugEnabled();
    if (debugEnabled)
        _log.debug("Checking for free space in buffer");
    int freeBufferThreshold = (int) (_sourcesConn.getConnectionConfig().getFreeBufferThreshold() * 100.0 / _pullerBufferUtilizationPct);
    try {
        curState.getDataEventsBuffer().waitForFreeSpace(freeBufferThreshold);
    } catch (InterruptedException ie) {
        //loop
        enqueueMessage(curState);
        return;
    }
    Checkpoint cp = curState.getCheckpoint();
    if (debugEnabled)
        _log.debug("Checkpoint at RequestDataEvents: " + cp.toString());
    if (null == _relayFilter) {
        if (debugEnabled)
            _log.debug("Initializing relay filter config");
        _relayFilter = new DbusKeyCompositeFilter();
        Map<String, IdNamePair> srcNameIdMap = curState.getSourcesNameMap();
        for (DbusKeyCompositeFilterConfig conf : _relayFilterConfigs) {
            Map<String, KeyFilterConfigHolder> cMap = conf.getConfigMap();
            Map<Long, KeyFilterConfigHolder> fConfMap = new HashMap<Long, KeyFilterConfigHolder>();
            for (Entry<String, KeyFilterConfigHolder> e : cMap.entrySet()) {
                IdNamePair idName = srcNameIdMap.get(e.getKey());
                if (null != idName) {
                    fConfMap.put(idName.getId(), e.getValue());
                }
            }
            if (debugEnabled)
                _log.debug("FilterConfMap is :" + fConfMap);
            _relayFilter.merge(new DbusKeyCompositeFilter(fConfMap));
        }
        if (debugEnabled)
            _log.debug("Merged Filter (before deduping) is :" + _relayFilter);
        _relayFilter.dedupe();
        if (debugEnabled)
            _log.debug("Merged Filter (after deduping) is :" + _relayFilter);
    }
    _streamCallStartMs = System.currentTimeMillis();
    if (null != _relayCallsStats)
        _relayCallsStats.registerStreamRequest(cp, EMPTY_STREAM_LIST);
    int fetchSize = (int) ((curState.getDataEventsBuffer().getBufferFreeReadSpace() / 100.0) * _pullerBufferUtilizationPct);
    fetchSize = Math.max(freeBufferThreshold, fetchSize);
    CheckpointMult cpMult = new CheckpointMult();
    String args;
    if (curState.getRelayConnection().getProtocolVersion() >= 3) {
        // for version 3 and higher we pass subscriptions
        args = curState.getSubsListString();
        for (DatabusSubscription sub : curState.getSubscriptions()) {
            PhysicalPartition p = sub.getPhysicalPartition();
            cpMult.addCheckpoint(p, cp);
        }
    } else {
        args = curState.getSourcesIdListString();
        cpMult.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
    }
    curState.switchToStreamRequestSent();
    sendHeartbeat(_sourcesConn.getUnifiedClientStats());
    curState.getRelayConnection().requestStream(args, _relayFilter, fetchSize, cpMult, _sourcesConn.getConnectionConfig().getKeyRange(), curState);
}

Example 10 with IdNamePair

use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.

the class TestMultiConsumerCallback method test1StreamConsumerHappyPath.

@Test(groups = { "small", "functional" })
public void test1StreamConsumerHappyPath() {
    LOG.info("\n\nstarting test1StreamConsumerHappyPath()");
    Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
    DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
    eventsBuf.start(0);
    eventsBuf.startEvents();
    initBufferWithEvents(eventsBuf, 1, 1, (short) 1, keyCounts);
    initBufferWithEvents(eventsBuf, 2, 2, (short) 3, keyCounts);
    eventsBuf.endEvents(100L);
    DatabusStreamConsumer mockConsumer = EasyMock.createStrictMock(DatabusStreamConsumer.class);
    SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
    List<String> sources = new ArrayList<String>();
    Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
    for (int i = 1; i <= 3; ++i) {
        IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
        sources.add(sourcePair.getName());
        sourcesMap.put(sourcePair.getId(), sourcePair);
    }
    DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
    List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
    MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 60000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
    callback.setSourceMap(sourcesMap);
    DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
    assert iter.hasNext() : "unable to read event";
    DbusEvent event1 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event2 = iter.next();
    assert iter.hasNext() : "unable to read event";
    DbusEvent event3 = iter.next();
    initMockStreamConsumer3EventFullLifecycle(mockConsumer, event1, event2, event3, keyCounts);
    assert3EventFullLifecycle(callback, event1, event2, event3);
    EasyMock.verify(mockConsumer);
    assert keyCounts.get(1L).get() == 1 : "invalid number of event(1) calls: " + keyCounts.get(1L).get();
    assert keyCounts.get(2L).get() == 1 : "invalid number of event(2) calls:" + keyCounts.get(2L).get();
    assert keyCounts.get(3L).get() == 1 : "invalid number of event(3) calls:" + keyCounts.get(3L).get();
}