Examples with DbusKeyCompositeFilterConfig com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig used on opensource projects

Example 1 with DbusKeyCompositeFilterConfig

use of com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig in project databus by linkedin.

the class DbusModPartitionedFilterFactory method createServerSideFilter.

@Override
public DbusKeyCompositeFilterConfig createServerSideFilter(DbusClusterInfo cluster, DbusPartitionInfo partition) throws InvalidConfigException {
    DbusKeyCompositeFilterConfig.Config compositeConfig = new DbusKeyCompositeFilterConfig.Config();
    for (DatabusSubscription s : _subscriptions) {
        KeyFilterConfigHolder.Config filterConfig = new KeyFilterConfigHolder.Config();
        filterConfig.setType(PartitionType.MOD.toString());
        filterConfig.getMod().setNumBuckets(cluster.getNumTotalPartitions());
        filterConfig.getMod().setBuckets("[" + partition.getPartitionId() + "]");
        compositeConfig.setFilter(s.getLogicalSource().getName(), filterConfig);
    }
    DbusKeyCompositeFilterConfig c = new DbusKeyCompositeFilterConfig(compositeConfig.build());
    LOG.info("Generated Mod Partitioned Config for partition (" + partition + ") of cluster (" + cluster + ") is :" + c);
    return c;
}

Example 2 with DbusKeyCompositeFilterConfig

use of com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig in project databus by linkedin.

the class RelayPullThread method doRequestStream.

protected void doRequestStream(ConnectionState curState) {
    boolean debugEnabled = _log.isDebugEnabled();
    if (debugEnabled)
        _log.debug("Checking for free space in buffer");
    int freeBufferThreshold = (int) (_sourcesConn.getConnectionConfig().getFreeBufferThreshold() * 100.0 / _pullerBufferUtilizationPct);
    try {
        curState.getDataEventsBuffer().waitForFreeSpace(freeBufferThreshold);
    } catch (InterruptedException ie) {
        //loop
        enqueueMessage(curState);
        return;
    }
    Checkpoint cp = curState.getCheckpoint();
    if (debugEnabled)
        _log.debug("Checkpoint at RequestDataEvents: " + cp.toString());
    if (null == _relayFilter) {
        if (debugEnabled)
            _log.debug("Initializing relay filter config");
        _relayFilter = new DbusKeyCompositeFilter();
        Map<String, IdNamePair> srcNameIdMap = curState.getSourcesNameMap();
        for (DbusKeyCompositeFilterConfig conf : _relayFilterConfigs) {
            Map<String, KeyFilterConfigHolder> cMap = conf.getConfigMap();
            Map<Long, KeyFilterConfigHolder> fConfMap = new HashMap<Long, KeyFilterConfigHolder>();
            for (Entry<String, KeyFilterConfigHolder> e : cMap.entrySet()) {
                IdNamePair idName = srcNameIdMap.get(e.getKey());
                if (null != idName) {
                    fConfMap.put(idName.getId(), e.getValue());
                }
            }
            if (debugEnabled)
                _log.debug("FilterConfMap is :" + fConfMap);
            _relayFilter.merge(new DbusKeyCompositeFilter(fConfMap));
        }
        if (debugEnabled)
            _log.debug("Merged Filter (before deduping) is :" + _relayFilter);
        _relayFilter.dedupe();
        if (debugEnabled)
            _log.debug("Merged Filter (after deduping) is :" + _relayFilter);
    }
    _streamCallStartMs = System.currentTimeMillis();
    if (null != _relayCallsStats)
        _relayCallsStats.registerStreamRequest(cp, EMPTY_STREAM_LIST);
    int fetchSize = (int) ((curState.getDataEventsBuffer().getBufferFreeReadSpace() / 100.0) * _pullerBufferUtilizationPct);
    fetchSize = Math.max(freeBufferThreshold, fetchSize);
    CheckpointMult cpMult = new CheckpointMult();
    String args;
    if (curState.getRelayConnection().getProtocolVersion() >= 3) {
        // for version 3 and higher we pass subscriptions
        args = curState.getSubsListString();
        for (DatabusSubscription sub : curState.getSubscriptions()) {
            PhysicalPartition p = sub.getPhysicalPartition();
            cpMult.addCheckpoint(p, cp);
        }
    } else {
        args = curState.getSourcesIdListString();
        cpMult.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
    }
    curState.switchToStreamRequestSent();
    sendHeartbeat(_sourcesConn.getUnifiedClientStats());
    curState.getRelayConnection().requestStream(args, _relayFilter, fetchSize, cpMult, _sourcesConn.getConnectionConfig().getKeyRange(), curState);
}

Example 3 with DbusKeyCompositeFilterConfig

use of com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig in project databus by linkedin.

the class BootstrapPullThread method doRequestBootstrapStream.

protected void doRequestBootstrapStream(ConnectionState curState) {
    boolean debugEnabled = _log.isDebugEnabled();
    if (debugEnabled)
        _log.debug("Checking for free space");
    //curState.getDataEventsBuffer().waitForFreeSpace(FREE_BUFFER_THRESHOLD);
    int freeBufferThreshold = (int) (_sourcesConn.getConnectionConfig().getFreeBufferThreshold() * 100.0 / _pullerBufferUtilizationPct);
    int freeSpace = curState.getDataEventsBuffer().getBufferFreeReadSpace();
    if (freeSpace >= freeBufferThreshold) {
        Checkpoint cp = curState.getCheckpoint();
        if (debugEnabled)
            _log.debug("Checkpoint at RequestBootstrapData: " + cp.toString());
        _log.debug("Sending /bootstrap request");
        Map<String, IdNamePair> srcNameMap = curState.getSourcesNameMap();
        String curSrcName = null;
        if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT) {
            curSrcName = cp.getSnapshotSource();
        } else {
            curSrcName = cp.getCatchupSource();
        }
        if (null == _bootstrapFilter) {
            _bootstrapFilter = new DbusKeyCompositeFilter();
            Map<String, IdNamePair> srcNameIdMap = curState.getSourcesNameMap();
            for (DbusKeyCompositeFilterConfig conf : _bootstrapFilterConfigs) {
                Map<String, KeyFilterConfigHolder> cMap = conf.getConfigMap();
                Map<Long, KeyFilterConfigHolder> fConfMap = new HashMap<Long, KeyFilterConfigHolder>();
                for (Entry<String, KeyFilterConfigHolder> e : cMap.entrySet()) {
                    IdNamePair idName = srcNameIdMap.get(e.getKey());
                    if (null != idName) {
                        fConfMap.put(idName.getId(), e.getValue());
                    }
                }
                _bootstrapFilter.merge(new DbusKeyCompositeFilter(fConfMap));
            }
            _bootstrapFilter.dedupe();
        }
        DbusKeyFilter filter = null;
        IdNamePair srcEntry = srcNameMap.get(curSrcName);
        if (null != srcEntry) {
            Map<Long, DbusKeyFilter> fMap = _bootstrapFilter.getFilterMap();
            if (null != fMap)
                filter = fMap.get(srcEntry.getId());
        }
        int fetchSize = (int) ((curState.getDataEventsBuffer().getBufferFreeReadSpace() / 100.0) * _pullerBufferUtilizationPct);
        fetchSize = Math.max(freeBufferThreshold, fetchSize);
        curState.switchToStreamRequestSent();
        sendHeartbeat(_sourcesConn.getUnifiedClientStats());
        curState.getBootstrapConnection().requestStream(curState.getSourcesIdListString(), filter, fetchSize, cp, curState);
    } else {
        try {
            Thread.sleep(50);
        } catch (InterruptedException ie) {
        }
        enqueueMessage(curState);
    }
}

Example 4 with DbusKeyCompositeFilterConfig

use of com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig in project databus by linkedin.

the class MockBootstrapConnection method testBootstrapPendingEvent.

// Make sure that we suspend on error when we get the x-dbus-pending-size header with a size that is
// larger than our dbusevent size.
@Test
public void testBootstrapPendingEvent() throws Exception {
    List<String> sources = Arrays.asList("source1");
    Properties clientProps = new Properties();
    clientProps.setProperty("client.container.httpPort", "0");
    clientProps.setProperty("client.container.jmx.rmiEnabled", "false");
    clientProps.setProperty("client.runtime.bootstrap.enabled", "true");
    clientProps.setProperty("client.runtime.bootstrap.service(1).name", "bs1");
    clientProps.setProperty("client.runtime.bootstrap.service(1).host", "localhost");
    clientProps.setProperty("client.runtime.bootstrap.service(1).port", "10001");
    clientProps.setProperty("client.runtime.bootstrap.service(1).sources", "source1");
    clientProps.setProperty("client.runtime.relay(1).name", "relay1");
    clientProps.setProperty("client.runtime.relay(1).port", "10001");
    clientProps.setProperty("client.runtime.relay(1).sources", "source1");
    clientProps.setProperty("client.connectionDefaults.eventBuffer.maxSize", "100000");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.maxRetryNum", "3");
    DatabusHttpClientImpl.Config clientConfBuilder = new DatabusHttpClientImpl.Config();
    ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>("client.", clientConfBuilder);
    configLoader.loadConfig(clientProps);
    DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build();
    DatabusSourcesConnection.StaticConfig srcConnConf = clientConf.getConnectionDefaults();
    DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConf);
    client.registerDatabusBootstrapListener(new LoggingConsumer(), null, "source1");
    Assert.assertNotNull(client, "client instantiation failed");
    DatabusHttpClientImpl.RuntimeConfig clientRtConf = clientConf.getRuntime().build();
    //we keep the index of the next server we expect to see
    AtomicInteger serverIdx = new AtomicInteger(-1);
    List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
    sourcesResponse.add(new IdNamePair(1L, "source1"));
    Map<Long, List<RegisterResponseEntry>> registerResponse = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> regResponse = new ArrayList<RegisterResponseEntry>();
    regResponse.add(new RegisterResponseEntry(1L, (short) 1, SCHEMA$.toString()));
    registerResponse.put(1L, regResponse);
    ChunkedBodyReadableByteChannel channel = EasyMock.createMock(ChunkedBodyReadableByteChannel.class);
    // getting the pending-event-size header is called twice, once for checking and once for logging.
    EasyMock.expect(channel.getMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE)).andReturn("1000000").times(2);
    EasyMock.expect(channel.getMetadata("x-dbus-error-cause")).andReturn(null).times(2);
    EasyMock.expect(channel.getMetadata("x-dbus-error")).andReturn(null).times(2);
    EasyMock.replay(channel);
    DbusEventBuffer dbusBuffer = EasyMock.createMock(DbusEventBuffer.class);
    dbusBuffer.endEvents(false, -1, false, false, null);
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(dbusBuffer.injectEvent(EasyMock.<DbusEventInternalReadable>notNull())).andReturn(true).anyTimes();
    EasyMock.expect(dbusBuffer.getEventSerializationVersion()).andReturn(DbusEventFactory.DBUS_EVENT_V1).anyTimes();
    EasyMock.expect(dbusBuffer.getMaxReadBufferCapacity()).andReturn(600).times(2);
    EasyMock.expect(dbusBuffer.getBufferFreeReadSpace()).andReturn(600000).times(2);
    EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull())).andReturn(1).times(1);
    EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull(), EasyMock.<List<InternalDatabusEventsListener>>notNull(), EasyMock.<DbusEventsStatisticsCollector>isNull())).andReturn(0).times(1);
    EasyMock.replay(dbusBuffer);
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sources);
    //This guy succeeds on /sources but fails on /register
    MockBootstrapConnection mockSuccessConn = new MockBootstrapConnection(10, 10, channel, serverIdx, false);
    DatabusBootstrapConnectionFactory mockConnFactory = org.easymock.EasyMock.createMock("mockRelayFactory", DatabusBootstrapConnectionFactory.class);
    //each server should be tried MAX_RETRIES time until all retries are exhausted
    EasyMock.expect(mockConnFactory.createConnection(EasyMock.<ServerInfo>notNull(), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockSuccessConn).anyTimes();
    List<DatabusSubscription> sourcesSubList = DatabusSubscription.createSubscriptionList(sources);
    DatabusSourcesConnection sourcesConn2 = EasyMock.createMock(DatabusSourcesConnection.class);
    EasyMock.expect(sourcesConn2.getSourcesNames()).andReturn(Arrays.asList("source1")).anyTimes();
    EasyMock.expect(sourcesConn2.getSubscriptions()).andReturn(sourcesSubList).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionConfig()).andReturn(srcConnConf).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionStatus()).andReturn(new DatabusComponentStatus("dummy")).anyTimes();
    EasyMock.expect(sourcesConn2.getLocalRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getUnifiedClientStats()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapConnFactory()).andReturn(mockConnFactory).anyTimes();
    EasyMock.expect(sourcesConn2.loadPersistentCheckpoint()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getDataEventsBuffer()).andReturn(dbusBuffer).anyTimes();
    EasyMock.expect(sourcesConn2.isBootstrapEnabled()).andReturn(true).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapRegistrations()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapServices()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapEventsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.makeThreadSafe(mockConnFactory, true);
    EasyMock.makeThreadSafe(sourcesConn2, true);
    EasyMock.replay(mockConnFactory);
    EasyMock.replay(sourcesConn2);
    BootstrapPullThread bsPuller = new BootstrapPullThread("RelayPuller", sourcesConn2, dbusBuffer, connStateFactory, clientRtConf.getBootstrap().getServicesSet(), new ArrayList<DbusKeyCompositeFilterConfig>(), clientConf.getPullerBufferUtilizationPct(), ManagementFactory.getPlatformMBeanServer(), new DbusEventV2Factory(), null, null);
    mockSuccessConn.setCallback(bsPuller);
    bsPuller.getComponentStatus().start();
    Checkpoint cp = _ckptHandlerSource1.createInitialBootstrapCheckpoint(null, 0L);
    //TODO remove
    //cp.setSnapshotSource("source1");
    //cp.setCatchupSource("source1");
    //cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    ConnectionState connState = bsPuller.getConnectionState();
    connState.switchToBootstrap(cp);
    testTransitionCase(bsPuller, StateId.BOOTSTRAP, StateId.REQUEST_START_SCN, cp);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.REQUEST_START_SCN, StateId.START_SCN_RESPONSE_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    Map<Long, List<RegisterResponseEntry>> entries = new HashMap<Long, List<RegisterResponseEntry>>();
    entries.put(1L, new ArrayList<RegisterResponseEntry>());
    connState.setSourcesSchemas(entries);
    connState.setCurrentBSServerInfo(bsPuller.getCurentServer());
    testTransitionCase(bsPuller, StateId.START_SCN_RESPONSE_SUCCESS, StateId.REQUEST_STREAM, null);
    bsPuller.getMessageQueue().clear();
    connState.getSourcesNameMap().put("source1", new IdNamePair(1L, "source1"));
    connState.getSourceIdMap().put(1L, new IdNamePair(1L, "source1"));
    testTransitionCase(bsPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_REQUEST_SUCCESS, "SUSPEND_ON_ERROR", null);
    EasyMock.verify(channel);
    EasyMock.verify(sourcesConn2);
    EasyMock.verify(dbusBuffer);
    EasyMock.verify(channel);
    EasyMock.verify(mockConnFactory);
}

Example 5 with DbusKeyCompositeFilterConfig

use of com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig in project databus by linkedin.

the class DatabusV2ClusterRegistrationImpl method activateOnePartition.

/**
   * Callback to activate one partition that was added
   * @param partition
   * @throws DatabusException
   */
private synchronized void activateOnePartition(DbusPartitionInfo partition) throws DatabusClientException {
    _log.info("Trying to activate partition :" + partition);
    try {
        if (regMap.containsKey(partition)) {
            _log.info("Partition (" + partition + ") is already added and is currently in state : " + regMap.get(partition).getState() + " skipping !!");
            return;
        }
        // Call factories to get consumer callbacks and server-side filter config.
        Collection<DatabusCombinedConsumer> consumers = _consumerFactory.createPartitionedConsumers(_clusterInfo, partition);
        DbusKeyCompositeFilterConfig filterConfig = null;
        if (_serverSideFilterFactory != null)
            filterConfig = _serverSideFilterFactory.createServerSideFilter(_clusterInfo, partition);
        if ((null == consumers) || (consumers.isEmpty())) {
            _log.error("ConsumerFactory for cluster (" + _clusterInfo + ") returned null or empty consumers ");
            throw new DatabusClientException("ConsumerFactory for cluster (" + _clusterInfo + ") returned null or empty consumers");
        }
        // Create Registration
        RegistrationId id = new RegistrationId(_id + "-" + partition.getPartitionId());
        CheckpointPersistenceProvider ckptProvider = createCheckpointPersistenceProvider(partition);
        DatabusV2RegistrationImpl reg = createChildRegistration(id, _client, ckptProvider);
        reg.addDatabusConsumers(consumers);
        String[] srcs = new String[_sources.size()];
        srcs = _sources.toArray(srcs);
        reg.addSubscriptions(srcs);
        regMap.put(partition, reg);
        reg.onRegister();
        // Add Server-Side Filter
        if (null != filterConfig)
            reg.withServerSideFilter(filterConfig);
        // Notify Listener
        if (null != _partitionListener)
            _partitionListener.onAddPartition(partition, reg);
        // Start the registration
        try {
            reg.start();
        } catch (DatabusClientException e) {
            _log.error("Got exception while starting the registration for partition (" + partition + ")", e);
            throw e;
        }
        //Add partition Specific metrics to cluster-merge
        _relayEventStatsMerger.addStatsCollector(id.getId(), (DbusEventsStatisticsCollector) reg.getRelayEventStats());
        _bootstrapEventStatsMerger.addStatsCollector(id.getId(), (DbusEventsStatisticsCollector) reg.getBootstrapEventStats());
        _relayCallbackStatsMerger.addStatsCollector(id.getId(), (ConsumerCallbackStats) reg.getRelayCallbackStats());
        _bootstrapCallbackStatsMerger.addStatsCollector(id.getId(), (ConsumerCallbackStats) reg.getBootstrapCallbackStats());
        _log.info("Partition (" + partition + ") started !!");
    } catch (DatabusException e) {
        _log.error("Got exception while activating partition(" + partition + ")", e);
        throw new DatabusClientException(e);
    } catch (ClusterCheckpointException e) {
        _log.error("Got exception while activating partition(" + partition + ")", e);
        throw new DatabusClientException(e);
    }
}