Examples with DbusKeyCompositeFilter com.linkedin.databus2.core.filter.DbusKeyCompositeFilter used on opensource projects

Example 1 with DbusKeyCompositeFilter

use of com.linkedin.databus2.core.filter.DbusKeyCompositeFilter in project databus by linkedin.

the class RelayPullThread method doRequestStream.

protected void doRequestStream(ConnectionState curState) {
    boolean debugEnabled = _log.isDebugEnabled();
    if (debugEnabled)
        _log.debug("Checking for free space in buffer");
    int freeBufferThreshold = (int) (_sourcesConn.getConnectionConfig().getFreeBufferThreshold() * 100.0 / _pullerBufferUtilizationPct);
    try {
        curState.getDataEventsBuffer().waitForFreeSpace(freeBufferThreshold);
    } catch (InterruptedException ie) {
        //loop
        enqueueMessage(curState);
        return;
    }
    Checkpoint cp = curState.getCheckpoint();
    if (debugEnabled)
        _log.debug("Checkpoint at RequestDataEvents: " + cp.toString());
    if (null == _relayFilter) {
        if (debugEnabled)
            _log.debug("Initializing relay filter config");
        _relayFilter = new DbusKeyCompositeFilter();
        Map<String, IdNamePair> srcNameIdMap = curState.getSourcesNameMap();
        for (DbusKeyCompositeFilterConfig conf : _relayFilterConfigs) {
            Map<String, KeyFilterConfigHolder> cMap = conf.getConfigMap();
            Map<Long, KeyFilterConfigHolder> fConfMap = new HashMap<Long, KeyFilterConfigHolder>();
            for (Entry<String, KeyFilterConfigHolder> e : cMap.entrySet()) {
                IdNamePair idName = srcNameIdMap.get(e.getKey());
                if (null != idName) {
                    fConfMap.put(idName.getId(), e.getValue());
                }
            }
            if (debugEnabled)
                _log.debug("FilterConfMap is :" + fConfMap);
            _relayFilter.merge(new DbusKeyCompositeFilter(fConfMap));
        }
        if (debugEnabled)
            _log.debug("Merged Filter (before deduping) is :" + _relayFilter);
        _relayFilter.dedupe();
        if (debugEnabled)
            _log.debug("Merged Filter (after deduping) is :" + _relayFilter);
    }
    _streamCallStartMs = System.currentTimeMillis();
    if (null != _relayCallsStats)
        _relayCallsStats.registerStreamRequest(cp, EMPTY_STREAM_LIST);
    int fetchSize = (int) ((curState.getDataEventsBuffer().getBufferFreeReadSpace() / 100.0) * _pullerBufferUtilizationPct);
    fetchSize = Math.max(freeBufferThreshold, fetchSize);
    CheckpointMult cpMult = new CheckpointMult();
    String args;
    if (curState.getRelayConnection().getProtocolVersion() >= 3) {
        // for version 3 and higher we pass subscriptions
        args = curState.getSubsListString();
        for (DatabusSubscription sub : curState.getSubscriptions()) {
            PhysicalPartition p = sub.getPhysicalPartition();
            cpMult.addCheckpoint(p, cp);
        }
    } else {
        args = curState.getSourcesIdListString();
        cpMult.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
    }
    curState.switchToStreamRequestSent();
    sendHeartbeat(_sourcesConn.getUnifiedClientStats());
    curState.getRelayConnection().requestStream(args, _relayFilter, fetchSize, cpMult, _sourcesConn.getConnectionConfig().getKeyRange(), curState);
}

Example 2 with DbusKeyCompositeFilter

use of com.linkedin.databus2.core.filter.DbusKeyCompositeFilter in project databus by linkedin.

the class BootstrapPullThread method doRequestBootstrapStream.

protected void doRequestBootstrapStream(ConnectionState curState) {
    boolean debugEnabled = _log.isDebugEnabled();
    if (debugEnabled)
        _log.debug("Checking for free space");
    //curState.getDataEventsBuffer().waitForFreeSpace(FREE_BUFFER_THRESHOLD);
    int freeBufferThreshold = (int) (_sourcesConn.getConnectionConfig().getFreeBufferThreshold() * 100.0 / _pullerBufferUtilizationPct);
    int freeSpace = curState.getDataEventsBuffer().getBufferFreeReadSpace();
    if (freeSpace >= freeBufferThreshold) {
        Checkpoint cp = curState.getCheckpoint();
        if (debugEnabled)
            _log.debug("Checkpoint at RequestBootstrapData: " + cp.toString());
        _log.debug("Sending /bootstrap request");
        Map<String, IdNamePair> srcNameMap = curState.getSourcesNameMap();
        String curSrcName = null;
        if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT) {
            curSrcName = cp.getSnapshotSource();
        } else {
            curSrcName = cp.getCatchupSource();
        }
        if (null == _bootstrapFilter) {
            _bootstrapFilter = new DbusKeyCompositeFilter();
            Map<String, IdNamePair> srcNameIdMap = curState.getSourcesNameMap();
            for (DbusKeyCompositeFilterConfig conf : _bootstrapFilterConfigs) {
                Map<String, KeyFilterConfigHolder> cMap = conf.getConfigMap();
                Map<Long, KeyFilterConfigHolder> fConfMap = new HashMap<Long, KeyFilterConfigHolder>();
                for (Entry<String, KeyFilterConfigHolder> e : cMap.entrySet()) {
                    IdNamePair idName = srcNameIdMap.get(e.getKey());
                    if (null != idName) {
                        fConfMap.put(idName.getId(), e.getValue());
                    }
                }
                _bootstrapFilter.merge(new DbusKeyCompositeFilter(fConfMap));
            }
            _bootstrapFilter.dedupe();
        }
        DbusKeyFilter filter = null;
        IdNamePair srcEntry = srcNameMap.get(curSrcName);
        if (null != srcEntry) {
            Map<Long, DbusKeyFilter> fMap = _bootstrapFilter.getFilterMap();
            if (null != fMap)
                filter = fMap.get(srcEntry.getId());
        }
        int fetchSize = (int) ((curState.getDataEventsBuffer().getBufferFreeReadSpace() / 100.0) * _pullerBufferUtilizationPct);
        fetchSize = Math.max(freeBufferThreshold, fetchSize);
        curState.switchToStreamRequestSent();
        sendHeartbeat(_sourcesConn.getUnifiedClientStats());
        curState.getBootstrapConnection().requestStream(curState.getSourcesIdListString(), filter, fetchSize, cp, curState);
    } else {
        try {
            Thread.sleep(50);
        } catch (InterruptedException ie) {
        }
        enqueueMessage(curState);
    }
}

Example 3 with DbusKeyCompositeFilter

use of com.linkedin.databus2.core.filter.DbusKeyCompositeFilter in project databus by linkedin.

the class ReadEventsRequestProcessor method process.

@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException, DatabusException {
    boolean isDebug = LOG.isDebugEnabled();
    try {
        ObjectMapper objMapper = new ObjectMapper();
        String checkpointString = request.getParams().getProperty(CHECKPOINT_PARAM, null);
        String checkpointStringMult = request.getParams().getProperty(CHECKPOINT_PARAM_MULT, null);
        int fetchSize = request.getRequiredIntParam(FETCH_SIZE_PARAM);
        String formatStr = request.getRequiredStringParam(OUTPUT_FORMAT_PARAM);
        Encoding enc = Encoding.valueOf(formatStr.toUpperCase());
        String sourcesListStr = request.getParams().getProperty(SOURCES_PARAM, null);
        String subsStr = request.getParams().getProperty(SUBS_PARAM, null);
        String partitionInfoStr = request.getParams().getProperty(PARTITION_INFO_STRING);
        String streamFromLatestSCNStr = request.getParams().getProperty(STREAM_FROM_LATEST_SCN);
        String clientMaxEventVersionStr = request.getParams().getProperty(DatabusHttpHeaders.MAX_EVENT_VERSION);
        int clientEventVersion = (clientMaxEventVersionStr != null) ? Integer.parseInt(clientMaxEventVersionStr) : DbusEventFactory.DBUS_EVENT_V1;
        if (clientEventVersion < 0 || clientEventVersion == 1 || clientEventVersion > DbusEventFactory.DBUS_EVENT_V2) {
            throw new InvalidRequestParamValueException(COMMAND_NAME, DatabusHttpHeaders.MAX_EVENT_VERSION, clientMaxEventVersionStr);
        }
        if (null == sourcesListStr && null == subsStr) {
            throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM + "|" + SUBS_PARAM, "null");
        }
        //TODO for now we separte the code paths to limit the impact on existing Databus 2 deployments (DDSDBUS-79)
        //We have to get rid of this eventually and have a single data path.
        boolean v2Mode = null == subsStr;
        DbusKeyCompositeFilter keyCompositeFilter = null;
        if (null != partitionInfoStr) {
            try {
                Map<Long, DbusKeyFilter> fMap = KeyFilterConfigJSONFactory.parseSrcIdFilterConfigMap(partitionInfoStr);
                keyCompositeFilter = new DbusKeyCompositeFilter();
                keyCompositeFilter.setFilterMap(fMap);
                if (isDebug)
                    LOG.debug("keyCompositeFilter is :" + keyCompositeFilter);
            } catch (Exception ex) {
                String msg = "Got exception while parsing partition Configs. PartitionInfo is:" + partitionInfoStr;
                LOG.error(msg, ex);
                throw new InvalidRequestParamValueException(COMMAND_NAME, PARTITION_INFO_STRING, partitionInfoStr);
            }
        }
        boolean streamFromLatestSCN = false;
        if (null != streamFromLatestSCNStr) {
            streamFromLatestSCN = Boolean.valueOf(streamFromLatestSCNStr);
        }
        long start = System.currentTimeMillis();
        List<DatabusSubscription> subs = null;
        //parse source ids
        SourceIdNameRegistry srcRegistry = _relay.getSourcesIdNameRegistry();
        HashSet<Integer> sourceIds = new HashSet<Integer>();
        if (null != sourcesListStr) {
            String[] sourcesList = sourcesListStr.split(",");
            for (String sourceId : sourcesList) {
                try {
                    Integer srcId = Integer.valueOf(sourceId);
                    sourceIds.add(srcId);
                } catch (NumberFormatException nfe) {
                    HttpStatisticsCollector globalHttpStatsCollector = _relay.getHttpStatisticsCollector();
                    if (null != globalHttpStatsCollector) {
                        globalHttpStatsCollector.registerInvalidStreamRequest();
                    }
                    throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, sourceId);
                }
            }
        }
        //process explicit subscriptions and generate respective logical partition filters
        NavigableSet<PhysicalPartitionKey> ppartKeys = null;
        if (null != subsStr) {
            List<DatabusSubscription.Builder> subsBuilder = null;
            subsBuilder = objMapper.readValue(subsStr, new TypeReference<List<DatabusSubscription.Builder>>() {
            });
            subs = new ArrayList<DatabusSubscription>(subsBuilder.size());
            for (DatabusSubscription.Builder subBuilder : subsBuilder) {
                subs.add(subBuilder.build());
            }
            ppartKeys = new TreeSet<PhysicalPartitionKey>();
            for (DatabusSubscription sub : subs) {
                PhysicalPartition ppart = sub.getPhysicalPartition();
                if (ppart.isAnyPartitionWildcard()) {
                    ppartKeys = _eventBuffer.getAllPhysicalPartitionKeys();
                    break;
                } else {
                    ppartKeys.add(new PhysicalPartitionKey(ppart));
                }
            }
        }
        // Need to make sure that we don't have tests that send requests in this form.
        if (subs != null && checkpointStringMult == null && checkpointString != null) {
            throw new RequestProcessingException("Both Subscriptions and CheckpointMult should be present");
        }
        //convert source ids into subscriptions
        if (null == subs)
            subs = new ArrayList<DatabusSubscription>();
        for (Integer srcId : sourceIds) {
            LogicalSource lsource = srcRegistry.getSource(srcId);
            if (lsource == null)
                throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, srcId.toString());
            if (isDebug)
                LOG.debug("registry returns " + lsource + " for srcid=" + srcId);
            DatabusSubscription newSub = DatabusSubscription.createSimpleSourceSubscription(lsource);
            subs.add(newSub);
        }
        DbusFilter ppartFilters = null;
        if (subs.size() > 0) {
            try {
                ppartFilters = _eventBuffer.constructFilters(subs);
            } catch (DatabusException de) {
                throw new RequestProcessingException("unable to generate physical partitions filters:" + de.getMessage(), de);
            }
        }
        ConjunctionDbusFilter filters = new ConjunctionDbusFilter();
        // Source filter comes first
        if (v2Mode)
            filters.addFilter(new SourceDbusFilter(sourceIds));
        else if (null != ppartFilters)
            filters.addFilter(ppartFilters);
        /*
      // Key range filter comes next
      if ((keyMin >0) && (keyMax > 0))
      {
        filters.addFilter(new KeyRangeFilter(keyMin, keyMax));
      }
      */
        if (null != keyCompositeFilter) {
            filters.addFilter(keyCompositeFilter);
        }
        // need to update registerStreamRequest to support Mult checkpoint TODO (DDSDBUS-80)
        // temp solution
        // 3 options:
        // 1. checkpointStringMult not null - generate checkpoint from it
        // 2. checkpointStringMult null, checkpointString not null - create empty CheckpointMult
        // and add create Checkpoint(checkpointString) and add it to cpMult;
        // 3 both are null - create empty CheckpointMult and add empty Checkpoint to it for each ppartition
        PhysicalPartition pPartition;
        Checkpoint cp = null;
        CheckpointMult cpMult = null;
        if (checkpointStringMult != null) {
            try {
                cpMult = new CheckpointMult(checkpointStringMult);
            } catch (InvalidParameterSpecException e) {
                LOG.error("Invalid CheckpointMult:" + checkpointStringMult, e);
                throw new InvalidRequestParamValueException("stream", "CheckpointMult", checkpointStringMult);
            }
        } else {
            // there is no checkpoint - create an empty one
            cpMult = new CheckpointMult();
            Iterator<Integer> it = sourceIds.iterator();
            while (it.hasNext()) {
                Integer srcId = it.next();
                pPartition = _eventBuffer.getPhysicalPartition(srcId);
                if (pPartition == null)
                    throw new RequestProcessingException("unable to find physical partitions for source:" + srcId);
                if (checkpointString != null) {
                    cp = new Checkpoint(checkpointString);
                } else {
                    cp = new Checkpoint();
                    cp.setFlexible();
                }
                cpMult.addCheckpoint(pPartition, cp);
            }
        }
        if (isDebug)
            LOG.debug("checkpointStringMult = " + checkpointStringMult + ";singlecheckpointString=" + checkpointString + ";CPM=" + cpMult);
        // of the server context.
        if (cpMult.getCursorPartition() == null) {
            cpMult.setCursorPartition(request.getCursorPartition());
        }
        if (isDebug) {
            if (cpMult.getCursorPartition() != null) {
                LOG.debug("Using physical paritition cursor " + cpMult.getCursorPartition());
            }
        }
        // for registerStreamRequest we need a single Checkpoint (TODO - fix it) (DDSDBUS-81)
        if (cp == null) {
            Iterator<Integer> it = sourceIds.iterator();
            if (it.hasNext()) {
                Integer srcId = it.next();
                pPartition = _eventBuffer.getPhysicalPartition(srcId);
                cp = cpMult.getCheckpoint(pPartition);
            } else {
                cp = new Checkpoint();
                cp.setFlexible();
            }
        }
        if (null != checkpointString && isDebug)
            LOG.debug("About to stream from cp: " + checkpointString.toString());
        HttpStatisticsCollector globalHttpStatsCollector = _relay.getHttpStatisticsCollector();
        HttpStatisticsCollector connHttpStatsCollector = null;
        if (null != globalHttpStatsCollector) {
            connHttpStatsCollector = (HttpStatisticsCollector) request.getParams().get(globalHttpStatsCollector.getName());
        }
        if (null != globalHttpStatsCollector)
            globalHttpStatsCollector.registerStreamRequest(cp, sourceIds);
        StatsCollectors<DbusEventsStatisticsCollector> statsCollectors = _relay.getOutBoundStatsCollectors();
        try {
            DbusEventBufferBatchReadable bufRead = v2Mode ? _eventBuffer.getDbusEventBufferBatchReadable(sourceIds, cpMult, statsCollectors) : _eventBuffer.getDbusEventBufferBatchReadable(cpMult, ppartKeys, statsCollectors);
            int eventsRead = 0;
            int minPendingEventSize = 0;
            StreamEventsResult result = null;
            bufRead.setClientMaxEventVersion(clientEventVersion);
            if (v2Mode) {
                result = bufRead.streamEvents(streamFromLatestSCN, fetchSize, request.getResponseContent(), enc, filters);
                eventsRead = result.getNumEventsStreamed();
                minPendingEventSize = result.getSizeOfPendingEvent();
                if (isDebug) {
                    LOG.debug("Process: streamed " + eventsRead + " from sources " + Arrays.toString(sourceIds.toArray()));
                    //can be used for debugging to stream from a cp
                    LOG.debug("CP=" + cpMult);
                }
            //if (null != statsCollectors) statsCollectors.mergeStatsCollectors();
            } else {
                result = bufRead.streamEvents(streamFromLatestSCN, fetchSize, request.getResponseContent(), enc, filters);
                eventsRead = result.getNumEventsStreamed();
                minPendingEventSize = result.getSizeOfPendingEvent();
                if (isDebug)
                    LOG.debug("Process: streamed " + eventsRead + " with subscriptions " + subs);
                cpMult = bufRead.getCheckpointMult();
                if (cpMult != null) {
                    request.setCursorPartition(cpMult.getCursorPartition());
                }
            }
            if (eventsRead == 0 && minPendingEventSize > 0) {
                // Append a header to indicate to the client that we do have at least one event to
                // send, but it is too large to fit into client's offered buffer.
                request.getResponseContent().addMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE, minPendingEventSize);
                LOG.debug("Returning 0 events but have pending event of size " + minPendingEventSize);
            }
        } catch (ScnNotFoundException snfe) {
            if (null != globalHttpStatsCollector) {
                globalHttpStatsCollector.registerScnNotFoundStreamResponse();
            }
            throw new RequestProcessingException(snfe);
        } catch (OffsetNotFoundException snfe) {
            LOG.error("OffsetNotFound", snfe);
            if (null != globalHttpStatsCollector) {
                globalHttpStatsCollector.registerScnNotFoundStreamResponse();
            }
            throw new RequestProcessingException(snfe);
        }
        if (null != connHttpStatsCollector) {
            connHttpStatsCollector.registerStreamResponse(System.currentTimeMillis() - start);
            globalHttpStatsCollector.merge(connHttpStatsCollector);
            connHttpStatsCollector.reset();
        } else if (null != globalHttpStatsCollector) {
            globalHttpStatsCollector.registerStreamResponse(System.currentTimeMillis() - start);
        }
    } catch (InvalidRequestParamValueException e) {
        HttpStatisticsCollector globalHttpStatsCollector = _relay.getHttpStatisticsCollector();
        if (null != globalHttpStatsCollector) {
            globalHttpStatsCollector.registerInvalidStreamRequest();
        }
        throw e;
    }
    return request;
}

Example 4 with DbusKeyCompositeFilter

use of com.linkedin.databus2.core.filter.DbusKeyCompositeFilter in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyCompositeFilter.

@Test
public void testDbusKeyCompositeFilter() throws Exception {
    KeyFilterConfigHolder.Config partConf1 = new KeyFilterConfigHolder.Config();
    partConf1.setType("MOD");
    KeyModFilterConfig.Config modConf1 = new KeyModFilterConfig.Config();
    modConf1.setNumBuckets(100);
    modConf1.setBuckets("[0,0]");
    partConf1.setMod(modConf1);
    KeyFilterConfigHolder.Config partConf2 = new KeyFilterConfigHolder.Config();
    partConf2.setType("RANGE");
    KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
    rangeConf.setSize(100);
    rangeConf.setPartitions("[3-4,3-4]");
    partConf2.setRange(rangeConf);
    HashMap<Long, KeyFilterConfigHolder> partConfigMap = new HashMap<Long, KeyFilterConfigHolder>();
    partConfigMap.put(1L, new KeyFilterConfigHolder(partConf1.build()));
    partConfigMap.put(2L, new KeyFilterConfigHolder(partConf2.build()));
    List<Long> keys1 = new ArrayList<Long>();
    List<Long> keys2 = new ArrayList<Long>();
    List<Long> keys3 = new ArrayList<Long>();
    for (long i = 0; i < 1000; i++) {
        keys1.add(i);
        keys2.add(i);
        keys3.add(i);
    }
    List<DbusEvent> dbusEvents = new ArrayList<DbusEvent>();
    generateEvents(1000, (short) 1, keys1, dbusEvents);
    generateEvents(1000, (short) 2, keys2, dbusEvents);
    generateEvents(1000, (short) 3, keys3, dbusEvents);
    List<DbusEvent> expPassedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> expFailedEvents = new ArrayList<DbusEvent>();
    System.out.println("TOTAL Events :" + dbusEvents.size());
    for (DbusEvent event : dbusEvents) {
        long key = event.key();
        int srcId = event.srcId();
        long bktId = key % 100;
        if ((srcId == 1) && (bktId == 0))
            expPassedEvents.add(event);
        else if ((srcId == 2) && ((key >= 300) && (key < 500)))
            expPassedEvents.add(event);
        else if (srcId == 3)
            expPassedEvents.add(event);
        else
            expFailedEvents.add(event);
    }
    DbusKeyCompositeFilter filter = new DbusKeyCompositeFilter(partConfigMap);
    filter.dedupe();
    List<DbusEvent> passedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> failedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        if (filter.allow(event)) {
            passedEvents.add(event);
        } else {
            failedEvents.add(event);
        }
    }
    System.out.println("Passed Event Size :" + passedEvents.size());
    System.out.println("Failed Event Size :" + failedEvents.size());
    assertEquals("Passed Size", expPassedEvents.size(), passedEvents.size());
    assertEquals("Failed Size", expFailedEvents.size(), failedEvents.size());
    for (int i = 0; i < passedEvents.size(); i++) {
        assertEquals("Passed Element " + i, expPassedEvents.get(i), passedEvents.get(i));
    }
    for (int i = 0; i < passedEvents.size(); i++) {
        assertEquals("Failed Element " + i, expFailedEvents.get(i), failedEvents.get(i));
    }
    ObjectMapper objMapper = new ObjectMapper();
    String objStr = objMapper.writeValueAsString(filter.getFilterMap());
    System.out.println("CompositeKeyFilter :" + objStr);
    Map<Long, DbusKeyFilter> map2 = KeyFilterConfigJSONFactory.parseSrcIdFilterConfigMap(objStr);
    String objStr2 = objMapper.writeValueAsString(filter.getFilterMap());
    System.out.println("CompositeKeyFilter2 :" + objStr2);
    assertEquals("CompositeKeys: JSON Serialization Test", objStr, objStr2);
    //String objStr3 = "{\"filterMap\":{\"40\":{\"partitionType\":\"RANGE\",\"filters\":[{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}},{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}}]}}}";
    //DbusKeyCompositeFilter f = KeyFilterJSONFactory.parseKeyCompositeFilter(objStr3);
    //System.out.println("Deserialized Filter is :" + f);
    String objStr4 = "{\"40\":{\"partitionType\":\"RANGE\",\"filters\":[{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}},{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}}]}}}";
    Map<Long, DbusKeyFilter> map3 = KeyFilterConfigJSONFactory.parseSrcIdFilterConfigMap(objStr4);
    DbusKeyCompositeFilter f2 = new DbusKeyCompositeFilter();
    f2.setFilterMap(map3);
    System.out.println("Deserialized Filter is (before dedupe): " + f2);
    f2.dedupe();
    System.out.println("Deserialized Filter is (after dedupe): " + f2);
}

Example 5 with DbusKeyCompositeFilter

use of com.linkedin.databus2.core.filter.DbusKeyCompositeFilter in project databus by linkedin.

the class MockRemoteExceptionHandler method testRelayPullerThreadV3WithFilter.

// make sure the filter is passed along to the doRequestStream call
@Test
public void testRelayPullerThreadV3WithFilter() throws Exception {
    RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
    // create filter
    DbusKeyCompositeFilterConfig filterConfig = createFakeFilter();
    bldr.setFilterConfig(filterConfig);
    RelayPullThread relayPuller = bldr.createRelayPullThread();
    relayPuller.getComponentStatus().start();
    ConnectionState connState = relayPuller.getConnectionState();
    connState.switchToPickServer();
    testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
    // get the created connection
    MockRelayConnection conn = (MockRelayConnection) relayPuller.getLastOpenConnection();
    conn.setProtocolVersion(3);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
    DbusKeyCompositeFilter filter = conn.getRelayFilter();
    Assert.assertNotNull(filter);
    // get DbusKeyFilter that we put in
    KeyFilterConfigHolder configHolder = filterConfig.getConfigMap().values().iterator().next();
    DbusKeyFilter inFilter = new DbusKeyFilter(configHolder);
    // get filter from the connection
    Assert.assertEquals(filter.getFilterMap().size(), 1, "1 filter in the composite filter");
    Assert.assertTrue(filter.getFilterMap().entrySet().iterator().hasNext(), "has one filter");
    DbusKeyFilter newFilter = filter.getFilterMap().values().iterator().next();
    //compare them
    boolean eq = inFilter.getPartitionType() == newFilter.getPartitionType() && inFilter.getFilters().size() == newFilter.getFilters().size();
    Assert.assertTrue(eq, "Same Filter");
}