Examples with BitsArray org.apache.rocketmq.filter.util.BitsArray used on opensource projects

Example 6 with BitsArray

use of org.apache.rocketmq.filter.util.BitsArray in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class CommitLogDispatcherCalcBitMap method dispatch.

@Override
public void dispatch(DispatchRequest request) {
    if (!this.brokerConfig.isEnableCalcFilterBitMap()) {
        return;
    }
    try {
        Collection<ConsumerFilterData> filterDatas = consumerFilterManager.get(request.getTopic());
        if (filterDatas == null || filterDatas.isEmpty()) {
            return;
        }
        Iterator<ConsumerFilterData> iterator = filterDatas.iterator();
        BitsArray filterBitMap = BitsArray.create(this.consumerFilterManager.getBloomFilter().getM());
        long startTime = System.currentTimeMillis();
        while (iterator.hasNext()) {
            ConsumerFilterData filterData = iterator.next();
            if (filterData.getCompiledExpression() == null) {
                log.error("[BUG] Consumer in filter manager has no compiled expression! {}", filterData);
                continue;
            }
            if (filterData.getBloomFilterData() == null) {
                log.error("[BUG] Consumer in filter manager has no bloom data! {}", filterData);
                continue;
            }
            Object ret = null;
            try {
                MessageEvaluationContext context = new MessageEvaluationContext(request.getPropertiesMap());
                ret = filterData.getCompiledExpression().evaluate(context);
            } catch (Throwable e) {
                log.error("Calc filter bit map error!commitLogOffset={}, consumer={}, {}", request.getCommitLogOffset(), filterData, e);
            }
            log.debug("Result of Calc bit map:ret={}, data={}, props={}, offset={}", ret, filterData, request.getPropertiesMap(), request.getCommitLogOffset());
            // eval true
            if (ret != null && ret instanceof Boolean && (Boolean) ret) {
                consumerFilterManager.getBloomFilter().hashTo(filterData.getBloomFilterData(), filterBitMap);
            }
        }
        request.setBitMap(filterBitMap.bytes());
        long eclipseTime = System.currentTimeMillis() - startTime;
        // 1ms
        if (eclipseTime >= 1) {
            log.warn("Spend {} ms to calc bit map, consumerNum={}, topic={}", eclipseTime, filterDatas.size(), request.getTopic());
        }
    } catch (Throwable e) {
        log.error("Calc bit map error! topic={}, offset={}, queueId={}, {}", request.getTopic(), request.getCommitLogOffset(), request.getQueueId(), e);
    }
}

Example 7 with BitsArray

use of org.apache.rocketmq.filter.util.BitsArray in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class ExpressionMessageFilter method isMatchedByConsumeQueue.

@Override
public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
    if (null == subscriptionData) {
        return true;
    }
    if (subscriptionData.isClassFilterMode()) {
        return true;
    }
    // by tags code.
    if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {
        if (tagsCode == null || tagsCode < 0L) {
            return true;
        }
        if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) {
            return true;
        }
        return subscriptionData.getCodeSet().contains(tagsCode.intValue());
    } else {
        // no expression or no bloom
        if (consumerFilterData == null || consumerFilterData.getExpression() == null || consumerFilterData.getCompiledExpression() == null || consumerFilterData.getBloomFilterData() == null) {
            return true;
        }
        // message is before consumer
        if (cqExtUnit == null || !consumerFilterData.isMsgInLive(cqExtUnit.getMsgStoreTime())) {
            log.debug("Pull matched because not in live: {}, {}", consumerFilterData, cqExtUnit);
            return true;
        }
        byte[] filterBitMap = cqExtUnit.getFilterBitMap();
        BloomFilter bloomFilter = this.consumerFilterManager.getBloomFilter();
        if (filterBitMap == null || !this.bloomDataValid || filterBitMap.length * Byte.SIZE != consumerFilterData.getBloomFilterData().getBitNum()) {
            return true;
        }
        BitsArray bitsArray = null;
        try {
            bitsArray = BitsArray.create(filterBitMap);
            boolean ret = bloomFilter.isHit(consumerFilterData.getBloomFilterData(), bitsArray);
            log.debug("Pull {} by bit map:{}, {}, {}", ret, consumerFilterData, bitsArray, cqExtUnit);
            return ret;
        } catch (Throwable e) {
            log.error("bloom filter error, sub=" + subscriptionData + ", filter=" + consumerFilterData + ", bitMap=" + bitsArray, e);
        }
    }
    return true;
}

Example 8 with BitsArray

use of org.apache.rocketmq.filter.util.BitsArray in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class CommitLogDispatcherCalcBitMapTest method testDispatch_filterDataIllegal.

@Test
public void testDispatch_filterDataIllegal() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    ConsumerFilterManager filterManager = new ConsumerFilterManager();
    filterManager.register("topic0", "CID_0", "a is not null and a >= 5", ExpressionType.SQL92, System.currentTimeMillis());
    filterManager.register("topic0", "CID_1", "a is not null and a >= 15", ExpressionType.SQL92, System.currentTimeMillis());
    ConsumerFilterData nullExpression = filterManager.get("topic0", "CID_0");
    nullExpression.setExpression(null);
    nullExpression.setCompiledExpression(null);
    ConsumerFilterData nullBloomData = filterManager.get("topic0", "CID_1");
    nullBloomData.setBloomFilterData(null);
    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager);
    for (int i = 0; i < 1; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));
        String topic = "topic" + i;
        DispatchRequest dispatchRequest = new DispatchRequest(topic, 0, i * 100 + 123, 100, (long) ("tags" + i).hashCode(), System.currentTimeMillis(), i, null, UUID.randomUUID().toString(), 0, 0, properties);
        calcBitMap.dispatch(dispatchRequest);
        assertThat(dispatchRequest.getBitMap()).isNotNull();
        BitsArray bitsArray = BitsArray.create(dispatchRequest.getBitMap(), filterManager.getBloomFilter().getM());
        for (int j = 0; j < bitsArray.bitLength(); j++) {
            assertThat(bitsArray.getBit(j)).isFalse();
        }
    }
}

Example 9 with BitsArray

use of org.apache.rocketmq.filter.util.BitsArray in project rocketmq by apache.

the class BitsArrayTest method testOr.

@Test
public void testOr() {
    BitsArray b1 = BitsArray.create(new byte[] { (byte) 0xff, 0x00 });
    BitsArray b2 = BitsArray.create(new byte[] { 0x00, (byte) 0xff });
    b1.or(b2);
    for (int i = 0; i < b1.bitLength(); i++) {
        assertThat(b1.getBit(i)).isTrue();
    }
}

Example 10 with BitsArray

use of org.apache.rocketmq.filter.util.BitsArray in project rocketmq by apache.

the class BitsArrayTest method testNot.

@Test
public void testNot() {
    BitsArray bitsArray = gen(bitLength);
    BitsArray backUp = bitsArray.clone();
    bitsArray.not(2);
    bitsArray.xor(backUp);
    assertThat(bitsArray.getBit(2)).isTrue();
}