Examples with BitsArray org.apache.rocketmq.filter.util.BitsArray used on opensource projects

Example 21 with BitsArray

use of org.apache.rocketmq.filter.util.BitsArray in project rocketmq by apache.

the class BitsArrayTest method testXor.

@Test
public void testXor() {
    BitsArray bitsArray = gen(bitLength);
    boolean val = bitsArray.getBit(2);
    bitsArray.xor(2, !val);
    assertThat(bitsArray.getBit(2)).isTrue();
}

Example 22 with BitsArray

use of org.apache.rocketmq.filter.util.BitsArray in project rocketmq by apache.

the class BitsArrayTest method testConstructor.

@Test
public void testConstructor() {
    BitsArray bitsArray = BitsArray.create(8);
    assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 8).isTrue();
    bitsArray = BitsArray.create(9);
    assertThat(bitsArray.byteLength() == 2 && bitsArray.bitLength() == 9).isTrue();
    bitsArray = BitsArray.create(7);
    assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 7).isTrue();
}

Example 23 with BitsArray

use of org.apache.rocketmq.filter.util.BitsArray in project rocketmq by apache.

the class BloomFilterTest method testCheckFalseHit.

@Test
public void testCheckFalseHit() {
    BloomFilter bloomFilter = BloomFilter.createByFn(1, 300);
    BitsArray bits = BitsArray.create(bloomFilter.getM());
    int falseHit = 0;
    for (int i = 0; i < bloomFilter.getN(); i++) {
        String str = randomString((new Random(System.nanoTime())).nextInt(127) + 10);
        int[] bitPos = bloomFilter.calcBitPositions(str);
        if (bloomFilter.checkFalseHit(bitPos, bits)) {
            falseHit++;
        }
        bloomFilter.hashTo(bitPos, bits);
    }
    assertThat(falseHit).isLessThanOrEqualTo(bloomFilter.getF() * bloomFilter.getN() / 100);
}

Example 24 with BitsArray

use of org.apache.rocketmq.filter.util.BitsArray in project rocketmq by apache.

the class CommitLogDispatcherCalcBitMapTest method testDispatch.

@Test
public void testDispatch() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    ConsumerFilterManager filterManager = ConsumerFilterManagerTest.gen(10, 10);
    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager);
    for (int i = 0; i < 10; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));
        String topic = "topic" + i;
        DispatchRequest dispatchRequest = new DispatchRequest(topic, 0, i * 100 + 123, 100, (long) ("tags" + i).hashCode(), System.currentTimeMillis(), i, null, UUID.randomUUID().toString(), 0, 0, properties);
        calcBitMap.dispatch(dispatchRequest);
        assertThat(dispatchRequest.getBitMap()).isNotNull();
        BitsArray bits = BitsArray.create(dispatchRequest.getBitMap());
        Collection<ConsumerFilterData> filterDatas = filterManager.get(topic);
        for (ConsumerFilterData filterData : filterDatas) {
            if (filterManager.getBloomFilter().isHit(filterData.getBloomFilterData(), bits)) {
                try {
                    assertThat((Boolean) filterData.getCompiledExpression().evaluate(new MessageEvaluationContext(properties))).isTrue();
                } catch (Exception e) {
                    e.printStackTrace();
                    assertThat(true).isFalse();
                }
            } else {
                try {
                    assertThat((Boolean) filterData.getCompiledExpression().evaluate(new MessageEvaluationContext(properties))).isFalse();
                } catch (Exception e) {
                    e.printStackTrace();
                    assertThat(true).isFalse();
                }
            }
        }
    }
}

Example 25 with BitsArray

use of org.apache.rocketmq.filter.util.BitsArray in project rocketmq by apache.

the class CommitLogDispatcherCalcBitMap method dispatch.

@Override
public void dispatch(DispatchRequest request) {
    if (!this.brokerConfig.isEnableCalcFilterBitMap()) {
        return;
    }
    try {
        Collection<ConsumerFilterData> filterDatas = consumerFilterManager.get(request.getTopic());
        if (filterDatas == null || filterDatas.isEmpty()) {
            return;
        }
        Iterator<ConsumerFilterData> iterator = filterDatas.iterator();
        BitsArray filterBitMap = BitsArray.create(this.consumerFilterManager.getBloomFilter().getM());
        long startTime = System.currentTimeMillis();
        while (iterator.hasNext()) {
            ConsumerFilterData filterData = iterator.next();
            if (filterData.getCompiledExpression() == null) {
                log.error("[BUG] Consumer in filter manager has no compiled expression! {}", filterData);
                continue;
            }
            if (filterData.getBloomFilterData() == null) {
                log.error("[BUG] Consumer in filter manager has no bloom data! {}", filterData);
                continue;
            }
            Object ret = null;
            try {
                MessageEvaluationContext context = new MessageEvaluationContext(request.getPropertiesMap());
                ret = filterData.getCompiledExpression().evaluate(context);
            } catch (Throwable e) {
                log.error("Calc filter bit map error!commitLogOffset={}, consumer={}, {}", request.getCommitLogOffset(), filterData, e);
            }
            log.debug("Result of Calc bit map:ret={}, data={}, props={}, offset={}", ret, filterData, request.getPropertiesMap(), request.getCommitLogOffset());
            // eval true
            if (ret != null && ret instanceof Boolean && (Boolean) ret) {
                consumerFilterManager.getBloomFilter().hashTo(filterData.getBloomFilterData(), filterBitMap);
            }
        }
        request.setBitMap(filterBitMap.bytes());
        long eclipseTime = System.currentTimeMillis() - startTime;
        // 1ms
        if (eclipseTime >= 1) {
            log.warn("Spend {} ms to calc bit map, consumerNum={}, topic={}", eclipseTime, filterDatas.size(), request.getTopic());
        }
    } catch (Throwable e) {
        log.error("Calc bit map error! topic={}, offset={}, queueId={}, {}", request.getTopic(), request.getCommitLogOffset(), request.getQueueId(), e);
    }
}