Examples with BloomFilter org.apache.rocketmq.filter.util.BloomFilter used on opensource projects

Example 11 with BloomFilter

use of org.apache.rocketmq.filter.util.BloomFilter in project rocketmq by apache.

the class BloomFilterTest method testIsHit.

@Test
public void testIsHit() {
    String cid = "CID_abc_efg";
    String cid2 = "CID_abc_123";
    BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);
    BitsArray bits = BitsArray.create(bloomFilter.getM());
    bloomFilter.hashTo(cid, bits);
    assertThat(bloomFilter.isHit(cid, bits)).isTrue();
    assertThat(!bloomFilter.isHit(cid2, bits)).isTrue();
    bloomFilter.hashTo(cid2, bits);
    assertThat(bloomFilter.isHit(cid, bits)).isTrue();
    assertThat(bloomFilter.isHit(cid2, bits)).isTrue();
}

Example 12 with BloomFilter

use of org.apache.rocketmq.filter.util.BloomFilter in project rocketmq by apache.

the class BloomFilterTest method testBloomFilterData.

@Test
public void testBloomFilterData() {
    BloomFilterData bloomFilterData = new BloomFilterData(new int[] { 1, 2, 3 }, 128);
    BloomFilterData bloomFilterData1 = new BloomFilterData(new int[] { 1, 2, 3 }, 128);
    BloomFilterData bloomFilterData2 = new BloomFilterData(new int[] { 1, 2, 3 }, 129);
    assertThat(bloomFilterData).isEqualTo(bloomFilterData1);
    assertThat(bloomFilterData2).isNotEqualTo(bloomFilterData);
    assertThat(bloomFilterData2).isNotEqualTo(bloomFilterData1);
    assertThat(bloomFilterData.hashCode()).isEqualTo(bloomFilterData1.hashCode());
    assertThat(bloomFilterData2.hashCode()).isNotEqualTo(bloomFilterData.hashCode());
    assertThat(bloomFilterData2.hashCode()).isNotEqualTo(bloomFilterData1.hashCode());
    assertThat(bloomFilterData.getBitPos()).isEqualTo(bloomFilterData2.getBitPos());
    assertThat(bloomFilterData.getBitNum()).isEqualTo(bloomFilterData1.getBitNum());
    assertThat(bloomFilterData.getBitNum()).isNotEqualTo(bloomFilterData2.getBitNum());
    bloomFilterData2.setBitNum(128);
    assertThat(bloomFilterData).isEqualTo(bloomFilterData2);
    bloomFilterData2.setBitPos(new int[] { 1, 2, 3, 4 });
    assertThat(bloomFilterData).isNotEqualTo(bloomFilterData2);
    BloomFilterData nullData = new BloomFilterData();
    assertThat(nullData.getBitNum()).isEqualTo(0);
    assertThat(nullData.getBitPos()).isNull();
    BloomFilter bloomFilter = BloomFilter.createByFn(1, 300);
    assertThat(bloomFilter).isNotNull();
    assertThat(bloomFilter.isValid(bloomFilterData)).isFalse();
}

Example 13 with BloomFilter

use of org.apache.rocketmq.filter.util.BloomFilter in project rocketmq by apache.

the class BloomFilterTest method testCheckFalseHit.

@Test
public void testCheckFalseHit() {
    BloomFilter bloomFilter = BloomFilter.createByFn(1, 300);
    BitsArray bits = BitsArray.create(bloomFilter.getM());
    int falseHit = 0;
    for (int i = 0; i < bloomFilter.getN(); i++) {
        String str = randomString((new Random(System.nanoTime())).nextInt(127) + 10);
        int[] bitPos = bloomFilter.calcBitPositions(str);
        if (bloomFilter.checkFalseHit(bitPos, bits)) {
            falseHit++;
        }
        bloomFilter.hashTo(bitPos, bits);
    }
    assertThat(falseHit).isLessThanOrEqualTo(bloomFilter.getF() * bloomFilter.getN() / 100);
}

Example 14 with BloomFilter

use of org.apache.rocketmq.filter.util.BloomFilter in project rocketmq by apache.

the class ExpressionMessageFilter method isMatchedByConsumeQueue.

@Override
public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
    if (null == subscriptionData) {
        return true;
    }
    if (subscriptionData.isClassFilterMode()) {
        return true;
    }
    // by tags code.
    if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {
        if (tagsCode == null) {
            return true;
        }
        if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) {
            return true;
        }
        return subscriptionData.getCodeSet().contains(tagsCode.intValue());
    } else {
        // no expression or no bloom
        if (consumerFilterData == null || consumerFilterData.getExpression() == null || consumerFilterData.getCompiledExpression() == null || consumerFilterData.getBloomFilterData() == null) {
            return true;
        }
        // message is before consumer
        if (cqExtUnit == null || !consumerFilterData.isMsgInLive(cqExtUnit.getMsgStoreTime())) {
            log.debug("Pull matched because not in live: {}, {}", consumerFilterData, cqExtUnit);
            return true;
        }
        byte[] filterBitMap = cqExtUnit.getFilterBitMap();
        BloomFilter bloomFilter = this.consumerFilterManager.getBloomFilter();
        if (filterBitMap == null || !this.bloomDataValid || filterBitMap.length * Byte.SIZE != consumerFilterData.getBloomFilterData().getBitNum()) {
            return true;
        }
        BitsArray bitsArray = null;
        try {
            bitsArray = BitsArray.create(filterBitMap);
            boolean ret = bloomFilter.isHit(consumerFilterData.getBloomFilterData(), bitsArray);
            log.debug("Pull {} by bit map:{}, {}, {}", ret, consumerFilterData, bitsArray, cqExtUnit);
            return ret;
        } catch (Throwable e) {
            log.error("bloom filter error, sub=" + subscriptionData + ", filter=" + consumerFilterData + ", bitMap=" + bitsArray, e);
        }
    }
    return true;
}