Examples with BloomFilter org.apache.rocketmq.filter.util.BloomFilter used on opensource projects

Example 6 with BloomFilter

use of org.apache.rocketmq.filter.util.BloomFilter in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class BloomFilterTest method testBloomFilterData.

@Test
public void testBloomFilterData() {
    BloomFilterData bloomFilterData = new BloomFilterData(new int[] { 1, 2, 3 }, 128);
    BloomFilterData bloomFilterData1 = new BloomFilterData(new int[] { 1, 2, 3 }, 128);
    BloomFilterData bloomFilterData2 = new BloomFilterData(new int[] { 1, 2, 3 }, 129);
    assertThat(bloomFilterData).isEqualTo(bloomFilterData1);
    assertThat(bloomFilterData2).isNotEqualTo(bloomFilterData);
    assertThat(bloomFilterData2).isNotEqualTo(bloomFilterData1);
    assertThat(bloomFilterData.hashCode()).isEqualTo(bloomFilterData1.hashCode());
    assertThat(bloomFilterData2.hashCode()).isNotEqualTo(bloomFilterData.hashCode());
    assertThat(bloomFilterData2.hashCode()).isNotEqualTo(bloomFilterData1.hashCode());
    assertThat(bloomFilterData.getBitPos()).isEqualTo(bloomFilterData2.getBitPos());
    assertThat(bloomFilterData.getBitNum()).isEqualTo(bloomFilterData1.getBitNum());
    assertThat(bloomFilterData.getBitNum()).isNotEqualTo(bloomFilterData2.getBitNum());
    bloomFilterData2.setBitNum(128);
    assertThat(bloomFilterData).isEqualTo(bloomFilterData2);
    bloomFilterData2.setBitPos(new int[] { 1, 2, 3, 4 });
    assertThat(bloomFilterData).isNotEqualTo(bloomFilterData2);
    BloomFilterData nullData = new BloomFilterData();
    assertThat(nullData.getBitNum()).isEqualTo(0);
    assertThat(nullData.getBitPos()).isNull();
    BloomFilter bloomFilter = BloomFilter.createByFn(1, 300);
    assertThat(bloomFilter).isNotNull();
    assertThat(bloomFilter.isValid(bloomFilterData)).isFalse();
}

Example 7 with BloomFilter

use of org.apache.rocketmq.filter.util.BloomFilter in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class ExpressionMessageFilter method isMatchedByConsumeQueue.

@Override
public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
    if (null == subscriptionData) {
        return true;
    }
    if (subscriptionData.isClassFilterMode()) {
        return true;
    }
    // by tags code.
    if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {
        if (tagsCode == null || tagsCode < 0L) {
            return true;
        }
        if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) {
            return true;
        }
        return subscriptionData.getCodeSet().contains(tagsCode.intValue());
    } else {
        // no expression or no bloom
        if (consumerFilterData == null || consumerFilterData.getExpression() == null || consumerFilterData.getCompiledExpression() == null || consumerFilterData.getBloomFilterData() == null) {
            return true;
        }
        // message is before consumer
        if (cqExtUnit == null || !consumerFilterData.isMsgInLive(cqExtUnit.getMsgStoreTime())) {
            log.debug("Pull matched because not in live: {}, {}", consumerFilterData, cqExtUnit);
            return true;
        }
        byte[] filterBitMap = cqExtUnit.getFilterBitMap();
        BloomFilter bloomFilter = this.consumerFilterManager.getBloomFilter();
        if (filterBitMap == null || !this.bloomDataValid || filterBitMap.length * Byte.SIZE != consumerFilterData.getBloomFilterData().getBitNum()) {
            return true;
        }
        BitsArray bitsArray = null;
        try {
            bitsArray = BitsArray.create(filterBitMap);
            boolean ret = bloomFilter.isHit(consumerFilterData.getBloomFilterData(), bitsArray);
            log.debug("Pull {} by bit map:{}, {}, {}", ret, consumerFilterData, bitsArray, cqExtUnit);
            return ret;
        } catch (Throwable e) {
            log.error("bloom filter error, sub=" + subscriptionData + ", filter=" + consumerFilterData + ", bitMap=" + bitsArray, e);
        }
    }
    return true;
}

Example 8 with BloomFilter

use of org.apache.rocketmq.filter.util.BloomFilter in project rocketmq by apache.

the class BloomFilterTest method testEquals.

@Test
public void testEquals() {
    BloomFilter a = BloomFilter.createByFn(10, 20);
    BloomFilter b = BloomFilter.createByFn(10, 20);
    BloomFilter c = BloomFilter.createByFn(12, 20);
    BloomFilter d = BloomFilter.createByFn(10, 30);
    assertThat(a).isEqualTo(b);
    assertThat(a).isNotEqualTo(c);
    assertThat(a).isNotEqualTo(d);
    assertThat(d).isNotEqualTo(c);
    assertThat(a.hashCode()).isEqualTo(b.hashCode());
    assertThat(a.hashCode()).isNotEqualTo(c.hashCode());
    assertThat(a.hashCode()).isNotEqualTo(d.hashCode());
    assertThat(c.hashCode()).isNotEqualTo(d.hashCode());
}

Example 9 with BloomFilter

use of org.apache.rocketmq.filter.util.BloomFilter in project rocketmq by apache.

the class BloomFilterTest method testHashTo.

@Test
public void testHashTo() {
    String cid = "CID_abc_efg";
    BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);
    BitsArray bits = BitsArray.create(bloomFilter.getM());
    int[] bitPos = bloomFilter.calcBitPositions(cid);
    bloomFilter.hashTo(cid, bits);
    for (int bit : bitPos) {
        assertThat(bits.getBit(bit)).isTrue();
    }
}

Example 10 with BloomFilter

use of org.apache.rocketmq.filter.util.BloomFilter in project rocketmq by apache.

the class BloomFilterTest method testCalcBitPositions.

@Test
public void testCalcBitPositions() {
    String cid = "CID_abc_efg";
    BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);
    int[] bitPos = bloomFilter.calcBitPositions(cid);
    assertThat(bitPos).isNotNull();
    assertThat(bitPos.length).isEqualTo(bloomFilter.getK());
    int[] bitPos2 = bloomFilter.calcBitPositions(cid);
    assertThat(bitPos2).isNotNull();
    assertThat(bitPos2.length).isEqualTo(bloomFilter.getK());
    assertThat(bitPos).isEqualTo(bitPos2);
}