Examples with DbusKeyFilter com.linkedin.databus2.core.filter.DbusKeyFilter used on opensource projects

Example 11 with DbusKeyFilter

use of com.linkedin.databus2.core.filter.DbusKeyFilter in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyCompositeFilter.

@Test
public void testDbusKeyCompositeFilter() throws Exception {
    KeyFilterConfigHolder.Config partConf1 = new KeyFilterConfigHolder.Config();
    partConf1.setType("MOD");
    KeyModFilterConfig.Config modConf1 = new KeyModFilterConfig.Config();
    modConf1.setNumBuckets(100);
    modConf1.setBuckets("[0,0]");
    partConf1.setMod(modConf1);
    KeyFilterConfigHolder.Config partConf2 = new KeyFilterConfigHolder.Config();
    partConf2.setType("RANGE");
    KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
    rangeConf.setSize(100);
    rangeConf.setPartitions("[3-4,3-4]");
    partConf2.setRange(rangeConf);
    HashMap<Long, KeyFilterConfigHolder> partConfigMap = new HashMap<Long, KeyFilterConfigHolder>();
    partConfigMap.put(1L, new KeyFilterConfigHolder(partConf1.build()));
    partConfigMap.put(2L, new KeyFilterConfigHolder(partConf2.build()));
    List<Long> keys1 = new ArrayList<Long>();
    List<Long> keys2 = new ArrayList<Long>();
    List<Long> keys3 = new ArrayList<Long>();
    for (long i = 0; i < 1000; i++) {
        keys1.add(i);
        keys2.add(i);
        keys3.add(i);
    }
    List<DbusEvent> dbusEvents = new ArrayList<DbusEvent>();
    generateEvents(1000, (short) 1, keys1, dbusEvents);
    generateEvents(1000, (short) 2, keys2, dbusEvents);
    generateEvents(1000, (short) 3, keys3, dbusEvents);
    List<DbusEvent> expPassedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> expFailedEvents = new ArrayList<DbusEvent>();
    System.out.println("TOTAL Events :" + dbusEvents.size());
    for (DbusEvent event : dbusEvents) {
        long key = event.key();
        int srcId = event.srcId();
        long bktId = key % 100;
        if ((srcId == 1) && (bktId == 0))
            expPassedEvents.add(event);
        else if ((srcId == 2) && ((key >= 300) && (key < 500)))
            expPassedEvents.add(event);
        else if (srcId == 3)
            expPassedEvents.add(event);
        else
            expFailedEvents.add(event);
    }
    DbusKeyCompositeFilter filter = new DbusKeyCompositeFilter(partConfigMap);
    filter.dedupe();
    List<DbusEvent> passedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> failedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        if (filter.allow(event)) {
            passedEvents.add(event);
        } else {
            failedEvents.add(event);
        }
    }
    System.out.println("Passed Event Size :" + passedEvents.size());
    System.out.println("Failed Event Size :" + failedEvents.size());
    assertEquals("Passed Size", expPassedEvents.size(), passedEvents.size());
    assertEquals("Failed Size", expFailedEvents.size(), failedEvents.size());
    for (int i = 0; i < passedEvents.size(); i++) {
        assertEquals("Passed Element " + i, expPassedEvents.get(i), passedEvents.get(i));
    }
    for (int i = 0; i < passedEvents.size(); i++) {
        assertEquals("Failed Element " + i, expFailedEvents.get(i), failedEvents.get(i));
    }
    ObjectMapper objMapper = new ObjectMapper();
    String objStr = objMapper.writeValueAsString(filter.getFilterMap());
    System.out.println("CompositeKeyFilter :" + objStr);
    Map<Long, DbusKeyFilter> map2 = KeyFilterConfigJSONFactory.parseSrcIdFilterConfigMap(objStr);
    String objStr2 = objMapper.writeValueAsString(filter.getFilterMap());
    System.out.println("CompositeKeyFilter2 :" + objStr2);
    assertEquals("CompositeKeys: JSON Serialization Test", objStr, objStr2);
    //String objStr3 = "{\"filterMap\":{\"40\":{\"partitionType\":\"RANGE\",\"filters\":[{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}},{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}}]}}}";
    //DbusKeyCompositeFilter f = KeyFilterJSONFactory.parseKeyCompositeFilter(objStr3);
    //System.out.println("Deserialized Filter is :" + f);
    String objStr4 = "{\"40\":{\"partitionType\":\"RANGE\",\"filters\":[{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}},{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}}]}}}";
    Map<Long, DbusKeyFilter> map3 = KeyFilterConfigJSONFactory.parseSrcIdFilterConfigMap(objStr4);
    DbusKeyCompositeFilter f2 = new DbusKeyCompositeFilter();
    f2.setFilterMap(map3);
    System.out.println("Deserialized Filter is (before dedupe): " + f2);
    f2.dedupe();
    System.out.println("Deserialized Filter is (after dedupe): " + f2);
}

Example 12 with DbusKeyFilter

use of com.linkedin.databus2.core.filter.DbusKeyFilter in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyModFilterWithStringKeys1.

@Test
public // Case when string keys have numeric values
void testDbusKeyModFilterWithStringKeys1() throws Exception {
    KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
    partConf.setType("MOD");
    KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
    modConf.setNumBuckets(100);
    modConf.setBuckets("[0,3-4]");
    partConf.setMod(modConf);
    DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    // String Keys with numeric values
    List<DbusEvent> dbusEvents = new ArrayList<DbusEvent>();
    List<String> keys = new ArrayList<String>();
    for (long i = 0; i < 1000; ++i) {
        keys.add(new Long(i).toString());
    }
    generateStringEvents(1000, (short) 1, keys, dbusEvents);
    List<DbusEvent> expPassedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> expFailedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        long bkt = new Long(new String(event.keyBytes())) % 100;
        if ((bkt == 0) || ((bkt >= 3) && (bkt < 5)))
            expPassedEvents.add(event);
        else
            expFailedEvents.add(event);
    }
    List<DbusEvent> passedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> failedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        if (filter.allow(event)) {
            passedEvents.add(event);
        } else {
            failedEvents.add(event);
        }
    }
    System.out.println("Passed Event Size :" + passedEvents.size());
    System.out.println("Failed Event Size :" + failedEvents.size());
    assertEquals("Passed Size", expPassedEvents.size(), passedEvents.size());
    assertEquals("Failed Size", expFailedEvents.size(), failedEvents.size());
    for (int i = 0; i < passedEvents.size(); ++i) {
        assertEquals("Passed Element " + i, expPassedEvents.get(i), passedEvents.get(i));
    }
    for (int i = 0; i < passedEvents.size(); ++i) {
        assertEquals("Failed Element " + i, expFailedEvents.get(i), failedEvents.get(i));
    }
    ObjectMapper objMapper = new ObjectMapper();
    String objStr = objMapper.writeValueAsString(filter);
    System.out.println("KeyModFilter :" + objStr);
    DbusKeyFilter filter2 = KeyFilterConfigJSONFactory.parseDbusKeyFilter(objStr);
    String objStr2 = objMapper.writeValueAsString(filter2);
    System.out.println("KeyModFilter2 :" + objStr2);
    assertEquals("KeyModFilter JSON Serialization Test", objStr, objStr2);
}

Example 13 with DbusKeyFilter

use of com.linkedin.databus2.core.filter.DbusKeyFilter in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyModFilterWithStringKeys2.

@Test
public // Case when string keys have non-numeric values
void testDbusKeyModFilterWithStringKeys2() throws Exception {
    KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
    partConf.setType("MOD");
    KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
    modConf.setNumBuckets(100);
    modConf.setBuckets("[0,3-4]");
    partConf.setMod(modConf);
    DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    // String Keys with numeric values
    List<DbusEvent> dbusEvents = new ArrayList<DbusEvent>();
    List<String> keys = new ArrayList<String>();
    for (long i = 0; i < 1000; ++i) {
        //contains non-numeric char
        keys.add(i + "_1000");
    }
    generateStringEvents(1000, (short) 1, keys, dbusEvents);
    List<DbusEvent> expPassedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> expFailedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        long bkt = new Long(Math.abs(new String(event.keyBytes()).hashCode())) % 100;
        if ((bkt == 0) || ((bkt >= 3) && (bkt < 5)))
            expPassedEvents.add(event);
        else
            expFailedEvents.add(event);
    }
    List<DbusEvent> passedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> failedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        if (filter.allow(event)) {
            passedEvents.add(event);
        } else {
            failedEvents.add(event);
        }
    }
    System.out.println("Passed Event Size :" + passedEvents.size());
    System.out.println("Failed Event Size :" + failedEvents.size());
    assertEquals("Passed Size", expPassedEvents.size(), passedEvents.size());
    assertEquals("Failed Size", expFailedEvents.size(), failedEvents.size());
    for (int i = 0; i < passedEvents.size(); ++i) {
        assertEquals("Passed Element " + i, expPassedEvents.get(i), passedEvents.get(i));
    }
    for (int i = 0; i < passedEvents.size(); ++i) {
        assertEquals("Failed Element " + i, expFailedEvents.get(i), failedEvents.get(i));
    }
    ObjectMapper objMapper = new ObjectMapper();
    String objStr = objMapper.writeValueAsString(filter);
    System.out.println("KeyModFilter :" + objStr);
    DbusKeyFilter filter2 = KeyFilterConfigJSONFactory.parseDbusKeyFilter(objStr);
    String objStr2 = objMapper.writeValueAsString(filter2);
    System.out.println("KeyModFilter2 :" + objStr2);
    assertEquals("KeyModFilter JSON Serialization Test", objStr, objStr2);
}

Example 14 with DbusKeyFilter

use of com.linkedin.databus2.core.filter.DbusKeyFilter in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyModFilter.

@Test
public void testDbusKeyModFilter() throws Exception {
    KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
    partConf.setType("MOD");
    KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
    modConf.setNumBuckets(100);
    modConf.setBuckets("[0,3-4]");
    partConf.setMod(modConf);
    DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    List<DbusEvent> dbusEvents = new ArrayList<DbusEvent>();
    List<Long> keys = new ArrayList<Long>();
    for (long i = 0; i < 1000; i++) {
        keys.add(i);
    }
    generateEvents(1000, (short) 1, keys, dbusEvents);
    List<DbusEvent> expPassedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> expFailedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        long bkt = event.key() % 100;
        if ((bkt == 0) || ((bkt >= 3) && (bkt < 5)))
            expPassedEvents.add(event);
        else
            expFailedEvents.add(event);
    }
    List<DbusEvent> passedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> failedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        if (filter.allow(event)) {
            passedEvents.add(event);
        } else {
            failedEvents.add(event);
        }
    }
    System.out.println("Passed Event Size :" + passedEvents.size());
    System.out.println("Failed Event Size :" + failedEvents.size());
    assertEquals("Passed Size", expPassedEvents.size(), passedEvents.size());
    assertEquals("Failed Size", expFailedEvents.size(), failedEvents.size());
    for (int i = 0; i < passedEvents.size(); ++i) {
        assertEquals("Passed Element " + i, expPassedEvents.get(i), passedEvents.get(i));
    }
    for (int i = 0; i < passedEvents.size(); ++i) {
        assertEquals("Failed Element " + i, expFailedEvents.get(i), failedEvents.get(i));
    }
    ObjectMapper objMapper = new ObjectMapper();
    String objStr = objMapper.writeValueAsString(filter);
    System.out.println("KeyModFilter :" + objStr);
    DbusKeyFilter filter2 = KeyFilterConfigJSONFactory.parseDbusKeyFilter(objStr);
    String objStr2 = objMapper.writeValueAsString(filter2);
    System.out.println("KeyModFilter2 :" + objStr2);
    assertEquals("KeyModFilter JSON Serialization Test", objStr, objStr2);
}

Example 15 with DbusKeyFilter

use of com.linkedin.databus2.core.filter.DbusKeyFilter in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyNoneFilter.

@Test
public void testDbusKeyNoneFilter() throws Exception {
    KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
    partConf.setType("NONE");
    DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    List<DbusEvent> dbusEvents = new ArrayList<DbusEvent>();
    List<Long> keys = new ArrayList<Long>();
    for (long i = 0; i < 1000; i++) {
        keys.add(i);
    }
    generateEvents(1000, (short) 1, keys, dbusEvents);
    List<DbusEvent> passedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> failedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        if (filter.allow(event)) {
            passedEvents.add(event);
        } else {
            failedEvents.add(event);
        }
    }
    System.out.println("Passed Event Size :" + passedEvents.size());
    System.out.println("Failed Event Size :" + failedEvents.size());
    assertEquals("Passed Size", dbusEvents.size(), passedEvents.size());
    assertEquals("Failed Size", 0, failedEvents.size());
    for (int i = 0; i < passedEvents.size(); i++) {
        assertEquals("Passed Element " + i, dbusEvents.get(i), passedEvents.get(i));
    }
    ObjectMapper objMapper = new ObjectMapper();
    String objStr = objMapper.writeValueAsString(filter);
    System.out.println("KeyFilter :" + objStr);
    DbusKeyFilter filter2 = KeyFilterConfigJSONFactory.parseDbusKeyFilter(objStr);
    String objStr2 = objMapper.writeValueAsString(filter2);
    System.out.println("KeyFilter2 :" + objStr2);
    assertEquals("KeyNoneFilter: JSON Serialization Test", objStr, objStr2);
}