Examples with DbusKeyFilter com.linkedin.databus2.core.filter.DbusKeyFilter used on opensource projects

Example 1 with DbusKeyFilter

use of com.linkedin.databus2.core.filter.DbusKeyFilter in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyRangeFilter.

@Test
public void testDbusKeyRangeFilter() throws Exception {
    KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
    partConf.setType("RANGE");
    KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
    rangeConf.setSize(100);
    rangeConf.setPartitions("[0,3-4]");
    partConf.setRange(rangeConf);
    DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    List<DbusEvent> dbusEvents = new ArrayList<DbusEvent>();
    List<Long> keys = new ArrayList<Long>();
    for (long i = 0; i < 1000; ++i) {
        keys.add(i);
    }
    generateEvents(1000, (short) 1, keys, dbusEvents);
    List<DbusEvent> expPassedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> expFailedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        long key = event.key();
        if ((key < 100) || ((key >= 300) && (key < 500)))
            expPassedEvents.add(event);
        else
            expFailedEvents.add(event);
    }
    List<DbusEvent> passedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> failedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        if (filter.allow(event)) {
            passedEvents.add(event);
        } else {
            failedEvents.add(event);
        }
    }
    System.out.println("Passed Event Size :" + passedEvents.size());
    System.out.println("Failed Event Size :" + failedEvents.size());
    assertEquals("Passed Size", expPassedEvents.size(), passedEvents.size());
    assertEquals("Failed Size", expFailedEvents.size(), failedEvents.size());
    for (int i = 0; i < passedEvents.size(); ++i) {
        assertEquals("Passed Element " + i, expPassedEvents.get(i), passedEvents.get(i));
    }
    for (int i = 0; i < passedEvents.size(); ++i) {
        assertEquals("Failed Element " + i, expFailedEvents.get(i), failedEvents.get(i));
    }
    ObjectMapper objMapper = new ObjectMapper();
    String objStr = objMapper.writeValueAsString(filter);
    System.out.println("KeyRangeFilter :" + objStr);
    DbusKeyFilter filter2 = KeyFilterConfigJSONFactory.parseDbusKeyFilter(objStr);
    String objStr2 = objMapper.writeValueAsString(filter2);
    System.out.println("KeyRangeFilter2 :" + objStr2);
    assertEquals("KeyRangeFilter JSON Serialization Test", objStr, objStr2);
}

Example 2 with DbusKeyFilter

use of com.linkedin.databus2.core.filter.DbusKeyFilter in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyRangeFilterErrors.

@Test
public void testDbusKeyRangeFilterErrors() throws Exception {
    boolean isException = false;
    //Error Case: Range Size is negative
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("RANGE");
        KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
        rangeConf.setSize(-1);
        rangeConf.setPartitions("[0,3-4]");
        partConf.setRange(rangeConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ice) {
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (Range Size is negative) ", true, isException);
    isException = false;
    //Error Case: min is greater than max
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("RANGE");
        KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
        rangeConf.setSize(100);
        rangeConf.setPartitions("[0,5-4]");
        partConf.setRange(rangeConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ice) {
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (min is greater than max) ", true, isException);
    isException = false;
    //Error Case: min is -ve
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("RANGE");
        KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
        rangeConf.setSize(100);
        rangeConf.setPartitions("[-3,2-4]");
        partConf.setRange(rangeConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ice) {
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (min is greater than max) ", true, isException);
}

Example 3 with DbusKeyFilter

use of com.linkedin.databus2.core.filter.DbusKeyFilter in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyModFilterErrors.

@Test
public void testDbusKeyModFilterErrors() throws Exception {
    //Error Config : MaxBucket is more than numBuckets
    boolean isException = false;
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(5);
        //invalid config
        modConf.setBuckets("[0,3-9]");
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (MaxBucket is more than numBuckets) ", true, isException);
    //Error Case : Min Bucket is more than maxBucket
    isException = false;
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(50);
        //invalid config
        modConf.setBuckets("[0,9-3]");
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (Min Bucket is more than maxBucket) ", true, isException);
    //Error Case : numBuckets is negative
    isException = false;
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(-5);
        //invalid config
        modConf.setBuckets("[0]");
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
    }
    assertEquals("Got Exception for invalid Config numBuckets is negative) ", true, isException);
    //Error Case : minBucket is negative
    isException = false;
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(50);
        //invalid config
        modConf.setBuckets("[-5,1-3]");
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (minBucket is negative) ", true, isException);
}

Example 4 with DbusKeyFilter

use of com.linkedin.databus2.core.filter.DbusKeyFilter in project databus by linkedin.

the class TestFilterToSQL method testModFilter.

@Test
public void testModFilter() throws InvalidConfigException, InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException {
    partConf = new KeyFilterConfigHolder.Config();
    partConf.setType("MOD");
    KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
    modConf.setNumBuckets(100);
    modConf.setBuckets("[0,3-4]");
    partConf.setMod(modConf);
    DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    processor.setKeyFilter(filter);
    LOG.info("Testing multiple mod filters");
    LOG.info("CATCHUP TABLE: " + processor.getCatchupSQLString("catchuptab"));
    LOG.info("SNAPSHOT TABLE: " + processor.getSnapshotSQLString("snapshotTable"));
    String catchUpString = processor.getCatchupSQLString("catchuptab").replaceAll("\\s+", " ");
    String snapshotString = processor.getSnapshotSQLString("snapshotTable").replaceAll("\\s+", " ");
    String catchUpExpectedString = "Select id, scn, windowscn, val, CAST(srckey as SIGNED) as srckey from catchuptab where  id > ?  and windowscn >= ? and windowscn <= ?  and windowscn >= ? AND  ( srckey%100 >= 0 AND srckey%100 < 1 OR srckey%100 >= 3 AND srckey%100 < 5 )  order by id limit ?".replaceAll("\\s+", " ");
    String snapshotExpectedString = "Select id, scn,  CAST(srckey as SIGNED) as srckey, val from snapshotTable where  id > ?  and scn < ?  and scn >= ? AND  ( srckey%100 >= 0 AND srckey%100 < 1 OR srckey%100 >= 3 AND srckey%100 < 5 )  order by id limit ?".replaceAll("\\s+", " ");
    Assert.assertEquals(catchUpString, catchUpExpectedString);
    Assert.assertEquals(snapshotString, snapshotExpectedString);
    partConf.setType("MOD");
    modConf = new KeyModFilterConfig.Config();
    modConf.setNumBuckets(100);
    modConf.setBuckets("[0]");
    partConf.setMod(modConf);
    filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    processor.setKeyFilter(filter);
    LOG.info("Testing single mod filter");
    LOG.info("CATCHUP TABLE: " + processor.getCatchupSQLString("catchuptab"));
    LOG.info("SNAPSHOT TABLE: " + processor.getSnapshotSQLString("snapshotTable"));
    catchUpString = processor.getCatchupSQLString("catchuptab").replaceAll("\\s+", " ");
    snapshotString = processor.getSnapshotSQLString("snapshotTable").replaceAll("\\s+", " ");
    catchUpExpectedString = "Select id, scn, windowscn, val, CAST(srckey as SIGNED) as srckey from catchuptab where  id > ?  and windowscn >= ? and windowscn <= ?  and windowscn >= ? AND  ( srckey%100 >= 0 AND srckey%100 < 1 )  order by id limit ?".replaceAll("\\s+", " ");
    snapshotExpectedString = "Select id, scn,  CAST(srckey as SIGNED) as srckey, val from snapshotTable where  id > ?  and scn < ?  and scn >= ? AND  ( srckey%100 >= 0 AND srckey%100 < 1 )  order by id limit ?".replaceAll("\\s+", " ");
    Assert.assertEquals(catchUpString, catchUpExpectedString);
    Assert.assertEquals(snapshotString, snapshotExpectedString);
}

Example 5 with DbusKeyFilter

use of com.linkedin.databus2.core.filter.DbusKeyFilter in project databus by linkedin.

the class BootstrapPullThread method doRequestBootstrapStream.

protected void doRequestBootstrapStream(ConnectionState curState) {
    boolean debugEnabled = _log.isDebugEnabled();
    if (debugEnabled)
        _log.debug("Checking for free space");
    //curState.getDataEventsBuffer().waitForFreeSpace(FREE_BUFFER_THRESHOLD);
    int freeBufferThreshold = (int) (_sourcesConn.getConnectionConfig().getFreeBufferThreshold() * 100.0 / _pullerBufferUtilizationPct);
    int freeSpace = curState.getDataEventsBuffer().getBufferFreeReadSpace();
    if (freeSpace >= freeBufferThreshold) {
        Checkpoint cp = curState.getCheckpoint();
        if (debugEnabled)
            _log.debug("Checkpoint at RequestBootstrapData: " + cp.toString());
        _log.debug("Sending /bootstrap request");
        Map<String, IdNamePair> srcNameMap = curState.getSourcesNameMap();
        String curSrcName = null;
        if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT) {
            curSrcName = cp.getSnapshotSource();
        } else {
            curSrcName = cp.getCatchupSource();
        }
        if (null == _bootstrapFilter) {
            _bootstrapFilter = new DbusKeyCompositeFilter();
            Map<String, IdNamePair> srcNameIdMap = curState.getSourcesNameMap();
            for (DbusKeyCompositeFilterConfig conf : _bootstrapFilterConfigs) {
                Map<String, KeyFilterConfigHolder> cMap = conf.getConfigMap();
                Map<Long, KeyFilterConfigHolder> fConfMap = new HashMap<Long, KeyFilterConfigHolder>();
                for (Entry<String, KeyFilterConfigHolder> e : cMap.entrySet()) {
                    IdNamePair idName = srcNameIdMap.get(e.getKey());
                    if (null != idName) {
                        fConfMap.put(idName.getId(), e.getValue());
                    }
                }
                _bootstrapFilter.merge(new DbusKeyCompositeFilter(fConfMap));
            }
            _bootstrapFilter.dedupe();
        }
        DbusKeyFilter filter = null;
        IdNamePair srcEntry = srcNameMap.get(curSrcName);
        if (null != srcEntry) {
            Map<Long, DbusKeyFilter> fMap = _bootstrapFilter.getFilterMap();
            if (null != fMap)
                filter = fMap.get(srcEntry.getId());
        }
        int fetchSize = (int) ((curState.getDataEventsBuffer().getBufferFreeReadSpace() / 100.0) * _pullerBufferUtilizationPct);
        fetchSize = Math.max(freeBufferThreshold, fetchSize);
        curState.switchToStreamRequestSent();
        sendHeartbeat(_sourcesConn.getUnifiedClientStats());
        curState.getBootstrapConnection().requestStream(curState.getSourcesIdListString(), filter, fetchSize, cp, curState);
    } else {
        try {
            Thread.sleep(50);
        } catch (InterruptedException ie) {
        }
        enqueueMessage(curState);
    }
}