Examples with KeyFilterConfigHolder com.linkedin.databus2.core.filter.KeyFilterConfigHolder used on opensource projects

Example 1 with KeyFilterConfigHolder

use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyRangeFilter.

@Test
public void testDbusKeyRangeFilter() throws Exception {
    KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
    partConf.setType("RANGE");
    KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
    rangeConf.setSize(100);
    rangeConf.setPartitions("[0,3-4]");
    partConf.setRange(rangeConf);
    DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    List<DbusEvent> dbusEvents = new ArrayList<DbusEvent>();
    List<Long> keys = new ArrayList<Long>();
    for (long i = 0; i < 1000; ++i) {
        keys.add(i);
    }
    generateEvents(1000, (short) 1, keys, dbusEvents);
    List<DbusEvent> expPassedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> expFailedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        long key = event.key();
        if ((key < 100) || ((key >= 300) && (key < 500)))
            expPassedEvents.add(event);
        else
            expFailedEvents.add(event);
    }
    List<DbusEvent> passedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> failedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        if (filter.allow(event)) {
            passedEvents.add(event);
        } else {
            failedEvents.add(event);
        }
    }
    System.out.println("Passed Event Size :" + passedEvents.size());
    System.out.println("Failed Event Size :" + failedEvents.size());
    assertEquals("Passed Size", expPassedEvents.size(), passedEvents.size());
    assertEquals("Failed Size", expFailedEvents.size(), failedEvents.size());
    for (int i = 0; i < passedEvents.size(); ++i) {
        assertEquals("Passed Element " + i, expPassedEvents.get(i), passedEvents.get(i));
    }
    for (int i = 0; i < passedEvents.size(); ++i) {
        assertEquals("Failed Element " + i, expFailedEvents.get(i), failedEvents.get(i));
    }
    ObjectMapper objMapper = new ObjectMapper();
    String objStr = objMapper.writeValueAsString(filter);
    System.out.println("KeyRangeFilter :" + objStr);
    DbusKeyFilter filter2 = KeyFilterConfigJSONFactory.parseDbusKeyFilter(objStr);
    String objStr2 = objMapper.writeValueAsString(filter2);
    System.out.println("KeyRangeFilter2 :" + objStr2);
    assertEquals("KeyRangeFilter JSON Serialization Test", objStr, objStr2);
}

Example 2 with KeyFilterConfigHolder

use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyRangeFilterErrors.

@Test
public void testDbusKeyRangeFilterErrors() throws Exception {
    boolean isException = false;
    //Error Case: Range Size is negative
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("RANGE");
        KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
        rangeConf.setSize(-1);
        rangeConf.setPartitions("[0,3-4]");
        partConf.setRange(rangeConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ice) {
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (Range Size is negative) ", true, isException);
    isException = false;
    //Error Case: min is greater than max
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("RANGE");
        KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
        rangeConf.setSize(100);
        rangeConf.setPartitions("[0,5-4]");
        partConf.setRange(rangeConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ice) {
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (min is greater than max) ", true, isException);
    isException = false;
    //Error Case: min is -ve
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("RANGE");
        KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
        rangeConf.setSize(100);
        rangeConf.setPartitions("[-3,2-4]");
        partConf.setRange(rangeConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ice) {
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (min is greater than max) ", true, isException);
}

Example 3 with KeyFilterConfigHolder

use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyModFilterErrors.

@Test
public void testDbusKeyModFilterErrors() throws Exception {
    //Error Config : MaxBucket is more than numBuckets
    boolean isException = false;
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(5);
        //invalid config
        modConf.setBuckets("[0,3-9]");
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (MaxBucket is more than numBuckets) ", true, isException);
    //Error Case : Min Bucket is more than maxBucket
    isException = false;
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(50);
        //invalid config
        modConf.setBuckets("[0,9-3]");
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (Min Bucket is more than maxBucket) ", true, isException);
    //Error Case : numBuckets is negative
    isException = false;
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(-5);
        //invalid config
        modConf.setBuckets("[0]");
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
    }
    assertEquals("Got Exception for invalid Config numBuckets is negative) ", true, isException);
    //Error Case : minBucket is negative
    isException = false;
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(50);
        //invalid config
        modConf.setBuckets("[-5,1-3]");
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (minBucket is negative) ", true, isException);
}

Example 4 with KeyFilterConfigHolder

use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.

the class TestFilterToSQL method testModFilter.

@Test
public void testModFilter() throws InvalidConfigException, InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException {
    partConf = new KeyFilterConfigHolder.Config();
    partConf.setType("MOD");
    KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
    modConf.setNumBuckets(100);
    modConf.setBuckets("[0,3-4]");
    partConf.setMod(modConf);
    DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    processor.setKeyFilter(filter);
    LOG.info("Testing multiple mod filters");
    LOG.info("CATCHUP TABLE: " + processor.getCatchupSQLString("catchuptab"));
    LOG.info("SNAPSHOT TABLE: " + processor.getSnapshotSQLString("snapshotTable"));
    String catchUpString = processor.getCatchupSQLString("catchuptab").replaceAll("\\s+", " ");
    String snapshotString = processor.getSnapshotSQLString("snapshotTable").replaceAll("\\s+", " ");
    String catchUpExpectedString = "Select id, scn, windowscn, val, CAST(srckey as SIGNED) as srckey from catchuptab where  id > ?  and windowscn >= ? and windowscn <= ?  and windowscn >= ? AND  ( srckey%100 >= 0 AND srckey%100 < 1 OR srckey%100 >= 3 AND srckey%100 < 5 )  order by id limit ?".replaceAll("\\s+", " ");
    String snapshotExpectedString = "Select id, scn,  CAST(srckey as SIGNED) as srckey, val from snapshotTable where  id > ?  and scn < ?  and scn >= ? AND  ( srckey%100 >= 0 AND srckey%100 < 1 OR srckey%100 >= 3 AND srckey%100 < 5 )  order by id limit ?".replaceAll("\\s+", " ");
    Assert.assertEquals(catchUpString, catchUpExpectedString);
    Assert.assertEquals(snapshotString, snapshotExpectedString);
    partConf.setType("MOD");
    modConf = new KeyModFilterConfig.Config();
    modConf.setNumBuckets(100);
    modConf.setBuckets("[0]");
    partConf.setMod(modConf);
    filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    processor.setKeyFilter(filter);
    LOG.info("Testing single mod filter");
    LOG.info("CATCHUP TABLE: " + processor.getCatchupSQLString("catchuptab"));
    LOG.info("SNAPSHOT TABLE: " + processor.getSnapshotSQLString("snapshotTable"));
    catchUpString = processor.getCatchupSQLString("catchuptab").replaceAll("\\s+", " ");
    snapshotString = processor.getSnapshotSQLString("snapshotTable").replaceAll("\\s+", " ");
    catchUpExpectedString = "Select id, scn, windowscn, val, CAST(srckey as SIGNED) as srckey from catchuptab where  id > ?  and windowscn >= ? and windowscn <= ?  and windowscn >= ? AND  ( srckey%100 >= 0 AND srckey%100 < 1 )  order by id limit ?".replaceAll("\\s+", " ");
    snapshotExpectedString = "Select id, scn,  CAST(srckey as SIGNED) as srckey, val from snapshotTable where  id > ?  and scn < ?  and scn >= ? AND  ( srckey%100 >= 0 AND srckey%100 < 1 )  order by id limit ?".replaceAll("\\s+", " ");
    Assert.assertEquals(catchUpString, catchUpExpectedString);
    Assert.assertEquals(snapshotString, snapshotExpectedString);
}

Example 5 with KeyFilterConfigHolder

use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.

the class RelayPullThread method doRequestStream.

protected void doRequestStream(ConnectionState curState) {
    boolean debugEnabled = _log.isDebugEnabled();
    if (debugEnabled)
        _log.debug("Checking for free space in buffer");
    int freeBufferThreshold = (int) (_sourcesConn.getConnectionConfig().getFreeBufferThreshold() * 100.0 / _pullerBufferUtilizationPct);
    try {
        curState.getDataEventsBuffer().waitForFreeSpace(freeBufferThreshold);
    } catch (InterruptedException ie) {
        //loop
        enqueueMessage(curState);
        return;
    }
    Checkpoint cp = curState.getCheckpoint();
    if (debugEnabled)
        _log.debug("Checkpoint at RequestDataEvents: " + cp.toString());
    if (null == _relayFilter) {
        if (debugEnabled)
            _log.debug("Initializing relay filter config");
        _relayFilter = new DbusKeyCompositeFilter();
        Map<String, IdNamePair> srcNameIdMap = curState.getSourcesNameMap();
        for (DbusKeyCompositeFilterConfig conf : _relayFilterConfigs) {
            Map<String, KeyFilterConfigHolder> cMap = conf.getConfigMap();
            Map<Long, KeyFilterConfigHolder> fConfMap = new HashMap<Long, KeyFilterConfigHolder>();
            for (Entry<String, KeyFilterConfigHolder> e : cMap.entrySet()) {
                IdNamePair idName = srcNameIdMap.get(e.getKey());
                if (null != idName) {
                    fConfMap.put(idName.getId(), e.getValue());
                }
            }
            if (debugEnabled)
                _log.debug("FilterConfMap is :" + fConfMap);
            _relayFilter.merge(new DbusKeyCompositeFilter(fConfMap));
        }
        if (debugEnabled)
            _log.debug("Merged Filter (before deduping) is :" + _relayFilter);
        _relayFilter.dedupe();
        if (debugEnabled)
            _log.debug("Merged Filter (after deduping) is :" + _relayFilter);
    }
    _streamCallStartMs = System.currentTimeMillis();
    if (null != _relayCallsStats)
        _relayCallsStats.registerStreamRequest(cp, EMPTY_STREAM_LIST);
    int fetchSize = (int) ((curState.getDataEventsBuffer().getBufferFreeReadSpace() / 100.0) * _pullerBufferUtilizationPct);
    fetchSize = Math.max(freeBufferThreshold, fetchSize);
    CheckpointMult cpMult = new CheckpointMult();
    String args;
    if (curState.getRelayConnection().getProtocolVersion() >= 3) {
        // for version 3 and higher we pass subscriptions
        args = curState.getSubsListString();
        for (DatabusSubscription sub : curState.getSubscriptions()) {
            PhysicalPartition p = sub.getPhysicalPartition();
            cpMult.addCheckpoint(p, cp);
        }
    } else {
        args = curState.getSourcesIdListString();
        cpMult.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
    }
    curState.switchToStreamRequestSent();
    sendHeartbeat(_sourcesConn.getUnifiedClientStats());
    curState.getRelayConnection().requestStream(args, _relayFilter, fetchSize, cpMult, _sourcesConn.getConnectionConfig().getKeyRange(), curState);
}