Examples with KeyFilterConfigHolder com.linkedin.databus2.core.filter.KeyFilterConfigHolder used on opensource projects

Example 6 with KeyFilterConfigHolder

use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyModFilterErrors.

@Test
public void testDbusKeyModFilterErrors() throws Exception {
    //Error Config : MaxBucket is more than numBuckets
    boolean isException = false;
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(5);
        //invalid config
        modConf.setBuckets("[0,3-9]");
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (MaxBucket is more than numBuckets) ", true, isException);
    //Error Case : Min Bucket is more than maxBucket
    isException = false;
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(50);
        //invalid config
        modConf.setBuckets("[0,9-3]");
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (Min Bucket is more than maxBucket) ", true, isException);
    //Error Case : numBuckets is negative
    isException = false;
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(-5);
        //invalid config
        modConf.setBuckets("[0]");
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
    }
    assertEquals("Got Exception for invalid Config numBuckets is negative) ", true, isException);
    //Error Case : minBucket is negative
    isException = false;
    try {
        KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
        partConf.setType("MOD");
        KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
        modConf.setNumBuckets(50);
        //invalid config
        modConf.setBuckets("[-5,1-3]");
        partConf.setMod(modConf);
        DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    } catch (InvalidConfigException ie) {
        ie.printStackTrace();
        isException = true;
    }
    assertEquals("Got Exception for invalid Config (minBucket is negative) ", true, isException);
}

Example 7 with KeyFilterConfigHolder

use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.

the class MockRemoteExceptionHandler method testRelayPullerThreadV3WithFilter.

// make sure the filter is passed along to the doRequestStream call
@Test
public void testRelayPullerThreadV3WithFilter() throws Exception {
    RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
    // create filter
    DbusKeyCompositeFilterConfig filterConfig = createFakeFilter();
    bldr.setFilterConfig(filterConfig);
    RelayPullThread relayPuller = bldr.createRelayPullThread();
    relayPuller.getComponentStatus().start();
    ConnectionState connState = relayPuller.getConnectionState();
    connState.switchToPickServer();
    testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
    // get the created connection
    MockRelayConnection conn = (MockRelayConnection) relayPuller.getLastOpenConnection();
    conn.setProtocolVersion(3);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
    relayPuller.getMessageQueue().clear();
    testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
    DbusKeyCompositeFilter filter = conn.getRelayFilter();
    Assert.assertNotNull(filter);
    // get DbusKeyFilter that we put in
    KeyFilterConfigHolder configHolder = filterConfig.getConfigMap().values().iterator().next();
    DbusKeyFilter inFilter = new DbusKeyFilter(configHolder);
    // get filter from the connection
    Assert.assertEquals(filter.getFilterMap().size(), 1, "1 filter in the composite filter");
    Assert.assertTrue(filter.getFilterMap().entrySet().iterator().hasNext(), "has one filter");
    DbusKeyFilter newFilter = filter.getFilterMap().values().iterator().next();
    //compare them
    boolean eq = inFilter.getPartitionType() == newFilter.getPartitionType() && inFilter.getFilters().size() == newFilter.getFilters().size();
    Assert.assertTrue(eq, "Same Filter");
}

Example 8 with KeyFilterConfigHolder

use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.

the class TestFilterToSQL method testNonePartition.

@Test
public void testNonePartition() throws IOException, InvalidConfigException, InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException {
    partConf.setType("NONE");
    DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    processor.setKeyFilter(filter);
    LOG.info(" Testing no parition filter (None parition");
    LOG.info("CATCHUP TABLE: " + processor.getCatchupSQLString("catchuptab"));
    LOG.info("SNAPSHOT TABLE: " + processor.getSnapshotSQLString("snapshotTable"));
    String catchUpString = processor.getCatchupSQLString("catchuptab").replaceAll("\\s+", " ");
    String snapshotString = processor.getSnapshotSQLString("snapshotTable").replaceAll("\\s+", " ");
    String catchUpExpectedString = "Select id, scn, windowscn, val from catchuptab where  id > ?  and windowscn >= ? and windowscn <= ?  and windowscn >= ?  order by id limit ?".replaceAll("\\s+", " ");
    String snapshotExpectedString = "Select id, scn, srckey, val from snapshotTable where  id > ?  and scn < ?  and scn >= ?  order by id limit ?".replaceAll("\\s+", " ");
    Assert.assertEquals(catchUpString, catchUpExpectedString);
    Assert.assertEquals(snapshotString, snapshotExpectedString);
}

Example 9 with KeyFilterConfigHolder

use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.

the class TestFilterToSQL method testRangeFilter.

@Test
public void testRangeFilter() throws InvalidConfigException, InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException {
    partConf = new KeyFilterConfigHolder.Config();
    partConf.setType("RANGE");
    KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
    rangeConf.setSize(100);
    rangeConf.setPartitions("[0,3-4]");
    partConf.setRange(rangeConf);
    DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    processor.setKeyFilter(filter);
    LOG.info("Testing multiple range filters: ");
    LOG.info("CATCHUP TABLE: " + processor.getCatchupSQLString("catchuptab"));
    LOG.info("SNAPSHOT TABLE: " + processor.getSnapshotSQLString("snapshotTable"));
    String catchUpString = processor.getCatchupSQLString("catchuptab").replaceAll("\\s+", " ");
    String snapshotString = processor.getSnapshotSQLString("snapshotTable").replaceAll("\\s+", " ");
    String catchUpExpectedString = "Select id, scn, windowscn, val, CAST(srckey as SIGNED) as srckey from catchuptab where  id > ?  and windowscn >= ? and windowscn <= ?  and windowscn >= ? AND  ( srckey >= 0 AND srckey < 100 OR srckey >= 300 AND srckey < 500 )  order by id limit ?".replaceAll("\\s+", " ");
    String snapshotExpectedString = "Select id, scn,  CAST(srckey as SIGNED) as srckey, val from snapshotTable where  id > ?  and scn < ?  and scn >= ? AND  ( srckey >= 0 AND srckey < 100 OR srckey >= 300 AND srckey < 500 )  order by id limit ?".replaceAll("\\s+", " ");
    Assert.assertEquals(catchUpString, catchUpExpectedString);
    Assert.assertEquals(snapshotString, snapshotExpectedString);
    partConf = new KeyFilterConfigHolder.Config();
    partConf.setType("RANGE");
    rangeConf = new KeyRangeFilterConfig.Config();
    rangeConf.setSize(100);
    rangeConf.setPartitions("[0]");
    partConf.setRange(rangeConf);
    filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
    processor.setKeyFilter(filter);
    LOG.info("Testing single range filter");
    LOG.info("CATCHUP TABLE: " + processor.getCatchupSQLString("catchuptab"));
    LOG.info("SNAPSHOT TABLE: " + processor.getSnapshotSQLString("snapshotTable"));
    catchUpString = processor.getCatchupSQLString("catchuptab").replaceAll("\\s+", " ");
    snapshotString = processor.getSnapshotSQLString("snapshotTable").replaceAll("\\s+", " ");
    catchUpExpectedString = "Select id, scn, windowscn, val, CAST(srckey as SIGNED) as srckey from catchuptab where  id > ?  and windowscn >= ? and windowscn <= ?  and windowscn >= ? AND  ( srckey >= 0 AND srckey < 100 )  order by id limit ?".replaceAll("\\s+", " ");
    snapshotExpectedString = "Select id, scn,  CAST(srckey as SIGNED) as srckey, val from snapshotTable where  id > ?  and scn < ?  and scn >= ? AND  ( srckey >= 0 AND srckey < 100 )  order by id limit ?".replaceAll("\\s+", " ");
    Assert.assertEquals(catchUpString, catchUpExpectedString);
    Assert.assertEquals(snapshotString, snapshotExpectedString);
}

Example 10 with KeyFilterConfigHolder

use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.

the class TestDbusKeyCompositeFilter method testDbusKeyCompositeFilter.

@Test
public void testDbusKeyCompositeFilter() throws Exception {
    KeyFilterConfigHolder.Config partConf1 = new KeyFilterConfigHolder.Config();
    partConf1.setType("MOD");
    KeyModFilterConfig.Config modConf1 = new KeyModFilterConfig.Config();
    modConf1.setNumBuckets(100);
    modConf1.setBuckets("[0,0]");
    partConf1.setMod(modConf1);
    KeyFilterConfigHolder.Config partConf2 = new KeyFilterConfigHolder.Config();
    partConf2.setType("RANGE");
    KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
    rangeConf.setSize(100);
    rangeConf.setPartitions("[3-4,3-4]");
    partConf2.setRange(rangeConf);
    HashMap<Long, KeyFilterConfigHolder> partConfigMap = new HashMap<Long, KeyFilterConfigHolder>();
    partConfigMap.put(1L, new KeyFilterConfigHolder(partConf1.build()));
    partConfigMap.put(2L, new KeyFilterConfigHolder(partConf2.build()));
    List<Long> keys1 = new ArrayList<Long>();
    List<Long> keys2 = new ArrayList<Long>();
    List<Long> keys3 = new ArrayList<Long>();
    for (long i = 0; i < 1000; i++) {
        keys1.add(i);
        keys2.add(i);
        keys3.add(i);
    }
    List<DbusEvent> dbusEvents = new ArrayList<DbusEvent>();
    generateEvents(1000, (short) 1, keys1, dbusEvents);
    generateEvents(1000, (short) 2, keys2, dbusEvents);
    generateEvents(1000, (short) 3, keys3, dbusEvents);
    List<DbusEvent> expPassedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> expFailedEvents = new ArrayList<DbusEvent>();
    System.out.println("TOTAL Events :" + dbusEvents.size());
    for (DbusEvent event : dbusEvents) {
        long key = event.key();
        int srcId = event.srcId();
        long bktId = key % 100;
        if ((srcId == 1) && (bktId == 0))
            expPassedEvents.add(event);
        else if ((srcId == 2) && ((key >= 300) && (key < 500)))
            expPassedEvents.add(event);
        else if (srcId == 3)
            expPassedEvents.add(event);
        else
            expFailedEvents.add(event);
    }
    DbusKeyCompositeFilter filter = new DbusKeyCompositeFilter(partConfigMap);
    filter.dedupe();
    List<DbusEvent> passedEvents = new ArrayList<DbusEvent>();
    List<DbusEvent> failedEvents = new ArrayList<DbusEvent>();
    for (DbusEvent event : dbusEvents) {
        if (filter.allow(event)) {
            passedEvents.add(event);
        } else {
            failedEvents.add(event);
        }
    }
    System.out.println("Passed Event Size :" + passedEvents.size());
    System.out.println("Failed Event Size :" + failedEvents.size());
    assertEquals("Passed Size", expPassedEvents.size(), passedEvents.size());
    assertEquals("Failed Size", expFailedEvents.size(), failedEvents.size());
    for (int i = 0; i < passedEvents.size(); i++) {
        assertEquals("Passed Element " + i, expPassedEvents.get(i), passedEvents.get(i));
    }
    for (int i = 0; i < passedEvents.size(); i++) {
        assertEquals("Failed Element " + i, expFailedEvents.get(i), failedEvents.get(i));
    }
    ObjectMapper objMapper = new ObjectMapper();
    String objStr = objMapper.writeValueAsString(filter.getFilterMap());
    System.out.println("CompositeKeyFilter :" + objStr);
    Map<Long, DbusKeyFilter> map2 = KeyFilterConfigJSONFactory.parseSrcIdFilterConfigMap(objStr);
    String objStr2 = objMapper.writeValueAsString(filter.getFilterMap());
    System.out.println("CompositeKeyFilter2 :" + objStr2);
    assertEquals("CompositeKeys: JSON Serialization Test", objStr, objStr2);
    //String objStr3 = "{\"filterMap\":{\"40\":{\"partitionType\":\"RANGE\",\"filters\":[{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}},{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}}]}}}";
    //DbusKeyCompositeFilter f = KeyFilterJSONFactory.parseKeyCompositeFilter(objStr3);
    //System.out.println("Deserialized Filter is :" + f);
    String objStr4 = "{\"40\":{\"partitionType\":\"RANGE\",\"filters\":[{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}},{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}}]}}}";
    Map<Long, DbusKeyFilter> map3 = KeyFilterConfigJSONFactory.parseSrcIdFilterConfigMap(objStr4);
    DbusKeyCompositeFilter f2 = new DbusKeyCompositeFilter();
    f2.setFilterMap(map3);
    System.out.println("Deserialized Filter is (before dedupe): " + f2);
    f2.dedupe();
    System.out.println("Deserialized Filter is (after dedupe): " + f2);
}