use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.
the class TestDbusKeyCompositeFilter method testDbusKeyModFilterErrors.
@Test
public void testDbusKeyModFilterErrors() throws Exception {
//Error Config : MaxBucket is more than numBuckets
boolean isException = false;
try {
KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
partConf.setType("MOD");
KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
modConf.setNumBuckets(5);
//invalid config
modConf.setBuckets("[0,3-9]");
partConf.setMod(modConf);
DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
} catch (InvalidConfigException ie) {
ie.printStackTrace();
isException = true;
}
assertEquals("Got Exception for invalid Config (MaxBucket is more than numBuckets) ", true, isException);
//Error Case : Min Bucket is more than maxBucket
isException = false;
try {
KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
partConf.setType("MOD");
KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
modConf.setNumBuckets(50);
//invalid config
modConf.setBuckets("[0,9-3]");
partConf.setMod(modConf);
DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
} catch (InvalidConfigException ie) {
ie.printStackTrace();
isException = true;
}
assertEquals("Got Exception for invalid Config (Min Bucket is more than maxBucket) ", true, isException);
//Error Case : numBuckets is negative
isException = false;
try {
KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
partConf.setType("MOD");
KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
modConf.setNumBuckets(-5);
//invalid config
modConf.setBuckets("[0]");
partConf.setMod(modConf);
DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
} catch (InvalidConfigException ie) {
ie.printStackTrace();
isException = true;
}
assertEquals("Got Exception for invalid Config numBuckets is negative) ", true, isException);
//Error Case : minBucket is negative
isException = false;
try {
KeyFilterConfigHolder.Config partConf = new KeyFilterConfigHolder.Config();
partConf.setType("MOD");
KeyModFilterConfig.Config modConf = new KeyModFilterConfig.Config();
modConf.setNumBuckets(50);
//invalid config
modConf.setBuckets("[-5,1-3]");
partConf.setMod(modConf);
DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
} catch (InvalidConfigException ie) {
ie.printStackTrace();
isException = true;
}
assertEquals("Got Exception for invalid Config (minBucket is negative) ", true, isException);
}
use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.
the class MockRemoteExceptionHandler method testRelayPullerThreadV3WithFilter.
// make sure the filter is passed along to the doRequestStream call
@Test
public void testRelayPullerThreadV3WithFilter() throws Exception {
RelayPullThreadBuilder bldr = new RelayPullThreadBuilder(false, false);
// create filter
DbusKeyCompositeFilterConfig filterConfig = createFakeFilter();
bldr.setFilterConfig(filterConfig);
RelayPullThread relayPuller = bldr.createRelayPullThread();
relayPuller.getComponentStatus().start();
ConnectionState connState = relayPuller.getConnectionState();
connState.switchToPickServer();
testTransitionCase(relayPuller, StateId.PICK_SERVER, StateId.REQUEST_SOURCES);
// get the created connection
MockRelayConnection conn = (MockRelayConnection) relayPuller.getLastOpenConnection();
conn.setProtocolVersion(3);
relayPuller.getMessageQueue().clear();
testTransitionCase(relayPuller, StateId.REQUEST_SOURCES, StateId.SOURCES_RESPONSE_SUCCESS);
relayPuller.getMessageQueue().clear();
testTransitionCase(relayPuller, StateId.SOURCES_RESPONSE_SUCCESS, StateId.REQUEST_REGISTER);
relayPuller.getMessageQueue().clear();
testTransitionCase(relayPuller, StateId.REQUEST_REGISTER, StateId.REGISTER_RESPONSE_SUCCESS);
relayPuller.getMessageQueue().clear();
testTransitionCase(relayPuller, StateId.REGISTER_RESPONSE_SUCCESS, StateId.REQUEST_STREAM);
relayPuller.getMessageQueue().clear();
testTransitionCase(relayPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS);
DbusKeyCompositeFilter filter = conn.getRelayFilter();
Assert.assertNotNull(filter);
// get DbusKeyFilter that we put in
KeyFilterConfigHolder configHolder = filterConfig.getConfigMap().values().iterator().next();
DbusKeyFilter inFilter = new DbusKeyFilter(configHolder);
// get filter from the connection
Assert.assertEquals(filter.getFilterMap().size(), 1, "1 filter in the composite filter");
Assert.assertTrue(filter.getFilterMap().entrySet().iterator().hasNext(), "has one filter");
DbusKeyFilter newFilter = filter.getFilterMap().values().iterator().next();
//compare them
boolean eq = inFilter.getPartitionType() == newFilter.getPartitionType() && inFilter.getFilters().size() == newFilter.getFilters().size();
Assert.assertTrue(eq, "Same Filter");
}
use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.
the class TestFilterToSQL method testNonePartition.
@Test
public void testNonePartition() throws IOException, InvalidConfigException, InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException {
partConf.setType("NONE");
DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
processor.setKeyFilter(filter);
LOG.info(" Testing no parition filter (None parition");
LOG.info("CATCHUP TABLE: " + processor.getCatchupSQLString("catchuptab"));
LOG.info("SNAPSHOT TABLE: " + processor.getSnapshotSQLString("snapshotTable"));
String catchUpString = processor.getCatchupSQLString("catchuptab").replaceAll("\\s+", " ");
String snapshotString = processor.getSnapshotSQLString("snapshotTable").replaceAll("\\s+", " ");
String catchUpExpectedString = "Select id, scn, windowscn, val from catchuptab where id > ? and windowscn >= ? and windowscn <= ? and windowscn >= ? order by id limit ?".replaceAll("\\s+", " ");
String snapshotExpectedString = "Select id, scn, srckey, val from snapshotTable where id > ? and scn < ? and scn >= ? order by id limit ?".replaceAll("\\s+", " ");
Assert.assertEquals(catchUpString, catchUpExpectedString);
Assert.assertEquals(snapshotString, snapshotExpectedString);
}
use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.
the class TestFilterToSQL method testRangeFilter.
@Test
public void testRangeFilter() throws InvalidConfigException, InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException {
partConf = new KeyFilterConfigHolder.Config();
partConf.setType("RANGE");
KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
rangeConf.setSize(100);
rangeConf.setPartitions("[0,3-4]");
partConf.setRange(rangeConf);
DbusKeyFilter filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
processor.setKeyFilter(filter);
LOG.info("Testing multiple range filters: ");
LOG.info("CATCHUP TABLE: " + processor.getCatchupSQLString("catchuptab"));
LOG.info("SNAPSHOT TABLE: " + processor.getSnapshotSQLString("snapshotTable"));
String catchUpString = processor.getCatchupSQLString("catchuptab").replaceAll("\\s+", " ");
String snapshotString = processor.getSnapshotSQLString("snapshotTable").replaceAll("\\s+", " ");
String catchUpExpectedString = "Select id, scn, windowscn, val, CAST(srckey as SIGNED) as srckey from catchuptab where id > ? and windowscn >= ? and windowscn <= ? and windowscn >= ? AND ( srckey >= 0 AND srckey < 100 OR srckey >= 300 AND srckey < 500 ) order by id limit ?".replaceAll("\\s+", " ");
String snapshotExpectedString = "Select id, scn, CAST(srckey as SIGNED) as srckey, val from snapshotTable where id > ? and scn < ? and scn >= ? AND ( srckey >= 0 AND srckey < 100 OR srckey >= 300 AND srckey < 500 ) order by id limit ?".replaceAll("\\s+", " ");
Assert.assertEquals(catchUpString, catchUpExpectedString);
Assert.assertEquals(snapshotString, snapshotExpectedString);
partConf = new KeyFilterConfigHolder.Config();
partConf.setType("RANGE");
rangeConf = new KeyRangeFilterConfig.Config();
rangeConf.setSize(100);
rangeConf.setPartitions("[0]");
partConf.setRange(rangeConf);
filter = new DbusKeyFilter(new KeyFilterConfigHolder(partConf.build()));
processor.setKeyFilter(filter);
LOG.info("Testing single range filter");
LOG.info("CATCHUP TABLE: " + processor.getCatchupSQLString("catchuptab"));
LOG.info("SNAPSHOT TABLE: " + processor.getSnapshotSQLString("snapshotTable"));
catchUpString = processor.getCatchupSQLString("catchuptab").replaceAll("\\s+", " ");
snapshotString = processor.getSnapshotSQLString("snapshotTable").replaceAll("\\s+", " ");
catchUpExpectedString = "Select id, scn, windowscn, val, CAST(srckey as SIGNED) as srckey from catchuptab where id > ? and windowscn >= ? and windowscn <= ? and windowscn >= ? AND ( srckey >= 0 AND srckey < 100 ) order by id limit ?".replaceAll("\\s+", " ");
snapshotExpectedString = "Select id, scn, CAST(srckey as SIGNED) as srckey, val from snapshotTable where id > ? and scn < ? and scn >= ? AND ( srckey >= 0 AND srckey < 100 ) order by id limit ?".replaceAll("\\s+", " ");
Assert.assertEquals(catchUpString, catchUpExpectedString);
Assert.assertEquals(snapshotString, snapshotExpectedString);
}
use of com.linkedin.databus2.core.filter.KeyFilterConfigHolder in project databus by linkedin.
the class TestDbusKeyCompositeFilter method testDbusKeyCompositeFilter.
@Test
public void testDbusKeyCompositeFilter() throws Exception {
KeyFilterConfigHolder.Config partConf1 = new KeyFilterConfigHolder.Config();
partConf1.setType("MOD");
KeyModFilterConfig.Config modConf1 = new KeyModFilterConfig.Config();
modConf1.setNumBuckets(100);
modConf1.setBuckets("[0,0]");
partConf1.setMod(modConf1);
KeyFilterConfigHolder.Config partConf2 = new KeyFilterConfigHolder.Config();
partConf2.setType("RANGE");
KeyRangeFilterConfig.Config rangeConf = new KeyRangeFilterConfig.Config();
rangeConf.setSize(100);
rangeConf.setPartitions("[3-4,3-4]");
partConf2.setRange(rangeConf);
HashMap<Long, KeyFilterConfigHolder> partConfigMap = new HashMap<Long, KeyFilterConfigHolder>();
partConfigMap.put(1L, new KeyFilterConfigHolder(partConf1.build()));
partConfigMap.put(2L, new KeyFilterConfigHolder(partConf2.build()));
List<Long> keys1 = new ArrayList<Long>();
List<Long> keys2 = new ArrayList<Long>();
List<Long> keys3 = new ArrayList<Long>();
for (long i = 0; i < 1000; i++) {
keys1.add(i);
keys2.add(i);
keys3.add(i);
}
List<DbusEvent> dbusEvents = new ArrayList<DbusEvent>();
generateEvents(1000, (short) 1, keys1, dbusEvents);
generateEvents(1000, (short) 2, keys2, dbusEvents);
generateEvents(1000, (short) 3, keys3, dbusEvents);
List<DbusEvent> expPassedEvents = new ArrayList<DbusEvent>();
List<DbusEvent> expFailedEvents = new ArrayList<DbusEvent>();
System.out.println("TOTAL Events :" + dbusEvents.size());
for (DbusEvent event : dbusEvents) {
long key = event.key();
int srcId = event.srcId();
long bktId = key % 100;
if ((srcId == 1) && (bktId == 0))
expPassedEvents.add(event);
else if ((srcId == 2) && ((key >= 300) && (key < 500)))
expPassedEvents.add(event);
else if (srcId == 3)
expPassedEvents.add(event);
else
expFailedEvents.add(event);
}
DbusKeyCompositeFilter filter = new DbusKeyCompositeFilter(partConfigMap);
filter.dedupe();
List<DbusEvent> passedEvents = new ArrayList<DbusEvent>();
List<DbusEvent> failedEvents = new ArrayList<DbusEvent>();
for (DbusEvent event : dbusEvents) {
if (filter.allow(event)) {
passedEvents.add(event);
} else {
failedEvents.add(event);
}
}
System.out.println("Passed Event Size :" + passedEvents.size());
System.out.println("Failed Event Size :" + failedEvents.size());
assertEquals("Passed Size", expPassedEvents.size(), passedEvents.size());
assertEquals("Failed Size", expFailedEvents.size(), failedEvents.size());
for (int i = 0; i < passedEvents.size(); i++) {
assertEquals("Passed Element " + i, expPassedEvents.get(i), passedEvents.get(i));
}
for (int i = 0; i < passedEvents.size(); i++) {
assertEquals("Failed Element " + i, expFailedEvents.get(i), failedEvents.get(i));
}
ObjectMapper objMapper = new ObjectMapper();
String objStr = objMapper.writeValueAsString(filter.getFilterMap());
System.out.println("CompositeKeyFilter :" + objStr);
Map<Long, DbusKeyFilter> map2 = KeyFilterConfigJSONFactory.parseSrcIdFilterConfigMap(objStr);
String objStr2 = objMapper.writeValueAsString(filter.getFilterMap());
System.out.println("CompositeKeyFilter2 :" + objStr2);
assertEquals("CompositeKeys: JSON Serialization Test", objStr, objStr2);
//String objStr3 = "{\"filterMap\":{\"40\":{\"partitionType\":\"RANGE\",\"filters\":[{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}},{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}}]}}}";
//DbusKeyCompositeFilter f = KeyFilterJSONFactory.parseKeyCompositeFilter(objStr3);
//System.out.println("Deserialized Filter is :" + f);
String objStr4 = "{\"40\":{\"partitionType\":\"RANGE\",\"filters\":[{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}},{\"keyRange\":{\"start\":100,\"end\":200}},{\"keyRange\":{\"start\":300,\"end\":500}}]}}}";
Map<Long, DbusKeyFilter> map3 = KeyFilterConfigJSONFactory.parseSrcIdFilterConfigMap(objStr4);
DbusKeyCompositeFilter f2 = new DbusKeyCompositeFilter();
f2.setFilterMap(map3);
System.out.println("Deserialized Filter is (before dedupe): " + f2);
f2.dedupe();
System.out.println("Deserialized Filter is (after dedupe): " + f2);
}
Aggregations