use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.
the class StreamExpressionTest method testHashJoinStream.
@Test
public void testHashJoinStream() throws Exception {
new UpdateRequest().add(id, "1", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", // 8, 9
"left_1").add(id, "15", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", // 8, 9
"left_1").add(id, "2", "side_s", "left", "join1_i", "0", "join2_s", "b", "ident_s", "left_2").add(id, "3", "side_s", "left", "join1_i", "1", "join2_s", "a", "ident_s", // 10
"left_3").add(id, "4", "side_s", "left", "join1_i", "1", "join2_s", "b", "ident_s", // 11
"left_4").add(id, "5", "side_s", "left", "join1_i", "1", "join2_s", "c", "ident_s", // 12
"left_5").add(id, "6", "side_s", "left", "join1_i", "2", "join2_s", "d", "ident_s", "left_6").add(id, "7", "side_s", "left", "join1_i", "3", "join2_s", "e", "ident_s", // 14
"left_7").add(id, "8", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_1", "join3_i", // 1,15
"0").add(id, "9", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_2", "join3_i", // 1,15
"0").add(id, "10", "side_s", "right", "join1_i", "1", "join2_s", "a", "ident_s", "right_3", "join3_i", // 3
"1").add(id, "11", "side_s", "right", "join1_i", "1", "join2_s", "b", "ident_s", "right_4", "join3_i", // 4
"1").add(id, "12", "side_s", "right", "join1_i", "1", "join2_s", "c", "ident_s", "right_5", "join3_i", // 5
"1").add(id, "13", "side_s", "right", "join1_i", "2", "join2_s", "dad", "ident_s", "right_6", "join3_i", "2").add(id, "14", "side_s", "right", "join1_i", "3", "join2_s", "e", "ident_s", "right_7", "join3_i", // 7
"3").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
StreamExpression expression;
TupleStream stream;
List<Tuple> tuples;
StreamContext streamContext = new StreamContext();
SolrClientCache solrClientCache = new SolrClientCache();
streamContext.setSolrClientCache(solrClientCache);
StreamFactory factory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("hashJoin", HashJoinStream.class);
try {
// Basic test
expression = StreamExpressionParser.parse("hashJoin(" + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\")," + "hashed=search(collection1, q=\"side_s:right\", fl=\"join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc\")," + "on=\"join1_i, join2_s\")");
stream = new HashJoinStream(expression, factory);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assert (tuples.size() == 8);
assertOrder(tuples, 1, 1, 15, 15, 3, 4, 5, 7);
// Basic desc
expression = StreamExpressionParser.parse("hashJoin(" + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i desc, join2_s asc\")," + "hashed=search(collection1, q=\"side_s:right\", fl=\"join1_i,join2_s,ident_s\", sort=\"join1_i desc, join2_s asc\")," + "on=\"join1_i, join2_s\")");
stream = new HashJoinStream(expression, factory);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assert (tuples.size() == 8);
assertOrder(tuples, 7, 3, 4, 5, 1, 1, 15, 15);
// Results in both searches, no join matches
expression = StreamExpressionParser.parse("hashJoin(" + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"ident_s asc\")," + "hashed=search(collection1, q=\"side_s:right\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"ident_s asc\")," + "on=\"ident_s\")");
stream = new HashJoinStream(expression, factory);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assert (tuples.size() == 0);
// Basic test with "on" mapping
expression = StreamExpressionParser.parse("hashJoin(" + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join3_i,ident_s\", sort=\"join1_i asc, join3_i asc, id asc\")," + "hashed=search(collection1, q=\"side_s:right\", fl=\"join1_i,join3_i,ident_s\", sort=\"join1_i asc, join3_i asc\")," + "on=\"join1_i=join3_i\")");
stream = new HashJoinStream(expression, factory);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assertEquals(17, tuples.size());
//Does a lexical sort
assertOrder(tuples, 1, 1, 15, 15, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 7);
} finally {
solrClientCache.close();
}
}
use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.
the class StreamExpressionTest method testParallelFetchStream.
@Test
public void testParallelFetchStream() throws Exception {
new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1", "subject", "blah blah blah 0").add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2", "subject", "blah blah blah 2").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "subject", "blah blah blah 3").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "subject", "blah blah blah 4").add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5", "subject", "blah blah blah 1").add(id, "5", "a_s", "hello3", "a_i", "5", "a_f", "6", "subject", "blah blah blah 5").add(id, "6", "a_s", "hello4", "a_i", "6", "a_f", "7", "subject", "blah blah blah 6").add(id, "7", "a_s", "hello3", "a_i", "7", "a_f", "8", "subject", "blah blah blah 7").add(id, "8", "a_s", "hello3", "a_i", "8", "a_f", "9", "subject", "blah blah blah 8").add(id, "9", "a_s", "hello0", "a_i", "9", "a_f", "10", "subject", "blah blah blah 9").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
StreamContext streamContext = new StreamContext();
SolrClientCache solrClientCache = new SolrClientCache();
streamContext.setSolrClientCache(solrClientCache);
TupleStream stream;
List<Tuple> tuples;
StreamFactory factory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("parallel", ParallelStream.class).withFunctionName("fetch", FetchStream.class);
try {
stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\", fetch(" + COLLECTIONORALIAS + ", search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=\"id\"), on=\"id=a_i\", batchSize=\"2\", fl=\"subject\"))");
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assert (tuples.size() == 10);
Tuple t = tuples.get(0);
assertTrue("blah blah blah 0".equals(t.getString("subject")));
t = tuples.get(1);
assertTrue("blah blah blah 2".equals(t.getString("subject")));
t = tuples.get(2);
assertTrue("blah blah blah 3".equals(t.getString("subject")));
t = tuples.get(3);
assertTrue("blah blah blah 4".equals(t.getString("subject")));
t = tuples.get(4);
assertTrue("blah blah blah 1".equals(t.getString("subject")));
t = tuples.get(5);
assertTrue("blah blah blah 5".equals(t.getString("subject")));
t = tuples.get(6);
assertTrue("blah blah blah 6".equals(t.getString("subject")));
t = tuples.get(7);
assertTrue("blah blah blah 7".equals(t.getString("subject")));
t = tuples.get(8);
assertTrue("blah blah blah 8".equals(t.getString("subject")));
t = tuples.get(9);
assertTrue("blah blah blah 9".equals(t.getString("subject")));
stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\", fetch(" + COLLECTIONORALIAS + ", search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=\"id\"), on=\"id=a_i\", batchSize=\"3\", fl=\"subject\"))");
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assert (tuples.size() == 10);
t = tuples.get(0);
assertTrue("blah blah blah 0".equals(t.getString("subject")));
t = tuples.get(1);
assertTrue("blah blah blah 2".equals(t.getString("subject")));
t = tuples.get(2);
assertTrue("blah blah blah 3".equals(t.getString("subject")));
t = tuples.get(3);
assertTrue("blah blah blah 4".equals(t.getString("subject")));
t = tuples.get(4);
assertTrue("blah blah blah 1".equals(t.getString("subject")));
t = tuples.get(5);
assertTrue("blah blah blah 5".equals(t.getString("subject")));
t = tuples.get(6);
assertTrue("blah blah blah 6".equals(t.getString("subject")));
t = tuples.get(7);
assertTrue("blah blah blah 7".equals(t.getString("subject")));
t = tuples.get(8);
assertTrue("blah blah blah 8".equals(t.getString("subject")));
t = tuples.get(9);
assertTrue("blah blah blah 9".equals(t.getString("subject")));
} finally {
solrClientCache.close();
}
}
use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.
the class StreamExpressionTest method testSelectStream.
@Test
public void testSelectStream() throws Exception {
new UpdateRequest().add(id, "1", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", // 8, 9
"left_1").add(id, "15", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", // 8, 9
"left_1").add(id, "2", "side_s", "left", "join1_i", "0", "join2_s", "b", "ident_s", "left_2").add(id, "3", "side_s", "left", "join1_i", "1", "join2_s", "a", "ident_s", // 10
"left_3").add(id, "4", "side_s", "left", "join1_i", "1", "join2_s", "b", "ident_s", // 11
"left_4").add(id, "5", "side_s", "left", "join1_i", "1", "join2_s", "c", "ident_s", // 12
"left_5").add(id, "6", "side_s", "left", "join1_i", "2", "join2_s", "d", "ident_s", "left_6").add(id, "7", "side_s", "left", "join1_i", "3", "join2_s", "e", "ident_s", // 14
"left_7").add(id, "8", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_1", "join3_i", // 1,15
"0").add(id, "9", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_2", "join3_i", // 1,15
"0").add(id, "10", "side_s", "right", "join1_i", "1", "join2_s", "a", "ident_s", "right_3", "join3_i", // 3
"1").add(id, "11", "side_s", "right", "join1_i", "1", "join2_s", "b", "ident_s", "right_4", "join3_i", // 4
"1").add(id, "12", "side_s", "right", "join1_i", "1", "join2_s", "c", "ident_s", "right_5", "join3_i", // 5
"1").add(id, "13", "side_s", "right", "join1_i", "2", "join2_s", "dad", "ident_s", "right_6", "join3_i", "2").add(id, "14", "side_s", "right", "join1_i", "3", "join2_s", "e", "ident_s", "right_7", "join3_i", // 7
"3").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
String clause;
TupleStream stream;
List<Tuple> tuples;
StreamContext streamContext = new StreamContext();
SolrClientCache solrClientCache = new SolrClientCache();
streamContext.setSolrClientCache(solrClientCache);
StreamFactory factory = new StreamFactory().withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("innerJoin", InnerJoinStream.class).withFunctionName("select", SelectStream.class).withFunctionName("replace", ReplaceOperation.class).withFunctionName("concat", ConcatOperation.class).withFunctionName("add", AddEvaluator.class).withFunctionName("if", IfThenElseEvaluator.class).withFunctionName("gt", GreaterThanEvaluator.class);
try {
// Basic test
clause = "select(" + "id, join1_i as join1, join2_s as join2, ident_s as identity," + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\")" + ")";
stream = factory.constructStream(clause);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assertFields(tuples, "id", "join1", "join2", "identity");
assertNotFields(tuples, "join1_i", "join2_s", "ident_s");
// Basic with replacements test
clause = "select(" + "id, join1_i as join1, join2_s as join2, ident_s as identity," + "replace(join1, 0, withValue=12), replace(join1, 3, withValue=12), replace(join1, 2, withField=join2)," + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\")" + ")";
stream = factory.constructStream(clause);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assertFields(tuples, "id", "join1", "join2", "identity");
assertNotFields(tuples, "join1_i", "join2_s", "ident_s");
assertLong(tuples.get(0), "join1", 12);
assertLong(tuples.get(1), "join1", 12);
assertLong(tuples.get(2), "join1", 12);
assertLong(tuples.get(7), "join1", 12);
assertString(tuples.get(6), "join1", "d");
// Basic with replacements and concat test
clause = "select(" + "id, join1_i as join1, join2_s as join2, ident_s as identity," + "replace(join1, 0, withValue=12), replace(join1, 3, withValue=12), replace(join1, 2, withField=join2)," + "concat(fields=\"identity,join1\", as=\"newIdentity\",delim=\"-\")," + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\")" + ")";
stream = factory.constructStream(clause);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assertFields(tuples, "id", "join1", "join2", "identity", "newIdentity");
assertNotFields(tuples, "join1_i", "join2_s", "ident_s");
assertLong(tuples.get(0), "join1", 12);
assertString(tuples.get(0), "newIdentity", "left_1-12");
assertLong(tuples.get(1), "join1", 12);
assertString(tuples.get(1), "newIdentity", "left_1-12");
assertLong(tuples.get(2), "join1", 12);
assertString(tuples.get(2), "newIdentity", "left_2-12");
assertLong(tuples.get(7), "join1", 12);
assertString(tuples.get(7), "newIdentity", "left_7-12");
assertString(tuples.get(6), "join1", "d");
assertString(tuples.get(6), "newIdentity", "left_6-d");
// Inner stream test
clause = "innerJoin(" + "select(" + "id, join1_i as left.join1, join2_s as left.join2, ident_s as left.ident," + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\")" + ")," + "select(" + "join3_i as right.join1, join2_s as right.join2, ident_s as right.ident," + "search(collection1, q=\"side_s:right\", fl=\"join3_i,join2_s,ident_s\", sort=\"join3_i asc, join2_s asc\")," + ")," + "on=\"left.join1=right.join1, left.join2=right.join2\"" + ")";
stream = factory.constructStream(clause);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assertFields(tuples, "id", "left.join1", "left.join2", "left.ident", "right.join1", "right.join2", "right.ident");
// Wrapped select test
clause = "select(" + "id, left.ident, right.ident," + "innerJoin(" + "select(" + "id, join1_i as left.join1, join2_s as left.join2, ident_s as left.ident," + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\")" + ")," + "select(" + "join3_i as right.join1, join2_s as right.join2, ident_s as right.ident," + "search(collection1, q=\"side_s:right\", fl=\"join3_i,join2_s,ident_s\", sort=\"join3_i asc, join2_s asc\")," + ")," + "on=\"left.join1=right.join1, left.join2=right.join2\"" + ")" + ")";
stream = factory.constructStream(clause);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assertFields(tuples, "id", "left.ident", "right.ident");
assertNotFields(tuples, "left.join1", "left.join2", "right.join1", "right.join2");
} finally {
solrClientCache.close();
}
}
use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.
the class StreamExpressionTest method testParallelReducerStream.
@Test
public void testParallelReducerStream() throws Exception {
new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1").add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5").add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6").add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7").add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8").add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9").add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
StreamContext streamContext = new StreamContext();
SolrClientCache solrClientCache = new SolrClientCache();
streamContext.setSolrClientCache(solrClientCache);
String zkHost = cluster.getZkServer().getZkAddress();
StreamFactory streamFactory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, zkHost).withFunctionName("search", CloudSolrStream.class).withFunctionName("group", GroupOperation.class).withFunctionName("reduce", ReducerStream.class).withFunctionName("parallel", ParallelStream.class);
try {
ParallelStream pstream = (ParallelStream) streamFactory.constructStream("parallel(" + COLLECTIONORALIAS + ", " + "reduce(" + "search(" + COLLECTIONORALIAS + ", q=\"*:*\", fl=\"id,a_s,a_i,a_f\", sort=\"a_s asc,a_f asc\", partitionKeys=\"a_s\"), " + "by=\"a_s\"," + "group(sort=\"a_i asc\", n=\"5\")), " + "workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_s asc\")");
pstream.setStreamContext(streamContext);
List<Tuple> tuples = getTuples(pstream);
assert (tuples.size() == 3);
Tuple t0 = tuples.get(0);
List<Map> maps0 = t0.getMaps("group");
assertMaps(maps0, 0, 1, 2, 9);
Tuple t1 = tuples.get(1);
List<Map> maps1 = t1.getMaps("group");
assertMaps(maps1, 3, 5, 7, 8);
Tuple t2 = tuples.get(2);
List<Map> maps2 = t2.getMaps("group");
assertMaps(maps2, 4, 6);
pstream = (ParallelStream) streamFactory.constructStream("parallel(" + COLLECTIONORALIAS + ", " + "reduce(" + "search(" + COLLECTIONORALIAS + ", q=\"*:*\", fl=\"id,a_s,a_i,a_f\", sort=\"a_s desc,a_f asc\", partitionKeys=\"a_s\"), " + "by=\"a_s\", " + "group(sort=\"a_i desc\", n=\"5\"))," + "workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_s desc\")");
pstream.setStreamContext(streamContext);
tuples = getTuples(pstream);
assert (tuples.size() == 3);
t0 = tuples.get(0);
maps0 = t0.getMaps("group");
assertMaps(maps0, 6, 4);
t1 = tuples.get(1);
maps1 = t1.getMaps("group");
assertMaps(maps1, 8, 7, 5, 3);
t2 = tuples.get(2);
maps2 = t2.getMaps("group");
assertMaps(maps2, 9, 2, 1, 0);
} finally {
solrClientCache.close();
}
}
use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.
the class JDBCStreamTest method testJDBCSolrMerge.
@Test
public void testJDBCSolrMerge() throws Exception {
// Load Database Data
try (Connection connection = DriverManager.getConnection("jdbc:hsqldb:mem:.");
Statement statement = connection.createStatement()) {
statement.executeUpdate("insert into COUNTRIES (CODE,COUNTRY_NAME) values ('US', 'United States')");
statement.executeUpdate("insert into COUNTRIES (CODE,COUNTRY_NAME) values ('NL', 'Netherlands')");
statement.executeUpdate("insert into COUNTRIES (CODE,COUNTRY_NAME) values ('NP', 'Nepal')");
statement.executeUpdate("insert into COUNTRIES (CODE,COUNTRY_NAME) values ('NO', 'Norway')");
statement.executeUpdate("insert into COUNTRIES (CODE,COUNTRY_NAME) values ('AL', 'Algeria')");
}
StreamContext streamContext = new StreamContext();
SolrClientCache solrClientCache = new SolrClientCache();
streamContext.setSolrClientCache(solrClientCache);
// Load Solr
new UpdateRequest().add(id, "0", "code_s", "GB", "name_s", "Great Britian").add(id, "1", "code_s", "CA", "name_s", "Canada").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
StreamFactory factory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class);
List<Tuple> tuples;
try {
// Simple 1
TupleStream jdbcStream = new JDBCStream("jdbc:hsqldb:mem:.", "select CODE,COUNTRY_NAME from COUNTRIES order by CODE", new FieldComparator("CODE", ComparatorOrder.ASCENDING));
TupleStream selectStream = new SelectStream(jdbcStream, new HashMap<String, String>() {
{
put("CODE", "code_s");
put("COUNTRY_NAME", "name_s");
}
});
TupleStream searchStream = factory.constructStream("search(" + COLLECTIONORALIAS + ", fl=\"code_s,name_s\",q=\"*:*\",sort=\"code_s asc\")");
TupleStream mergeStream = new MergeStream(new FieldComparator("code_s", ComparatorOrder.ASCENDING), new TupleStream[] { selectStream, searchStream });
mergeStream.setStreamContext(streamContext);
tuples = getTuples(mergeStream);
assertEquals(7, tuples.size());
assertOrderOf(tuples, "code_s", "AL", "CA", "GB", "NL", "NO", "NP", "US");
assertOrderOf(tuples, "name_s", "Algeria", "Canada", "Great Britian", "Netherlands", "Norway", "Nepal", "United States");
} finally {
solrClientCache.close();
}
}
Aggregations