use of org.apache.solr.client.solrj.io.stream.expr.StreamExpression in project lucene-solr by apache.
the class StreamExpressionTest method testIntersectStream.
////////////////////////////////////////////
@Test
public void testIntersectStream() throws Exception {
new UpdateRequest().add(id, "0", "a_s", "setA", "a_i", "0").add(id, "2", "a_s", "setA", "a_i", "1").add(id, "3", "a_s", "setA", "a_i", "2").add(id, "4", "a_s", "setA", "a_i", "3").add(id, "5", "a_s", "setB", "a_i", "2").add(id, "6", "a_s", "setB", "a_i", "3").add(id, "7", "a_s", "setAB", "a_i", "0").add(id, "8", "a_s", "setAB", "a_i", "6").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
StreamExpression expression;
TupleStream stream;
List<Tuple> tuples;
StreamContext streamContext = new StreamContext();
SolrClientCache solrClientCache = new SolrClientCache();
streamContext.setSolrClientCache(solrClientCache);
StreamFactory factory = new StreamFactory().withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("intersect", IntersectStream.class);
try {
// basic
expression = StreamExpressionParser.parse("intersect(" + "search(collection1, q=a_s:(setA || setAB), fl=\"id,a_s,a_i\", sort=\"a_i asc, a_s asc\")," + "search(collection1, q=a_s:(setB || setAB), fl=\"id,a_s,a_i\", sort=\"a_i asc\")," + "on=\"a_i\")");
stream = new IntersectStream(expression, factory);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assert (tuples.size() == 5);
assertOrder(tuples, 0, 7, 3, 4, 8);
} finally {
solrClientCache.close();
}
}
use of org.apache.solr.client.solrj.io.stream.expr.StreamExpression in project lucene-solr by apache.
the class StreamExpressionTest method testParallelDaemonUpdateStream.
@Test
public void testParallelDaemonUpdateStream() throws Exception {
CollectionAdminRequest.createCollection("parallelDestinationCollection1", "conf", 2, 1).process(cluster.getSolrClient());
AbstractDistribZkTestBase.waitForRecoveriesToFinish("parallelDestinationCollection1", cluster.getSolrClient().getZkStateReader(), false, true, TIMEOUT);
new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0", "s_multi", "aaaa", "s_multi", "bbbb", "i_multi", "4", "i_multi", "7").add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0", "s_multi", "aaaa1", "s_multi", "bbbb1", "i_multi", "44", "i_multi", "77").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "s_multi", "aaaa2", "s_multi", "bbbb2", "i_multi", "444", "i_multi", "777").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "s_multi", "aaaa3", "s_multi", "bbbb3", "i_multi", "4444", "i_multi", "7777").add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1", "s_multi", "aaaa4", "s_multi", "bbbb4", "i_multi", "44444", "i_multi", "77777").commit(cluster.getSolrClient(), "collection1");
StreamExpression expression;
TupleStream stream;
Tuple t;
StreamContext streamContext = new StreamContext();
SolrClientCache solrClientCache = new SolrClientCache();
streamContext.setSolrClientCache(solrClientCache);
String zkHost = cluster.getZkServer().getZkAddress();
StreamFactory factory = new StreamFactory().withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()).withCollectionZkHost("parallelDestinationCollection1", cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("update", UpdateStream.class).withFunctionName("parallel", ParallelStream.class).withFunctionName("daemon", DaemonStream.class);
try {
//Copy all docs to destinationCollection
String updateExpression = "daemon(update(parallelDestinationCollection1, batchSize=2, search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f,s_multi,i_multi\", sort=\"a_f asc, a_i asc\", partitionKeys=\"a_f\")), runInterval=\"1000\", id=\"test\")";
TupleStream parallelUpdateStream = factory.constructStream("parallel(collection1, " + updateExpression + ", workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"batchNumber asc\")");
parallelUpdateStream.setStreamContext(streamContext);
List<Tuple> tuples = getTuples(parallelUpdateStream);
assert (tuples.size() == 2);
//Lets sleep long enough for daemon updates to run.
//Lets stop the daemons
ModifiableSolrParams sParams = new ModifiableSolrParams(StreamingTest.mapParams(CommonParams.QT, "/stream", "action", "list"));
int workersComplete = 0;
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
int iterations = 0;
INNER: while (iterations == 0) {
SolrStream solrStream = new SolrStream(jetty.getBaseUrl().toString() + "/collection1", sParams);
solrStream.setStreamContext(streamContext);
solrStream.open();
Tuple tupleResponse = solrStream.read();
if (tupleResponse.EOF) {
solrStream.close();
break INNER;
} else {
long l = tupleResponse.getLong("iterations");
if (l > 0) {
++workersComplete;
} else {
try {
Thread.sleep(1000);
} catch (Exception e) {
}
}
iterations = (int) l;
solrStream.close();
}
}
}
assertEquals(cluster.getJettySolrRunners().size(), workersComplete);
cluster.getSolrClient().commit("parallelDestinationCollection1");
//Lets stop the daemons
sParams = new ModifiableSolrParams();
sParams.set(CommonParams.QT, "/stream");
sParams.set("action", "stop");
sParams.set("id", "test");
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
SolrStream solrStream = new SolrStream(jetty.getBaseUrl() + "/collection1", sParams);
solrStream.setStreamContext(streamContext);
solrStream.open();
Tuple tupleResponse = solrStream.read();
solrStream.close();
}
sParams = new ModifiableSolrParams();
sParams.set(CommonParams.QT, "/stream");
sParams.set("action", "list");
workersComplete = 0;
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
long stopTime = 0;
INNER: while (stopTime == 0) {
SolrStream solrStream = new SolrStream(jetty.getBaseUrl() + "/collection1", sParams);
solrStream.setStreamContext(streamContext);
solrStream.open();
Tuple tupleResponse = solrStream.read();
if (tupleResponse.EOF) {
solrStream.close();
break INNER;
} else {
stopTime = tupleResponse.getLong("stopTime");
if (stopTime > 0) {
++workersComplete;
} else {
try {
Thread.sleep(1000);
} catch (Exception e) {
}
}
solrStream.close();
}
}
}
assertEquals(cluster.getJettySolrRunners().size(), workersComplete);
//Ensure that destinationCollection actually has the new docs.
expression = StreamExpressionParser.parse("search(parallelDestinationCollection1, q=*:*, fl=\"id,a_s,a_i,a_f,s_multi,i_multi\", sort=\"a_i asc\")");
stream = new CloudSolrStream(expression, factory);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assertEquals(5, tuples.size());
Tuple tuple = tuples.get(0);
assert (tuple.getLong("id") == 0);
assert (tuple.get("a_s").equals("hello0"));
assert (tuple.getLong("a_i") == 0);
assert (tuple.getDouble("a_f") == 0.0);
assertList(tuple.getStrings("s_multi"), "aaaa", "bbbb");
assertList(tuple.getLongs("i_multi"), Long.parseLong("4"), Long.parseLong("7"));
tuple = tuples.get(1);
assert (tuple.getLong("id") == 1);
assert (tuple.get("a_s").equals("hello1"));
assert (tuple.getLong("a_i") == 1);
assert (tuple.getDouble("a_f") == 1.0);
assertList(tuple.getStrings("s_multi"), "aaaa4", "bbbb4");
assertList(tuple.getLongs("i_multi"), Long.parseLong("44444"), Long.parseLong("77777"));
tuple = tuples.get(2);
assert (tuple.getLong("id") == 2);
assert (tuple.get("a_s").equals("hello2"));
assert (tuple.getLong("a_i") == 2);
assert (tuple.getDouble("a_f") == 0.0);
assertList(tuple.getStrings("s_multi"), "aaaa1", "bbbb1");
assertList(tuple.getLongs("i_multi"), Long.parseLong("44"), Long.parseLong("77"));
tuple = tuples.get(3);
assert (tuple.getLong("id") == 3);
assert (tuple.get("a_s").equals("hello3"));
assert (tuple.getLong("a_i") == 3);
assert (tuple.getDouble("a_f") == 3.0);
assertList(tuple.getStrings("s_multi"), "aaaa2", "bbbb2");
assertList(tuple.getLongs("i_multi"), Long.parseLong("444"), Long.parseLong("777"));
tuple = tuples.get(4);
assert (tuple.getLong("id") == 4);
assert (tuple.get("a_s").equals("hello4"));
assert (tuple.getLong("a_i") == 4);
assert (tuple.getDouble("a_f") == 4.0);
assertList(tuple.getStrings("s_multi"), "aaaa3", "bbbb3");
assertList(tuple.getLongs("i_multi"), Long.parseLong("4444"), Long.parseLong("7777"));
} finally {
CollectionAdminRequest.deleteCollection("parallelDestinationCollection1").process(cluster.getSolrClient());
solrClientCache.close();
}
}
use of org.apache.solr.client.solrj.io.stream.expr.StreamExpression in project lucene-solr by apache.
the class StreamExpressionTest method testCartesianProductStream.
@Test
public void testCartesianProductStream() throws Exception {
new UpdateRequest().add(id, "0", "a_ss", "a", "a_ss", "b", "a_ss", "c", "a_ss", "d", "a_ss", "e", "b_ls", "1", "b_ls", "2", "b_ls", "3").add(id, "1", "a_ss", "a", "a_ss", "b", "a_ss", "c", "a_ss", "d", "a_ss", "e").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
StreamExpression expression;
TupleStream stream;
List<Tuple> tuples;
StreamContext streamContext = new StreamContext();
SolrClientCache solrClientCache = new SolrClientCache();
streamContext.setSolrClientCache(solrClientCache);
StreamFactory factory = new StreamFactory().withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("cartesian", CartesianProductStream.class);
// single selection, no sort
try {
stream = factory.constructStream("cartesian(" + "search(collection1, q=*:*, fl=\"id,a_ss\", sort=\"id asc\")," + "a_ss" + ")");
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assertEquals(10, tuples.size());
assertOrder(tuples, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1);
assertEquals("a", tuples.get(0).get("a_ss"));
assertEquals("c", tuples.get(2).get("a_ss"));
assertEquals("a", tuples.get(5).get("a_ss"));
assertEquals("c", tuples.get(7).get("a_ss"));
// single selection, sort
stream = factory.constructStream("cartesian(" + "search(collection1, q=*:*, fl=\"id,a_ss\", sort=\"id asc\")," + "a_ss," + "productSort=\"a_ss DESC\"" + ")");
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assertEquals(10, tuples.size());
assertOrder(tuples, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1);
assertEquals("e", tuples.get(0).get("a_ss"));
assertEquals("c", tuples.get(2).get("a_ss"));
assertEquals("e", tuples.get(5).get("a_ss"));
assertEquals("c", tuples.get(7).get("a_ss"));
// multi selection, sort
stream = factory.constructStream("cartesian(" + "search(collection1, q=*:*, fl=\"id,a_ss,b_ls\", sort=\"id asc\")," + "a_ss," + "b_ls," + "productSort=\"a_ss ASC\"" + ")");
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
// (5 * 3) + 5
assertEquals(20, tuples.size());
assertOrder(tuples, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1);
assertEquals("a", tuples.get(0).get("a_ss"));
assertEquals(1L, tuples.get(0).get("b_ls"));
assertEquals("a", tuples.get(1).get("a_ss"));
assertEquals(2L, tuples.get(1).get("b_ls"));
assertEquals("a", tuples.get(2).get("a_ss"));
assertEquals(3L, tuples.get(2).get("b_ls"));
assertEquals("b", tuples.get(3).get("a_ss"));
assertEquals(1L, tuples.get(3).get("b_ls"));
assertEquals("b", tuples.get(4).get("a_ss"));
assertEquals(2L, tuples.get(4).get("b_ls"));
assertEquals("b", tuples.get(5).get("a_ss"));
assertEquals(3L, tuples.get(5).get("b_ls"));
// multi selection, sort
stream = factory.constructStream("cartesian(" + "search(collection1, q=*:*, fl=\"id,a_ss,b_ls\", sort=\"id asc\")," + "a_ss," + "b_ls," + "productSort=\"a_ss ASC, b_ls DESC\"" + ")");
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
// (5 * 3) + 5
assertEquals(20, tuples.size());
assertOrder(tuples, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1);
assertEquals("a", tuples.get(0).get("a_ss"));
assertEquals(3L, tuples.get(0).get("b_ls"));
assertEquals("a", tuples.get(1).get("a_ss"));
assertEquals(2L, tuples.get(1).get("b_ls"));
assertEquals("a", tuples.get(2).get("a_ss"));
assertEquals(1L, tuples.get(2).get("b_ls"));
assertEquals("b", tuples.get(3).get("a_ss"));
assertEquals(3L, tuples.get(3).get("b_ls"));
assertEquals("b", tuples.get(4).get("a_ss"));
assertEquals(2L, tuples.get(4).get("b_ls"));
assertEquals("b", tuples.get(5).get("a_ss"));
assertEquals(1L, tuples.get(5).get("b_ls"));
// multi selection, sort
stream = factory.constructStream("cartesian(" + "search(collection1, q=*:*, fl=\"id,a_ss,b_ls\", sort=\"id asc\")," + "a_ss," + "b_ls," + "productSort=\"b_ls DESC\"" + ")");
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
// (5 * 3) + 5
assertEquals(20, tuples.size());
assertOrder(tuples, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1);
assertEquals("a", tuples.get(0).get("a_ss"));
assertEquals(3L, tuples.get(0).get("b_ls"));
assertEquals("b", tuples.get(1).get("a_ss"));
assertEquals(3L, tuples.get(1).get("b_ls"));
assertEquals("c", tuples.get(2).get("a_ss"));
assertEquals(3L, tuples.get(2).get("b_ls"));
assertEquals("d", tuples.get(3).get("a_ss"));
assertEquals(3L, tuples.get(3).get("b_ls"));
assertEquals("e", tuples.get(4).get("a_ss"));
assertEquals(3L, tuples.get(4).get("b_ls"));
assertEquals("a", tuples.get(5).get("a_ss"));
assertEquals(2L, tuples.get(5).get("b_ls"));
assertEquals("b", tuples.get(6).get("a_ss"));
assertEquals(2L, tuples.get(6).get("b_ls"));
assertEquals("c", tuples.get(7).get("a_ss"));
assertEquals(2L, tuples.get(7).get("b_ls"));
assertEquals("d", tuples.get(8).get("a_ss"));
assertEquals(2L, tuples.get(8).get("b_ls"));
assertEquals("e", tuples.get(9).get("a_ss"));
assertEquals(2L, tuples.get(9).get("b_ls"));
} finally {
solrClientCache.close();
}
}
use of org.apache.solr.client.solrj.io.stream.expr.StreamExpression in project lucene-solr by apache.
the class StreamExpressionTest method testPriorityStream.
@Test
public void testPriorityStream() throws Exception {
Assume.assumeTrue(!useAlias);
new UpdateRequest().add(id, "0", "a_s", "hello1", "a_i", "0", "a_f", "1").add(id, "2", "a_s", "hello1", "a_i", "2", "a_f", "2").add(id, "3", "a_s", "hello1", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello1", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "5").add(id, "5", "a_s", "hello", "a_i", "10", "a_f", "6").add(id, "6", "a_s", "hello", "a_i", "11", "a_f", "7").add(id, "7", "a_s", "hello", "a_i", "12", "a_f", "8").add(id, "8", "a_s", "hello", "a_i", "13", "a_f", "9").add(id, "9", "a_s", "hello1", "a_i", "14", "a_f", "10").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
StreamFactory factory = new StreamFactory().withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()).withFunctionName("topic", TopicStream.class).withFunctionName("priority", PriorityStream.class);
StreamExpression expression;
TupleStream stream;
List<Tuple> tuples;
SolrClientCache cache = new SolrClientCache();
try {
FieldComparator comp = new FieldComparator("a_i", ComparatorOrder.ASCENDING);
expression = StreamExpressionParser.parse("priority(topic(collection1, collection1, q=\"a_s:hello\", fl=\"id,a_i\", id=1000000, initialCheckpoint=0)," + "topic(collection1, collection1, q=\"a_s:hello1\", fl=\"id,a_i\", id=2000000, initialCheckpoint=0))");
stream = factory.constructStream(expression);
StreamContext context = new StreamContext();
context.setSolrClientCache(cache);
stream.setStreamContext(context);
tuples = getTuples(stream);
Collections.sort(tuples, comp);
//The tuples from the first topic (high priority) should be returned.
assertEquals(tuples.size(), 4);
assertOrder(tuples, 5, 6, 7, 8);
expression = StreamExpressionParser.parse("priority(topic(collection1, collection1, q=\"a_s:hello\", fl=\"id,a_i\", id=1000000, initialCheckpoint=0)," + "topic(collection1, collection1, q=\"a_s:hello1\", fl=\"id,a_i\", id=2000000, initialCheckpoint=0))");
stream = factory.constructStream(expression);
context = new StreamContext();
context.setSolrClientCache(cache);
stream.setStreamContext(context);
tuples = getTuples(stream);
Collections.sort(tuples, comp);
//The Tuples from the second topic (Low priority) should be returned.
assertEquals(tuples.size(), 6);
assertOrder(tuples, 0, 1, 2, 3, 4, 9);
expression = StreamExpressionParser.parse("priority(topic(collection1, collection1, q=\"a_s:hello\", fl=\"id,a_i\", id=1000000, initialCheckpoint=0)," + "topic(collection1, collection1, q=\"a_s:hello1\", fl=\"id,a_i\", id=2000000, initialCheckpoint=0))");
stream = factory.constructStream(expression);
context = new StreamContext();
context.setSolrClientCache(cache);
stream.setStreamContext(context);
tuples = getTuples(stream);
//Both queus are empty.
assertEquals(tuples.size(), 0);
} finally {
cache.close();
}
}
use of org.apache.solr.client.solrj.io.stream.expr.StreamExpression in project lucene-solr by apache.
the class StreamExpressionTest method testCloudSolrStreamWithZkHost.
@Test
public void testCloudSolrStreamWithZkHost() throws Exception {
new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0").add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
StreamFactory factory = new StreamFactory();
StreamExpression expression;
CloudSolrStream stream;
StreamContext streamContext = new StreamContext();
SolrClientCache solrClientCache = new SolrClientCache();
streamContext.setSolrClientCache(solrClientCache);
List<Tuple> tuples;
try {
// Basic test
expression = StreamExpressionParser.parse("search(" + COLLECTIONORALIAS + ", zkHost=" + cluster.getZkServer().getZkAddress() + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\")");
stream = new CloudSolrStream(expression, factory);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assert (tuples.size() == 5);
assertOrder(tuples, 0, 2, 1, 3, 4);
assertLong(tuples.get(0), "a_i", 0);
// Basic w/aliases
expression = StreamExpressionParser.parse("search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\", aliases=\"a_i=alias.a_i, a_s=name\", zkHost=" + cluster.getZkServer().getZkAddress() + ")");
stream = new CloudSolrStream(expression, factory);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assert (tuples.size() == 5);
assertOrder(tuples, 0, 2, 1, 3, 4);
assertLong(tuples.get(0), "alias.a_i", 0);
assertString(tuples.get(0), "name", "hello0");
// Basic filtered test
expression = StreamExpressionParser.parse("search(" + COLLECTIONORALIAS + ", q=\"id:(0 3 4)\", fl=\"id,a_s,a_i,a_f\", zkHost=" + cluster.getZkServer().getZkAddress() + ", sort=\"a_f asc, a_i asc\")");
stream = new CloudSolrStream(expression, factory);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assert (tuples.size() == 3);
assertOrder(tuples, 0, 3, 4);
assertLong(tuples.get(1), "a_i", 3);
// Test a couple of multile field lists.
expression = StreamExpressionParser.parse("search(collection1, fq=\"a_s:hello0\", fq=\"a_s:hello1\", q=\"id:(*)\", " + "zkHost=" + cluster.getZkServer().getZkAddress() + ", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\")");
stream = new CloudSolrStream(expression, factory);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assertEquals("fq clauses should have prevented any docs from coming back", tuples.size(), 0);
expression = StreamExpressionParser.parse("search(collection1, fq=\"a_s:(hello0 OR hello1)\", q=\"id:(*)\", " + "zkHost=" + cluster.getZkServer().getZkAddress() + ", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\")");
stream = new CloudSolrStream(expression, factory);
stream.setStreamContext(streamContext);
tuples = getTuples(stream);
assertEquals("Combining an f1 clause should show us 2 docs", tuples.size(), 2);
} finally {
solrClientCache.close();
}
}
Aggregations