use of org.apache.rya.api.RdfCloudTripleStoreConfiguration in project incubator-rya by apache.
the class RdfCloudTripleStoreSelectivityEvaluationStatisticsTest method testOptimizeQ1ZeroCard.
@Test
public void testOptimizeQ1ZeroCard() throws Exception {
RdfEvalStatsDAO<RdfCloudTripleStoreConfiguration> res = new ProspectorServiceEvalStatsDAO(conn, arc);
AccumuloSelectivityEvalDAO accc = new AccumuloSelectivityEvalDAO();
accc.setConf(arc);
accc.setConnector(conn);
accc.setRdfEvalDAO(res);
accc.init();
BatchWriter bw1 = conn.createBatchWriter("rya_prospects", config);
BatchWriter bw2 = conn.createBatchWriter("rya_selectivity", config);
String s1 = "predicateobject" + DELIM + "http://www.w3.org/2000/01/rdf-schema#label" + DELIM + "uri:dog";
String s2 = "predicateobject" + DELIM + "uri:barksAt" + DELIM + "uri:cat";
String s3 = "predicateobject" + DELIM + "uri:peesOn" + DELIM + "uri:hydrant";
List<Mutation> mList = new ArrayList<Mutation>();
List<Mutation> mList2 = new ArrayList<Mutation>();
List<String> sList = Arrays.asList("subjectobject", "subjectpredicate", "subjectsubject", "predicateobject", "predicatepredicate", "predicatesubject");
Mutation m1, m2, m3, m4;
m1 = new Mutation(s1 + DELIM + "1");
m1.put(new Text("count"), new Text(""), new Value("1".getBytes()));
m2 = new Mutation(s2 + DELIM + "2");
m2.put(new Text("count"), new Text(""), new Value("2".getBytes()));
// m3 = new Mutation(s3 + DELIM + "3");
// m3.put(new Text("count"), new Text(""), new Value("3".getBytes()));
mList.add(m1);
mList.add(m2);
// mList.add(m3);
bw1.addMutations(mList);
bw1.close();
// Scanner scan = conn.createScanner("rya_prospects", new Authorizations());
// scan.setRange(new Range());
// for (Map.Entry<Key, Value> entry : scan) {
// System.out.println("Key row string is " + entry.getKey().getRow().toString());
// System.out.println("Key is " + entry.getKey());
// System.out.println("Value is " + (new String(entry.getValue().get())));
// }
m1 = new Mutation(s1);
m2 = new Mutation(s2);
m3 = new Mutation(s3);
m4 = new Mutation(new Text("subjectpredicateobject" + DELIM + "FullTableCardinality"));
m4.put(new Text("FullTableCardinality"), new Text("100"), EMPTY_VAL);
int i = 2;
int j = 3;
int k = 4;
Long count1;
Long count2;
Long count3;
for (String s : sList) {
count1 = (long) i;
count2 = (long) j;
count3 = (long) k;
m1.put(new Text(s), new Text(count1.toString()), EMPTY_VAL);
m2.put(new Text(s), new Text(count2.toString()), EMPTY_VAL);
m3.put(new Text(s), new Text(count3.toString()), EMPTY_VAL);
i = 2 * i;
j = 2 * j;
k = 2 * k;
}
mList2.add(m1);
mList2.add(m2);
mList2.add(m3);
mList2.add(m4);
bw2.addMutations(mList2);
bw2.close();
// scan = conn.createScanner("rya_selectivity", new Authorizations());
// scan.setRange(new Range());
// for (Map.Entry<Key, Value> entry : scan) {
// System.out.println("Key row string is " + entry.getKey().getRow().toString());
// System.out.println("Key is " + entry.getKey());
// System.out.println("Value is " + (new String(entry.getKey().getColumnQualifier().toString())));
//
// }
TupleExpr te = getTupleExpr(q1);
System.out.println(te);
RdfCloudTripleStoreSelectivityEvaluationStatistics ars = new RdfCloudTripleStoreSelectivityEvaluationStatistics(arc, res, accc);
double card = ars.getCardinality(te);
Assert.assertEquals(4.04, card, .0001);
}
use of org.apache.rya.api.RdfCloudTripleStoreConfiguration in project incubator-rya by apache.
the class QueryJoinSelectOptimizerTest method testOptimizeQ3.
@Test
public void testOptimizeQ3() throws Exception {
RdfEvalStatsDAO<RdfCloudTripleStoreConfiguration> res = new ProspectorServiceEvalStatsDAO(conn, arc);
AccumuloSelectivityEvalDAO accc = new AccumuloSelectivityEvalDAO();
accc.setConf(arc);
accc.setConnector(conn);
accc.setRdfEvalDAO(res);
accc.init();
BatchWriter bw1 = conn.createBatchWriter("rya_prospects", config);
BatchWriter bw2 = conn.createBatchWriter("rya_selectivity", config);
String s1 = "predicateobject" + DELIM + "http://www.w3.org/2000/01/rdf-schema#label" + DELIM + "uri:dog";
String s2 = "predicateobject" + DELIM + "uri:barksAt" + DELIM + "uri:cat";
String s3 = "predicateobject" + DELIM + "uri:peesOn" + DELIM + "uri:hydrant";
String s5 = "predicateobject" + DELIM + "uri:scratches" + DELIM + "uri:ears";
String s4 = "predicateobject" + DELIM + "uri:eats" + DELIM + "uri:chickens";
String s6 = "predicateobject" + DELIM + "uri:eats" + DELIM + "uri:kibble";
String s7 = "predicateobject" + DELIM + "uri:rollsIn" + DELIM + "uri:mud";
String s8 = "predicateobject" + DELIM + "uri:runsIn" + DELIM + "uri:field";
String s9 = "predicateobject" + DELIM + "uri:smells" + DELIM + "uri:butt";
String s10 = "predicateobject" + DELIM + "uri:eats" + DELIM + "uri:sticks";
List<Mutation> mList = new ArrayList<Mutation>();
List<Mutation> mList2 = new ArrayList<Mutation>();
List<String> sList = Arrays.asList("subjectobject", "subjectpredicate", "subjectsubject", "predicateobject", "predicatepredicate", "predicatesubject");
Mutation m1, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11;
m1 = new Mutation(s1 + DELIM + "3");
m1.put(new Text("count"), new Text(""), new Value("5".getBytes()));
m2 = new Mutation(s2 + DELIM + "2");
m2.put(new Text("count"), new Text(""), new Value("3".getBytes()));
m3 = new Mutation(s3 + DELIM + "1");
m3.put(new Text("count"), new Text(""), new Value("2".getBytes()));
m4 = new Mutation(s4 + DELIM + "1");
m4.put(new Text("count"), new Text(""), new Value("3".getBytes()));
m5 = new Mutation(s5 + DELIM + "1");
m5.put(new Text("count"), new Text(""), new Value("5".getBytes()));
m6 = new Mutation(s6 + DELIM + "1");
m6.put(new Text("count"), new Text(""), new Value("3".getBytes()));
m7 = new Mutation(s7 + DELIM + "1");
m7.put(new Text("count"), new Text(""), new Value("2".getBytes()));
m8 = new Mutation(s8 + DELIM + "1");
m8.put(new Text("count"), new Text(""), new Value("3".getBytes()));
m9 = new Mutation(s9 + DELIM + "1");
m9.put(new Text("count"), new Text(""), new Value("1".getBytes()));
m10 = new Mutation(s10 + DELIM + "1");
m10.put(new Text("count"), new Text(""), new Value("1".getBytes()));
mList.add(m1);
mList.add(m2);
mList.add(m3);
mList.add(m4);
mList.add(m5);
mList.add(m6);
mList.add(m7);
mList.add(m8);
mList.add(m9);
mList.add(m10);
bw1.addMutations(mList);
bw1.close();
Scanner scan = conn.createScanner("rya_prospects", new Authorizations());
scan.setRange(new Range());
for (Map.Entry<Key, Value> entry : scan) {
System.out.println("Key row string is " + entry.getKey().getRow().toString());
System.out.println("Key is " + entry.getKey());
System.out.println("Value is " + (new String(entry.getValue().get())));
}
m1 = new Mutation(s1);
m2 = new Mutation(s2);
m3 = new Mutation(s3);
m4 = new Mutation(s4);
m5 = new Mutation(s5);
m6 = new Mutation(s6);
m7 = new Mutation(s7);
m8 = new Mutation(s8);
m9 = new Mutation(s9);
m10 = new Mutation(s10);
m11 = new Mutation(new Text("subjectpredicateobject" + DELIM + "FullTableCardinality"));
m11.put(new Text("FullTableCardinality"), new Text("100"), EMPTY_VAL);
int i = 2;
int j = 3;
int k = 4;
int l = 5;
Long count1;
Long count2;
Long count3;
Long count4;
for (String s : sList) {
count1 = (long) i;
count2 = (long) j;
count3 = (long) k;
count4 = (long) l;
m1.put(new Text(s), new Text(count4.toString()), EMPTY_VAL);
m2.put(new Text(s), new Text(count2.toString()), EMPTY_VAL);
m3.put(new Text(s), new Text(count1.toString()), EMPTY_VAL);
m4.put(new Text(s), new Text(count3.toString()), EMPTY_VAL);
m5.put(new Text(s), new Text(count1.toString()), EMPTY_VAL);
m6.put(new Text(s), new Text(count2.toString()), EMPTY_VAL);
m7.put(new Text(s), new Text(count1.toString()), EMPTY_VAL);
m8.put(new Text(s), new Text(count4.toString()), EMPTY_VAL);
m9.put(new Text(s), new Text(count3.toString()), EMPTY_VAL);
m10.put(new Text(s), new Text(count1.toString()), EMPTY_VAL);
i = 2 * i;
j = 2 * j;
k = 2 * k;
l = 2 * l;
}
mList2.add(m1);
mList2.add(m2);
mList2.add(m3);
mList2.add(m5);
mList2.add(m4);
mList2.add(m6);
mList2.add(m7);
mList2.add(m8);
mList2.add(m9);
mList2.add(m10);
mList2.add(m11);
bw2.addMutations(mList2);
bw2.close();
scan = conn.createScanner("rya_selectivity", new Authorizations());
scan.setRange(new Range());
for (Map.Entry<Key, Value> entry : scan) {
System.out.println("Key row string is " + entry.getKey().getRow().toString());
System.out.println("Key is " + entry.getKey());
System.out.println("Value is " + (new String(entry.getKey().getColumnQualifier().toString())));
}
TupleExpr te = getTupleExpr(q3);
RdfCloudTripleStoreSelectivityEvaluationStatistics ars = new RdfCloudTripleStoreSelectivityEvaluationStatistics(arc, res, accc);
QueryJoinSelectOptimizer qjs = new QueryJoinSelectOptimizer(ars, accc);
System.out.println("Originial query is " + te);
qjs.optimize(te, null, null);
System.out.print("Optimized query is " + te);
}
use of org.apache.rya.api.RdfCloudTripleStoreConfiguration in project incubator-rya by apache.
the class QueryJoinSelectOptimizerTest method testOptimizeQ1.
@Test
public void testOptimizeQ1() throws Exception {
RdfEvalStatsDAO<RdfCloudTripleStoreConfiguration> res = new ProspectorServiceEvalStatsDAO(conn, arc);
AccumuloSelectivityEvalDAO accc = new AccumuloSelectivityEvalDAO();
accc.setConf(arc);
accc.setConnector(conn);
accc.setRdfEvalDAO(res);
accc.init();
BatchWriter bw1 = conn.createBatchWriter("rya_prospects", config);
BatchWriter bw2 = conn.createBatchWriter("rya_selectivity", config);
String s1 = "predicateobject" + DELIM + "http://www.w3.org/2000/01/rdf-schema#label" + DELIM + "uri:dog";
String s2 = "predicateobject" + DELIM + "uri:barksAt" + DELIM + "uri:cat";
String s3 = "predicateobject" + DELIM + "uri:peesOn" + DELIM + "uri:hydrant";
List<Mutation> mList = new ArrayList<Mutation>();
List<Mutation> mList2 = new ArrayList<Mutation>();
List<String> sList = Arrays.asList("subjectobject", "subjectpredicate", "subjectsubject", "predicateobject", "predicatepredicate", "predicatesubject");
Mutation m1, m2, m3, m4;
m1 = new Mutation(s1 + DELIM + "3");
m1.put(new Text("count"), new Text(""), new Value("3".getBytes()));
m2 = new Mutation(s2 + DELIM + "2");
m2.put(new Text("count"), new Text(""), new Value("2".getBytes()));
m3 = new Mutation(s3 + DELIM + "1");
m3.put(new Text("count"), new Text(""), new Value("1".getBytes()));
mList.add(m1);
mList.add(m2);
mList.add(m3);
bw1.addMutations(mList);
bw1.close();
Scanner scan = conn.createScanner("rya_prospects", new Authorizations());
scan.setRange(new Range());
for (Map.Entry<Key, Value> entry : scan) {
System.out.println("Key row string is " + entry.getKey().getRow().toString());
System.out.println("Key is " + entry.getKey());
System.out.println("Value is " + (new String(entry.getValue().get())));
}
m1 = new Mutation(s1);
m2 = new Mutation(s2);
m3 = new Mutation(s3);
m4 = new Mutation(new Text("subjectpredicateobject" + DELIM + "FullTableCardinality"));
m4.put(new Text("FullTableCardinality"), new Text("100"), EMPTY_VAL);
int i = 2;
int j = 3;
int k = 4;
Long count1;
Long count2;
Long count3;
for (String s : sList) {
count1 = (long) i;
count2 = (long) j;
count3 = (long) k;
m1.put(new Text(s), new Text(count1.toString()), EMPTY_VAL);
m2.put(new Text(s), new Text(count2.toString()), EMPTY_VAL);
m3.put(new Text(s), new Text(count3.toString()), EMPTY_VAL);
i = 2 * i;
j = 2 * j;
k = 2 * k;
}
mList2.add(m1);
mList2.add(m2);
mList2.add(m3);
mList2.add(m4);
bw2.addMutations(mList2);
bw2.close();
scan = conn.createScanner("rya_selectivity", new Authorizations());
scan.setRange(new Range());
for (Map.Entry<Key, Value> entry : scan) {
System.out.println("Key row string is " + entry.getKey().getRow().toString());
System.out.println("Key is " + entry.getKey());
System.out.println("Value is " + (new String(entry.getKey().getColumnQualifier().toString())));
}
TupleExpr te = getTupleExpr(q1);
RdfCloudTripleStoreSelectivityEvaluationStatistics ars = new RdfCloudTripleStoreSelectivityEvaluationStatistics(arc, res, accc);
QueryJoinSelectOptimizer qjs = new QueryJoinSelectOptimizer(ars, accc);
System.out.println("Originial query is " + te);
qjs.optimize(te, null, null);
Assert.assertTrue(te.equals(getTupleExpr(Q1)));
}
use of org.apache.rya.api.RdfCloudTripleStoreConfiguration in project incubator-rya by apache.
the class QueryJoinSelectOptimizerTest method testOptimizeQ4.
@Test
public void testOptimizeQ4() throws Exception {
RdfEvalStatsDAO<RdfCloudTripleStoreConfiguration> res = new ProspectorServiceEvalStatsDAO(conn, arc);
AccumuloSelectivityEvalDAO accc = new AccumuloSelectivityEvalDAO();
accc.setConf(arc);
accc.setConnector(conn);
accc.setRdfEvalDAO(res);
accc.init();
BatchWriter bw1 = conn.createBatchWriter("rya_prospects", config);
BatchWriter bw2 = conn.createBatchWriter("rya_selectivity", config);
String s1 = "predicateobject" + DELIM + "http://www.w3.org/2000/01/rdf-schema#label" + DELIM + "uri:dog";
String s2 = "predicateobject" + DELIM + "uri:barksAt" + DELIM + "uri:cat";
String s3 = "predicateobject" + DELIM + "uri:peesOn" + DELIM + "uri:hydrant";
String s5 = "predicateobject" + DELIM + "uri:scratches" + DELIM + "uri:ears";
String s4 = "predicateobject" + DELIM + "uri:eats" + DELIM + "uri:chickens";
List<Mutation> mList = new ArrayList<Mutation>();
List<Mutation> mList2 = new ArrayList<Mutation>();
List<String> sList = Arrays.asList("subjectobject", "subjectpredicate", "subjectsubject", "predicateobject", "predicatepredicate", "predicatesubject");
Mutation m1, m2, m3, m4, m5, m6;
m1 = new Mutation(s1 + DELIM + "3");
m1.put(new Text("count"), new Text(""), new Value("4".getBytes()));
m2 = new Mutation(s2 + DELIM + "2");
m2.put(new Text("count"), new Text(""), new Value("0".getBytes()));
m3 = new Mutation(s3 + DELIM + "1");
m3.put(new Text("count"), new Text(""), new Value("8".getBytes()));
m4 = new Mutation(s4 + DELIM + "1");
m4.put(new Text("count"), new Text(""), new Value("3".getBytes()));
m5 = new Mutation(s5 + DELIM + "1");
m5.put(new Text("count"), new Text(""), new Value("0".getBytes()));
mList.add(m1);
mList.add(m2);
mList.add(m3);
mList.add(m4);
mList.add(m5);
bw1.addMutations(mList);
bw1.close();
Scanner scan = conn.createScanner("rya_prospects", new Authorizations());
scan.setRange(new Range());
for (Map.Entry<Key, Value> entry : scan) {
System.out.println("Key row string is " + entry.getKey().getRow().toString());
System.out.println("Key is " + entry.getKey());
System.out.println("Value is " + (new String(entry.getValue().get())));
}
m1 = new Mutation(s1);
m2 = new Mutation(s2);
m3 = new Mutation(s3);
m4 = new Mutation(s4);
m5 = new Mutation(s5);
m6 = new Mutation(new Text("subjectpredicateobject" + DELIM + "FullTableCardinality"));
m6.put(new Text("FullTableCardinality"), new Text("100"), EMPTY_VAL);
int i = 2;
int j = 3;
int k = 4;
Long count1;
Long count2;
Long count3;
for (String s : sList) {
count1 = (long) i;
count2 = (long) j;
count3 = (long) k;
m1.put(new Text(s), new Text(count1.toString()), EMPTY_VAL);
m2.put(new Text(s), new Text(count2.toString()), EMPTY_VAL);
m3.put(new Text(s), new Text(count1.toString()), EMPTY_VAL);
m4.put(new Text(s), new Text(count3.toString()), EMPTY_VAL);
m5.put(new Text(s), new Text(count1.toString()), EMPTY_VAL);
i = 2 * i;
j = 2 * j;
k = 2 * k;
}
mList2.add(m1);
mList2.add(m2);
mList2.add(m3);
mList2.add(m5);
mList2.add(m4);
mList2.add(m6);
bw2.addMutations(mList2);
bw2.close();
scan = conn.createScanner("rya_selectivity", new Authorizations());
scan.setRange(new Range());
for (Map.Entry<Key, Value> entry : scan) {
System.out.println("Key row string is " + entry.getKey().getRow().toString());
System.out.println("Key is " + entry.getKey());
System.out.println("Value is " + (new String(entry.getKey().getColumnQualifier().toString())));
}
TupleExpr te = getTupleExpr(q2);
RdfCloudTripleStoreSelectivityEvaluationStatistics ars = new RdfCloudTripleStoreSelectivityEvaluationStatistics(arc, res, accc);
QueryJoinSelectOptimizer qjs = new QueryJoinSelectOptimizer(ars, accc);
System.out.println("Originial query is " + te);
qjs.optimize(te, null, null);
TupleExpr what = getTupleExpr(Q4);
System.out.println("lolol: \n" + what);
Assert.assertTrue(te.equals(getTupleExpr(Q4)));
System.out.print("Optimized query is " + te);
}
use of org.apache.rya.api.RdfCloudTripleStoreConfiguration in project incubator-rya by apache.
the class GeoRyaSailFactory method getRyaSail.
private static Sail getRyaSail(final Configuration config) throws InferenceEngineException, RyaDAOException, AccumuloException, AccumuloSecurityException, SailException {
final RdfCloudTripleStore store = new RdfCloudTripleStore();
final RyaDAO<?> dao;
final RdfCloudTripleStoreConfiguration rdfConfig;
final String user;
final String pswd;
// XXX Should(?) be MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX inside the if below. RYA-135
final String ryaInstance = config.get(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX);
Objects.requireNonNull(ryaInstance, "RyaInstance or table prefix is missing from configuration." + RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX);
if (ConfigUtils.getUseMongo(config)) {
// Get a reference to a Mongo DB configuration object.
final MongoDBRdfConfiguration mongoConfig = (config instanceof MongoDBRdfConfiguration) ? (MongoDBRdfConfiguration) config : new MongoDBRdfConfiguration(config);
// Create the MongoClient that will be used by the Sail object's components.
final MongoClient client = createMongoClient(mongoConfig);
// Add the Indexer and Optimizer names to the configuration object that are configured to be used.
OptionalConfigUtils.setIndexers(mongoConfig);
// Populate the configuration using previously stored Rya Details if this instance uses them.
try {
final MongoRyaInstanceDetailsRepository ryaDetailsRepo = new MongoRyaInstanceDetailsRepository(client, mongoConfig.getRyaInstanceName());
RyaDetailsToConfiguration.addRyaDetailsToConfiguration(ryaDetailsRepo.getRyaInstanceDetails(), mongoConfig);
} catch (final RyaDetailsRepositoryException e) {
LOG.info("Instance does not have a rya details collection, skipping.");
}
// Set the configuration to the stateful configuration that is used to pass the constructed objects around.
final StatefulMongoDBRdfConfiguration statefulConfig = new StatefulMongoDBRdfConfiguration(mongoConfig, client);
final List<MongoSecondaryIndex> indexers = statefulConfig.getInstances(AccumuloRdfConfiguration.CONF_ADDITIONAL_INDEXERS, MongoSecondaryIndex.class);
statefulConfig.setIndexers(indexers);
rdfConfig = statefulConfig;
// Create the DAO that is able to interact with MongoDB.
final MongoDBRyaDAO mongoDao = new MongoDBRyaDAO();
mongoDao.setConf(statefulConfig);
mongoDao.init();
dao = mongoDao;
} else {
rdfConfig = new AccumuloRdfConfiguration(config);
user = rdfConfig.get(ConfigUtils.CLOUDBASE_USER);
pswd = rdfConfig.get(ConfigUtils.CLOUDBASE_PASSWORD);
Objects.requireNonNull(user, "Accumulo user name is missing from configuration." + ConfigUtils.CLOUDBASE_USER);
Objects.requireNonNull(pswd, "Accumulo user password is missing from configuration." + ConfigUtils.CLOUDBASE_PASSWORD);
rdfConfig.setTableLayoutStrategy(new TablePrefixLayoutStrategy(ryaInstance));
RyaSailFactory.updateAccumuloConfig((AccumuloRdfConfiguration) rdfConfig, user, pswd, ryaInstance);
dao = getAccumuloDAO((AccumuloRdfConfiguration) rdfConfig);
}
store.setRyaDAO(dao);
rdfConfig.setTablePrefix(ryaInstance);
if (rdfConfig.isInfer()) {
final InferenceEngine inferenceEngine = new InferenceEngine();
inferenceEngine.setConf(rdfConfig);
inferenceEngine.setRyaDAO(dao);
inferenceEngine.init();
store.setInferenceEngine(inferenceEngine);
}
store.initialize();
return store;
}
Aggregations