Examples with ExternalTupleSet org.apache.rya.indexing.external.tupleSet.ExternalTupleSet used on opensource projects

Example 76 with ExternalTupleSet

use of org.apache.rya.indexing.external.tupleSet.ExternalTupleSet in project incubator-rya by apache.

the class MongoPcjIndexSetProvider method getIndices.

@Override
protected List<ExternalTupleSet> getIndices() throws PcjIndexSetException {
    try {
        final StatefulMongoDBRdfConfiguration mongoConf = (StatefulMongoDBRdfConfiguration) conf;
        final MongoClient client = mongoConf.getMongoClient();
        final MongoPcjDocuments pcjDocs = new MongoPcjDocuments(client, mongoConf.getRyaInstanceName());
        List<String> documents = null;
        documents = mongoConf.getPcjTables();
        // this maps associates pcj document name with pcj sparql query
        final Map<String, String> indexDocuments = Maps.newLinkedHashMap();
        try (final PrecomputedJoinStorage storage = new MongoPcjStorage(client, mongoConf.getRyaInstanceName())) {
            final boolean docsProvided = documents != null && !documents.isEmpty();
            if (docsProvided) {
                // if tables provided, associate table name with sparql
                for (final String doc : documents) {
                    indexDocuments.put(doc, storage.getPcjMetadata(doc).getSparql());
                }
            } else if (hasRyaDetails()) {
                // If this is a newer install of Rya, and it has PCJ Details, then
                // use those.
                final List<String> ids = storage.listPcjs();
                for (final String pcjId : ids) {
                    indexDocuments.put(pcjId, storage.getPcjMetadata(pcjId).getSparql());
                }
            } else {
                // Otherwise figure it out by getting document IDs.
                documents = pcjDocs.listPcjDocuments();
                for (final String pcjId : documents) {
                    if (pcjId.startsWith("INDEX")) {
                        indexDocuments.put(pcjId, pcjDocs.getPcjMetadata(pcjId).getSparql());
                    }
                }
            }
        }
        final List<ExternalTupleSet> index = Lists.newArrayList();
        if (indexDocuments.isEmpty()) {
            log.info("No Index found");
        } else {
            for (final String pcjID : indexDocuments.keySet()) {
                final String indexSparqlString = indexDocuments.get(pcjID);
                index.add(new MongoPcjQueryNode(indexSparqlString, pcjID, pcjDocs));
            }
        }
        return index;
    } catch (final PCJStorageException | MalformedQueryException e) {
        throw new PcjIndexSetException("Failed to get indicies for this PCJ index.", e);
    }
}

Example 77 with ExternalTupleSet

use of org.apache.rya.indexing.external.tupleSet.ExternalTupleSet in project incubator-rya by apache.

the class AbstractPcjIndexSetProvider method getExternalSets.

/**
 * @param segment - QuerySegment used to get relevant queries form index cache for matching
 * @return List of PCJs for matching
 */
@Override
public List<ExternalTupleSet> getExternalSets(final QuerySegment<ExternalTupleSet> segment) {
    try {
        if (!init) {
            indexCache = PCJOptimizerUtilities.getValidPCJs(getIndices());
            init = true;
        }
        final TupleExpr query = segment.getQuery().getTupleExpr();
        final IndexedExecutionPlanGenerator iep = new IndexedExecutionPlanGenerator(query, indexCache);
        final List<ExternalTupleSet> pcjs = iep.getNormalizedIndices();
        final List<ExternalTupleSet> tuples = new ArrayList<>();
        for (final ExternalTupleSet tuple : pcjs) {
            final QuerySegment<ExternalTupleSet> pcj = converter.setToSegment(tuple);
            if (segment.containsQuerySegment(pcj)) {
                tuples.add(tuple);
            }
        }
        return tuples;
    } catch (final Exception e) {
        throw new RuntimeException(e);
    }
}

Example 78 with ExternalTupleSet

use of org.apache.rya.indexing.external.tupleSet.ExternalTupleSet in project incubator-rya by apache.

the class AccumuloPcjIT method testEvaluateTwoIndexValidate.

@Test
public void testEvaluateTwoIndexValidate() throws Exception {
    final URI superclass = new URIImpl("uri:superclass");
    final URI superclass2 = new URIImpl("uri:superclass2");
    conn.add(subclass, RDF.TYPE, superclass);
    conn.add(subclass2, RDF.TYPE, superclass2);
    conn.add(obj, RDFS.LABEL, new LiteralImpl("label"));
    conn.add(obj2, RDFS.LABEL, new LiteralImpl("label2"));
    conn.add(obj, RDFS.LABEL, new LiteralImpl("label"));
    conn.add(obj2, RDFS.LABEL, new LiteralImpl("label2"));
    final String indexSparqlString = // 
    "" + // 
    "SELECT ?dog ?pig ?duck  " + // 
    "{" + // 
    "  ?pig a ?dog . " + // 
    "  ?pig <http://www.w3.org/2000/01/rdf-schema#label> ?duck " + // 
    "}";
    final String indexSparqlString2 = // 
    "" + // 
    "SELECT ?o ?f ?e ?c ?l  " + // 
    "{" + // 
    "  ?e <uri:talksTo> ?o . " + // 
    "  ?o <http://www.w3.org/2000/01/rdf-schema#label> ?l. " + // 
    "  ?c a ?f . " + // 
    "}";
    final String queryString = // 
    "" + // 
    "SELECT ?e ?c ?l ?f ?o " + // 
    "{" + // 
    "  ?e a ?c . " + // 
    "  ?e <http://www.w3.org/2000/01/rdf-schema#label> ?l. " + // 
    "  ?e <uri:talksTo> ?o . " + // 
    "  ?o <http://www.w3.org/2000/01/rdf-schema#label> ?l. " + // 
    "  ?c a ?f . " + // 
    "}";
    PcjIntegrationTestingUtil.createAndPopulatePcj(conn, accCon, tablename + 1, indexSparqlString, new String[] { "dog", "pig", "duck" }, Optional.<PcjVarOrderFactory>absent());
    final AccumuloIndexSet ais1 = new AccumuloIndexSet(conf, tablename + 1);
    PcjIntegrationTestingUtil.createAndPopulatePcj(conn, accCon, tablename + 2, indexSparqlString2, new String[] { "o", "f", "e", "c", "l" }, Optional.<PcjVarOrderFactory>absent());
    final AccumuloIndexSet ais2 = new AccumuloIndexSet(conf, tablename + 2);
    final List<ExternalTupleSet> index = new ArrayList<>();
    index.add(ais1);
    index.add(ais2);
    ParsedQuery pq = null;
    final SPARQLParser sp = new SPARQLParser();
    pq = sp.parseQuery(queryString, null);
    final List<TupleExpr> teList = Lists.newArrayList();
    final TupleExpr te = pq.getTupleExpr();
    final PCJOptimizer pcj = new PCJOptimizer(index, false, new AccumuloIndexSetProvider(conf));
    pcj.optimize(te, null, null);
    teList.add(te);
    final IndexPlanValidator ipv = new IndexPlanValidator(false);
    Assert.assertTrue(ipv.isValid(te));
}

Example 79 with ExternalTupleSet

use of org.apache.rya.indexing.external.tupleSet.ExternalTupleSet in project incubator-rya by apache.

the class PrecompJoinOptimizerTest method testSixIndex.

@Test
public void testSixIndex() throws Exception {
    final String q1 = // 
    "" + // 
    "SELECT ?f ?m ?d ?h ?i " + // 
    "{" + // 
    "  ?f a ?m ." + // 
    "  ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ." + // 
    "  ?d <uri:talksTo> ?f . " + // 
    "  ?d <uri:hangOutWith> ?f ." + // 
    "  ?f <uri:hangOutWith> ?h ." + // 
    "  ?f <uri:associatesWith> ?i ." + // 
    "  ?i <uri:associatesWith> ?h ." + // 
    "}";
    final String q2 = // 
    "" + // 
    "SELECT ?t ?s ?u " + // 
    "{" + // 
    "  ?s a ?t ." + // 
    "  ?t <http://www.w3.org/2000/01/rdf-schema#label> ?u ." + // 
    "  ?u <uri:talksTo> ?s . " + // 
    "}";
    final String q3 = // 
    "" + // 
    "SELECT ?s ?t ?u " + // 
    "{" + // 
    "  ?s <uri:hangOutWith> ?t ." + // 
    "  ?t <uri:hangOutWith> ?u ." + // 
    "}";
    final String q4 = // 
    "" + // 
    "SELECT ?s ?t ?u " + // 
    "{" + // 
    "  ?s <uri:associatesWith> ?t ." + // 
    "  ?t <uri:associatesWith> ?u ." + // 
    "}";
    final String q5 = // 
    "" + // 
    "SELECT ?m ?f ?d " + // 
    "{" + // 
    "  ?f a ?m ." + // 
    "  ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ." + // 
    "  ?d <uri:talksTo> ?f . " + // 
    "}";
    final String q6 = // 
    "" + // 
    "SELECT ?d ?f ?h " + // 
    "{" + // 
    "  ?d <uri:hangOutWith> ?f ." + // 
    "  ?f <uri:hangOutWith> ?h ." + // 
    "}";
    final String q7 = // 
    "" + // 
    "SELECT ?f ?i ?h " + // 
    "{" + // 
    "  ?f <uri:associatesWith> ?i ." + // 
    "  ?i <uri:associatesWith> ?h ." + // 
    "}";
    final SPARQLParser parser = new SPARQLParser();
    final ParsedQuery pq1 = parser.parseQuery(q1, null);
    final ParsedQuery pq2 = parser.parseQuery(q2, null);
    final ParsedQuery pq3 = parser.parseQuery(q3, null);
    final ParsedQuery pq4 = parser.parseQuery(q4, null);
    final ParsedQuery pq5 = parser.parseQuery(q5, null);
    final ParsedQuery pq6 = parser.parseQuery(q6, null);
    final ParsedQuery pq7 = parser.parseQuery(q7, null);
    final SimpleExternalTupleSet extTup1 = new SimpleExternalTupleSet((Projection) pq2.getTupleExpr());
    final SimpleExternalTupleSet extTup2 = new SimpleExternalTupleSet((Projection) pq3.getTupleExpr());
    final SimpleExternalTupleSet extTup3 = new SimpleExternalTupleSet((Projection) pq4.getTupleExpr());
    final SimpleExternalTupleSet extTup4 = new SimpleExternalTupleSet((Projection) pq5.getTupleExpr());
    final SimpleExternalTupleSet extTup5 = new SimpleExternalTupleSet((Projection) pq6.getTupleExpr());
    final SimpleExternalTupleSet extTup6 = new SimpleExternalTupleSet((Projection) pq7.getTupleExpr());
    final List<ExternalTupleSet> list = new ArrayList<ExternalTupleSet>();
    list.add(extTup2);
    list.add(extTup1);
    list.add(extTup3);
    final List<QueryModelNode> optTupNodes = Lists.newArrayList();
    optTupNodes.add(extTup4);
    optTupNodes.add(extTup6);
    optTupNodes.add(extTup5);
    provider.setIndices(list);
    final PCJOptimizer pcj = new PCJOptimizer(list, true, provider);
    final TupleExpr te = pq1.getTupleExpr();
    pcj.optimize(te, null, null);
    System.out.println(te);
    final NodeCollector nc = new NodeCollector();
    te.visit(nc);
    Assert.assertEquals(nc.qNodes, Sets.newHashSet(optTupNodes));
}

Example 80 with ExternalTupleSet

use of org.apache.rya.indexing.external.tupleSet.ExternalTupleSet in project incubator-rya by apache.

the class PrecompJoinOptimizerTest method testThreeIndex2.

@Test
public void testThreeIndex2() throws Exception {
    final String q1 = // 
    "" + // 
    "SELECT ?f ?m ?d ?e ?l ?c " + // 
    "{" + // 
    "  ?f a ?m ." + // 
    "  ?c a ?l ." + // 
    "  ?d <uri:talksTo> ?f . " + // 
    "  ?e <uri:talksTo> ?c . " + // 
    "  ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ." + // 
    "  ?l <http://www.w3.org/2000/01/rdf-schema#label> ?e ." + // 
    "  ?m <uri:talksTo> ?e . " + // 
    "}";
    final String q2 = // 
    "" + // 
    "SELECT ?u ?s ?t " + // 
    "{" + // 
    "  ?s a ?t ." + // 
    "  ?t <http://www.w3.org/2000/01/rdf-schema#label> ?u ." + // 
    "  ?u <uri:talksTo> ?s . " + // 
    "}";
    final String q3 = // 
    "" + // 
    "SELECT ?e ?c ?l " + // 
    "{" + // 
    "  ?c a ?l ." + // 
    "  ?l <http://www.w3.org/2000/01/rdf-schema#label> ?e ." + // 
    "  ?e <uri:talksTo> ?c . " + // 
    "}";
    final String q4 = // 
    "" + // 
    "SELECT ?d ?f ?m " + // 
    "{" + // 
    "  ?f a ?m ." + // 
    "  ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ." + // 
    "  ?d <uri:talksTo> ?f . " + // 
    "}";
    final SPARQLParser parser = new SPARQLParser();
    final ParsedQuery pq1 = parser.parseQuery(q1, null);
    final ParsedQuery pq2 = parser.parseQuery(q2, null);
    final ParsedQuery pq3 = parser.parseQuery(q3, null);
    final ParsedQuery pq4 = parser.parseQuery(q4, null);
    final SimpleExternalTupleSet extTup1 = new SimpleExternalTupleSet((Projection) pq2.getTupleExpr());
    final SimpleExternalTupleSet extTup2 = new SimpleExternalTupleSet((Projection) pq3.getTupleExpr());
    final SimpleExternalTupleSet extTup3 = new SimpleExternalTupleSet((Projection) pq4.getTupleExpr());
    final List<ExternalTupleSet> list = new ArrayList<ExternalTupleSet>();
    list.add(extTup1);
    final List<StatementPattern> spList = StatementPatternCollector.process(pq1.getTupleExpr());
    final List<QueryModelNode> optTupNodes = Lists.newArrayList();
    optTupNodes.add(extTup3);
    optTupNodes.add(spList.get(6));
    optTupNodes.add(extTup2);
    provider.setIndices(list);
    final PCJOptimizer pcj = new PCJOptimizer(list, true, provider);
    final TupleExpr te = pq1.getTupleExpr();
    pcj.optimize(te, null, null);
    final NodeCollector nc = new NodeCollector();
    te.visit(nc);
    Assert.assertEquals(nc.qNodes, Sets.newHashSet(optTupNodes));
}