Examples with StatementPatternMetadata org.apache.rya.indexing.pcj.fluo.app.query.StatementPatternMetadata used on opensource projects

Example 1 with StatementPatternMetadata

use of org.apache.rya.indexing.pcj.fluo.app.query.StatementPatternMetadata in project incubator-rya by apache.

the class GetQueryReport method getReport.

/**
 * Get a report that indicates how many biniding sets have been emitted for
 * a query that is being managed by the fluo application.
 *
 * @param fluo - The connection to Fluo that will be used to fetch the metadata. (not null)
 * @param queryId - The ID of the query to fetch. (not null)
 * @return A report that was built for the query.
 * @throws UnsupportedQueryException
 */
public QueryReport getReport(final FluoClient fluo, final String queryId) throws UnsupportedQueryException {
    checkNotNull(fluo);
    checkNotNull(queryId);
    final QueryReport.Builder reportBuilder = QueryReport.builder();
    try (Snapshot sx = fluo.newSnapshot()) {
        final FluoQuery fluoQuery = metadataDao.readFluoQuery(sx, queryId);
        reportBuilder.setFluoQuery(fluoQuery);
        // Query results.
        BigInteger count = countBindingSets(sx, queryId, FluoQueryColumns.QUERY_BINDING_SET);
        reportBuilder.setCount(queryId, count);
        // Filter results.
        for (final FilterMetadata filter : fluoQuery.getFilterMetadata()) {
            final String filterId = filter.getNodeId();
            count = countBindingSets(sx, filterId, FluoQueryColumns.FILTER_BINDING_SET);
            reportBuilder.setCount(filterId, count);
        }
        // Join results.
        for (final JoinMetadata join : fluoQuery.getJoinMetadata()) {
            final String joinId = join.getNodeId();
            count = countBindingSets(sx, joinId, FluoQueryColumns.JOIN_BINDING_SET);
            reportBuilder.setCount(joinId, count);
        }
        // Statement Pattern results.
        for (final StatementPatternMetadata statementPattern : fluoQuery.getStatementPatternMetadata()) {
            final String patternId = statementPattern.getNodeId();
            count = countBindingSets(sx, patternId, FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET);
            reportBuilder.setCount(patternId, count);
        }
    }
    return reportBuilder.build();
}

Example 2 with StatementPatternMetadata

use of org.apache.rya.indexing.pcj.fluo.app.query.StatementPatternMetadata in project incubator-rya by apache.

the class TripleObserver method process.

@Override
public void process(final TransactionBase tx, final Bytes brow, final Column column) {
    // Get string representation of triple.
    final RyaStatement ryaStatement = IncUpdateDAO.deserializeTriple(brow);
    log.trace("Transaction ID: {}\nRya Statement: {}\n", tx.getStartTimestamp(), ryaStatement);
    log.trace("Beginging to process triple.");
    final String triple = IncUpdateDAO.getTripleString(ryaStatement);
    Set<String> spIDs = SP_ID_CACHE.getStatementPatternIds(tx);
    // see if triple matches conditions of any of the SP
    for (String spID : spIDs) {
        // Fetch its metadata.
        final StatementPatternMetadata spMetadata = QUERY_METADATA_DAO.readStatementPatternMetadata(tx, spID);
        log.trace("Retrieved metadata: {}", spMetadata);
        // Attempt to match the triple against the pattern.
        final String pattern = spMetadata.getStatementPattern();
        final VariableOrder varOrder = spMetadata.getVariableOrder();
        final String bindingSetString = getBindingSet(triple, pattern, varOrder);
        log.trace("Created binding set match string: {}", bindingSetString);
        // Statement matches to a binding set.
        if (bindingSetString.length() != 0) {
            // Fetch the triple's visibility label.
            final String visibility = tx.gets(brow.toString(), FluoQueryColumns.TRIPLES, "");
            // Make BindingSet and sharded row
            final VisibilityBindingSet visBindingSet = VIS_BS_CONVERTER.convert(bindingSetString, varOrder);
            visBindingSet.setVisibility(visibility);
            Bytes row = BindingHashShardingFunction.addShard(spID, varOrder, visBindingSet);
            // If this is a new Binding Set, then emit it.
            if (tx.get(row, FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET) == null) {
                try {
                    final Bytes valueBytes = BS_SERDE.serialize(visBindingSet);
                    log.trace("Transaction ID: {}\nMatched Statement Pattern: {}\nBinding Set: {}\n", tx.getStartTimestamp(), spID, visBindingSet);
                    tx.set(row, FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET, valueBytes);
                } catch (final Exception e) {
                    log.error("Couldn't serialize a Binding Set. This value will be skipped.", e);
                }
            }
        }
    }
    // Once the triple has been handled, it may be deleted.
    tx.delete(brow, column);
}

Example 3 with StatementPatternMetadata

use of org.apache.rya.indexing.pcj.fluo.app.query.StatementPatternMetadata in project incubator-rya by apache.

the class QueryReportRenderer method render.

/**
 * Pretty render a {@link QueryReport}.
 *
 * @param queryReport - The report that will be rendered. (not null)
 * @return A pretty render of the report.
 * @throws Exception Indicates the SPARQL could not be rendered for some reason.
 */
public String render(final QueryReport queryReport) throws Exception {
    checkNotNull(queryReport);
    final Report.Builder builder = Report.builder();
    final FluoQuery metadata = queryReport.getFluoQuery();
    QueryMetadata queryMetadata = metadata.getQueryMetadata();
    builder.appendItem(new ReportItem(""));
    builder.appendItem(new ReportItem("QUERY NODE"));
    builder.appendItem(new ReportItem("Node ID", queryMetadata.getNodeId()));
    builder.appendItem(new ReportItem("Variable Order", queryMetadata.getVariableOrder().toString()));
    builder.appendItem(new ReportItem("SPARQL", queryMetadata.getSparql()));
    builder.appendItem(new ReportItem("Child Node ID", queryMetadata.getChildNodeId()));
    builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(queryMetadata.getNodeId())));
    if (metadata.getQueryType() == QueryType.CONSTRUCT) {
        builder.appendItem(new ReportItem(""));
        final ConstructQueryMetadata constructMetadata = metadata.getConstructQueryMetadata().get();
        builder.appendItem(new ReportItem("CONSTRUCT QUERY NODE"));
        builder.appendItem(new ReportItem("Node ID", constructMetadata.getNodeId()));
        builder.appendItem(new ReportItem("Variable Order", constructMetadata.getVariableOrder().toString()));
        builder.appendItem(new ReportItem("Parent Node ID", constructMetadata.getParentNodeId()));
        builder.appendItem(new ReportItem("Child Node ID", constructMetadata.getChildNodeId()));
        builder.appendItem(new ReportItem("Construct Graph", constructMetadata.getConstructGraph().toString()));
        builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(constructMetadata.getNodeId())));
    }
    for (ProjectionMetadata projectionMetadata : metadata.getProjectionMetadata()) {
        builder.appendItem(new ReportItem(""));
        builder.appendItem(new ReportItem("PROJECTION NODE"));
        builder.appendItem(new ReportItem("Node ID", projectionMetadata.getNodeId()));
        builder.appendItem(new ReportItem("Variable Order", projectionMetadata.getVariableOrder().toString()));
        builder.appendItem(new ReportItem("Parent Node ID", projectionMetadata.getParentNodeId()));
        builder.appendItem(new ReportItem("Child Node ID", projectionMetadata.getChildNodeId()));
        builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(projectionMetadata.getNodeId())));
    }
    for (final FilterMetadata filterMetadata : metadata.getFilterMetadata()) {
        builder.appendItem(new ReportItem(""));
        builder.appendItem(new ReportItem("FILTER NODE"));
        builder.appendItem(new ReportItem("Node ID", filterMetadata.getNodeId()));
        builder.appendItem(new ReportItem("Variable Order", filterMetadata.getVariableOrder().toString()));
        builder.appendItem(new ReportItem("Filter SPARQL", prettyFormatSparql(filterMetadata.getFilterSparql())));
        builder.appendItem(new ReportItem("Parent Node ID", filterMetadata.getParentNodeId()));
        builder.appendItem(new ReportItem("Child Node ID", filterMetadata.getChildNodeId()));
        builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(filterMetadata.getNodeId())));
    }
    for (final JoinMetadata joinMetadata : metadata.getJoinMetadata()) {
        builder.appendItem(new ReportItem(""));
        builder.appendItem(new ReportItem("JOIN NODE"));
        builder.appendItem(new ReportItem("Node ID", joinMetadata.getNodeId()));
        builder.appendItem(new ReportItem("Variable Order", joinMetadata.getVariableOrder().toString()));
        builder.appendItem(new ReportItem("Parent Node ID", joinMetadata.getParentNodeId()));
        builder.appendItem(new ReportItem("Left Child Node ID", joinMetadata.getLeftChildNodeId()));
        builder.appendItem(new ReportItem("Right Child Node ID", joinMetadata.getRightChildNodeId()));
        builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(joinMetadata.getNodeId())));
    }
    for (final StatementPatternMetadata spMetadata : metadata.getStatementPatternMetadata()) {
        builder.appendItem(new ReportItem(""));
        builder.appendItem(new ReportItem("STATEMENT PATTERN NODE"));
        builder.appendItem(new ReportItem("Node ID", spMetadata.getNodeId()));
        builder.appendItem(new ReportItem("Variable Order", spMetadata.getVariableOrder().toString()));
        builder.appendItem(new ReportItem("Statement Pattern", spMetadata.getStatementPattern()));
        builder.appendItem(new ReportItem("Parent Node ID", spMetadata.getParentNodeId()));
        builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(spMetadata.getNodeId())));
    }
    return builder.build().toString();
}

Example 4 with StatementPatternMetadata

use of org.apache.rya.indexing.pcj.fluo.app.query.StatementPatternMetadata in project incubator-rya by apache.

the class StatementPatternObserver method parseObservation.

@Override
public Observation parseObservation(final TransactionBase tx, final Bytes row) throws Exception {
    requireNonNull(tx);
    requireNonNull(row);
    // Make nodeId and get the Statement Pattern metadata.
    final String spNodeId = BindingSetRow.makeFromShardedRow(Bytes.of(SP_PREFIX), row).getNodeId();
    final StatementPatternMetadata spMetadata = queryDao.readStatementPatternMetadata(tx, spNodeId);
    // Read the Visibility Binding Set from the value.
    final Bytes valueBytes = tx.get(row, FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET);
    final VisibilityBindingSet spBindingSet = BS_SERDE.deserialize(valueBytes);
    // Figure out which node needs to handle the new metadata.
    final String parentNodeId = spMetadata.getParentNodeId();
    return new Observation(spNodeId, spBindingSet, parentNodeId);
}

Example 5 with StatementPatternMetadata

use of org.apache.rya.indexing.pcj.fluo.app.query.StatementPatternMetadata in project incubator-rya by apache.

the class GetQueryReportIT method getReport.

@Test
public void getReport() throws Exception {
    final String sparql = "SELECT ?worker ?company ?city" + "{ " + "FILTER(?worker = <http://Alice>) " + "?worker <http://worksAt> ?company . " + "?worker <http://livesIn> ?city ." + "}";
    // Triples that will be streamed into Fluo after the PCJ has been created.
    final Set<RyaStatement> streamedTriples = Sets.newHashSet(new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://worksAt"), new RyaURI("http://Taco Shop")), new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://worksAt"), new RyaURI("http://Burger Join")), new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://worksAt"), new RyaURI("http://Pastery Shop")), new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://worksAt"), new RyaURI("http://Burrito Place")), new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://livesIn"), new RyaURI("http://Lost County")), new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://livesIn"), new RyaURI("http://Big City")), new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://worksAt"), new RyaURI("http://Burrito Place")), new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://livesIn"), new RyaURI("http://Big City")), new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://worksAt"), new RyaURI("http://Burrito Place")), new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://livesIn"), new RyaURI("http://Big City")), new RyaStatement(new RyaURI("http://David"), new RyaURI("http://worksAt"), new RyaURI("http://Burrito Place")), new RyaStatement(new RyaURI("http://David"), new RyaURI("http://livesIn"), new RyaURI("http://Lost County")), new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://worksAt"), new RyaURI("http://Burrito Place")), new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://livesIn"), new RyaURI("http://Big City")), new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://worksAt"), new RyaURI("http://Burrito Place")), new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://livesIn"), new RyaURI("http://Lost County")));
    // Create the PCJ table.
    final Connector accumuloConn = super.getAccumuloConnector();
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    final String pcjId = pcjStorage.createPcj(sparql);
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Tell the Fluo app to maintain the PCJ.
        new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Stream the data into Fluo.
        new InsertTriples().insert(fluoClient, streamedTriples, Optional.<String>absent());
        // Wait for the results to finish processing.
        super.getMiniFluo().waitForObservers();
        // Fetch the report.
        final Map<String, PcjMetadata> metadata = new GetPcjMetadata().getMetadata(pcjStorage, fluoClient);
        final Set<String> queryIds = metadata.keySet();
        assertEquals(1, queryIds.size());
        final String queryId = queryIds.iterator().next();
        final QueryReport report = new GetQueryReport().getReport(fluoClient, queryId);
        // Build the expected counts map.
        final Map<String, BigInteger> expectedCounts = new HashMap<>();
        final FluoQuery fluoQuery = report.getFluoQuery();
        final String queryNodeId = fluoQuery.getQueryMetadata().getNodeId();
        expectedCounts.put(queryNodeId, BigInteger.valueOf(8));
        final String filterNodeId = fluoQuery.getFilterMetadata().iterator().next().getNodeId();
        expectedCounts.put(filterNodeId, BigInteger.valueOf(8));
        final String joinNodeId = fluoQuery.getJoinMetadata().iterator().next().getNodeId();
        expectedCounts.put(joinNodeId, BigInteger.valueOf(13));
        final Iterator<StatementPatternMetadata> patterns = fluoQuery.getStatementPatternMetadata().iterator();
        final StatementPatternMetadata sp1 = patterns.next();
        final StatementPatternMetadata sp2 = patterns.next();
        if (sp1.getStatementPattern().contains("http://worksAt")) {
            expectedCounts.put(sp1.getNodeId(), BigInteger.valueOf(9));
            expectedCounts.put(sp2.getNodeId(), BigInteger.valueOf(7));
        } else {
            expectedCounts.put(sp2.getNodeId(), BigInteger.valueOf(9));
            expectedCounts.put(sp1.getNodeId(), BigInteger.valueOf(7));
        }
        assertEquals(expectedCounts, report.getCounts());
    }
}