Examples with JoinMetadata org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata used on opensource projects

Example 1 with JoinMetadata

use of org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata in project incubator-rya by apache.

the class GetQueryReport method getReport.

/**
 * Get a report that indicates how many biniding sets have been emitted for
 * a query that is being managed by the fluo application.
 *
 * @param fluo - The connection to Fluo that will be used to fetch the metadata. (not null)
 * @param queryId - The ID of the query to fetch. (not null)
 * @return A report that was built for the query.
 * @throws UnsupportedQueryException
 */
public QueryReport getReport(final FluoClient fluo, final String queryId) throws UnsupportedQueryException {
    checkNotNull(fluo);
    checkNotNull(queryId);
    final QueryReport.Builder reportBuilder = QueryReport.builder();
    try (Snapshot sx = fluo.newSnapshot()) {
        final FluoQuery fluoQuery = metadataDao.readFluoQuery(sx, queryId);
        reportBuilder.setFluoQuery(fluoQuery);
        // Query results.
        BigInteger count = countBindingSets(sx, queryId, FluoQueryColumns.QUERY_BINDING_SET);
        reportBuilder.setCount(queryId, count);
        // Filter results.
        for (final FilterMetadata filter : fluoQuery.getFilterMetadata()) {
            final String filterId = filter.getNodeId();
            count = countBindingSets(sx, filterId, FluoQueryColumns.FILTER_BINDING_SET);
            reportBuilder.setCount(filterId, count);
        }
        // Join results.
        for (final JoinMetadata join : fluoQuery.getJoinMetadata()) {
            final String joinId = join.getNodeId();
            count = countBindingSets(sx, joinId, FluoQueryColumns.JOIN_BINDING_SET);
            reportBuilder.setCount(joinId, count);
        }
        // Statement Pattern results.
        for (final StatementPatternMetadata statementPattern : fluoQuery.getStatementPatternMetadata()) {
            final String patternId = statementPattern.getNodeId();
            count = countBindingSets(sx, patternId, FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET);
            reportBuilder.setCount(patternId, count);
        }
    }
    return reportBuilder.build();
}

Example 2 with JoinMetadata

use of org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata in project incubator-rya by apache.

the class JoinBatchBindingSetUpdater method processBatch.

/**
 * Processes {@link JoinBatchInformation}. Updates the BindingSets
 * associated with the specified nodeId. The BindingSets are processed in
 * batch fashion, where the number of results is indicated by
 * {@link JoinBatchInformation#getBatchSize()}. BindingSets are either
 * Added, Deleted, or Updated according to
 * {@link JoinBatchInformation#getTask()}. In the event that the number of
 * entries that need to be updated exceeds the batch size, the row of the
 * first unprocessed BindingSets is used to create a new JoinBatch job to
 * process the remaining BindingSets.
 * @throws Exception
 */
@Override
public void processBatch(final TransactionBase tx, final Bytes row, final BatchInformation batch) throws Exception {
    super.processBatch(tx, row, batch);
    final String nodeId = BatchRowKeyUtil.getNodeId(row);
    Preconditions.checkArgument(batch instanceof JoinBatchInformation);
    final JoinBatchInformation joinBatch = (JoinBatchInformation) batch;
    final Task task = joinBatch.getTask();
    // Figure out which join algorithm we are going to use.
    final IterativeJoin joinAlgorithm;
    switch(joinBatch.getJoinType()) {
        case NATURAL_JOIN:
            joinAlgorithm = new NaturalJoin();
            break;
        case LEFT_OUTER_JOIN:
            joinAlgorithm = new LeftOuterJoin();
            break;
        default:
            throw new RuntimeException("Unsupported JoinType: " + joinBatch.getJoinType());
    }
    final Set<VisibilityBindingSet> bsSet = new HashSet<>();
    final Optional<RowColumn> rowCol = fillSiblingBatch(tx, joinBatch, bsSet);
    // Iterates over the resulting BindingSets from the join.
    final Iterator<VisibilityBindingSet> newJoinResults;
    final VisibilityBindingSet bs = joinBatch.getBs();
    if (joinBatch.getSide() == Side.LEFT) {
        newJoinResults = joinAlgorithm.newLeftResult(bs, bsSet.iterator());
    } else {
        newJoinResults = joinAlgorithm.newRightResult(bsSet.iterator(), bs);
    }
    // Read join metadata, create new join BindingSets and insert them into the Fluo table.
    final JoinMetadata joinMetadata = CACHE.readJoinMetadata(tx, nodeId);
    final VariableOrder joinVarOrder = joinMetadata.getVariableOrder();
    while (newJoinResults.hasNext()) {
        final VisibilityBindingSet newJoinResult = newJoinResults.next();
        // create BindingSet value
        final Bytes bsBytes = BS_SERDE.serialize(newJoinResult);
        // make rowId
        Bytes rowKey = BindingHashShardingFunction.addShard(nodeId, joinVarOrder, newJoinResult);
        final Column col = FluoQueryColumns.JOIN_BINDING_SET;
        processTask(tx, task, rowKey, col, bsBytes);
    }
    // update the span and register updated batch job
    if (rowCol.isPresent()) {
        final Span newSpan = getNewSpan(rowCol.get(), joinBatch.getSpan());
        joinBatch.setSpan(newSpan);
        BatchInformationDAO.addBatch(tx, nodeId, joinBatch);
    }
}

Example 3 with JoinMetadata

use of org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata in project incubator-rya by apache.

the class JoinObserver method parseObservation.

@Override
public Observation parseObservation(final TransactionBase tx, final Bytes row) throws Exception {
    requireNonNull(tx);
    requireNonNull(row);
    // Read the Join metadata.
    final String joinNodeId = BindingSetRow.makeFromShardedRow(Bytes.of(JOIN_PREFIX), row).getNodeId();
    final JoinMetadata joinMetadata = queryDao.readJoinMetadata(tx, joinNodeId);
    // Read the Visibility Binding Set from the value.
    final Bytes valueBytes = tx.get(row, FluoQueryColumns.JOIN_BINDING_SET);
    final VisibilityBindingSet joinBindingSet = BS_SERDE.deserialize(valueBytes);
    // Figure out which node needs to handle the new metadata.
    final String parentNodeId = joinMetadata.getParentNodeId();
    return new Observation(joinNodeId, joinBindingSet, parentNodeId);
}

Example 4 with JoinMetadata

use of org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata in project incubator-rya by apache.

the class QueryReportRenderer method render.

/**
 * Pretty render a {@link QueryReport}.
 *
 * @param queryReport - The report that will be rendered. (not null)
 * @return A pretty render of the report.
 * @throws Exception Indicates the SPARQL could not be rendered for some reason.
 */
public String render(final QueryReport queryReport) throws Exception {
    checkNotNull(queryReport);
    final Report.Builder builder = Report.builder();
    final FluoQuery metadata = queryReport.getFluoQuery();
    QueryMetadata queryMetadata = metadata.getQueryMetadata();
    builder.appendItem(new ReportItem(""));
    builder.appendItem(new ReportItem("QUERY NODE"));
    builder.appendItem(new ReportItem("Node ID", queryMetadata.getNodeId()));
    builder.appendItem(new ReportItem("Variable Order", queryMetadata.getVariableOrder().toString()));
    builder.appendItem(new ReportItem("SPARQL", queryMetadata.getSparql()));
    builder.appendItem(new ReportItem("Child Node ID", queryMetadata.getChildNodeId()));
    builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(queryMetadata.getNodeId())));
    if (metadata.getQueryType() == QueryType.CONSTRUCT) {
        builder.appendItem(new ReportItem(""));
        final ConstructQueryMetadata constructMetadata = metadata.getConstructQueryMetadata().get();
        builder.appendItem(new ReportItem("CONSTRUCT QUERY NODE"));
        builder.appendItem(new ReportItem("Node ID", constructMetadata.getNodeId()));
        builder.appendItem(new ReportItem("Variable Order", constructMetadata.getVariableOrder().toString()));
        builder.appendItem(new ReportItem("Parent Node ID", constructMetadata.getParentNodeId()));
        builder.appendItem(new ReportItem("Child Node ID", constructMetadata.getChildNodeId()));
        builder.appendItem(new ReportItem("Construct Graph", constructMetadata.getConstructGraph().toString()));
        builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(constructMetadata.getNodeId())));
    }
    for (ProjectionMetadata projectionMetadata : metadata.getProjectionMetadata()) {
        builder.appendItem(new ReportItem(""));
        builder.appendItem(new ReportItem("PROJECTION NODE"));
        builder.appendItem(new ReportItem("Node ID", projectionMetadata.getNodeId()));
        builder.appendItem(new ReportItem("Variable Order", projectionMetadata.getVariableOrder().toString()));
        builder.appendItem(new ReportItem("Parent Node ID", projectionMetadata.getParentNodeId()));
        builder.appendItem(new ReportItem("Child Node ID", projectionMetadata.getChildNodeId()));
        builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(projectionMetadata.getNodeId())));
    }
    for (final FilterMetadata filterMetadata : metadata.getFilterMetadata()) {
        builder.appendItem(new ReportItem(""));
        builder.appendItem(new ReportItem("FILTER NODE"));
        builder.appendItem(new ReportItem("Node ID", filterMetadata.getNodeId()));
        builder.appendItem(new ReportItem("Variable Order", filterMetadata.getVariableOrder().toString()));
        builder.appendItem(new ReportItem("Filter SPARQL", prettyFormatSparql(filterMetadata.getFilterSparql())));
        builder.appendItem(new ReportItem("Parent Node ID", filterMetadata.getParentNodeId()));
        builder.appendItem(new ReportItem("Child Node ID", filterMetadata.getChildNodeId()));
        builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(filterMetadata.getNodeId())));
    }
    for (final JoinMetadata joinMetadata : metadata.getJoinMetadata()) {
        builder.appendItem(new ReportItem(""));
        builder.appendItem(new ReportItem("JOIN NODE"));
        builder.appendItem(new ReportItem("Node ID", joinMetadata.getNodeId()));
        builder.appendItem(new ReportItem("Variable Order", joinMetadata.getVariableOrder().toString()));
        builder.appendItem(new ReportItem("Parent Node ID", joinMetadata.getParentNodeId()));
        builder.appendItem(new ReportItem("Left Child Node ID", joinMetadata.getLeftChildNodeId()));
        builder.appendItem(new ReportItem("Right Child Node ID", joinMetadata.getRightChildNodeId()));
        builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(joinMetadata.getNodeId())));
    }
    for (final StatementPatternMetadata spMetadata : metadata.getStatementPatternMetadata()) {
        builder.appendItem(new ReportItem(""));
        builder.appendItem(new ReportItem("STATEMENT PATTERN NODE"));
        builder.appendItem(new ReportItem("Node ID", spMetadata.getNodeId()));
        builder.appendItem(new ReportItem("Variable Order", spMetadata.getVariableOrder().toString()));
        builder.appendItem(new ReportItem("Statement Pattern", spMetadata.getStatementPattern()));
        builder.appendItem(new ReportItem("Parent Node ID", spMetadata.getParentNodeId()));
        builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(spMetadata.getNodeId())));
    }
    return builder.build().toString();
}

Example 5 with JoinMetadata

use of org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata in project incubator-rya by apache.

the class BindingSetUpdater method process.

@Override
public final void process(final TransactionBase tx, final Bytes row, final Column col) {
    checkNotNull(tx);
    checkNotNull(row);
    checkNotNull(col);
    final Observation observation;
    try {
        observation = parseObservation(tx, row);
    } catch (final Exception e) {
        log.error("Unable to parse an Observation from a Row and Column pair, so this notification will be skipped. " + "Row: " + row + " Column: " + col, e);
        return;
    }
    final String observedNodeId = observation.getObservedNodeId();
    final VisibilityBindingSet observedBindingSet = observation.getObservedBindingSet();
    final String parentNodeId = observation.getParentId();
    // Figure out which node needs to handle the new metadata.
    final NodeType parentNodeType = NodeType.fromNodeId(parentNodeId).get();
    switch(parentNodeType) {
        case QUERY:
            final QueryMetadata parentQuery = queryDao.readQueryMetadata(tx, parentNodeId);
            try {
                queryUpdater.updateQueryResults(tx, observedBindingSet, parentQuery);
            } catch (final Exception e) {
                throw new RuntimeException("Could not process a Query node.", e);
            }
            break;
        case PROJECTION:
            final ProjectionMetadata projectionQuery = queryDao.readProjectionMetadata(tx, parentNodeId);
            try {
                projectionUpdater.updateProjectionResults(tx, observedBindingSet, projectionQuery);
            } catch (final Exception e) {
                throw new RuntimeException("Could not process a Query node.", e);
            }
            break;
        case CONSTRUCT:
            final ConstructQueryMetadata constructQuery = queryDao.readConstructQueryMetadata(tx, parentNodeId);
            try {
                constructUpdater.updateConstructQueryResults(tx, observedBindingSet, constructQuery);
            } catch (final Exception e) {
                throw new RuntimeException("Could not process a Query node.", e);
            }
            break;
        case FILTER:
            final FilterMetadata parentFilter = queryDao.readFilterMetadata(tx, parentNodeId);
            try {
                filterUpdater.updateFilterResults(tx, observedBindingSet, parentFilter);
            } catch (final Exception e) {
                throw new RuntimeException("Could not process a Filter node.", e);
            }
            break;
        case JOIN:
            final JoinMetadata parentJoin = queryDao.readJoinMetadata(tx, parentNodeId);
            try {
                joinUpdater.updateJoinResults(tx, observedNodeId, observedBindingSet, parentJoin);
            } catch (final Exception e) {
                throw new RuntimeException("Could not process a Join node.", e);
            }
            break;
        case PERIODIC_QUERY:
            final PeriodicQueryMetadata parentPeriodicQuery = queryDao.readPeriodicQueryMetadata(tx, parentNodeId);
            try {
                periodicQueryUpdater.updatePeriodicBinResults(tx, observedBindingSet, parentPeriodicQuery);
            } catch (Exception e) {
                throw new RuntimeException("Could not process PeriodicBin node.", e);
            }
            break;
        case AGGREGATION:
            final AggregationMetadata parentAggregation = queryDao.readAggregationMetadata(tx, parentNodeId);
            try {
                aggregationUpdater.updateAggregateResults(tx, observedBindingSet, parentAggregation);
            } catch (final Exception e) {
                throw new RuntimeException("Could not process an Aggregation node.", e);
            }
            break;
        default:
            throw new IllegalArgumentException("The parent node's NodeType must be of type Aggregation, Projection, ConstructQuery, Filter, Join, PeriodicBin or Query, but was " + parentNodeType);
    }
}