Examples with VariableOrder org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder used on opensource projects

Example 41 with VariableOrder

use of org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder in project incubator-rya by apache.

the class PeriodicQueryUtilTest method testFluoQueryVarOrders.

@Test
public void testFluoQueryVarOrders() throws MalformedQueryException, UnsupportedQueryException {
    String query = // n
    "prefix function: <http://org.apache.rya/function#> " + // n
    "prefix time: <http://www.w3.org/2006/time#> " + // n
    "select (count(?obs) as ?total) where {" + // n
    "Filter(function:periodic(?time, 12.4, 6.2,time:hours)) " + // n
    "?obs <uri:hasTime> ?time. " + // n
    "?obs <uri:hasLattitude> ?lat }";
    SparqlFluoQueryBuilder builder = new SparqlFluoQueryBuilder();
    builder.setSparql(query);
    builder.setFluoQueryId(NodeType.generateNewFluoIdForType(NodeType.QUERY));
    FluoQuery fluoQuery = builder.build();
    PeriodicQueryMetadata periodicMeta = fluoQuery.getPeriodicQueryMetadata().orNull();
    Assert.assertEquals(true, periodicMeta != null);
    VariableOrder periodicVars = periodicMeta.getVariableOrder();
    Assert.assertEquals(IncrementalUpdateConstants.PERIODIC_BIN_ID, periodicVars.getVariableOrders().get(0));
    QueryMetadata queryMeta = fluoQuery.getQueryMetadata();
    VariableOrder queryVars = queryMeta.getVariableOrder();
    Assert.assertEquals(IncrementalUpdateConstants.PERIODIC_BIN_ID, queryVars.getVariableOrders().get(0));
    Collection<AggregationMetadata> aggMetaCollection = fluoQuery.getAggregationMetadata();
    Assert.assertEquals(1, aggMetaCollection.size());
    AggregationMetadata aggMeta = aggMetaCollection.iterator().next();
    VariableOrder aggVars = aggMeta.getVariableOrder();
    Assert.assertEquals(IncrementalUpdateConstants.PERIODIC_BIN_ID, aggVars.getVariableOrders().get(0));
    System.out.println(fluoQuery);
}

Example 42 with VariableOrder

use of org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder in project incubator-rya by apache.

the class BindingHashShardingFunction method addShard.

/**
 * Generates a sharded rowId. The rowId is of the form: node_prefix:shardId:nodeId//Binding_values. For
 * example, the row key generated from nodeId = SP_123, varOrder = a;b, bs = [a = uri:Bob, b = uri:Doug] would be
 * SP:HASH(uri:Bob):123//uri:Bob;uri:Doug, where HASH(uri:Bob) indicates the shard id hash generated from the
 * Binding value "uri:Bob".
 *
 * @param nodeId - Node Id with type and UUID
 * @param varOrder - VarOrder used to order BindingSet values
 * @param bs - BindingSet with partially formed query values
 * @return - serialized Bytes rowId for storing BindingSet results in Fluo
 */
public static Bytes addShard(String nodeId, VariableOrder varOrder, VisibilityBindingSet bs) {
    checkNotNull(nodeId);
    checkNotNull(varOrder);
    checkNotNull(bs);
    String[] rowPrefixAndId = nodeId.split("_");
    Preconditions.checkArgument(rowPrefixAndId.length == 2);
    String prefix = rowPrefixAndId[0];
    String id = rowPrefixAndId[1];
    String firstBindingString = "";
    Bytes rowSuffix = Bytes.of(id);
    if (varOrder.getVariableOrders().size() > 0) {
        VariableOrder first = new VariableOrder(varOrder.getVariableOrders().get(0));
        firstBindingString = BS_CONVERTER.convert(bs, first);
        rowSuffix = RowKeyUtil.makeRowKey(id, varOrder, bs);
    }
    BytesBuilder builder = Bytes.builder();
    builder.append(Bytes.of(prefix + ":"));
    builder.append(genHash(Bytes.of(id + NODEID_BS_DELIM + firstBindingString)));
    builder.append(":");
    builder.append(rowSuffix);
    return builder.toBytes();
}

Example 43 with VariableOrder

use of org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder in project incubator-rya by apache.

the class ProjectionResultUpdater method updateProjectionResults.

/**
 * Updates the results of a Projection node when one of its children has added a
 * new Binding Set to its results.
 *
 * @param tx - The transaction all Fluo queries will use. (not null)
 * @param childBindingSet - A binding set that the query's child node has emmitted. (not null)
 * @param projectionMetadata - The metadata of the Query whose results will be updated. (not null)
 * @throws Exception A problem caused the update to fail.
 */
public void updateProjectionResults(final TransactionBase tx, final VisibilityBindingSet childBindingSet, final ProjectionMetadata projectionMetadata) throws Exception {
    checkNotNull(tx);
    checkNotNull(childBindingSet);
    checkNotNull(projectionMetadata);
    log.trace("Transaction ID: " + tx.getStartTimestamp() + "\n" + "Node ID: " + projectionMetadata.getNodeId() + "\n" + "Parent Node ID: " + projectionMetadata.getParentNodeId() + "\n" + "Child Node ID: " + projectionMetadata.getChildNodeId() + "\n" + "Child Binding Set:\n" + childBindingSet + "\n");
    // Create the query's Binding Set from the child node's binding set.
    final VariableOrder queryVarOrder = projectionMetadata.getVariableOrder();
    final VariableOrder projectionVarOrder = projectionMetadata.getProjectedVars();
    final BindingSet queryBindingSet = BindingSetUtil.keepBindings(projectionVarOrder, childBindingSet);
    VisibilityBindingSet projectedBs = new VisibilityBindingSet(queryBindingSet, childBindingSet.getVisibility());
    // Create the Row Key for the result. If the child node groups results, then the key must only contain the Group By variables.
    Bytes resultRow = makeRowKey(projectionMetadata.getNodeId(), queryVarOrder, projectedBs);
    // Create the Binding Set that goes in the Node Value. It does contain visibilities.
    final Bytes nodeValueBytes = BS_SERDE.serialize(projectedBs);
    log.trace("Transaction ID: " + tx.getStartTimestamp() + "\n" + "New Binding Set: " + childBindingSet + "\n");
    tx.set(resultRow, FluoQueryColumns.PROJECTION_BINDING_SET, nodeValueBytes);
}

Example 44 with VariableOrder

use of org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder in project incubator-rya by apache.

the class BindingSetSerDe method fromBytes.

private BindingSet fromBytes(final byte[] bsBytes) {
    try {
        final int firstIndex = Bytes.indexOf(bsBytes, DELIM_BYTE);
        final byte[] varOrderBytes = Arrays.copyOf(bsBytes, firstIndex);
        final byte[] bsBytesNoVarOrder = Arrays.copyOfRange(bsBytes, firstIndex + 1, bsBytes.length);
        final VariableOrder varOrder = new VariableOrder(new String(varOrderBytes, StandardCharsets.UTF_8).split(";"));
        return getBindingSet(varOrder, bsBytesNoVarOrder);
    } catch (final Exception e) {
        log.trace("Unable to deserialize BindingSet: " + bsBytes);
        return new QueryBindingSet();
    }
}

Example 45 with VariableOrder

use of org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder in project incubator-rya by apache.

the class KafkaExportIT method nestedGroupByManyBindings_averages.

@Test
public void nestedGroupByManyBindings_averages() throws Exception {
    // A query that groups what is aggregated by two of the keys.
    final String sparql = "SELECT ?type ?location ?averagePrice {" + "FILTER(?averagePrice > 4) " + "{SELECT ?type ?location (avg(?price) as ?averagePrice) {" + "?id <urn:type> ?type . " + "?id <urn:location> ?location ." + "?id <urn:price> ?price ." + "} " + "GROUP BY ?type ?location }}";
    // Create the Statements that will be loaded into Rya.
    final ValueFactory vf = new ValueFactoryImpl();
    final Collection<Statement> statements = Sets.newHashSet(// American items that will be averaged.
    vf.createStatement(vf.createURI("urn:1"), vf.createURI("urn:type"), vf.createLiteral("apple")), vf.createStatement(vf.createURI("urn:1"), vf.createURI("urn:location"), vf.createLiteral("USA")), vf.createStatement(vf.createURI("urn:1"), vf.createURI("urn:price"), vf.createLiteral(2.50)), vf.createStatement(vf.createURI("urn:2"), vf.createURI("urn:type"), vf.createLiteral("cheese")), vf.createStatement(vf.createURI("urn:2"), vf.createURI("urn:location"), vf.createLiteral("USA")), vf.createStatement(vf.createURI("urn:2"), vf.createURI("urn:price"), vf.createLiteral(4.25)), vf.createStatement(vf.createURI("urn:3"), vf.createURI("urn:type"), vf.createLiteral("cheese")), vf.createStatement(vf.createURI("urn:3"), vf.createURI("urn:location"), vf.createLiteral("USA")), vf.createStatement(vf.createURI("urn:3"), vf.createURI("urn:price"), vf.createLiteral(5.25)), // French items that will be averaged.
    vf.createStatement(vf.createURI("urn:4"), vf.createURI("urn:type"), vf.createLiteral("cheese")), vf.createStatement(vf.createURI("urn:4"), vf.createURI("urn:location"), vf.createLiteral("France")), vf.createStatement(vf.createURI("urn:4"), vf.createURI("urn:price"), vf.createLiteral(8.5)), vf.createStatement(vf.createURI("urn:5"), vf.createURI("urn:type"), vf.createLiteral("cigarettes")), vf.createStatement(vf.createURI("urn:5"), vf.createURI("urn:location"), vf.createLiteral("France")), vf.createStatement(vf.createURI("urn:5"), vf.createURI("urn:price"), vf.createLiteral(3.99)), vf.createStatement(vf.createURI("urn:6"), vf.createURI("urn:type"), vf.createLiteral("cigarettes")), vf.createStatement(vf.createURI("urn:6"), vf.createURI("urn:location"), vf.createLiteral("France")), vf.createStatement(vf.createURI("urn:6"), vf.createURI("urn:price"), vf.createLiteral(4.99)));
    // Create the PCJ in Fluo and load the statements into Rya.
    final String pcjId = loadDataAndCreateQuery(sparql, statements);
    // Create the expected results of the SPARQL query once the PCJ has been computed.
    final Set<VisibilityBindingSet> expectedResults = new HashSet<>();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("type", vf.createLiteral("cheese", XMLSchema.STRING));
    bs.addBinding("location", vf.createLiteral("France", XMLSchema.STRING));
    bs.addBinding("averagePrice", vf.createLiteral("8.5", XMLSchema.DECIMAL));
    expectedResults.add(new VisibilityBindingSet(bs));
    bs = new MapBindingSet();
    bs.addBinding("type", vf.createLiteral("cigarettes", XMLSchema.STRING));
    bs.addBinding("location", vf.createLiteral("France", XMLSchema.STRING));
    bs.addBinding("averagePrice", vf.createLiteral("4.49", XMLSchema.DECIMAL));
    expectedResults.add(new VisibilityBindingSet(bs));
    bs = new MapBindingSet();
    bs.addBinding("type", vf.createLiteral("cheese", XMLSchema.STRING));
    bs.addBinding("location", vf.createLiteral("USA", XMLSchema.STRING));
    bs.addBinding("averagePrice", vf.createLiteral("4.75", XMLSchema.DECIMAL));
    expectedResults.add(new VisibilityBindingSet(bs));
    // Verify the end results of the query match the expected results.
    final Set<VisibilityBindingSet> results = readGroupedResults(pcjId, new VariableOrder("type", "location"));
    assertEquals(expectedResults, results);
}