Examples with VariableOrder org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder used on opensource projects

Example 1 with VariableOrder

use of org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder in project incubator-rya by apache.

the class PeriodicNotificationProcessorIT method periodicProcessorTest.

@Test
public void periodicProcessorTest() throws Exception {
    String id = UUID.randomUUID().toString().replace("-", "");
    BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
    BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
    BlockingQueue<BindingSetRecord> bindingSets = new LinkedBlockingQueue<>();
    TimestampedNotification ts1 = new TimestampedNotification(PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
    long binId1 = (ts1.getTimestamp().getTime() / ts1.getPeriod()) * ts1.getPeriod();
    Thread.sleep(2000);
    TimestampedNotification ts2 = new TimestampedNotification(PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
    long binId2 = (ts2.getTimestamp().getTime() / ts2.getPeriod()) * ts2.getPeriod();
    Set<NodeBin> expectedBins = new HashSet<>();
    expectedBins.add(new NodeBin(id, binId1));
    expectedBins.add(new NodeBin(id, binId2));
    Set<BindingSet> expected = new HashSet<>();
    Set<VisibilityBindingSet> storageResults = new HashSet<>();
    QueryBindingSet bs1 = new QueryBindingSet();
    bs1.addBinding("periodicBinId", vf.createLiteral(binId1));
    bs1.addBinding("id", vf.createLiteral(1));
    expected.add(bs1);
    storageResults.add(new VisibilityBindingSet(bs1));
    QueryBindingSet bs2 = new QueryBindingSet();
    bs2.addBinding("periodicBinId", vf.createLiteral(binId1));
    bs2.addBinding("id", vf.createLiteral(2));
    expected.add(bs2);
    storageResults.add(new VisibilityBindingSet(bs2));
    QueryBindingSet bs3 = new QueryBindingSet();
    bs3.addBinding("periodicBinId", vf.createLiteral(binId2));
    bs3.addBinding("id", vf.createLiteral(3));
    expected.add(bs3);
    storageResults.add(new VisibilityBindingSet(bs3));
    QueryBindingSet bs4 = new QueryBindingSet();
    bs4.addBinding("periodicBinId", vf.createLiteral(binId2));
    bs4.addBinding("id", vf.createLiteral(4));
    expected.add(bs4);
    storageResults.add(new VisibilityBindingSet(bs4));
    PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(), RYA_INSTANCE_NAME);
    periodicStorage.createPeriodicQuery(id, "select ?id where {?obs <urn:hasId> ?id.}", new VariableOrder("periodicBinId", "id"));
    periodicStorage.addPeriodicQueryResults(id, storageResults);
    NotificationProcessorExecutor processor = new NotificationProcessorExecutor(periodicStorage, notifications, bins, bindingSets, 1);
    processor.start();
    notifications.add(ts1);
    notifications.add(ts2);
    Thread.sleep(5000);
    Assert.assertEquals(expectedBins.size(), bins.size());
    Assert.assertEquals(true, bins.containsAll(expectedBins));
    Set<BindingSet> actual = new HashSet<>();
    bindingSets.forEach(x -> actual.add(x.getBindingSet()));
    Assert.assertEquals(expected, actual);
    processor.stop();
}

Example 2 with VariableOrder

use of org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder in project incubator-rya by apache.

the class AccumuloPeriodicQueryResultStorageIT method testCreateAndMeta.

@Test
public void testCreateAndMeta() throws PeriodicQueryStorageException {
    String sparql = "select ?x where { ?x <urn:pred> ?y.}";
    VariableOrder varOrder = new VariableOrder("periodicBinId", "x");
    PeriodicQueryStorageMetadata expectedMeta = new PeriodicQueryStorageMetadata(sparql, varOrder);
    String id = periodicStorage.createPeriodicQuery(sparql);
    Assert.assertEquals(expectedMeta, periodicStorage.getPeriodicQueryMetadata(id));
    Assert.assertEquals(Arrays.asList(nameFactory.makeTableName(RYA, id)), periodicStorage.listPeriodicTables());
    periodicStorage.deletePeriodicQuery(id);
}

Example 3 with VariableOrder

use of org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder in project incubator-rya by apache.

the class MongoPcjStorageIT method purge.

@Test
public void purge() throws Exception {
    try (final PrecomputedJoinStorage pcjStorage = new MongoPcjStorage(getMongoClient(), conf.getRyaInstanceName())) {
        final MongoRyaInstanceDetailsRepository detailsRepo = new MongoRyaInstanceDetailsRepository(getMongoClient(), conf.getRyaInstanceName());
        detailsRepo.initialize(RyaDetails.builder().setRyaInstanceName(conf.getRyaInstanceName()).setRyaVersion("test").setEntityCentricIndexDetails(new EntityCentricIndexDetails(false)).setTemporalIndexDetails(new TemporalIndexDetails(false)).setFreeTextDetails(new FreeTextIndexDetails(false)).setProspectorDetails(new ProspectorDetails(Optional.absent())).setJoinSelectivityDetails(new JoinSelectivityDetails(Optional.absent())).setPCJIndexDetails(PCJIndexDetails.builder().setEnabled(true)).build());
        // Create a PCJ.
        final String sparql = "SELECT * WHERE { ?a <http://isA> ?b }";
        final String pcjId = pcjStorage.createPcj(sparql);
        // Add some binding sets to it.
        final Set<VisibilityBindingSet> expectedResults = new HashSet<>();
        final MapBindingSet aliceBS = new MapBindingSet();
        aliceBS.addBinding("a", new URIImpl("http://Alice"));
        aliceBS.addBinding("b", new URIImpl("http://Person"));
        expectedResults.add(new VisibilityBindingSet(aliceBS, ""));
        final MapBindingSet charlieBS = new MapBindingSet();
        charlieBS.addBinding("a", new URIImpl("http://Charlie"));
        charlieBS.addBinding("b", new URIImpl("http://Comedian"));
        expectedResults.add(new VisibilityBindingSet(charlieBS, ""));
        pcjStorage.addResults(pcjId, expectedResults);
        // Purge the PCJ.
        pcjStorage.purge(pcjId);
        // List the results that were stored.
        final Set<BindingSet> results = new HashSet<>();
        try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
            while (resultsIt.hasNext()) {
                results.add(resultsIt.next());
            }
        }
        assertTrue(results.isEmpty());
        // Make sure the PCJ metadata was updated.
        final PcjMetadata metadata = pcjStorage.getPcjMetadata(pcjId);
        final Set<VariableOrder> varOrders = new ShiftVarOrderFactory().makeVarOrders(sparql);
        final PcjMetadata expectedMetadata = new PcjMetadata(sparql, 0L, varOrders);
        assertEquals(expectedMetadata, metadata);
    }
}

Example 4 with VariableOrder

use of org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder in project incubator-rya by apache.

the class MongoPcjStorageIT method getPcjMetadata.

@Test
public void getPcjMetadata() throws Exception {
    try (final PrecomputedJoinStorage pcjStorage = new MongoPcjStorage(getMongoClient(), conf.getRyaInstanceName())) {
        final MongoRyaInstanceDetailsRepository detailsRepo = new MongoRyaInstanceDetailsRepository(getMongoClient(), conf.getRyaInstanceName());
        detailsRepo.initialize(RyaDetails.builder().setRyaInstanceName(conf.getRyaInstanceName()).setRyaVersion("test").setEntityCentricIndexDetails(new EntityCentricIndexDetails(false)).setTemporalIndexDetails(new TemporalIndexDetails(false)).setFreeTextDetails(new FreeTextIndexDetails(false)).setProspectorDetails(new ProspectorDetails(Optional.absent())).setJoinSelectivityDetails(new JoinSelectivityDetails(Optional.absent())).setPCJIndexDetails(PCJIndexDetails.builder().setEnabled(true)).build());
        // Create a PCJ.
        final String sparql = "SELECT * WHERE { ?a <http://isA> ?b }";
        final String pcjId = pcjStorage.createPcj(sparql);
        // Fetch the PCJ's metadata.
        final PcjMetadata metadata = pcjStorage.getPcjMetadata(pcjId);
        // Ensure it has the expected values.
        final Set<VariableOrder> varOrders = new ShiftVarOrderFactory().makeVarOrders(sparql);
        final PcjMetadata expectedMetadata = new PcjMetadata(sparql, 0L, varOrders);
        assertEquals(expectedMetadata, metadata);
    }
}

Example 5 with VariableOrder

use of org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder in project incubator-rya by apache.

the class JoinBatchBindingSetUpdater method processBatch.

/**
 * Processes {@link JoinBatchInformation}. Updates the BindingSets
 * associated with the specified nodeId. The BindingSets are processed in
 * batch fashion, where the number of results is indicated by
 * {@link JoinBatchInformation#getBatchSize()}. BindingSets are either
 * Added, Deleted, or Updated according to
 * {@link JoinBatchInformation#getTask()}. In the event that the number of
 * entries that need to be updated exceeds the batch size, the row of the
 * first unprocessed BindingSets is used to create a new JoinBatch job to
 * process the remaining BindingSets.
 * @throws Exception
 */
@Override
public void processBatch(final TransactionBase tx, final Bytes row, final BatchInformation batch) throws Exception {
    super.processBatch(tx, row, batch);
    final String nodeId = BatchRowKeyUtil.getNodeId(row);
    Preconditions.checkArgument(batch instanceof JoinBatchInformation);
    final JoinBatchInformation joinBatch = (JoinBatchInformation) batch;
    final Task task = joinBatch.getTask();
    // Figure out which join algorithm we are going to use.
    final IterativeJoin joinAlgorithm;
    switch(joinBatch.getJoinType()) {
        case NATURAL_JOIN:
            joinAlgorithm = new NaturalJoin();
            break;
        case LEFT_OUTER_JOIN:
            joinAlgorithm = new LeftOuterJoin();
            break;
        default:
            throw new RuntimeException("Unsupported JoinType: " + joinBatch.getJoinType());
    }
    final Set<VisibilityBindingSet> bsSet = new HashSet<>();
    final Optional<RowColumn> rowCol = fillSiblingBatch(tx, joinBatch, bsSet);
    // Iterates over the resulting BindingSets from the join.
    final Iterator<VisibilityBindingSet> newJoinResults;
    final VisibilityBindingSet bs = joinBatch.getBs();
    if (joinBatch.getSide() == Side.LEFT) {
        newJoinResults = joinAlgorithm.newLeftResult(bs, bsSet.iterator());
    } else {
        newJoinResults = joinAlgorithm.newRightResult(bsSet.iterator(), bs);
    }
    // Read join metadata, create new join BindingSets and insert them into the Fluo table.
    final JoinMetadata joinMetadata = CACHE.readJoinMetadata(tx, nodeId);
    final VariableOrder joinVarOrder = joinMetadata.getVariableOrder();
    while (newJoinResults.hasNext()) {
        final VisibilityBindingSet newJoinResult = newJoinResults.next();
        // create BindingSet value
        final Bytes bsBytes = BS_SERDE.serialize(newJoinResult);
        // make rowId
        Bytes rowKey = BindingHashShardingFunction.addShard(nodeId, joinVarOrder, newJoinResult);
        final Column col = FluoQueryColumns.JOIN_BINDING_SET;
        processTask(tx, task, rowKey, col, bsBytes);
    }
    // update the span and register updated batch job
    if (rowCol.isPresent()) {
        final Span newSpan = getNewSpan(rowCol.get(), joinBatch.getSpan());
        joinBatch.setSpan(newSpan);
        BatchInformationDAO.addBatch(tx, nodeId, joinBatch);
    }
}