Examples with Bytes org.apache.fluo.api.data.Bytes used on opensource projects

Example 6 with Bytes

use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.

the class CreateDeleteIT method getFluoTableEntries.

private List<Bytes> getFluoTableEntries(final FluoClient fluoClient) {
    try (Snapshot snapshot = fluoClient.newSnapshot()) {
        final List<Bytes> rows = new ArrayList<>();
        final RowScanner rscanner = snapshot.scanner().over(Span.prefix("")).byRow().build();
        for (final ColumnScanner cscanner : rscanner) {
            rows.add(cscanner.getRow());
        }
        return rows;
    }
}

Example 7 with Bytes

use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.

the class BatchIT method createSpanBatches.

private void createSpanBatches(FluoClient fluoClient, List<String> ids, List<String> prefixes, int batchSize) {
    Preconditions.checkArgument(ids.size() == prefixes.size());
    try (Transaction tx = fluoClient.newTransaction()) {
        for (int i = 0; i < ids.size(); i++) {
            String id = ids.get(i);
            String bsPrefix = prefixes.get(i);
            URI uri = vf.createURI(bsPrefix);
            Bytes prefixBytes = BindingHashShardingFunction.getShardedScanPrefix(id, uri);
            NodeType type = NodeType.fromNodeId(id).get();
            Column bsCol = type.getResultColumn();
            SpanBatchDeleteInformation.Builder builder = SpanBatchDeleteInformation.builder().setBatchSize(batchSize).setColumn(bsCol);
            if (type == NodeType.JOIN) {
                builder.setSpan(Span.prefix(type.getNodeTypePrefix()));
                builder.setNodeId(java.util.Optional.of(id));
            } else {
                builder.setSpan(Span.prefix(prefixBytes));
            }
            BatchInformationDAO.addBatch(tx, id, builder.build());
        }
        tx.commit();
    }
}

Example 8 with Bytes

use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.

the class BatchIT method countResults.

private int countResults(FluoClient fluoClient, String nodeId, Column bsColumn) {
    try (Transaction tx = fluoClient.newTransaction()) {
        int count = 0;
        Optional<NodeType> type = NodeType.fromNodeId(nodeId);
        Bytes prefixBytes = Bytes.of(type.get().getNodeTypePrefix());
        RowScanner scanner = tx.scanner().over(Span.prefix(prefixBytes)).fetch(bsColumn).byRow().build();
        Iterator<ColumnScanner> colScanners = scanner.iterator();
        while (colScanners.hasNext()) {
            ColumnScanner colScanner = colScanners.next();
            BindingSetRow bsRow = BindingSetRow.makeFromShardedRow(prefixBytes, colScanner.getRow());
            if (bsRow.getNodeId().equals(nodeId)) {
                Iterator<ColumnValue> vals = colScanner.iterator();
                while (vals.hasNext()) {
                    vals.next();
                    count++;
                }
            }
        }
        tx.commit();
        return count;
    }
}

Example 9 with Bytes

use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.

the class BatchIT method simpleJoinAdd.

@Test
public void simpleJoinAdd() throws Exception {
    final String sparql = "SELECT ?subject ?object1 ?object2 WHERE { ?subject <urn:predicate_1> ?object1; " + " <urn:predicate_2> ?object2 } ";
    try (FluoClient fluoClient = new FluoClientImpl(getFluoConfiguration())) {
        RyaURI subj = new RyaURI("urn:subject_1");
        RyaStatement statement2 = new RyaStatement(subj, new RyaURI("urn:predicate_2"), null);
        Set<RyaStatement> statements2 = getRyaStatements(statement2, 5);
        // Create the PCJ table.
        final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(getAccumuloConnector(), getRyaInstanceName());
        final String pcjId = pcjStorage.createPcj(sparql);
        // Tell the Fluo app to maintain the PCJ.
        String queryId = new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, getAccumuloConnector(), getRyaInstanceName()).getQueryId();
        List<String> ids = getNodeIdStrings(fluoClient, queryId);
        String joinId = ids.get(2);
        String rightSp = ids.get(4);
        QueryBindingSet bs = new QueryBindingSet();
        bs.addBinding("subject", vf.createURI("urn:subject_1"));
        bs.addBinding("object1", vf.createURI("urn:object_0"));
        VisibilityBindingSet vBs = new VisibilityBindingSet(bs);
        URI uri = vf.createURI("urn:subject_1");
        Bytes prefixBytes = BindingHashShardingFunction.getShardedScanPrefix(rightSp, uri);
        Span span = Span.prefix(prefixBytes);
        // Stream the data into Fluo.
        InsertTriples inserter = new InsertTriples();
        inserter.insert(fluoClient, statements2, Optional.absent());
        getMiniFluo().waitForObservers();
        verifyCounts(fluoClient, ids, Arrays.asList(0, 0, 0, 0, 5));
        JoinBatchInformation batch = JoinBatchInformation.builder().setBatchSize(1).setColumn(FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET).setSpan(span).setTask(Task.Add).setJoinType(JoinType.NATURAL_JOIN).setSide(Side.LEFT).setBs(vBs).build();
        // Verify the end results of the query match the expected results.
        createSpanBatch(fluoClient, joinId, batch);
        getMiniFluo().waitForObservers();
        verifyCounts(fluoClient, ids, Arrays.asList(5, 5, 5, 0, 5));
    }
}

Example 10 with Bytes

use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.

the class CreateDeletePeriodicPCJ method runTest.

private void runTest(String query, Collection<Statement> statements, int expectedEntries) throws Exception {
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        String topic = "notification_topic";
        PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(), RYA_INSTANCE_NAME);
        PeriodicNotificationClient notificationClient = new KafkaNotificationRegistrationClient(topic, getNotificationProducer("localhost:9092"));
        CreatePeriodicQuery periodicPCJ = new CreatePeriodicQuery(fluoClient, storage);
        String id = periodicPCJ.createPeriodicQuery(query, notificationClient).getQueryId();
        loadData(statements);
        // Ensure the data was loaded.
        final List<Bytes> rows = getFluoTableEntries(fluoClient);
        assertEquals(expectedEntries, rows.size());
        DeletePeriodicQuery deletePeriodic = new DeletePeriodicQuery(fluoClient, storage);
        deletePeriodic.deletePeriodicQuery(FluoQueryUtils.convertFluoQueryIdToPcjId(id), notificationClient);
        getMiniFluo().waitForObservers();
        // Ensure all data related to the query has been removed.
        final List<Bytes> empty_rows = getFluoTableEntries(fluoClient);
        assertEquals(1, empty_rows.size());
        // Ensure that Periodic Service notified to add and delete PeriodicNotification
        Set<CommandNotification> notifications;
        try (KafkaConsumer<String, CommandNotification> consumer = makeNotificationConsumer(topic)) {
            notifications = getKafkaNotifications(topic, 7000, consumer);
        }
        assertEquals(2, notifications.size());
        String notificationId = "";
        boolean addCalled = false;
        boolean deleteCalled = false;
        for (CommandNotification notification : notifications) {
            if (notificationId.length() == 0) {
                notificationId = notification.getId();
            } else {
                assertEquals(notificationId, notification.getId());
            }
            if (notification.getCommand() == Command.ADD) {
                addCalled = true;
            }
            if (notification.getCommand() == Command.DELETE) {
                deleteCalled = true;
            }
        }
        assertEquals(true, addCalled);
        assertEquals(true, deleteCalled);
    }
}