Examples with FluoClient org.apache.fluo.api.client.FluoClient used on opensource projects

Example 36 with FluoClient

use of org.apache.fluo.api.client.FluoClient in project incubator-rya by apache.

the class PcjVisibilityIT method createWithVisibilityFluo.

@Test
public void createWithVisibilityFluo() throws Exception {
    final String sparql = "SELECT ?customer ?worker ?city " + "{ " + "?customer <http://talksTo> ?worker. " + "?worker <http://livesIn> ?city. " + "?worker <http://worksAt> <http://Chipotle>. " + "}";
    // Triples that will be streamed into Fluo after the PCJ has been created.
    final Map<RyaStatement, String> streamedTriples = new HashMap<>();
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://Bob")), "A&B");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://livesIn"), new RyaURI("http://London")), "A");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "B");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://Charlie")), "B&C");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://livesIn"), new RyaURI("http://London")), "B");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "C");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://David")), "C&D");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://David"), new RyaURI("http://livesIn"), new RyaURI("http://London")), "C");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://David"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "D");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://Eve")), "D&E");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://livesIn"), new RyaURI("http://Leeds")), "D");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "E");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://talksTo"), new RyaURI("http://Alice")), "");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://livesIn"), new RyaURI("http://London")), "");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "");
    final Connector accumuloConn = super.getAccumuloConnector();
    // Create the PCJ Table in Accumulo.
    final PrecomputedJoinStorage rootStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    final String pcjId = rootStorage.createPcj(sparql);
    try (final FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Create the PCJ in Fluo.
        new CreateFluoPcj().withRyaIntegration(pcjId, rootStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Stream the data into Fluo.
        for (final RyaStatement statement : streamedTriples.keySet()) {
            final Optional<String> visibility = Optional.of(streamedTriples.get(statement));
            new InsertTriples().insert(fluoClient, statement, visibility);
        }
    }
    // Fetch the exported results from Accumulo once the observers finish working.
    super.getMiniFluo().waitForObservers();
    setupTestUsers(accumuloConn, getRyaInstanceName(), pcjId);
    // Verify ABCDE using root.
    final Set<BindingSet> rootResults = toSet(rootStorage.listResults(pcjId));
    final Set<BindingSet> rootExpected = Sets.newHashSet();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("customer", VF.createURI("http://Alice"));
    bs.addBinding("worker", VF.createURI("http://Bob"));
    bs.addBinding("city", VF.createURI("http://London"));
    rootExpected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("customer", VF.createURI("http://Alice"));
    bs.addBinding("worker", VF.createURI("http://Charlie"));
    bs.addBinding("city", VF.createURI("http://London"));
    rootExpected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("customer", VF.createURI("http://Alice"));
    bs.addBinding("worker", VF.createURI("http://Eve"));
    bs.addBinding("city", VF.createURI("http://Leeds"));
    rootExpected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("customer", VF.createURI("http://Alice"));
    bs.addBinding("worker", VF.createURI("http://David"));
    bs.addBinding("city", VF.createURI("http://London"));
    rootExpected.add(bs);
    assertEquals(rootExpected, rootResults);
    final MiniAccumuloCluster cluster = super.getMiniAccumuloCluster();
    // Verify AB
    final Connector abConn = cluster.getConnector("abUser", "password");
    try (final PrecomputedJoinStorage abStorage = new AccumuloPcjStorage(abConn, getRyaInstanceName())) {
        final Set<BindingSet> abResults = toSet(abStorage.listResults(pcjId));
        final Set<BindingSet> abExpected = Sets.newHashSet();
        bs = new MapBindingSet();
        bs.addBinding("customer", VF.createURI("http://Alice"));
        bs.addBinding("worker", VF.createURI("http://Bob"));
        bs.addBinding("city", VF.createURI("http://London"));
        abExpected.add(bs);
        assertEquals(abExpected, abResults);
    }
    // Verify ABC
    final Connector abcConn = cluster.getConnector("abcUser", "password");
    try (final PrecomputedJoinStorage abcStorage = new AccumuloPcjStorage(abcConn, getRyaInstanceName())) {
        final Set<BindingSet> abcResults = toSet(abcStorage.listResults(pcjId));
        final Set<BindingSet> abcExpected = Sets.newHashSet();
        bs = new MapBindingSet();
        bs.addBinding("customer", VF.createURI("http://Alice"));
        bs.addBinding("worker", VF.createURI("http://Bob"));
        bs.addBinding("city", VF.createURI("http://London"));
        abcExpected.add(bs);
        bs = new MapBindingSet();
        bs.addBinding("customer", VF.createURI("http://Alice"));
        bs.addBinding("worker", VF.createURI("http://Charlie"));
        bs.addBinding("city", VF.createURI("http://London"));
        abcExpected.add(bs);
        assertEquals(abcExpected, abcResults);
    }
    // Verify ADE
    final Connector adeConn = cluster.getConnector("adeUser", "password");
    try (final PrecomputedJoinStorage adeStorage = new AccumuloPcjStorage(adeConn, getRyaInstanceName())) {
        final Set<BindingSet> adeResults = toSet(adeStorage.listResults(pcjId));
        final Set<BindingSet> adeExpected = Sets.newHashSet();
        bs = new MapBindingSet();
        bs.addBinding("customer", VF.createURI("http://Alice"));
        bs.addBinding("worker", VF.createURI("http://Eve"));
        bs.addBinding("city", VF.createURI("http://Leeds"));
        adeExpected.add(bs);
        assertEquals(adeExpected, adeResults);
    }
    // Verify no auths.
    final Connector noAuthConn = cluster.getConnector("noAuth", "password");
    try (final PrecomputedJoinStorage noAuthStorage = new AccumuloPcjStorage(noAuthConn, getRyaInstanceName())) {
        final Set<BindingSet> noAuthResults = toSet(noAuthStorage.listResults(pcjId));
        assertTrue(noAuthResults.isEmpty());
    }
}

Example 37 with FluoClient

use of org.apache.fluo.api.client.FluoClient in project incubator-rya by apache.

the class PcjAdminClient method main.

public static void main(final String[] args) {
    log.trace("Starting up the PCJ Admin Client.");
    // If no command provided or the command isn't recognized, then print the usage.
    if (args.length == 0 || !commands.containsKey(args[0])) {
        System.out.println(usage);
        System.exit(-1);
    }
    // Load the properties file.
    final Properties props = new Properties();
    try (InputStream pin = Files.newInputStream(PROPERTIES_FILE)) {
        props.load(pin);
    } catch (final IOException e) {
        throw new RuntimeException("Could not load properties file: " + PROPERTIES_FILE, e);
    }
    // Fetch the command that will be executed.
    final String command = args[0];
    final String[] commandArgs = Arrays.copyOfRange(args, 1, args.length);
    final PcjAdminClientCommand pcjCommand = commands.get(command);
    RyaSailRepository rya = null;
    FluoClient fluo = null;
    try {
        // Connect to Accumulo, Rya, and Fluo.
        final PcjAdminClientProperties clientProps = new PcjAdminClientProperties(props);
        final Connector accumulo = createAccumuloConnector(clientProps);
        rya = makeRyaRepository(clientProps, accumulo);
        fluo = createFluoClient(clientProps);
        // Execute the command.
        pcjCommand.execute(accumulo, clientProps.getRyaTablePrefix(), rya, fluo, commandArgs);
    } catch (final AccumuloException | AccumuloSecurityException e) {
        System.err.println("Could not connect to the Accumulo instance that hosts the export PCJ tables.");
        e.printStackTrace();
        System.exit(-1);
    } catch (final RepositoryException e) {
        System.err.println("Could not connect to the Rya instance that hosts the historic RDF statements.");
        e.printStackTrace();
        System.exit(-1);
    } catch (final ArgumentsException e) {
        System.err.println(pcjCommand.getUsage());
        System.exit(-1);
    } catch (final ExecutionException e) {
        System.err.println("Could not execute the command.");
        e.printStackTrace();
        System.exit(-1);
    } catch (UnsupportedQueryException e) {
        System.err.println("Could not execute the command because the query is invalid.");
        e.printStackTrace();
    } finally {
        log.trace("Shutting down the PCJ Admin Client.");
        if (rya != null) {
            try {
                rya.shutDown();
            } catch (final RepositoryException e) {
                System.err.println("Problem while shutting down the Rya connection.");
                e.printStackTrace();
            }
        }
        if (fluo != null) {
            fluo.close();
        }
    }
}

Example 38 with FluoClient

use of org.apache.fluo.api.client.FluoClient in project incubator-rya by apache.

the class GetQueryReportIT method getReport.

@Test
public void getReport() throws Exception {
    final String sparql = "SELECT ?worker ?company ?city" + "{ " + "FILTER(?worker = <http://Alice>) " + "?worker <http://worksAt> ?company . " + "?worker <http://livesIn> ?city ." + "}";
    // Triples that will be streamed into Fluo after the PCJ has been created.
    final Set<RyaStatement> streamedTriples = Sets.newHashSet(new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://worksAt"), new RyaURI("http://Taco Shop")), new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://worksAt"), new RyaURI("http://Burger Join")), new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://worksAt"), new RyaURI("http://Pastery Shop")), new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://worksAt"), new RyaURI("http://Burrito Place")), new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://livesIn"), new RyaURI("http://Lost County")), new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://livesIn"), new RyaURI("http://Big City")), new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://worksAt"), new RyaURI("http://Burrito Place")), new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://livesIn"), new RyaURI("http://Big City")), new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://worksAt"), new RyaURI("http://Burrito Place")), new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://livesIn"), new RyaURI("http://Big City")), new RyaStatement(new RyaURI("http://David"), new RyaURI("http://worksAt"), new RyaURI("http://Burrito Place")), new RyaStatement(new RyaURI("http://David"), new RyaURI("http://livesIn"), new RyaURI("http://Lost County")), new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://worksAt"), new RyaURI("http://Burrito Place")), new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://livesIn"), new RyaURI("http://Big City")), new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://worksAt"), new RyaURI("http://Burrito Place")), new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://livesIn"), new RyaURI("http://Lost County")));
    // Create the PCJ table.
    final Connector accumuloConn = super.getAccumuloConnector();
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    final String pcjId = pcjStorage.createPcj(sparql);
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Tell the Fluo app to maintain the PCJ.
        new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Stream the data into Fluo.
        new InsertTriples().insert(fluoClient, streamedTriples, Optional.<String>absent());
        // Wait for the results to finish processing.
        super.getMiniFluo().waitForObservers();
        // Fetch the report.
        final Map<String, PcjMetadata> metadata = new GetPcjMetadata().getMetadata(pcjStorage, fluoClient);
        final Set<String> queryIds = metadata.keySet();
        assertEquals(1, queryIds.size());
        final String queryId = queryIds.iterator().next();
        final QueryReport report = new GetQueryReport().getReport(fluoClient, queryId);
        // Build the expected counts map.
        final Map<String, BigInteger> expectedCounts = new HashMap<>();
        final FluoQuery fluoQuery = report.getFluoQuery();
        final String queryNodeId = fluoQuery.getQueryMetadata().getNodeId();
        expectedCounts.put(queryNodeId, BigInteger.valueOf(8));
        final String filterNodeId = fluoQuery.getFilterMetadata().iterator().next().getNodeId();
        expectedCounts.put(filterNodeId, BigInteger.valueOf(8));
        final String joinNodeId = fluoQuery.getJoinMetadata().iterator().next().getNodeId();
        expectedCounts.put(joinNodeId, BigInteger.valueOf(13));
        final Iterator<StatementPatternMetadata> patterns = fluoQuery.getStatementPatternMetadata().iterator();
        final StatementPatternMetadata sp1 = patterns.next();
        final StatementPatternMetadata sp2 = patterns.next();
        if (sp1.getStatementPattern().contains("http://worksAt")) {
            expectedCounts.put(sp1.getNodeId(), BigInteger.valueOf(9));
            expectedCounts.put(sp2.getNodeId(), BigInteger.valueOf(7));
        } else {
            expectedCounts.put(sp2.getNodeId(), BigInteger.valueOf(9));
            expectedCounts.put(sp1.getNodeId(), BigInteger.valueOf(7));
        }
        assertEquals(expectedCounts, report.getCounts());
    }
}

Example 39 with FluoClient

use of org.apache.fluo.api.client.FluoClient in project incubator-rya by apache.

the class ListQueryIdsIT method getQueryIds.

/**
 * This test ensures that when there are PCJ tables in Accumulo as well as
 * the Fluo table's export destinations column, the command for fetching the
 * list of queries only includes queries that appear in both places.
 */
@Test
public void getQueryIds() throws AccumuloException, AccumuloSecurityException, TableExistsException {
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Store a few SPARQL/Query ID pairs in the Fluo table.
        try (Transaction tx = fluoClient.newTransaction()) {
            tx.set("SPARQL_3", QUERY_NODE_ID, "ID_3");
            tx.set("SPARQL_1", QUERY_NODE_ID, "ID_1");
            tx.set("SPARQL_4", QUERY_NODE_ID, "ID_4");
            tx.set("SPARQL_2", QUERY_NODE_ID, "ID_2");
            tx.commit();
        }
        // Ensure the correct list of Query IDs is retured.
        final List<String> expected = Lists.newArrayList("ID_1", "ID_2", "ID_3", "ID_4");
        final List<String> queryIds = new ListQueryIds().listQueryIds(fluoClient);
        assertEquals(expected, queryIds);
    }
}

Example 40 with FluoClient

use of org.apache.fluo.api.client.FluoClient in project incubator-rya by apache.

the class BatchIT method simpleScanDelete.

@Test
public void simpleScanDelete() throws Exception {
    final String sparql = "SELECT ?subject ?object1 ?object2 WHERE { ?subject <urn:predicate_1> ?object1; " + " <urn:predicate_2> ?object2 } ";
    try (FluoClient fluoClient = new FluoClientImpl(getFluoConfiguration())) {
        RyaURI subj = new RyaURI("urn:subject_1");
        RyaStatement statement1 = new RyaStatement(subj, new RyaURI("urn:predicate_1"), null);
        RyaStatement statement2 = new RyaStatement(subj, new RyaURI("urn:predicate_2"), null);
        Set<RyaStatement> statements1 = getRyaStatements(statement1, 10);
        Set<RyaStatement> statements2 = getRyaStatements(statement2, 10);
        // Create the PCJ table.
        final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(getAccumuloConnector(), getRyaInstanceName());
        final String pcjId = pcjStorage.createPcj(sparql);
        // Tell the Fluo app to maintain the PCJ.
        String queryId = new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, getAccumuloConnector(), getRyaInstanceName()).getQueryId();
        List<String> ids = getNodeIdStrings(fluoClient, queryId);
        List<String> prefixes = Arrays.asList("urn:subject_1", "urn:subject_1", "urn:object", "urn:subject_1", "urn:subject_1");
        // Stream the data into Fluo.
        InsertTriples inserter = new InsertTriples();
        inserter.insert(fluoClient, statements1, Optional.absent());
        inserter.insert(fluoClient, statements2, Optional.absent());
        // Verify the end results of the query match the expected results.
        getMiniFluo().waitForObservers();
        verifyCounts(fluoClient, ids, Arrays.asList(100, 100, 100, 10, 10));
        createSpanBatches(fluoClient, ids, prefixes, 10);
        getMiniFluo().waitForObservers();
        verifyCounts(fluoClient, ids, Arrays.asList(0, 0, 0, 0, 0));
    }
}