Examples with MiniAccumuloCluster org.apache.accumulo.minicluster.MiniAccumuloCluster used on opensource projects

Example 21 with MiniAccumuloCluster

use of org.apache.accumulo.minicluster.MiniAccumuloCluster in project incubator-rya by apache.

the class PcjVisibilityIT method createWithVisibilityFluo.

@Test
public void createWithVisibilityFluo() throws Exception {
    final String sparql = "SELECT ?customer ?worker ?city " + "{ " + "?customer <http://talksTo> ?worker. " + "?worker <http://livesIn> ?city. " + "?worker <http://worksAt> <http://Chipotle>. " + "}";
    // Triples that will be streamed into Fluo after the PCJ has been created.
    final Map<RyaStatement, String> streamedTriples = new HashMap<>();
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://Bob")), "A&B");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://livesIn"), new RyaURI("http://London")), "A");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "B");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://Charlie")), "B&C");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://livesIn"), new RyaURI("http://London")), "B");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "C");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://David")), "C&D");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://David"), new RyaURI("http://livesIn"), new RyaURI("http://London")), "C");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://David"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "D");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://Eve")), "D&E");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://livesIn"), new RyaURI("http://Leeds")), "D");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "E");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://talksTo"), new RyaURI("http://Alice")), "");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://livesIn"), new RyaURI("http://London")), "");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "");
    final Connector accumuloConn = super.getAccumuloConnector();
    // Create the PCJ Table in Accumulo.
    final PrecomputedJoinStorage rootStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    final String pcjId = rootStorage.createPcj(sparql);
    try (final FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Create the PCJ in Fluo.
        new CreateFluoPcj().withRyaIntegration(pcjId, rootStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Stream the data into Fluo.
        for (final RyaStatement statement : streamedTriples.keySet()) {
            final Optional<String> visibility = Optional.of(streamedTriples.get(statement));
            new InsertTriples().insert(fluoClient, statement, visibility);
        }
    }
    // Fetch the exported results from Accumulo once the observers finish working.
    super.getMiniFluo().waitForObservers();
    setupTestUsers(accumuloConn, getRyaInstanceName(), pcjId);
    // Verify ABCDE using root.
    final Set<BindingSet> rootResults = toSet(rootStorage.listResults(pcjId));
    final Set<BindingSet> rootExpected = Sets.newHashSet();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("customer", VF.createURI("http://Alice"));
    bs.addBinding("worker", VF.createURI("http://Bob"));
    bs.addBinding("city", VF.createURI("http://London"));
    rootExpected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("customer", VF.createURI("http://Alice"));
    bs.addBinding("worker", VF.createURI("http://Charlie"));
    bs.addBinding("city", VF.createURI("http://London"));
    rootExpected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("customer", VF.createURI("http://Alice"));
    bs.addBinding("worker", VF.createURI("http://Eve"));
    bs.addBinding("city", VF.createURI("http://Leeds"));
    rootExpected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("customer", VF.createURI("http://Alice"));
    bs.addBinding("worker", VF.createURI("http://David"));
    bs.addBinding("city", VF.createURI("http://London"));
    rootExpected.add(bs);
    assertEquals(rootExpected, rootResults);
    final MiniAccumuloCluster cluster = super.getMiniAccumuloCluster();
    // Verify AB
    final Connector abConn = cluster.getConnector("abUser", "password");
    try (final PrecomputedJoinStorage abStorage = new AccumuloPcjStorage(abConn, getRyaInstanceName())) {
        final Set<BindingSet> abResults = toSet(abStorage.listResults(pcjId));
        final Set<BindingSet> abExpected = Sets.newHashSet();
        bs = new MapBindingSet();
        bs.addBinding("customer", VF.createURI("http://Alice"));
        bs.addBinding("worker", VF.createURI("http://Bob"));
        bs.addBinding("city", VF.createURI("http://London"));
        abExpected.add(bs);
        assertEquals(abExpected, abResults);
    }
    // Verify ABC
    final Connector abcConn = cluster.getConnector("abcUser", "password");
    try (final PrecomputedJoinStorage abcStorage = new AccumuloPcjStorage(abcConn, getRyaInstanceName())) {
        final Set<BindingSet> abcResults = toSet(abcStorage.listResults(pcjId));
        final Set<BindingSet> abcExpected = Sets.newHashSet();
        bs = new MapBindingSet();
        bs.addBinding("customer", VF.createURI("http://Alice"));
        bs.addBinding("worker", VF.createURI("http://Bob"));
        bs.addBinding("city", VF.createURI("http://London"));
        abcExpected.add(bs);
        bs = new MapBindingSet();
        bs.addBinding("customer", VF.createURI("http://Alice"));
        bs.addBinding("worker", VF.createURI("http://Charlie"));
        bs.addBinding("city", VF.createURI("http://London"));
        abcExpected.add(bs);
        assertEquals(abcExpected, abcResults);
    }
    // Verify ADE
    final Connector adeConn = cluster.getConnector("adeUser", "password");
    try (final PrecomputedJoinStorage adeStorage = new AccumuloPcjStorage(adeConn, getRyaInstanceName())) {
        final Set<BindingSet> adeResults = toSet(adeStorage.listResults(pcjId));
        final Set<BindingSet> adeExpected = Sets.newHashSet();
        bs = new MapBindingSet();
        bs.addBinding("customer", VF.createURI("http://Alice"));
        bs.addBinding("worker", VF.createURI("http://Eve"));
        bs.addBinding("city", VF.createURI("http://Leeds"));
        adeExpected.add(bs);
        assertEquals(adeExpected, adeResults);
    }
    // Verify no auths.
    final Connector noAuthConn = cluster.getConnector("noAuth", "password");
    try (final PrecomputedJoinStorage noAuthStorage = new AccumuloPcjStorage(noAuthConn, getRyaInstanceName())) {
        final Set<BindingSet> noAuthResults = toSet(noAuthStorage.listResults(pcjId));
        assertTrue(noAuthResults.isEmpty());
    }
}

Example 22 with MiniAccumuloCluster

use of org.apache.accumulo.minicluster.MiniAccumuloCluster in project incubator-rya by apache.

the class KafkaExportITBase method teardownRya.

@After
public void teardownRya() {
    final MiniAccumuloCluster cluster = getMiniAccumuloCluster();
    final String instanceName = cluster.getInstanceName();
    final String zookeepers = cluster.getZooKeepers();
    // Uninstall the instance of Rya.
    final RyaClient ryaClient = AccumuloRyaClientFactory.build(new AccumuloConnectionDetails(ACCUMULO_USER, ACCUMULO_PASSWORD.toCharArray(), instanceName, zookeepers), super.getAccumuloConnector());
    try {
        ryaClient.getUninstall().uninstall(RYA_INSTANCE_NAME);
        // Shutdown the repo.
        if (ryaSailRepo != null) {
            ryaSailRepo.shutDown();
        }
        if (dao != null) {
            dao.destroy();
        }
    } catch (final Exception e) {
        System.out.println("Encountered the following Exception when shutting down Rya: " + e.getMessage());
    }
}

Example 23 with MiniAccumuloCluster

use of org.apache.accumulo.minicluster.MiniAccumuloCluster in project incubator-rya by apache.

the class RyaClientExample method main.

public static void main(final String[] args) throws Exception {
    final String accumuloUsername = "root";
    final String accumuloPassword = "password";
    MiniAccumuloCluster cluster = null;
    MiniFluo fluo = null;
    Sail ryaSail = null;
    try {
        // Setup a Mini Accumulo Cluster to host the Rya instance.
        log.info("Setting up the Mini Accumulo Cluster used by this example.");
        final File miniDataDir = Files.createTempDir();
        final MiniAccumuloConfig cfg = new MiniAccumuloConfig(miniDataDir, accumuloPassword);
        cluster = new MiniAccumuloCluster(cfg);
        cluster.start();
        // Setup a Mini Fluo application that will be used to incrementally update the PCJ indicies.
        log.info("Setting up the Mini Fluo application used by this example.");
        final String fluoAppName = "demoInstance_pcjUpdater";
        fluo = makeMiniFluo(accumuloUsername, accumuloPassword, cluster.getInstanceName(), cluster.getZooKeepers(), fluoAppName);
        // Give the root user the 'U' authorizations.
        final Connector connector = cluster.getConnector(accumuloUsername, accumuloPassword);
        connector.securityOperations().changeUserAuthorizations(accumuloUsername, new Authorizations("U"));
        // Setup a Rya Client that is able to interact with the mini cluster.
        final AccumuloConnectionDetails connectionDetails = new AccumuloConnectionDetails(accumuloUsername, accumuloPassword.toCharArray(), cluster.getInstanceName(), cluster.getZooKeepers());
        final RyaClient ryaClient = AccumuloRyaClientFactory.build(connectionDetails, connector);
        // Install an instance of Rya that has all of the secondary indexers turned on.
        final String ryaInstanceName = "demoInstance_";
        final InstallConfiguration installConfig = InstallConfiguration.builder().setEnableTableHashPrefix(true).setEnableEntityCentricIndex(true).setEnableGeoIndex(true).setEnableFreeTextIndex(true).setEnableTemporalIndex(true).setEnablePcjIndex(true).setFluoPcjAppName(fluoAppName).build();
        ryaClient.getInstall().install(ryaInstanceName, installConfig);
        // Add a PCJ index.
        final String sparql = "SELECT ?patron ?employee " + "WHERE { " + "?patron <http://talksTo> ?employee. " + "?employee <http://worksAt> <http://CoffeeShop>. " + "}";
        // Load some statements into the Rya instance.
        final AccumuloIndexingConfiguration conf = AccumuloIndexingConfiguration.builder().setAuths("U").setAccumuloUser(accumuloUsername).setAccumuloPassword(accumuloPassword).setAccumuloInstance(cluster.getInstanceName()).setAccumuloZooKeepers(cluster.getZooKeepers()).setRyaPrefix(ryaInstanceName).setPcjUpdaterFluoAppName(fluoAppName).build();
        ryaSail = RyaSailFactory.getInstance(conf);
        final ValueFactory vf = ryaSail.getValueFactory();
        final List<Statement> statements = Lists.newArrayList(vf.createStatement(vf.createURI("http://Eve"), vf.createURI("http://talksTo"), vf.createURI("http://Charlie")), vf.createStatement(vf.createURI("http://David"), vf.createURI("http://talksTo"), vf.createURI("http://Alice")), vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://worksAt"), vf.createURI("http://CoffeeShop")), vf.createStatement(vf.createURI("http://Bob"), vf.createURI("http://worksAt"), vf.createURI("http://CoffeeShop")), vf.createStatement(vf.createURI("http://George"), vf.createURI("http://talksTo"), vf.createURI("http://Frank")), vf.createStatement(vf.createURI("http://Frank"), vf.createURI("http://worksAt"), vf.createURI("http://CoffeeShop")), vf.createStatement(vf.createURI("http://Eve"), vf.createURI("http://talksTo"), vf.createURI("http://Bob")), vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://worksAt"), vf.createURI("http://CoffeeShop")));
        SailConnection ryaConn = ryaSail.getConnection();
        log.info("");
        log.info("Loading the following statements:");
        ryaConn.begin();
        for (final Statement statement : statements) {
            log.info("    " + statement.toString());
            ryaConn.addStatement(statement.getSubject(), statement.getPredicate(), statement.getObject());
        }
        log.info("");
        ryaConn.close();
        fluo.waitForObservers();
        // Execute the SPARQL query and print the results.
        log.info("Executing the following query: ");
        prettyLogSparql(sparql);
        log.info("");
        final ParsedQuery parsedQuery = new SPARQLParser().parseQuery(sparql, null);
        ryaConn = ryaSail.getConnection();
        final CloseableIteration<? extends BindingSet, QueryEvaluationException> result = ryaConn.evaluate(parsedQuery.getTupleExpr(), null, null, false);
        log.info("Results:");
        while (result.hasNext()) {
            log.info("    " + result.next());
        }
        log.info("");
    } finally {
        if (ryaSail != null) {
            log.info("Shutting down the Rya Sail instance.");
            ryaSail.shutDown();
        }
        if (fluo != null) {
            try {
                log.info("Shutting down the Mini Fluo instance.");
                fluo.close();
            } catch (final Exception e) {
                log.error("Could not shut down the Mini Fluo instance.", e);
            }
        }
        if (cluster != null) {
            log.info("Sutting down the Mini Accumulo Cluster.");
            cluster.stop();
        }
    }
}

Example 24 with MiniAccumuloCluster

use of org.apache.accumulo.minicluster.MiniAccumuloCluster in project incubator-rya by apache.

the class AccumuloInstanceDriver method setUpInstance.

/**
 * Sets up the {@link MiniAccumuloCluster} or the {@link MockInstance}.
 * @throws Exception
 */
public void setUpInstance() throws Exception {
    if (!isMock) {
        log.info("Setting up " + driverName + " MiniAccumulo cluster...");
        // Create and Run MiniAccumulo Cluster
        tempDir = Files.createTempDir();
        tempDir.deleteOnExit();
        miniAccumuloCluster = new MiniAccumuloCluster(tempDir, userpwd);
        copyHadoopHomeToTemp();
        miniAccumuloCluster.getConfig().setInstanceName(instanceName);
        log.info(driverName + " MiniAccumulo instance starting up...");
        miniAccumuloCluster.start();
        Thread.sleep(1000);
        log.info(driverName + " MiniAccumulo instance started");
        log.info("Creating connector to " + driverName + " MiniAccumulo instance...");
        zooKeeperInstance = new ZooKeeperInstance(miniAccumuloCluster.getClientConfig());
        instance = zooKeeperInstance;
        connector = zooKeeperInstance.getConnector(user, new PasswordToken(userpwd));
        log.info("Created connector to " + driverName + " MiniAccumulo instance");
    } else {
        log.info("Setting up " + driverName + " mock instance...");
        mockInstance = new MockInstance(instanceName);
        instance = mockInstance;
        connector = mockInstance.getConnector(user, new PasswordToken(userpwd));
        log.info("Created connector to " + driverName + " mock instance");
    }
    zooKeepers = instance.getZooKeepers();
}

Example 25 with MiniAccumuloCluster

use of org.apache.accumulo.minicluster.MiniAccumuloCluster in project incubator-rya by apache.

the class MiniAccumuloClusterInstance method startMiniAccumulo.

/**
 * Start the {@link MiniAccumuloCluster}.
 */
public void startMiniAccumulo() throws IOException, InterruptedException, AccumuloException, AccumuloSecurityException {
    final File miniDataDir = Files.createTempDir();
    // Setup and start the Mini Accumulo.
    final MiniAccumuloConfig cfg = new MiniAccumuloConfig(miniDataDir, PASSWORD);
    cluster = new MiniAccumuloCluster(cfg);
    cluster.start();
}