Examples with PcjException org.apache.rya.indexing.pcj.storage.PcjException used on opensource projects

Example 1 with PcjException

use of org.apache.rya.indexing.pcj.storage.PcjException in project incubator-rya by apache.

the class AccumuloCreatePCJ method createPCJ.

@Override
public String createPCJ(final String instanceName, final String sparql, Set<ExportStrategy> strategies) throws InstanceDoesNotExistException, RyaClientException {
    requireNonNull(instanceName);
    requireNonNull(sparql);
    final Optional<RyaDetails> ryaDetailsHolder = getInstanceDetails.getDetails(instanceName);
    final boolean ryaInstanceExists = ryaDetailsHolder.isPresent();
    if (!ryaInstanceExists) {
        throw new InstanceDoesNotExistException(String.format("The '%s' instance of Rya does not exist.", instanceName));
    }
    final PCJIndexDetails pcjIndexDetails = ryaDetailsHolder.get().getPCJIndexDetails();
    final boolean pcjIndexingEnabeld = pcjIndexDetails.isEnabled();
    if (!pcjIndexingEnabeld) {
        throw new RyaClientException(String.format("The '%s' instance of Rya does not have PCJ Indexing enabled.", instanceName));
    }
    // Create the PCJ table that will receive the index results.
    final String pcjId;
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(getConnector(), instanceName)) {
        pcjId = pcjStorage.createPcj(sparql);
        // If a Fluo application is being used, task it with updating the PCJ.
        final Optional<FluoDetails> fluoDetailsHolder = pcjIndexDetails.getFluoDetails();
        if (fluoDetailsHolder.isPresent()) {
            final String fluoAppName = fluoDetailsHolder.get().getUpdateAppName();
            try {
                updateFluoApp(instanceName, fluoAppName, pcjId, sparql, strategies);
            } catch (RepositoryException | MalformedQueryException | SailException | QueryEvaluationException | PcjException | RyaDAOException e) {
                throw new RyaClientException("Problem while initializing the Fluo application with the new PCJ.", e);
            } catch (UnsupportedQueryException e) {
                throw new RyaClientException("The new PCJ could not be initialized because it either contains an unsupported query node " + "or an invalid ExportStrategy for the given QueryType.  Projection queries can be exported to either Rya or Kafka," + "unless they contain an aggregation, in which case they can only be exported to Kafka.  Construct queries can be exported" + "to Rya and Kafka, and Periodic queries can only be exported to Rya.");
            }
            // Update the Rya Details to indicate the PCJ is being updated incrementally.
            final RyaDetailsRepository detailsRepo = new AccumuloRyaInstanceDetailsRepository(getConnector(), instanceName);
            try {
                new RyaDetailsUpdater(detailsRepo).update(new RyaDetailsMutator() {

                    @Override
                    public RyaDetails mutate(final RyaDetails originalDetails) throws CouldNotApplyMutationException {
                        // Update the original PCJ Details to indicate they are incrementally updated.
                        final PCJDetails originalPCJDetails = originalDetails.getPCJIndexDetails().getPCJDetails().get(pcjId);
                        final PCJDetails.Builder mutatedPCJDetails = PCJDetails.builder(originalPCJDetails).setUpdateStrategy(PCJUpdateStrategy.INCREMENTAL);
                        // Replace the old PCJ Details with the updated ones.
                        final RyaDetails.Builder builder = RyaDetails.builder(originalDetails);
                        builder.getPCJIndexDetails().addPCJDetails(mutatedPCJDetails);
                        return builder.build();
                    }
                });
            } catch (RyaDetailsRepositoryException | CouldNotApplyMutationException e) {
                throw new RyaClientException("Problem while updating the Rya instance's Details to indicate the PCJ is being incrementally updated.", e);
            }
        }
        // Return the ID that was assigned to the PCJ.
        return pcjId;
    } catch (final PCJStorageException e) {
        throw new RyaClientException("Problem while initializing the PCJ table.", e);
    }
}

Example 2 with PcjException

use of org.apache.rya.indexing.pcj.storage.PcjException in project incubator-rya by apache.

the class PcjIntegrationTestingUtil method populatePcj.

/**
 * Scan Rya for results that solve the PCJ's query and store them in the PCJ
 * table.
 * <p>
 * This method assumes the PCJ table has already been created.
 *
 * @param mongoClient - A connection to the mongoDB that hosts the PCJ table. (not null)
 * @param pcjTableName - The name of the PCJ table that will receive the results. (not null)
 * @param ryaConn - A connection to the Rya store that will be queried to find results. (not null)
 * @throws PcjException
 *             If results could not be written to the PCJ table, the PCJ
 *             table does not exist, or the query that is being execute was
 *             malformed.
 */
public static void populatePcj(final MongoPcjDocuments pcj, final String pcjTableName, final RepositoryConnection ryaConn) throws PcjException {
    checkNotNull(pcj);
    checkNotNull(pcjTableName);
    checkNotNull(ryaConn);
    try {
        // Fetch the query that needs to be executed from the PCJ table.
        final PcjMetadata pcjMetadata = pcj.getPcjMetadata(pcjTableName);
        final String sparql = pcjMetadata.getSparql();
        // Query Rya for results to the SPARQL query.
        final TupleQuery query = ryaConn.prepareTupleQuery(QueryLanguage.SPARQL, sparql);
        final TupleQueryResult results = query.evaluate();
        // Load batches of 1000 of them at a time into the PCJ table
        final Set<BindingSet> batch = new HashSet<>(1000);
        while (results.hasNext()) {
            batch.add(results.next());
            if (batch.size() == 1000) {
                writeResults(pcj, pcjTableName, batch);
                batch.clear();
            }
        }
        if (!batch.isEmpty()) {
            writeResults(pcj, pcjTableName, batch);
        }
    } catch (RepositoryException | MalformedQueryException | QueryEvaluationException e) {
        throw new PcjException("Could not populate a PCJ table with Rya results for the table named: " + pcjTableName, e);
    }
}

Example 3 with PcjException

use of org.apache.rya.indexing.pcj.storage.PcjException in project incubator-rya by apache.

the class GetPcjMetadata method getMetadata.

/**
 * Get the {@link PcjMetadata} of a query that is being maintained by the
 * Fluo app.
 *
 * @param pcjStorage - The PCJ Storage that will be searched. (not null)
 * @param fluo - The Fluo instance that will be searched. (not null)
 * @param queryId - The Query Id whose metadata will be fetched. (not null)
 * @return The {@link PcjMetadata} of the query.
 * @throws NotInFluoException The query Id does not have a PCJ export able
 *   associated with it in the Fluo table.
 * @throws NotInAccumuloException The PCJ export table that was found either
 *   does not exist in Accumulo or it is not a PCJ table.
 */
public PcjMetadata getMetadata(final PrecomputedJoinStorage pcjStorage, final FluoClient fluo, final String queryId) throws NotInFluoException, NotInAccumuloException {
    requireNonNull(pcjStorage);
    requireNonNull(fluo);
    requireNonNull(queryId);
    // Lookup the Rya PCJ ID associated with the query.
    String pcjId = null;
    try (Snapshot snap = fluo.newSnapshot()) {
        pcjId = FluoQueryUtils.convertFluoQueryIdToPcjId(queryId);
        if (pcjId == null) {
            throw new NotInFluoException("Could not get the PcjMetadata for queryId '" + queryId + "' because a Rya PCJ ID not stored in the Fluo table.");
        }
    }
    // Fetch the metadata from the storage.
    try {
        return pcjStorage.getPcjMetadata(pcjId);
    } catch (final PcjException e) {
        throw new NotInAccumuloException("Could not get the PcjMetadata for queryId '" + queryId + "' because the metadata was missing from the Rya storage.", e);
    }
}

Example 4 with PcjException

use of org.apache.rya.indexing.pcj.storage.PcjException in project incubator-rya by apache.

the class FluoAndHistoricPcjsDemo method execute.

@Override
public void execute(final MiniAccumuloCluster accumulo, final Connector accumuloConn, final String ryaTablePrefix, final RyaSailRepository ryaRepo, final RepositoryConnection ryaConn, final MiniFluo fluo, final FluoClient fluoClient) throws DemoExecutionException {
    log.setLevel(Level.INFO);
    // 1. Introduce some RDF Statements that we are going to start with and
    // pause so the presenter can introduce this information to the audience.
    final Set<RyaStatement> relevantHistoricStatements = Sets.newHashSet(new RyaStatement(eve, talksTo, charlie), new RyaStatement(david, talksTo, alice), new RyaStatement(alice, worksAt, coffeeShop), new RyaStatement(bob, worksAt, coffeeShop));
    log.info("We add some Statements that are relevant to the query we will compute:");
    prettyLogStatements(relevantHistoricStatements);
    waitForEnter();
    log.info("We also some more Satements that aren't realted to the query we will compute");
    final Set<RyaStatement> otherHistoricStatements = Sets.newHashSet(new RyaStatement(henry, worksAt, burgerShop), new RyaStatement(irene, worksAt, burgerShop), new RyaStatement(justin, worksAt, burgerShop), new RyaStatement(kristi, worksAt, burgerShop), new RyaStatement(luke, worksAt, burgerShop), new RyaStatement(manny, worksAt, cupcakeShop), new RyaStatement(nate, worksAt, cupcakeShop), new RyaStatement(olivia, worksAt, cupcakeShop), new RyaStatement(paul, worksAt, cupcakeShop), new RyaStatement(ross, worksAt, cupcakeShop), new RyaStatement(henry, talksTo, irene), new RyaStatement(henry, talksTo, justin), new RyaStatement(kristi, talksTo, irene), new RyaStatement(luke, talksTo, irene), new RyaStatement(sally, talksTo, paul), new RyaStatement(sally, talksTo, ross), new RyaStatement(sally, talksTo, kristi), new RyaStatement(tim, talksTo, nate), new RyaStatement(tim, talksTo, paul), new RyaStatement(tim, talksTo, kristi));
    log.info("Theese statements will also be inserted into the core Rya tables:");
    prettyLogStatements(otherHistoricStatements);
    waitForEnter();
    // 2. Load the statements into the core Rya tables.
    log.info("Loading the historic statements into Rya...");
    loadDataIntoRya(ryaConn, relevantHistoricStatements);
    loadDataIntoRya(ryaConn, otherHistoricStatements);
    log.info("");
    // 3. Introduce the query that we're going to load into Fluo and pause so that the
    // presenter may show what they believe the expected output should be.
    final String sparql = "SELECT ?patron ?employee " + "WHERE { " + "?patron <http://talksTo> ?employee. " + "?employee <http://worksAt> <http://CoffeeShop>. " + "}";
    log.info("The following SPARQL query will be loaded into the Fluo application for incremental updates:");
    prettyLogSparql(sparql);
    waitForEnter();
    // 4. Write the query to Fluo and import the historic matches. Wait for the app to finish exporting results.
    log.info("Telling Fluo to maintain the query and import the historic Statement Pattern matches.");
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, ryaTablePrefix);
    final String pcjId;
    try {
        // Create the PCJ Index in Rya.
        pcjId = pcjStorage.createPcj(sparql);
        // Tell the Fluo app to maintain it.
        new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, ryaTablePrefix);
    } catch (MalformedQueryException | PcjException | RyaDAOException | UnsupportedQueryException e) {
        throw new DemoExecutionException("Error while using Fluo to compute and export historic matches, so the demo can not continue. Exiting.", e);
    }
    log.info("Waiting for the fluo application to finish exporting the initial results...");
    fluo.waitForObservers();
    log.info("Historic result exporting finished.");
    log.info("");
    // 5. Show that the Fluo app exported the results to the PCJ table in Accumulo.
    log.info("The following Binding Sets were exported to the PCJ with ID '" + pcjId + "' in Rya:");
    try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
        while (resultsIt.hasNext()) {
            log.info("    " + resultsIt.next());
        }
    } catch (final Exception e) {
        throw new DemoExecutionException("Could not fetch the PCJ's reuslts from Accumulo. Exiting.", e);
    }
    waitForEnter();
    // 6. Introduce some new Statements that we will stream into the Fluo app.
    final RyaStatement newLeft = new RyaStatement(george, talksTo, frank);
    final RyaStatement newRight = new RyaStatement(frank, worksAt, coffeeShop);
    final RyaStatement joinLeft = new RyaStatement(eve, talksTo, bob);
    final RyaStatement joinRight = new RyaStatement(charlie, worksAt, coffeeShop);
    final Set<RyaStatement> relevantstreamedStatements = Sets.newHashSet(newLeft, newRight, joinLeft, joinRight);
    log.info("We stream these relevant Statements into Fluo and the core Rya tables:");
    log.info(prettyFormat(newLeft) + "          - Part of a new result");
    log.info(prettyFormat(newRight) + "      - Other part of a new result");
    log.info(prettyFormat(joinLeft) + "               - Joins with a historic <http://talksTo> statement");
    log.info(prettyFormat(joinRight) + "    - Joins with a historic <http://worksA>t statement");
    waitForEnter();
    final Set<RyaStatement> otherStreamedStatements = Sets.newHashSet(new RyaStatement(alice, talksTo, tim), new RyaStatement(bob, talksTo, tim), new RyaStatement(charlie, talksTo, tim), new RyaStatement(frank, talksTo, tim), new RyaStatement(david, talksTo, tim), new RyaStatement(eve, talksTo, sally), new RyaStatement(george, talksTo, sally), new RyaStatement(henry, talksTo, sally), new RyaStatement(irene, talksTo, sally), new RyaStatement(justin, talksTo, sally), new RyaStatement(kristi, talksTo, manny), new RyaStatement(luke, talksTo, manny), new RyaStatement(manny, talksTo, paul), new RyaStatement(nate, talksTo, manny), new RyaStatement(olivia, talksTo, manny), new RyaStatement(paul, talksTo, kristi), new RyaStatement(ross, talksTo, kristi), new RyaStatement(sally, talksTo, kristi), new RyaStatement(olivia, talksTo, kristi), new RyaStatement(olivia, talksTo, kristi));
    log.info("We also stream these irrelevant Statements into Fluo and the core Rya tables:");
    prettyLogStatements(otherStreamedStatements);
    waitForEnter();
    // 7. Insert the new triples into the core Rya tables and the Fluo app.
    loadDataIntoRya(ryaConn, relevantstreamedStatements);
    loadDataIntoFluo(fluoClient, relevantstreamedStatements);
    log.info("Waiting for the fluo application to finish exporting the newly streamed results...");
    fluo.waitForObservers();
    log.info("Streamed result exporting finished.");
    log.info("");
    // 8. Show the new results have been exported to the PCJ table in Accumulo.
    log.info("The following Binding Sets were exported to the PCJ with ID '" + pcjId + "' in Rya:");
    try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
        while (resultsIt.hasNext()) {
            log.info("    " + resultsIt.next());
        }
    } catch (final Exception e) {
        throw new DemoExecutionException("Could not fetch the PCJ's reuslts from Accumulo. Exiting.", e);
    }
    log.info("");
}

Example 5 with PcjException

use of org.apache.rya.indexing.pcj.storage.PcjException in project incubator-rya by apache.

the class NewQueryCommand method execute.

@Override
public void execute(final Connector accumulo, final String ryaTablePrefix, final RyaSailRepository rya, final FluoClient fluo, final String[] args) throws ArgumentsException, ExecutionException, UnsupportedQueryException {
    checkNotNull(accumulo);
    checkNotNull(fluo);
    checkNotNull(args);
    log.trace("Executing the New Query Command...");
    // Parse the command line arguments.
    final Parameters params = new Parameters();
    try {
        new JCommander(params, args);
    } catch (final ParameterException e) {
        throw new ArgumentsException("Could not create a new query because of invalid command line parameters.", e);
    }
    // Load the request from the file into memory.
    log.trace("Loading the query found in file '" + params.queryRequestFile + "' into the client app.");
    ParsedQueryRequest request = null;
    try {
        final Path requestFile = Paths.get(params.queryRequestFile);
        final String requestText = IOUtils.toString(Files.newInputStream(requestFile));
        request = ParsedQueryRequest.parse(requestText);
    } catch (final IOException e) {
        throw new ExecutionException("Could not load the query request into memory.", e);
    }
    // Load the query into the Fluo app.
    log.trace("SPARQL Query: " + request.getQuery());
    log.trace("Var Orders: " + request.getVarOrders());
    log.trace("Loading these values into the Fluo app.");
    final CreateFluoPcj createPcj = new CreateFluoPcj();
    try {
        // Create the PCJ in Rya.
        final String sparql = request.getQuery();
        final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumulo, ryaTablePrefix);
        final String pcjId = pcjStorage.createPcj(sparql);
        // Tell the Fluo PCJ Updater app to maintain the PCJ.
        createPcj.withRyaIntegration(pcjId, pcjStorage, fluo, accumulo, ryaTablePrefix);
    } catch (MalformedQueryException | PcjException | RyaDAOException e) {
        throw new ExecutionException("Could not create and load historic matches into the the Fluo app for the query.", e);
    }
    log.trace("Finished executing the New Query Command.");
}