Examples with PCJStorageException org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException used on opensource projects

Example 1 with PCJStorageException

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException in project incubator-rya by apache.

the class AccumuloCreatePCJ method createPCJ.

@Override
public String createPCJ(final String instanceName, final String sparql, Set<ExportStrategy> strategies) throws InstanceDoesNotExistException, RyaClientException {
    requireNonNull(instanceName);
    requireNonNull(sparql);
    final Optional<RyaDetails> ryaDetailsHolder = getInstanceDetails.getDetails(instanceName);
    final boolean ryaInstanceExists = ryaDetailsHolder.isPresent();
    if (!ryaInstanceExists) {
        throw new InstanceDoesNotExistException(String.format("The '%s' instance of Rya does not exist.", instanceName));
    }
    final PCJIndexDetails pcjIndexDetails = ryaDetailsHolder.get().getPCJIndexDetails();
    final boolean pcjIndexingEnabeld = pcjIndexDetails.isEnabled();
    if (!pcjIndexingEnabeld) {
        throw new RyaClientException(String.format("The '%s' instance of Rya does not have PCJ Indexing enabled.", instanceName));
    }
    // Create the PCJ table that will receive the index results.
    final String pcjId;
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(getConnector(), instanceName)) {
        pcjId = pcjStorage.createPcj(sparql);
        // If a Fluo application is being used, task it with updating the PCJ.
        final Optional<FluoDetails> fluoDetailsHolder = pcjIndexDetails.getFluoDetails();
        if (fluoDetailsHolder.isPresent()) {
            final String fluoAppName = fluoDetailsHolder.get().getUpdateAppName();
            try {
                updateFluoApp(instanceName, fluoAppName, pcjId, sparql, strategies);
            } catch (RepositoryException | MalformedQueryException | SailException | QueryEvaluationException | PcjException | RyaDAOException e) {
                throw new RyaClientException("Problem while initializing the Fluo application with the new PCJ.", e);
            } catch (UnsupportedQueryException e) {
                throw new RyaClientException("The new PCJ could not be initialized because it either contains an unsupported query node " + "or an invalid ExportStrategy for the given QueryType.  Projection queries can be exported to either Rya or Kafka," + "unless they contain an aggregation, in which case they can only be exported to Kafka.  Construct queries can be exported" + "to Rya and Kafka, and Periodic queries can only be exported to Rya.");
            }
            // Update the Rya Details to indicate the PCJ is being updated incrementally.
            final RyaDetailsRepository detailsRepo = new AccumuloRyaInstanceDetailsRepository(getConnector(), instanceName);
            try {
                new RyaDetailsUpdater(detailsRepo).update(new RyaDetailsMutator() {

                    @Override
                    public RyaDetails mutate(final RyaDetails originalDetails) throws CouldNotApplyMutationException {
                        // Update the original PCJ Details to indicate they are incrementally updated.
                        final PCJDetails originalPCJDetails = originalDetails.getPCJIndexDetails().getPCJDetails().get(pcjId);
                        final PCJDetails.Builder mutatedPCJDetails = PCJDetails.builder(originalPCJDetails).setUpdateStrategy(PCJUpdateStrategy.INCREMENTAL);
                        // Replace the old PCJ Details with the updated ones.
                        final RyaDetails.Builder builder = RyaDetails.builder(originalDetails);
                        builder.getPCJIndexDetails().addPCJDetails(mutatedPCJDetails);
                        return builder.build();
                    }
                });
            } catch (RyaDetailsRepositoryException | CouldNotApplyMutationException e) {
                throw new RyaClientException("Problem while updating the Rya instance's Details to indicate the PCJ is being incrementally updated.", e);
            }
        }
        // Return the ID that was assigned to the PCJ.
        return pcjId;
    } catch (final PCJStorageException e) {
        throw new RyaClientException("Problem while initializing the PCJ table.", e);
    }
}

Example 2 with PCJStorageException

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException in project incubator-rya by apache.

the class AccumuloDeletePCJ method deletePCJ.

@Override
public void deletePCJ(final String instanceName, final String pcjId) throws InstanceDoesNotExistException, RyaClientException {
    requireNonNull(instanceName);
    requireNonNull(pcjId);
    final Optional<RyaDetails> originalDetails = getInstanceDetails.getDetails(instanceName);
    final boolean ryaInstanceExists = originalDetails.isPresent();
    if (!ryaInstanceExists) {
        throw new InstanceDoesNotExistException(String.format("The '%s' instance of Rya does not exist.", instanceName));
    }
    final boolean pcjIndexingEnabeld = originalDetails.get().getPCJIndexDetails().isEnabled();
    if (!pcjIndexingEnabeld) {
        throw new RyaClientException(String.format("The '%s' instance of Rya does not have PCJ Indexing enabled.", instanceName));
    }
    final boolean pcjExists = originalDetails.get().getPCJIndexDetails().getPCJDetails().containsKey(pcjId);
    if (!pcjExists) {
        throw new RyaClientException(String.format("The '%s' instance of Rya does not have PCJ with ID '%s'.", instanceName, pcjId));
    }
    // If the PCJ was being maintained by a Fluo application, then stop that process.
    final PCJIndexDetails pcjIndexDetails = originalDetails.get().getPCJIndexDetails();
    final PCJDetails droppedPcjDetails = pcjIndexDetails.getPCJDetails().get(pcjId);
    if (droppedPcjDetails.getUpdateStrategy().isPresent()) {
        if (droppedPcjDetails.getUpdateStrategy().get() == PCJUpdateStrategy.INCREMENTAL) {
            final Optional<FluoDetails> fluoDetailsHolder = pcjIndexDetails.getFluoDetails();
            if (fluoDetailsHolder.isPresent()) {
                final String fluoAppName = pcjIndexDetails.getFluoDetails().get().getUpdateAppName();
                stopUpdatingPCJ(fluoAppName, pcjId);
            } else {
                log.error(String.format("Could not stop the Fluo application from updating the PCJ because the Fluo Details are " + "missing for the Rya instance named '%s'.", instanceName));
            }
        }
    }
    // Drop the table that holds the PCJ results from Accumulo.
    try (final PrecomputedJoinStorage pcjs = new AccumuloPcjStorage(getConnector(), instanceName)) {
        pcjs.dropPcj(pcjId);
    } catch (final PCJStorageException e) {
        throw new RyaClientException("Could not drop the PCJ's table from Accumulo.", e);
    }
}

Example 3 with PCJStorageException

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException in project incubator-rya by apache.

the class AccumuloIndexSetProvider method getIndices.

@Override
protected List<ExternalTupleSet> getIndices() throws PcjIndexSetException {
    requireNonNull(conf);
    try {
        final String tablePrefix = requireNonNull(conf.get(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX));
        final Connector conn = requireNonNull(ConfigUtils.getConnector(conf));
        List<String> tables = null;
        if (conf instanceof RdfCloudTripleStoreConfiguration) {
            tables = ((RdfCloudTripleStoreConfiguration) conf).getPcjTables();
        }
        // this maps associates pcj table name with pcj sparql query
        final Map<String, String> indexTables = Maps.newLinkedHashMap();
        try (final PrecomputedJoinStorage storage = new AccumuloPcjStorage(conn, tablePrefix)) {
            final PcjTableNameFactory pcjFactory = new PcjTableNameFactory();
            final boolean tablesProvided = tables != null && !tables.isEmpty();
            if (tablesProvided) {
                // if tables provided, associate table name with sparql
                for (final String table : tables) {
                    indexTables.put(table, storage.getPcjMetadata(pcjFactory.getPcjId(table)).getSparql());
                }
            } else if (hasRyaDetails(tablePrefix, conn)) {
                // If this is a newer install of Rya, and it has PCJ Details, then
                // use those.
                final List<String> ids = storage.listPcjs();
                for (final String id : ids) {
                    indexTables.put(pcjFactory.makeTableName(tablePrefix, id), storage.getPcjMetadata(id).getSparql());
                }
            } else {
                // Otherwise figure it out by scanning tables.
                final PcjTables pcjTables = new PcjTables();
                for (final String table : conn.tableOperations().list()) {
                    if (table.startsWith(tablePrefix + "INDEX")) {
                        indexTables.put(table, pcjTables.getPcjMetadata(conn, table).getSparql());
                    }
                }
            }
        }
        // use table name sparql map (indexTables) to create {@link
        // AccumuloIndexSet}
        final List<ExternalTupleSet> index = Lists.newArrayList();
        if (indexTables.isEmpty()) {
            log.info("No Index found");
        } else {
            for (final String table : indexTables.keySet()) {
                final String indexSparqlString = indexTables.get(table);
                index.add(new AccumuloIndexSet(indexSparqlString, conf, table));
            }
        }
        return index;
    } catch (final PCJStorageException | AccumuloException | AccumuloSecurityException | MalformedQueryException | SailException | QueryEvaluationException | TableNotFoundException e) {
        throw new PcjIndexSetException("Failed to retrieve the indicies.", e);
    }
}

Example 4 with PCJStorageException

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException in project incubator-rya by apache.

the class AccumuloAddUser method addUser.

@Override
public void addUser(final String instanceName, final String username) throws InstanceDoesNotExistException, RyaClientException {
    requireNonNull(instanceName);
    requireNonNull(username);
    // If the user has already been added, then return immediately.
    try {
        final RyaDetails details = new AccumuloRyaInstanceDetailsRepository(getConnector(), instanceName).getRyaInstanceDetails();
        final List<String> users = details.getUsers();
        if (users.contains(username)) {
            return;
        }
    } catch (final RyaDetailsRepositoryException e) {
        throw new RyaClientException("Could not fetch the RyaDetails for Rya instance named '" + instanceName + ".", e);
    }
    // Update the instance details
    final RyaDetailsUpdater updater = new RyaDetailsUpdater(new AccumuloRyaInstanceDetailsRepository(getConnector(), instanceName));
    try {
        updater.update(originalDetails -> RyaDetails.builder(originalDetails).addUser(username).build());
    } catch (RyaDetailsRepositoryException | CouldNotApplyMutationException e) {
        throw new RyaClientException("Could not add the user '" + username + "' to the Rya instance's details.", e);
    }
    // Grant all access to all the instance's tables.
    try {
        // Build the list of tables that are present within the Rya instance.
        final List<String> tables = new RyaTableNames().getTableNames(instanceName, getConnector());
        // Update the user permissions for those tables.
        for (final String table : tables) {
            try {
                TABLE_PERMISSIONS.grantAllPermissions(username, table, getConnector());
            } catch (AccumuloException | AccumuloSecurityException e) {
                throw new RyaClientException("Could not grant access to table '" + table + "' for user '" + username + "'.", e);
            }
        }
    } catch (PCJStorageException | RyaDetailsRepositoryException e) {
        throw new RyaClientException("Could not determine which tables exist for the '" + instanceName + "' instance of Rya.", e);
    }
}

Example 5 with PCJStorageException

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException in project incubator-rya by apache.

the class AccumuloBatchUpdatePCJ method updatePCJResults.

private void updatePCJResults(final String ryaInstanceName, final String pcjId) throws InstanceDoesNotExistException, PCJDoesNotExistException, RyaClientException {
    // Things that have to be closed before we exit.
    Sail sail = null;
    SailConnection sailConn = null;
    CloseableIteration<? extends BindingSet, QueryEvaluationException> results = null;
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(super.getConnector(), ryaInstanceName)) {
        // Create an instance of Sail backed by the Rya instance.
        sail = connectToRya(ryaInstanceName);
        // Purge the old results from the PCJ.
        try {
            pcjStorage.purge(pcjId);
        } catch (final PCJStorageException e) {
            throw new RyaClientException("Could not batch update PCJ with ID '" + pcjId + "' because the old " + "results could not be purged from it.", e);
        }
        // Parse the PCJ's SPARQL query.
        final PcjMetadata metadata = pcjStorage.getPcjMetadata(pcjId);
        final String sparql = metadata.getSparql();
        final SPARQLParser parser = new SPARQLParser();
        final ParsedQuery parsedQuery = parser.parseQuery(sparql, null);
        // Execute the query.
        sailConn = sail.getConnection();
        results = sailConn.evaluate(parsedQuery.getTupleExpr(), null, null, false);
        // Load the results into the PCJ table.
        final List<VisibilityBindingSet> batch = new ArrayList<>(1000);
        while (results.hasNext()) {
            final VisibilityBindingSet result = new VisibilityBindingSet(results.next(), "");
            batch.add(result);
            if (batch.size() == 1000) {
                pcjStorage.addResults(pcjId, batch);
                batch.clear();
            }
        }
        if (!batch.isEmpty()) {
            pcjStorage.addResults(pcjId, batch);
            batch.clear();
        }
    } catch (final MalformedQueryException | PCJStorageException | SailException | QueryEvaluationException e) {
        throw new RyaClientException("Fail to batch load new results into the PCJ with ID '" + pcjId + "'.", e);
    } finally {
        if (results != null) {
            try {
                results.close();
            } catch (final QueryEvaluationException e) {
                log.warn(e.getMessage(), e);
            }
        }
        if (sailConn != null) {
            try {
                sailConn.close();
            } catch (final SailException e) {
                log.warn(e.getMessage(), e);
            }
        }
        if (sail != null) {
            try {
                sail.shutDown();
            } catch (final SailException e) {
                log.warn(e.getMessage(), e);
            }
        }
    }
}