Examples with FluoDetails org.apache.rya.api.instance.RyaDetails.PCJIndexDetails.FluoDetails used on opensource projects

Example 1 with FluoDetails

use of org.apache.rya.api.instance.RyaDetails.PCJIndexDetails.FluoDetails in project incubator-rya by apache.

the class AccumuloInstall method makeRyaConfig.

/**
 * Builds a {@link AccumuloRdfConfiguration} object that will be used by the
 * Rya DAO to initialize all of the tables it will need.
 *
 * @param connectionDetails - Indicates how to connect to Accumulo. (not null)
 * @param details - Indicates what needs to be installed. (not null)
 * @return A Rya Configuration object that can be used to perform the install.
 */
private static AccumuloRdfConfiguration makeRyaConfig(final AccumuloConnectionDetails connectionDetails, final RyaDetails details) {
    final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration();
    // The Rya Instance Name is used as a prefix for the index tables in Accumulo.
    conf.setTablePrefix(details.getRyaInstanceName());
    // Enable the indexers that the instance is configured to use.
    /**
     * RYA-215
     * conf.set(ConfigUtils.USE_GEO, "" + details.getGeoIndexDetails().isEnabled() );
     */
    /**
     * RYA-322
     * conf.setPrefixRowsWithHash(details.getPrefixRowsWithHashDetails().isEnabled());
     */
    conf.set(ConfigUtils.USE_FREETEXT, "" + details.getFreeTextIndexDetails().isEnabled());
    conf.set(ConfigUtils.USE_TEMPORAL, "" + details.getTemporalIndexDetails().isEnabled());
    conf.set(ConfigUtils.USE_ENTITY, "" + details.getEntityCentricIndexDetails().isEnabled());
    conf.set(ConfigUtils.USE_PCJ, "" + details.getPCJIndexDetails().isEnabled());
    conf.set(ConfigUtils.PCJ_STORAGE_TYPE, PrecomputedJoinStorageType.ACCUMULO.toString());
    final Optional<FluoDetails> fluoHolder = details.getPCJIndexDetails().getFluoDetails();
    final PrecomputedJoinUpdaterType updaterType = fluoHolder.isPresent() ? PrecomputedJoinUpdaterType.FLUO : PrecomputedJoinUpdaterType.NO_UPDATE;
    conf.set(ConfigUtils.PCJ_UPDATER_TYPE, updaterType.toString());
    // XXX The Accumulo implementation of the secondary indices make need all
    // of the accumulo connector's parameters to initialize themselves, so
    // we need to include them here. It would be nice if the secondary
    // indexers used the connector that is provided to them instead of
    // building a new one.
    conf.set(ConfigUtils.CLOUDBASE_USER, connectionDetails.getUsername());
    conf.set(ConfigUtils.CLOUDBASE_PASSWORD, new String(connectionDetails.getUserPass()));
    conf.set(ConfigUtils.CLOUDBASE_INSTANCE, connectionDetails.getInstanceName());
    conf.set(ConfigUtils.CLOUDBASE_ZOOKEEPERS, connectionDetails.getZookeepers());
    // This initializes the living indexers that will be used by the application and
    // caches them within the configuration object so that they may be used later.
    ConfigUtils.setIndexers(conf);
    return conf;
}

Example 2 with FluoDetails

use of org.apache.rya.api.instance.RyaDetails.PCJIndexDetails.FluoDetails in project incubator-rya by apache.

the class AccumuloCreatePCJ method createPCJ.

@Override
public String createPCJ(final String instanceName, final String sparql, Set<ExportStrategy> strategies) throws InstanceDoesNotExistException, RyaClientException {
    requireNonNull(instanceName);
    requireNonNull(sparql);
    final Optional<RyaDetails> ryaDetailsHolder = getInstanceDetails.getDetails(instanceName);
    final boolean ryaInstanceExists = ryaDetailsHolder.isPresent();
    if (!ryaInstanceExists) {
        throw new InstanceDoesNotExistException(String.format("The '%s' instance of Rya does not exist.", instanceName));
    }
    final PCJIndexDetails pcjIndexDetails = ryaDetailsHolder.get().getPCJIndexDetails();
    final boolean pcjIndexingEnabeld = pcjIndexDetails.isEnabled();
    if (!pcjIndexingEnabeld) {
        throw new RyaClientException(String.format("The '%s' instance of Rya does not have PCJ Indexing enabled.", instanceName));
    }
    // Create the PCJ table that will receive the index results.
    final String pcjId;
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(getConnector(), instanceName)) {
        pcjId = pcjStorage.createPcj(sparql);
        // If a Fluo application is being used, task it with updating the PCJ.
        final Optional<FluoDetails> fluoDetailsHolder = pcjIndexDetails.getFluoDetails();
        if (fluoDetailsHolder.isPresent()) {
            final String fluoAppName = fluoDetailsHolder.get().getUpdateAppName();
            try {
                updateFluoApp(instanceName, fluoAppName, pcjId, sparql, strategies);
            } catch (RepositoryException | MalformedQueryException | SailException | QueryEvaluationException | PcjException | RyaDAOException e) {
                throw new RyaClientException("Problem while initializing the Fluo application with the new PCJ.", e);
            } catch (UnsupportedQueryException e) {
                throw new RyaClientException("The new PCJ could not be initialized because it either contains an unsupported query node " + "or an invalid ExportStrategy for the given QueryType.  Projection queries can be exported to either Rya or Kafka," + "unless they contain an aggregation, in which case they can only be exported to Kafka.  Construct queries can be exported" + "to Rya and Kafka, and Periodic queries can only be exported to Rya.");
            }
            // Update the Rya Details to indicate the PCJ is being updated incrementally.
            final RyaDetailsRepository detailsRepo = new AccumuloRyaInstanceDetailsRepository(getConnector(), instanceName);
            try {
                new RyaDetailsUpdater(detailsRepo).update(new RyaDetailsMutator() {

                    @Override
                    public RyaDetails mutate(final RyaDetails originalDetails) throws CouldNotApplyMutationException {
                        // Update the original PCJ Details to indicate they are incrementally updated.
                        final PCJDetails originalPCJDetails = originalDetails.getPCJIndexDetails().getPCJDetails().get(pcjId);
                        final PCJDetails.Builder mutatedPCJDetails = PCJDetails.builder(originalPCJDetails).setUpdateStrategy(PCJUpdateStrategy.INCREMENTAL);
                        // Replace the old PCJ Details with the updated ones.
                        final RyaDetails.Builder builder = RyaDetails.builder(originalDetails);
                        builder.getPCJIndexDetails().addPCJDetails(mutatedPCJDetails);
                        return builder.build();
                    }
                });
            } catch (RyaDetailsRepositoryException | CouldNotApplyMutationException e) {
                throw new RyaClientException("Problem while updating the Rya instance's Details to indicate the PCJ is being incrementally updated.", e);
            }
        }
        // Return the ID that was assigned to the PCJ.
        return pcjId;
    } catch (final PCJStorageException e) {
        throw new RyaClientException("Problem while initializing the PCJ table.", e);
    }
}

Example 3 with FluoDetails

use of org.apache.rya.api.instance.RyaDetails.PCJIndexDetails.FluoDetails in project incubator-rya by apache.

the class AccumuloDeletePCJ method deletePCJ.

@Override
public void deletePCJ(final String instanceName, final String pcjId) throws InstanceDoesNotExistException, RyaClientException {
    requireNonNull(instanceName);
    requireNonNull(pcjId);
    final Optional<RyaDetails> originalDetails = getInstanceDetails.getDetails(instanceName);
    final boolean ryaInstanceExists = originalDetails.isPresent();
    if (!ryaInstanceExists) {
        throw new InstanceDoesNotExistException(String.format("The '%s' instance of Rya does not exist.", instanceName));
    }
    final boolean pcjIndexingEnabeld = originalDetails.get().getPCJIndexDetails().isEnabled();
    if (!pcjIndexingEnabeld) {
        throw new RyaClientException(String.format("The '%s' instance of Rya does not have PCJ Indexing enabled.", instanceName));
    }
    final boolean pcjExists = originalDetails.get().getPCJIndexDetails().getPCJDetails().containsKey(pcjId);
    if (!pcjExists) {
        throw new RyaClientException(String.format("The '%s' instance of Rya does not have PCJ with ID '%s'.", instanceName, pcjId));
    }
    // If the PCJ was being maintained by a Fluo application, then stop that process.
    final PCJIndexDetails pcjIndexDetails = originalDetails.get().getPCJIndexDetails();
    final PCJDetails droppedPcjDetails = pcjIndexDetails.getPCJDetails().get(pcjId);
    if (droppedPcjDetails.getUpdateStrategy().isPresent()) {
        if (droppedPcjDetails.getUpdateStrategy().get() == PCJUpdateStrategy.INCREMENTAL) {
            final Optional<FluoDetails> fluoDetailsHolder = pcjIndexDetails.getFluoDetails();
            if (fluoDetailsHolder.isPresent()) {
                final String fluoAppName = pcjIndexDetails.getFluoDetails().get().getUpdateAppName();
                stopUpdatingPCJ(fluoAppName, pcjId);
            } else {
                log.error(String.format("Could not stop the Fluo application from updating the PCJ because the Fluo Details are " + "missing for the Rya instance named '%s'.", instanceName));
            }
        }
    }
    // Drop the table that holds the PCJ results from Accumulo.
    try (final PrecomputedJoinStorage pcjs = new AccumuloPcjStorage(getConnector(), instanceName)) {
        pcjs.dropPcj(pcjId);
    } catch (final PCJStorageException e) {
        throw new RyaClientException("Could not drop the PCJ's table from Accumulo.", e);
    }
}

Example 4 with FluoDetails

use of org.apache.rya.api.instance.RyaDetails.PCJIndexDetails.FluoDetails in project incubator-rya by apache.

the class MongoRyaDetailsRepositoryIT method initialize_alreadyInitialized.

@Test(expected = AlreadyInitializedException.class)
public void initialize_alreadyInitialized() throws AlreadyInitializedException, RyaDetailsRepositoryException {
    final String instanceName = "testInstance";
    // Create the metadata object the repository will be initialized with.
    final RyaDetails details = RyaDetails.builder().setRyaInstanceName(instanceName).setRyaVersion("1.2.3.4").setEntityCentricIndexDetails(new EntityCentricIndexDetails(true)).setTemporalIndexDetails(new TemporalIndexDetails(true)).setFreeTextDetails(new FreeTextIndexDetails(true)).setPCJIndexDetails(PCJIndexDetails.builder().setEnabled(true).setFluoDetails(new FluoDetails("test_instance_rya_pcj_updater")).addPCJDetails(PCJDetails.builder().setId("pcj 1").setUpdateStrategy(PCJUpdateStrategy.BATCH).setLastUpdateTime(new Date())).addPCJDetails(PCJDetails.builder().setId("pcj 2"))).setProspectorDetails(new ProspectorDetails(Optional.of(new Date()))).setJoinSelectivityDetails(new JoinSelectivityDetails(Optional.of(new Date()))).build();
    // Setup the repository that will be tested using a mock instance of MongoDB.
    final RyaDetailsRepository repo = new MongoRyaInstanceDetailsRepository(client, instanceName);
    // Initialize the repository
    repo.initialize(details);
    // Initialize it again.
    repo.initialize(details);
}

Example 5 with FluoDetails

use of org.apache.rya.api.instance.RyaDetails.PCJIndexDetails.FluoDetails in project incubator-rya by apache.

the class MongoRyaDetailsRepositoryIT method update_outOfDate.

@Test(expected = ConcurrentUpdateException.class)
public void update_outOfDate() throws AlreadyInitializedException, RyaDetailsRepositoryException {
    final String instanceName = "testInstance";
    // Create the metadata object the repository will be initialized with.
    final RyaDetails details = RyaDetails.builder().setRyaInstanceName(instanceName).setRyaVersion("1.2.3.4").setEntityCentricIndexDetails(new EntityCentricIndexDetails(true)).setTemporalIndexDetails(new TemporalIndexDetails(true)).setFreeTextDetails(new FreeTextIndexDetails(true)).setPCJIndexDetails(PCJIndexDetails.builder().setEnabled(true).setFluoDetails(new FluoDetails("test_instance_rya_pcj_updater")).addPCJDetails(PCJDetails.builder().setId("pcj 1").setUpdateStrategy(PCJUpdateStrategy.BATCH).setLastUpdateTime(new Date())).addPCJDetails(PCJDetails.builder().setId("pcj 2"))).setProspectorDetails(new ProspectorDetails(Optional.of(new Date()))).setJoinSelectivityDetails(new JoinSelectivityDetails(Optional.of(new Date()))).build();
    // Setup the repository that will be tested using a mock instance of MongoDB.
    final RyaDetailsRepository repo = new MongoRyaInstanceDetailsRepository(client, "testInstance");
    // Initialize the repository
    repo.initialize(details);
    // Create a new state for the details.
    final RyaDetails wrongOriginal = new RyaDetails.Builder(details).setTemporalIndexDetails(new TemporalIndexDetails(false)).build();
    final RyaDetails updated = new RyaDetails.Builder(details).setEntityCentricIndexDetails(new EntityCentricIndexDetails(false)).build();
    // Try to execute the update where the old state is not the currently stored state.
    repo.update(wrongOriginal, updated);
}