Examples with ClientConfiguration org.apache.accumulo.core.client.ClientConfiguration used on opensource projects

Example 41 with ClientConfiguration

use of org.apache.accumulo.core.client.ClientConfiguration in project incubator-rya by apache.

the class AbstractAccumuloMRTool method setupAccumuloInput.

/**
 * Sets up Accumulo input for a job: the job receives
 * ({@link org.apache.accumulo.core.data.Key},
 * {@link org.apache.accumulo.core.data.Value}) pairs from the table
 * specified by the configuration (using
 * {@link MRUtils#TABLE_PREFIX_PROPERTY} and
 * {@link MRUtils#TABLE_LAYOUT_PROP}).
 * @param   job     MapReduce Job to configure
 * @throws  AccumuloSecurityException if connecting to Accumulo with the
 *          given username and password fails.
 */
protected void setupAccumuloInput(Job job) throws AccumuloSecurityException {
    // set up accumulo input
    if (!hdfsInput) {
        job.setInputFormatClass(AccumuloInputFormat.class);
    } else {
        job.setInputFormatClass(AccumuloHDFSFileInputFormat.class);
    }
    AccumuloInputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd));
    String tableName = RdfCloudTripleStoreUtils.layoutPrefixToTable(rdfTableLayout, tablePrefix);
    AccumuloInputFormat.setInputTableName(job, tableName);
    AccumuloInputFormat.setScanAuthorizations(job, authorizations);
    if (mock) {
        AccumuloInputFormat.setMockInstance(job, instance);
    } else {
        ClientConfiguration clientConfig = ClientConfiguration.loadDefault().withInstance(instance).withZkHosts(zk);
        AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
    }
    if (ttl != null) {
        IteratorSetting setting = new IteratorSetting(1, "fi", AgeOffFilter.class.getName());
        AgeOffFilter.setTTL(setting, Long.valueOf(ttl));
        AccumuloInputFormat.addIterator(job, setting);
    }
}

Example 42 with ClientConfiguration

use of org.apache.accumulo.core.client.ClientConfiguration in project incubator-rya by apache.

the class AbstractAccumuloMRTool method setupRyaOutput.

/**
 * Sets up Rya output for a job: allows the job to write
 * {@link RyaStatementWritable} data, which will in turn be input into the
 * configured Rya instance. To perform secondary indexing, use the
 * configuration variables in {@link ConfigUtils}.
 * @param   job Job to configure
 * @throws  AccumuloSecurityException if connecting to Accumulo with the
 *          given username and password fails
 */
protected void setupRyaOutput(Job job) throws AccumuloSecurityException {
    job.setOutputFormatClass(RyaOutputFormat.class);
    job.setOutputValueClass(RyaStatementWritable.class);
    // Specify default visibility of output rows, if given
    RyaOutputFormat.setDefaultVisibility(job, conf.get(MRUtils.AC_CV_PROP));
    // Specify named graph, if given
    RyaOutputFormat.setDefaultContext(job, conf.get(MRUtils.NAMED_GRAPH_PROP));
    // Set the output prefix
    RyaOutputFormat.setTablePrefix(job, tablePrefix);
    // Determine which indexers to use based on the config
    RyaOutputFormat.setFreeTextEnabled(job, ConfigUtils.getUseFreeText(conf));
    RyaOutputFormat.setTemporalEnabled(job, ConfigUtils.getUseTemporal(conf));
    RyaOutputFormat.setEntityEnabled(job, ConfigUtils.getUseEntity(conf));
    // Configure the Accumulo connection
    AccumuloOutputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd));
    AccumuloOutputFormat.setCreateTables(job, true);
    AccumuloOutputFormat.setDefaultTableName(job, tablePrefix + RdfCloudTripleStoreConstants.TBL_SPO_SUFFIX);
    if (mock) {
        RyaOutputFormat.setMockInstance(job, instance);
    } else {
        ClientConfiguration clientConfig = ClientConfiguration.loadDefault().withInstance(instance).withZkHosts(zk);
        AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig);
    }
}

Example 43 with ClientConfiguration

use of org.apache.accumulo.core.client.ClientConfiguration in project incubator-rya by apache.

the class AccumuloRyaUtils method getScanner.

/**
 * Creates a {@link Scanner} of the provided table name using the specified {@link Configuration}.
 * @param tablename the name of the table to scan.
 * @param config the {@link Configuration}.
 * @param shouldAddCommonIterators {@code true} to add the common iterators to the table scanner.
 * {@code false} otherwise.
 * @return the {@link Scanner} for the table.
 * @throws IOException
 */
public static Scanner getScanner(final String tableName, final Configuration config, final boolean shouldAddCommonIterators) throws IOException {
    try {
        final String instanceName = config.get(ConfigUtils.CLOUDBASE_INSTANCE);
        final String zooKeepers = config.get(ConfigUtils.CLOUDBASE_ZOOKEEPERS);
        Instance instance;
        if (ConfigUtils.useMockInstance(config)) {
            instance = new MockInstance(instanceName);
        } else {
            instance = new ZooKeeperInstance(new ClientConfiguration().withInstance(instanceName).withZkHosts(zooKeepers));
        }
        final String username = ConfigUtils.getUsername(config);
        final String password = ConfigUtils.getPassword(config);
        final Connector connector = instance.getConnector(username, new PasswordToken(password));
        final Authorizations auths = ConfigUtils.getAuthorizations(config);
        final Scanner scanner = connector.createScanner(tableName, auths);
        if (shouldAddCommonIterators) {
            AccumuloRyaUtils.addCommonScannerIteratorsTo(scanner);
        }
        return scanner;
    } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
        log.error("Error connecting to " + tableName);
        throw new IOException(e);
    }
}

Example 44 with ClientConfiguration

use of org.apache.accumulo.core.client.ClientConfiguration in project incubator-rya by apache.

the class MergeTool method setupAccumuloInput.

@Override
protected void setupAccumuloInput(final Job job) throws AccumuloSecurityException {
    // set up accumulo input
    if (!hdfsInput) {
        job.setInputFormatClass(AccumuloInputFormat.class);
    } else {
        job.setInputFormatClass(AccumuloHDFSFileInputFormat.class);
    }
    AbstractInputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd));
    InputFormatBase.setInputTableName(job, RdfCloudTripleStoreUtils.layoutPrefixToTable(rdfTableLayout, tablePrefix));
    AbstractInputFormat.setScanAuthorizations(job, authorizations);
    if (!mock) {
        AbstractInputFormat.setZooKeeperInstance(job, new ClientConfiguration().withInstance(instance).withZkHosts(zk));
    } else {
        AbstractInputFormat.setMockInstance(job, instance);
    }
    if (ttl != null) {
        final IteratorSetting setting = new IteratorSetting(1, "fi", AgeOffFilter.class);
        AgeOffFilter.setTTL(setting, Long.valueOf(ttl));
        InputFormatBase.addIterator(job, setting);
    }
    for (final IteratorSetting iteratorSetting : AccumuloRyaUtils.COMMON_REG_EX_FILTER_SETTINGS) {
        InputFormatBase.addIterator(job, iteratorSetting);
    }
}

Example 45 with ClientConfiguration

use of org.apache.accumulo.core.client.ClientConfiguration in project incubator-rya by apache.

the class AccumuloRyaUtils method getScanner.

/**
 * Creates a {@link Scanner} of the provided table name using the specified {@link Configuration}.
 * @param tablename the name of the table to scan.
 * @param config the {@link Configuration}.
 * @param shouldAddCommonIterators {@code true} to add the common iterators to the table scanner.
 * {@code false} otherwise.
 * @return the {@link Scanner} for the table.
 * @throws IOException
 */
public static Scanner getScanner(final String tableName, final Configuration config, final boolean shouldAddCommonIterators) throws IOException {
    try {
        final String instanceName = config.get(ConfigUtils.CLOUDBASE_INSTANCE);
        final String zooKeepers = config.get(ConfigUtils.CLOUDBASE_ZOOKEEPERS);
        Instance instance;
        if (ConfigUtils.useMockInstance(config)) {
            instance = new MockInstance(config.get(ConfigUtils.CLOUDBASE_INSTANCE));
        } else {
            instance = new ZooKeeperInstance(new ClientConfiguration().withInstance(instanceName).withZkHosts(zooKeepers));
        }
        final String username = ConfigUtils.getUsername(config);
        final String password = ConfigUtils.getPassword(config);
        final Connector connector = instance.getConnector(username, new PasswordToken(password));
        final Authorizations auths = ConfigUtils.getAuthorizations(config);
        final Scanner scanner = connector.createScanner(tableName, auths);
        if (shouldAddCommonIterators) {
            AccumuloRyaUtils.addCommonScannerIteratorsTo(scanner);
        }
        return scanner;
    } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
        log.error("Error connecting to " + tableName);
        throw new IOException(e);
    }
}