Examples with InputTableConfig org.apache.accumulo.core.client.mapreduce.InputTableConfig used on opensource projects

Example 1 with InputTableConfig

use of org.apache.accumulo.core.client.mapreduce.InputTableConfig in project accumulo by apache.

the class InputConfigurator method setInputTableConfigs.

/**
 * Sets configurations for multiple tables at a time.
 *
 * @param implementingClass
 *          the class whose name will be used as a prefix for the property configuration key
 * @param conf
 *          the Hadoop configuration object to configure
 * @param configs
 *          an array of {@link InputTableConfig} objects to associate with the job
 * @since 1.6.0
 */
public static void setInputTableConfigs(Class<?> implementingClass, Configuration conf, Map<String, InputTableConfig> configs) {
    MapWritable mapWritable = new MapWritable();
    for (Map.Entry<String, InputTableConfig> tableConfig : configs.entrySet()) mapWritable.put(new Text(tableConfig.getKey()), tableConfig.getValue());
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try {
        mapWritable.write(new DataOutputStream(baos));
    } catch (IOException e) {
        throw new IllegalStateException("Table configuration could not be serialized.");
    }
    String confKey = enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS);
    conf.set(confKey, Base64.getEncoder().encodeToString(baos.toByteArray()));
}

Example 2 with InputTableConfig

use of org.apache.accumulo.core.client.mapreduce.InputTableConfig in project incubator-rya by apache.

the class CopyTool method setupMultiTableInputFormat.

/**
 * Set up job to use AccumuloMultiTableInput format, using the tables/ranges given by a ruleset.
 * @param job The Job to configure
 * @param rules The ruleset mapping a query to the appropriate tables and ranges
 */
protected void setupMultiTableInputFormat(final Job job, final AccumuloQueryRuleset rules) throws AccumuloSecurityException {
    AbstractInputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd));
    AbstractInputFormat.setScanAuthorizations(job, authorizations);
    if (!mock) {
        AbstractInputFormat.setZooKeeperInstance(job, new ClientConfiguration().withInstance(instance).withZkHosts(zk));
    } else {
        AbstractInputFormat.setMockInstance(job, instance);
    }
    final Map<String, InputTableConfig> configs = rules.getInputConfigs();
    // Add any relevant iterator settings
    final List<IteratorSetting> additionalSettings = new LinkedList<>(AccumuloRyaUtils.COMMON_REG_EX_FILTER_SETTINGS);
    if (ttl != null) {
        final IteratorSetting ttlSetting = new IteratorSetting(1, "fi", AgeOffFilter.class);
        AgeOffFilter.setTTL(ttlSetting, Long.valueOf(ttl));
        additionalSettings.add(ttlSetting);
    }
    if (startTime != null) {
        final IteratorSetting startTimeSetting = getStartTimeSetting(startTime);
        additionalSettings.add(startTimeSetting);
    }
    for (final Map.Entry<String, InputTableConfig> entry : configs.entrySet()) {
        final List<IteratorSetting> iterators = entry.getValue().getIterators();
        iterators.addAll(additionalSettings);
        entry.getValue().setIterators(iterators);
    }
    // Set the input format
    AccumuloMultiTableInputFormat.setInputTableConfigs(job, configs);
    job.setInputFormatClass(AccumuloMultiTableInputFormat.class);
}

Example 3 with InputTableConfig

use of org.apache.accumulo.core.client.mapreduce.InputTableConfig in project incubator-rya by apache.

the class AccumuloQueryRuleset method getInputConfigs.

/**
 * Get table names and input configurations for each range
 * @return A Map representing each table and {@link InputTableConfig} needed to get all the rows that match the rules.
 */
public Map<String, InputTableConfig> getInputConfigs() {
    final Map<String, InputTableConfig> configs = new HashMap<>();
    for (final TABLE_LAYOUT layout : tableRanges.keySet()) {
        final String parentTable = RdfCloudTripleStoreUtils.layoutPrefixToTable(layout, conf.getTablePrefix());
        final InputTableConfig config = new InputTableConfig();
        config.setRanges(tableRanges.get(layout));
        configs.put(parentTable, config);
    }
    for (final String tableName : entireTables) {
        final InputTableConfig config = new InputTableConfig();
        final List<Range> ranges = new LinkedList<>();
        ranges.add(new Range());
        config.setRanges(ranges);
        configs.put(tableName, config);
    }
    return configs;
}

Example 4 with InputTableConfig

use of org.apache.accumulo.core.client.mapreduce.InputTableConfig in project accumulo by apache.

the class InputConfigurator method getInputTableConfigs.

/**
 * Returns all {@link InputTableConfig} objects associated with this job.
 *
 * @param implementingClass
 *          the class whose name will be used as a prefix for the property configuration key
 * @param conf
 *          the Hadoop configuration object to configure
 * @return all of the table query configs for the job
 * @since 1.6.0
 */
public static Map<String, InputTableConfig> getInputTableConfigs(Class<?> implementingClass, Configuration conf) {
    Map<String, InputTableConfig> configs = new HashMap<>();
    Map.Entry<String, InputTableConfig> defaultConfig = getDefaultInputTableConfig(implementingClass, conf);
    if (defaultConfig != null)
        configs.put(defaultConfig.getKey(), defaultConfig.getValue());
    String configString = conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS));
    MapWritable mapWritable = new MapWritable();
    if (configString != null) {
        try {
            byte[] bytes = Base64.getDecoder().decode(configString);
            ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
            mapWritable.readFields(new DataInputStream(bais));
            bais.close();
        } catch (IOException e) {
            throw new IllegalStateException("The table query configurations could not be deserialized from the given configuration");
        }
    }
    for (Map.Entry<Writable, Writable> entry : mapWritable.entrySet()) configs.put(entry.getKey().toString(), (InputTableConfig) entry.getValue());
    return configs;
}

Example 5 with InputTableConfig

use of org.apache.accumulo.core.client.mapreduce.InputTableConfig in project accumulo by apache.

the class InputConfigurator method validateOptions.

// InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
/**
 * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
 *
 * <p>
 * The implementation (JobContext or JobConf which created the Configuration) needs to be used to extract the proper {@link AuthenticationToken} for
 * {@link DelegationTokenImpl} support.
 *
 * @param implementingClass
 *          the class whose name will be used as a prefix for the property configuration key
 * @param conf
 *          the Hadoop configuration object to configure
 * @throws IOException
 *           if the context is improperly configured
 * @since 1.6.0
 *
 * @see #validateInstance(Class, Configuration)
 * @see #validatePermissions(Class, Configuration, Connector)
 */
@Deprecated
public static void validateOptions(Class<?> implementingClass, Configuration conf) throws IOException {
    Map<String, InputTableConfig> inputTableConfigs = getInputTableConfigs(implementingClass, conf);
    if (!isConnectorInfoSet(implementingClass, conf))
        throw new IOException("Input info has not been set.");
    String instanceKey = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE));
    if (!"MockInstance".equals(instanceKey) && !"ZooKeeperInstance".equals(instanceKey))
        throw new IOException("Instance info has not been set.");
    // validate that we can connect as configured
    try {
        String principal = getPrincipal(implementingClass, conf);
        AuthenticationToken token = getAuthenticationToken(implementingClass, conf);
        Connector c = getInstance(implementingClass, conf).getConnector(principal, token);
        if (!c.securityOperations().authenticateUser(principal, token))
            throw new IOException("Unable to authenticate user");
        if (getInputTableConfigs(implementingClass, conf).size() == 0)
            throw new IOException("No table set.");
        for (Map.Entry<String, InputTableConfig> tableConfig : inputTableConfigs.entrySet()) {
            if (!c.securityOperations().hasTablePermission(getPrincipal(implementingClass, conf), tableConfig.getKey(), TablePermission.READ))
                throw new IOException("Unable to access table");
        }
        for (Map.Entry<String, InputTableConfig> tableConfigEntry : inputTableConfigs.entrySet()) {
            InputTableConfig tableConfig = tableConfigEntry.getValue();
            if (!tableConfig.shouldUseLocalIterators()) {
                if (tableConfig.getIterators() != null) {
                    for (IteratorSetting iter : tableConfig.getIterators()) {
                        if (!c.tableOperations().testClassLoad(tableConfigEntry.getKey(), iter.getIteratorClass(), SortedKeyValueIterator.class.getName()))
                            throw new AccumuloException("Servers are unable to load " + iter.getIteratorClass() + " as a " + SortedKeyValueIterator.class.getName());
                    }
                }
            }
        }
    } catch (AccumuloException | TableNotFoundException | AccumuloSecurityException e) {
        throw new IOException(e);
    }
}