Examples with Column org.apache.accumulo.core.client.IteratorSetting.Column used on opensource projects

Example 6 with Column

use of org.apache.accumulo.core.client.IteratorSetting.Column in project accumulo by apache.

the class Combiner method setColumns.

/**
 * A convenience method to set which columns a combiner should be applied to. For each column
 * specified, all versions of a Key which match that @{link IteratorSetting.Column} will be
 * combined individually in each row. This method is likely to be used in conjunction with
 * {@link ScannerBase#fetchColumnFamily(Text)} or {@link ScannerBase#fetchColumn(Text,Text)}.
 *
 * @param is
 *          iterator settings object to configure
 * @param columns
 *          a list of columns to encode as the value for the combiner column configuration
 */
public static void setColumns(IteratorSetting is, List<IteratorSetting.Column> columns) {
    String sep = "";
    StringBuilder sb = new StringBuilder();
    for (Column col : columns) {
        sb.append(sep);
        sep = ",";
        sb.append(ColumnSet.encodeColumns(col.getFirst(), col.getSecond()));
    }
    is.addOption(COLUMNS_OPTION, sb.toString());
}

Example 7 with Column

use of org.apache.accumulo.core.client.IteratorSetting.Column in project accumulo by apache.

the class MultiTableInputFormatTest method testManyTables.

@Test
public void testManyTables() throws Exception {
    Job job = Job.getInstance();
    Properties clientProps = org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormatTest.setupClientProperties();
    // if auths are not set client will try to get from server, we dont want that here
    Authorizations auths = Authorizations.EMPTY;
    // set the client properties once then loop over tables
    InputFormatBuilder.TableParams<Job> opts = AccumuloInputFormat.configure().clientProperties(clientProps);
    for (int i = 0; i < 10_000; i++) {
        List<Range> ranges = singletonList(new Range("a" + i, "b" + i));
        Set<Column> cols = singleton(new Column(new Text("CF" + i), new Text("CQ" + i)));
        IteratorSetting iter = new IteratorSetting(50, "iter" + i, "iterclass" + i);
        opts.table("table" + i).auths(auths).ranges(ranges).fetchColumns(cols).addIterator(iter);
    }
    opts.store(job);
    // verify
    Map<String, InputTableConfig> configs = InputConfigurator.getInputTableConfigs(CLASS, job.getConfiguration());
    assertEquals(10_000, configs.size());
    // create objects to test against
    for (int i = 0; i < 10_000; i++) {
        InputTableConfig t = new InputTableConfig();
        List<Range> ranges = singletonList(new Range("a" + i, "b" + i));
        Set<Column> cols = singleton(new Column(new Text("CF" + i), new Text("CQ" + i)));
        IteratorSetting iter = new IteratorSetting(50, "iter" + i, "iterclass" + i);
        t.setScanAuths(auths).setRanges(ranges).fetchColumns(cols).addIterator(iter);
        assertEquals(t, configs.get("table" + i));
    }
}

Example 8 with Column

use of org.apache.accumulo.core.client.IteratorSetting.Column in project accumulo by apache.

the class ReplicationTableUtil method configureMetadataTable.

public static synchronized void configureMetadataTable(AccumuloClient client, String tableName) {
    TableOperations tops = client.tableOperations();
    Map<String, EnumSet<IteratorScope>> iterators = null;
    try {
        iterators = tops.listIterators(tableName);
    } catch (AccumuloSecurityException | AccumuloException | TableNotFoundException e) {
        throw new RuntimeException(e);
    }
    if (!iterators.containsKey(COMBINER_NAME)) {
        // Set our combiner and combine all columns
        // Need to set the combiner beneath versioning since we don't want to turn it off
        @SuppressWarnings("deprecation") var statusCombinerClass = org.apache.accumulo.server.replication.StatusCombiner.class;
        IteratorSetting setting = new IteratorSetting(9, COMBINER_NAME, statusCombinerClass);
        Combiner.setColumns(setting, Collections.singletonList(new Column(ReplicationSection.COLF)));
        try {
            tops.attachIterator(tableName, setting);
        } catch (AccumuloSecurityException | AccumuloException | TableNotFoundException e) {
            throw new RuntimeException(e);
        }
    }
    // Make sure the StatusFormatter is set on the metadata table
    Map<String, String> properties;
    try {
        properties = tops.getConfiguration(tableName);
    } catch (AccumuloException | TableNotFoundException e) {
        throw new RuntimeException(e);
    }
    for (Entry<String, String> property : properties.entrySet()) {
        if (Property.TABLE_FORMATTER_CLASS.getKey().equals(property.getKey())) {
            if (!STATUS_FORMATTER_CLASS_NAME.equals(property.getValue())) {
                log.info("Setting formatter for {} from {} to {}", tableName, property.getValue(), STATUS_FORMATTER_CLASS_NAME);
                try {
                    tops.setProperty(tableName, Property.TABLE_FORMATTER_CLASS.getKey(), STATUS_FORMATTER_CLASS_NAME);
                } catch (AccumuloException | AccumuloSecurityException e) {
                    throw new RuntimeException(e);
                }
            }
            // for
            return;
        }
    }
    // Set the formatter on the table because it wasn't already there
    try {
        tops.setProperty(tableName, Property.TABLE_FORMATTER_CLASS.getKey(), STATUS_FORMATTER_CLASS_NAME);
    } catch (AccumuloException | AccumuloSecurityException e) {
        throw new RuntimeException(e);
    }
}

Example 9 with Column

use of org.apache.accumulo.core.client.IteratorSetting.Column in project accumulo by apache.

the class ReplicationIT method verifyReplicationTableConfig.

@Test
public void verifyReplicationTableConfig() throws AccumuloException, TableNotFoundException, AccumuloSecurityException {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        TableOperations tops = client.tableOperations();
        Map<String, EnumSet<IteratorScope>> iterators = tops.listIterators(ReplicationTable.NAME);
        // verify combiners are only iterators (no versioning)
        assertEquals(1, iterators.size());
        // look for combiner
        assertTrue(iterators.containsKey(ReplicationTable.COMBINER_NAME));
        assertTrue(iterators.get(ReplicationTable.COMBINER_NAME).containsAll(EnumSet.allOf(IteratorScope.class)));
        for (IteratorScope scope : EnumSet.allOf(IteratorScope.class)) {
            IteratorSetting is = tops.getIteratorSetting(ReplicationTable.NAME, ReplicationTable.COMBINER_NAME, scope);
            assertEquals(30, is.getPriority());
            assertEquals(StatusCombiner.class.getName(), is.getIteratorClass());
            assertEquals(1, is.getOptions().size());
            assertTrue(is.getOptions().containsKey("columns"));
            String cols = is.getOptions().get("columns");
            Column statusSectionCol = new Column(StatusSection.NAME);
            Column workSectionCol = new Column(WorkSection.NAME);
            assertEquals(ColumnSet.encodeColumns(statusSectionCol.getColumnFamily(), statusSectionCol.getColumnQualifier()) + "," + ColumnSet.encodeColumns(workSectionCol.getColumnFamily(), workSectionCol.getColumnQualifier()), cols);
        }
        boolean foundLocalityGroups = false;
        boolean foundLocalityGroupDef1 = false;
        boolean foundLocalityGroupDef2 = false;
        boolean foundFormatter = false;
        Joiner j = Joiner.on(",");
        for (Entry<String, String> p : tops.getProperties(ReplicationTable.NAME)) {
            String key = p.getKey();
            String val = p.getValue();
            // STATUS_LG_NAME, STATUS_LG_COLFAMS, WORK_LG_NAME, WORK_LG_COLFAMS
            if (key.equals(Property.TABLE_FORMATTER_CLASS.getKey()) && val.equals(StatusFormatter.class.getName())) {
                // look for formatter
                foundFormatter = true;
            } else if (key.equals(Property.TABLE_LOCALITY_GROUPS.getKey()) && val.equals(j.join(ReplicationTable.LOCALITY_GROUPS.keySet()))) {
                // look for locality groups enabled
                foundLocalityGroups = true;
            } else if (key.startsWith(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey())) {
                // look for locality group column family definitions
                if (key.equals(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + ReplicationTable.STATUS_LG_NAME) && val.equals(j.join(Iterables.transform(ReplicationTable.STATUS_LG_COLFAMS, Text::toString)))) {
                    foundLocalityGroupDef1 = true;
                } else if (key.equals(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + ReplicationTable.WORK_LG_NAME) && val.equals(j.join(Iterables.transform(ReplicationTable.WORK_LG_COLFAMS, Text::toString)))) {
                    foundLocalityGroupDef2 = true;
                }
            }
        }
        assertTrue(foundLocalityGroups);
        assertTrue(foundLocalityGroupDef1);
        assertTrue(foundLocalityGroupDef2);
        assertTrue(foundFormatter);
    }
}