Examples with IteratorSetting org.apache.accumulo.core.client.IteratorSetting used on opensource projects

Example 31 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project presto by prestodb.

the class TestIndexer method testMutationIndex.

@Test
public void testMutationIndex() throws Exception {
    Instance inst = new MockInstance();
    Connector conn = inst.getConnector("root", new PasswordToken(""));
    conn.tableOperations().create(table.getFullTableName());
    conn.tableOperations().create(table.getIndexTableName());
    conn.tableOperations().create(table.getMetricsTableName());
    for (IteratorSetting s : Indexer.getMetricIterators(table)) {
        conn.tableOperations().attachIterator(table.getMetricsTableName(), s);
    }
    Indexer indexer = new Indexer(conn, new Authorizations(), table, new BatchWriterConfig());
    indexer.index(m1);
    indexer.flush();
    Scanner scan = conn.createScanner(table.getIndexTableName(), new Authorizations());
    scan.setRange(new Range());
    Iterator<Entry<Key, Value>> iter = scan.iterator();
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", "");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", "");
    assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", "");
    assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", "");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", "");
    assertFalse(iter.hasNext());
    scan.close();
    scan = conn.createScanner(table.getMetricsTableName(), new Authorizations());
    scan.setRange(new Range());
    iter = scan.iterator();
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "1");
    assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___card___", "1");
    assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___first_row___", "row1");
    assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___last_row___", "row1");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "1");
    assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "1");
    assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "1");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "1");
    assertFalse(iter.hasNext());
    scan.close();
    indexer.index(m2);
    indexer.close();
    scan = conn.createScanner(table.getIndexTableName(), new Authorizations());
    scan.setRange(new Range());
    iter = scan.iterator();
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", "");
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row2", "");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", "");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row2", "");
    assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", "");
    assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "row2", "");
    assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", "");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", "");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row2", "");
    assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "row2", "");
    assertFalse(iter.hasNext());
    scan.close();
    scan = conn.createScanner(table.getMetricsTableName(), new Authorizations());
    scan.setRange(new Range());
    iter = scan.iterator();
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "2");
    assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___card___", "2");
    assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___first_row___", "row1");
    assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___last_row___", "row2");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "2");
    assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "1");
    assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "___card___", "1");
    assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "1");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "2");
    assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "___card___", "1");
    assertFalse(iter.hasNext());
    scan.close();
}

Example 32 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project presto by prestodb.

the class Indexer method getMetricIterators.

/**
     * Gets a collection of iterator settings that should be added to the metric table for the given Accumulo table. Don't forget! Please!
     *
     * @param table Table for retrieving metrics iterators, see AccumuloClient#getTable
     * @return Collection of iterator settings
     */
public static Collection<IteratorSetting> getMetricIterators(AccumuloTable table) {
    String cardQualifier = new String(CARDINALITY_CQ);
    String rowsFamily = new String(METRICS_TABLE_ROWS_CF.array());
    // Build a string for all columns where the summing combiner should be applied,
    // i.e. all indexed columns
    StringBuilder cardBuilder = new StringBuilder(rowsFamily + ":" + cardQualifier + ",");
    for (String s : getLocalityGroups(table).keySet()) {
        cardBuilder.append(s).append(":").append(cardQualifier).append(',');
    }
    cardBuilder.deleteCharAt(cardBuilder.length() - 1);
    // Configuration rows for the Min/Max combiners
    String firstRowColumn = rowsFamily + ":" + new String(METRICS_TABLE_FIRST_ROW_CQ.array());
    String lastRowColumn = rowsFamily + ":" + new String(METRICS_TABLE_LAST_ROW_CQ.array());
    // Summing combiner for cardinality columns
    IteratorSetting s1 = new IteratorSetting(1, SummingCombiner.class, ImmutableMap.of("columns", cardBuilder.toString(), "type", "STRING"));
    // Min/Max combiner for the first/last rows of the table
    IteratorSetting s2 = new IteratorSetting(2, MinByteArrayCombiner.class, ImmutableMap.of("columns", firstRowColumn));
    IteratorSetting s3 = new IteratorSetting(3, MaxByteArrayCombiner.class, ImmutableMap.of("columns", lastRowColumn));
    return ImmutableList.of(s1, s2, s3);
}

Example 33 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project presto by prestodb.

the class AccumuloClient method createIndexTables.

/**
     * Creates the index tables from the given Accumulo table. No op if
     * {@link AccumuloTable#isIndexed()} is false.
     *
     * @param table Table to create index tables
     */
private void createIndexTables(AccumuloTable table) {
    // Early-out if table is not indexed
    if (!table.isIndexed()) {
        return;
    }
    // Create index table if it does not exist (for 'external' table)
    if (!tableManager.exists(table.getIndexTableName())) {
        tableManager.createAccumuloTable(table.getIndexTableName());
    }
    // Create index metrics table if it does not exist
    if (!tableManager.exists(table.getMetricsTableName())) {
        tableManager.createAccumuloTable(table.getMetricsTableName());
    }
    // Set locality groups on index and metrics table
    Map<String, Set<Text>> indexGroups = Indexer.getLocalityGroups(table);
    tableManager.setLocalityGroups(table.getIndexTableName(), indexGroups);
    tableManager.setLocalityGroups(table.getMetricsTableName(), indexGroups);
    // Attach iterators to metrics table
    for (IteratorSetting setting : Indexer.getMetricIterators(table)) {
        tableManager.setIterator(table.getMetricsTableName(), setting);
    }
}

Example 34 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project gora by apache.

the class AccumuloStore method createScanner.

private Scanner createScanner(Query<K, T> query) throws TableNotFoundException {
    // TODO make isolated scanner optional?
    Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Authorizations.EMPTY));
    setFetchColumns(scanner, query.getFields());
    scanner.setRange(createRange(query));
    if (query.getStartTime() != -1 || query.getEndTime() != -1) {
        IteratorSetting is = new IteratorSetting(30, TimestampFilter.class);
        if (query.getStartTime() != -1)
            TimestampFilter.setStart(is, query.getStartTime(), true);
        if (query.getEndTime() != -1)
            TimestampFilter.setEnd(is, query.getEndTime(), true);
        scanner.addScanIterator(is);
    }
    return scanner;
}

Example 35 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project gora by apache.

the class AccumuloStore method deleteByQuery.

@Override
public long deleteByQuery(Query<K, T> query) {
    try {
        Scanner scanner = createScanner(query);
        // add iterator that drops values on the server side
        scanner.addScanIterator(new IteratorSetting(Integer.MAX_VALUE, SortedKeyIterator.class));
        RowIterator iterator = new RowIterator(scanner.iterator());
        long count = 0;
        while (iterator.hasNext()) {
            Iterator<Entry<Key, Value>> row = iterator.next();
            Mutation m = null;
            while (row.hasNext()) {
                Entry<Key, Value> entry = row.next();
                Key key = entry.getKey();
                if (m == null)
                    m = new Mutation(key.getRow());
                // TODO optimize to avoid continually creating column vis? prob does not matter for empty
                m.putDelete(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp());
            }
            getBatchWriter().addMutation(m);
            count++;
        }
        return count;
    } catch (TableNotFoundException e) {
        // TODO return 0?
        LOG.error(e.getMessage(), e);
        return 0;
    } catch (MutationsRejectedException e) {
        LOG.error(e.getMessage(), e);
        return 0;
    } catch (IOException e) {
        LOG.error(e.getMessage(), e);
        return 0;
    }
}