Examples with Summary org.apache.accumulo.core.client.summary.Summary used on opensource projects

Example 1 with Summary

use of org.apache.accumulo.core.client.summary.Summary in project accumulo by apache.

the class RFileTest method checkSummaries.

private void checkSummaries(Collection<Summary> summaries, Map<String, Long> expected, int extra) {
    Assert.assertEquals(1, summaries.size());
    for (Summary summary : summaries) {
        Assert.assertEquals(extra, summary.getFileStatistics().getInaccurate());
        Assert.assertEquals(extra, summary.getFileStatistics().getExtra());
        Assert.assertEquals(2, summary.getFileStatistics().getTotal());
        String className = summary.getSummarizerConfiguration().getClassName();
        CounterSummary counterSummary = new CounterSummary(summary);
        if (className.equals(VisibilitySummarizer.class.getName())) {
            Map<String, Long> counters = counterSummary.getCounters();
            Assert.assertEquals(expected, counters);
        } else {
            Assert.fail("Unexpected classname " + className);
        }
    }
}

Example 2 with Summary

use of org.apache.accumulo.core.client.summary.Summary in project accumulo by apache.

the class PermissionsIT method testGrantedTablePermission.

private void testGrantedTablePermission(Connector test_user_conn, ClusterUser normalUser, TablePermission perm, String tableName) throws AccumuloException, TableExistsException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
    BatchWriter writer;
    Mutation m;
    log.debug("Confirming that the presence of the {} permission properly permits the user", perm);
    // test permission after granting it
    switch(perm) {
        case READ:
            try (Scanner scanner = test_user_conn.createScanner(tableName, Authorizations.EMPTY)) {
                Iterator<Entry<Key, Value>> iter = scanner.iterator();
                while (iter.hasNext()) iter.next();
            }
            break;
        case WRITE:
            writer = test_user_conn.createBatchWriter(tableName, new BatchWriterConfig());
            m = new Mutation(new Text("row"));
            m.put(new Text("a"), new Text("b"), new Value("c".getBytes()));
            writer.addMutation(m);
            writer.close();
            break;
        case BULK_IMPORT:
            // test for bulk import permission would go here
            break;
        case ALTER_TABLE:
            Map<String, Set<Text>> groups = new HashMap<>();
            groups.put("tgroup", new HashSet<>(Arrays.asList(new Text("t1"), new Text("t2"))));
            break;
        case DROP_TABLE:
            test_user_conn.tableOperations().delete(tableName);
            break;
        case GRANT:
            test_user_conn.securityOperations().grantTablePermission(getAdminPrincipal(), tableName, TablePermission.GRANT);
            break;
        case GET_SUMMARIES:
            List<Summary> summaries = test_user_conn.tableOperations().summaries(tableName).retrieve();
            // just make sure it's not blocked by permissions, the actual summaries are tested in SummaryIT
            Assert.assertTrue(summaries.isEmpty());
            break;
        default:
            throw new IllegalArgumentException("Unrecognized table Permission: " + perm);
    }
}

Example 3 with Summary

use of org.apache.accumulo.core.client.summary.Summary in project accumulo by apache.

the class SummaryIT method basicSummaryTest.

@Test
public void basicSummaryTest() throws Exception {
    final String table = getUniqueNames(1)[0];
    Connector c = getConnector();
    NewTableConfiguration ntc = new NewTableConfiguration();
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(BasicSummarizer.class.getName()).build();
    ntc.enableSummarization(sc1);
    c.tableOperations().create(table, ntc);
    BatchWriter bw = writeData(table, c);
    Collection<Summary> summaries = c.tableOperations().summaries(table).flush(false).retrieve();
    Assert.assertEquals(0, summaries.size());
    LongSummaryStatistics stats = getTimestampStats(table, c);
    summaries = c.tableOperations().summaries(table).flush(true).retrieve();
    checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    Mutation m = new Mutation(String.format("r%09x", 999));
    m.put("f1", "q1", "999-0");
    m.putDelete("f1", "q2");
    bw.addMutation(m);
    bw.flush();
    c.tableOperations().flush(table, null, null, true);
    stats = getTimestampStats(table, c);
    summaries = c.tableOperations().summaries(table).retrieve();
    checkSummaries(summaries, sc1, 2, 0, 0, TOTAL_STAT, 100_002l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 1l);
    bw.close();
    c.tableOperations().compact(table, new CompactionConfig().setWait(true));
    summaries = c.tableOperations().summaries(table).retrieve();
    checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    // split tablet into two
    String sp1 = String.format("r%09x", 50_000);
    addSplits(table, c, sp1);
    summaries = c.tableOperations().summaries(table).retrieve();
    checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    // compact 2nd tablet
    c.tableOperations().compact(table, new CompactionConfig().setStartRow(new Text(sp1)).setWait(true));
    summaries = c.tableOperations().summaries(table).retrieve();
    checkSummaries(summaries, sc1, 2, 0, 1, TOTAL_STAT, 113_999l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    // get summaries for first tablet
    stats = getTimestampStats(table, c, sp1, null);
    summaries = c.tableOperations().summaries(table).startRow(sp1).retrieve();
    checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 49_999l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    // compact all tablets and regenerate all summaries
    c.tableOperations().compact(table, new CompactionConfig());
    summaries = c.tableOperations().summaries(table).retrieve();
    stats = getTimestampStats(table, c);
    checkSummaries(summaries, sc1, 2, 0, 0, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    summaries = c.tableOperations().summaries(table).startRow(String.format("r%09x", 75_000)).endRow(String.format("r%09x", 80_000)).retrieve();
    Summary summary = Iterables.getOnlyElement(summaries);
    Assert.assertEquals(1, summary.getFileStatistics().getTotal());
    Assert.assertEquals(1, summary.getFileStatistics().getExtra());
    long total = summary.getStatistics().get(TOTAL_STAT);
    Assert.assertTrue("Total " + total + " out of expected range", total > 0 && total <= 10_000);
    // test adding and removing
    c.tableOperations().removeSummarizers(table, sc -> sc.getClassName().contains("foo"));
    List<SummarizerConfiguration> summarizers = c.tableOperations().listSummarizers(table);
    Assert.assertEquals(1, summarizers.size());
    Assert.assertTrue(summarizers.contains(sc1));
    c.tableOperations().removeSummarizers(table, sc -> sc.getClassName().equals(BasicSummarizer.class.getName()));
    summarizers = c.tableOperations().listSummarizers(table);
    Assert.assertEquals(0, summarizers.size());
    c.tableOperations().compact(table, new CompactionConfig().setWait(true));
    summaries = c.tableOperations().summaries(table).retrieve();
    Assert.assertEquals(0, summaries.size());
    c.tableOperations().addSummarizers(table, sc1);
    c.tableOperations().compact(table, new CompactionConfig().setWait(true));
    summaries = c.tableOperations().summaries(table).retrieve();
    checkSummaries(summaries, sc1, 2, 0, 0, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
}

Example 4 with Summary

use of org.apache.accumulo.core.client.summary.Summary in project accumulo by apache.

the class SummaryIT method checkSummary.

private static void checkSummary(Collection<Summary> summaries, SummarizerConfiguration sc, Object... stats) {
    Map<String, Long> expected = new HashMap<>();
    for (int i = 0; i < stats.length; i += 2) {
        expected.put((String) stats[i], (Long) stats[i + 1]);
    }
    for (Summary summary : summaries) {
        if (summary.getSummarizerConfiguration().equals(sc)) {
            Assert.assertEquals(expected, summary.getStatistics());
            return;
        }
    }
    Assert.fail("Did not find summary with config : " + sc);
}

Example 5 with Summary

use of org.apache.accumulo.core.client.summary.Summary in project accumulo by apache.

the class SummaryIT method tooLargeTest.

@Test
public void tooLargeTest() throws Exception {
    final String table = getUniqueNames(1)[0];
    Connector c = getConnector();
    NewTableConfiguration ntc = new NewTableConfiguration();
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(BigSummarizer.class).build();
    ntc.enableSummarization(sc1);
    c.tableOperations().create(table, ntc);
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        write(bw, "a_large", "f1", "q1", "v1");
        write(bw, "v_small", "f1", "q1", "v2");
    }
    c.tableOperations().flush(table, null, null, true);
    Summary summary = c.tableOperations().summaries(table).retrieve().get(0);
    Assert.assertEquals(1, summary.getFileStatistics().getLarge());
    Assert.assertEquals(0, summary.getFileStatistics().getMissing());
    Assert.assertEquals(0, summary.getFileStatistics().getExtra());
    Assert.assertEquals(0, summary.getFileStatistics().getDeleted());
    Assert.assertEquals(1, summary.getFileStatistics().getInaccurate());
    Assert.assertEquals(1, summary.getFileStatistics().getTotal());
    Assert.assertEquals(Collections.emptyMap(), summary.getStatistics());
    // create situation where one tablet has summary data and one does not because the summary data was too large
    c.tableOperations().addSplits(table, new TreeSet<>(Collections.singleton(new Text("m"))));
    c.tableOperations().compact(table, new CompactionConfig().setWait(true));
    summary = c.tableOperations().summaries(table).retrieve().get(0);
    Assert.assertEquals(1, summary.getFileStatistics().getLarge());
    Assert.assertEquals(0, summary.getFileStatistics().getMissing());
    Assert.assertEquals(0, summary.getFileStatistics().getExtra());
    Assert.assertEquals(0, summary.getFileStatistics().getDeleted());
    Assert.assertEquals(1, summary.getFileStatistics().getInaccurate());
    Assert.assertEquals(2, summary.getFileStatistics().getTotal());
    HashMap<String, Long> expected = new HashMap<>();
    for (int i = 0; i < 10; i++) {
        expected.put(String.format("%09x", i), i * 19l);
    }
    Assert.assertEquals(expected, summary.getStatistics());
}