Examples with Summary org.apache.accumulo.core.client.summary.Summary used on opensource projects

Example 11 with Summary

use of org.apache.accumulo.core.client.summary.Summary in project accumulo by apache.

the class SummariesCommand method doTableOp.

@Override
protected void doTableOp(final Shell shellState, final String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException {
    Connector conn = shellState.getConnector();
    SummaryRetriever retriever = conn.tableOperations().summaries(tableName).withMatchingConfiguration(selectionRegex);
    if (startRow != null) {
        retriever.startRow(startRow);
    }
    if (endRow != null) {
        retriever.endRow(endRow);
    }
    Collection<Summary> summaries = retriever.retrieve();
    ArrayList<String> lines = new ArrayList<>();
    boolean addEmpty = false;
    for (Summary summary : summaries) {
        if (addEmpty)
            lines.add("");
        addEmpty = true;
        lines.add(String.format(" Summarizer         : %s", summary.getSummarizerConfiguration()));
        lines.add(String.format(" File Statistics    : %s", summary.getFileStatistics()));
        lines.add(String.format(" Summary Statistics : "));
        Map<String, Long> stats = summary.getStatistics();
        ArrayList<String> keys = new ArrayList<>(stats.keySet());
        Collections.sort(keys);
        for (String key : keys) {
            lines.add(String.format("    %-60s = %,d", key, stats.get(key)));
        }
    }
    shellState.printLines(lines.iterator(), paginate);
}

Example 12 with Summary

use of org.apache.accumulo.core.client.summary.Summary in project accumulo by apache.

the class SummaryIT method testLocalityGroups.

@Test
public void testLocalityGroups() throws Exception {
    final String table = getUniqueNames(1)[0];
    Connector c = getConnector();
    NewTableConfiguration ntc = new NewTableConfiguration();
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(FamilySummarizer.class).build();
    SummarizerConfiguration sc2 = SummarizerConfiguration.builder(BasicSummarizer.class).build();
    ntc.enableSummarization(sc1, sc2);
    c.tableOperations().create(table, ntc);
    Map<String, Set<Text>> lgroups = new HashMap<>();
    lgroups.put("lg1", ImmutableSet.of(new Text("chocolate"), new Text("coffee")));
    lgroups.put("lg2", ImmutableSet.of(new Text(" broccoli "), new Text("cabbage")));
    c.tableOperations().setLocalityGroups(table, lgroups);
    Map<Key, Value> expected = new HashMap<>();
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        write(bw, expected, "order:001", "chocolate", "dark", 3l, "99kg");
        write(bw, expected, "order:001", "chocolate", "light", 4l, "94kg");
        write(bw, expected, "order:001", "coffee", "dark", 5l, "33kg");
        write(bw, expected, "order:001", "broccoli", "crowns", 6l, "2kg");
        write(bw, expected, "order:001", "cheddar", "canadian", 7l, "40kg");
        write(bw, expected, "order:653", "chocolate", "dark", 3l, "3kg");
        write(bw, expected, "order:653", "chocolate", "light", 4l, "4kg");
        write(bw, expected, "order:653", "coffee", "dark", 5l, "2kg");
        write(bw, expected, "order:653", "broccoli", "crowns", 6l, "105kg");
        write(bw, expected, "order:653", "cabbage", "heads", 7l, "199kg");
        write(bw, expected, "order:653", "cheddar", "canadian", 8l, "43kg");
    }
    List<Summary> summaries = c.tableOperations().summaries(table).flush(true).retrieve();
    Assert.assertEquals(2, summaries.stream().map(Summary::getSummarizerConfiguration).distinct().count());
    for (Summary summary : summaries) {
        if (summary.getSummarizerConfiguration().equals(sc1)) {
            Map<String, Long> expectedStats = nm("c:chocolate", 4l, "c:coffee", 2l, "c:broccoli", 2l, "c:cheddar", 2l, "c:cabbage", 1l, TOO_LONG_STAT, 0l, TOO_MANY_STAT, 0l, SEEN_STAT, 11l, EMITTED_STAT, 11l, DELETES_IGNORED_STAT, 0l);
            Assert.assertEquals(expectedStats, summary.getStatistics());
            Assert.assertEquals(0, summary.getFileStatistics().getInaccurate());
            Assert.assertEquals(1, summary.getFileStatistics().getTotal());
        } else if (summary.getSummarizerConfiguration().equals(sc2)) {
            Map<String, Long> expectedStats = nm(DELETES_STAT, 0l, TOTAL_STAT, 11l, MIN_TIMESTAMP_STAT, 3l, MAX_TIMESTAMP_STAT, 8l);
            Assert.assertEquals(expectedStats, summary.getStatistics());
            Assert.assertEquals(0, summary.getFileStatistics().getInaccurate());
            Assert.assertEquals(1, summary.getFileStatistics().getTotal());
        } else {
            Assert.fail("unexpected summary config " + summary.getSummarizerConfiguration());
        }
    }
    Map<Key, Value> actual = new HashMap<>();
    c.createScanner(table, Authorizations.EMPTY).forEach(e -> actual.put(e.getKey(), e.getValue()));
    Assert.assertEquals(expected, actual);
}

Example 13 with Summary

use of org.apache.accumulo.core.client.summary.Summary in project accumulo by apache.

the class SummaryIT method testManyFiles.

@Test
public void testManyFiles() throws Exception {
    final String table = getUniqueNames(1)[0];
    Connector c = getConnector();
    NewTableConfiguration ntc = new NewTableConfiguration();
    ntc.enableSummarization(SummarizerConfiguration.builder(FamilySummarizer.class).build());
    c.tableOperations().create(table, ntc);
    Random rand = new Random(42);
    int q = 0;
    SortedSet<Text> partitionKeys = new TreeSet<>();
    for (int split = 100_000; split < 1_000_000; split += 100_000) {
        partitionKeys.add(new Text(String.format("%06d", split)));
    }
    c.tableOperations().addSplits(table, partitionKeys);
    Map<String, Long> famCounts = new HashMap<>();
    for (int t = 0; t < 20; t++) {
        // this loop should cause a varying number of files and compactions
        try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
            for (int i = 0; i < 10000; i++) {
                String row = String.format("%06d", rand.nextInt(1_000_000));
                String fam = String.format("%03d", rand.nextInt(100));
                String qual = String.format("%06d", q++);
                write(bw, row, fam, qual, "val");
                famCounts.merge(fam, 1L, Long::sum);
            }
        }
        List<Summary> summaries = c.tableOperations().summaries(table).flush(true).retrieve();
        Assert.assertEquals(1, summaries.size());
        CounterSummary cs = new CounterSummary(summaries.get(0));
        Assert.assertEquals(famCounts, cs.getCounters());
        FileStatistics fileStats = summaries.get(0).getFileStatistics();
        Assert.assertEquals(0, fileStats.getInaccurate());
        Assert.assertTrue("Saw " + fileStats.getTotal() + " files expected >=10", fileStats.getTotal() >= 10);
    }
}

Example 14 with Summary

use of org.apache.accumulo.core.client.summary.Summary in project accumulo by apache.

the class SummaryIT method selectionTest.

@Test
public void selectionTest() throws Exception {
    final String table = getUniqueNames(1)[0];
    Connector c = getConnector();
    NewTableConfiguration ntc = new NewTableConfiguration();
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(BasicSummarizer.class).build();
    SummarizerConfiguration sc2 = SummarizerConfiguration.builder(KeySizeSummarizer.class).addOption("maxLen", "512").build();
    ntc.enableSummarization(sc1, sc2);
    c.tableOperations().create(table, ntc);
    BatchWriter bw = writeData(table, c);
    bw.close();
    c.tableOperations().flush(table, null, null, true);
    LongSummaryStatistics stats = getTimestampStats(table, c);
    Collection<Summary> summaries = c.tableOperations().summaries(table).withConfiguration(sc2).retrieve();
    Assert.assertEquals(1, summaries.size());
    checkSummary(summaries, sc2, "len=14", 100_000l);
    summaries = c.tableOperations().summaries(table).withConfiguration(sc1).retrieve();
    Assert.assertEquals(1, summaries.size());
    checkSummary(summaries, sc1, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    // retrieve a non-existant summary
    SummarizerConfiguration sc3 = SummarizerConfiguration.builder(KeySizeSummarizer.class.getName()).addOption("maxLen", "256").build();
    summaries = c.tableOperations().summaries(table).withConfiguration(sc3).retrieve();
    Assert.assertEquals(0, summaries.size());
    summaries = c.tableOperations().summaries(table).withConfiguration(sc1, sc2).retrieve();
    Assert.assertEquals(2, summaries.size());
    checkSummary(summaries, sc1, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    checkSummary(summaries, sc2, "len=14", 100_000l);
    summaries = c.tableOperations().summaries(table).retrieve();
    Assert.assertEquals(2, summaries.size());
    checkSummary(summaries, sc1, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    checkSummary(summaries, sc2, "len=14", 100_000l);
    summaries = c.tableOperations().summaries(table).withMatchingConfiguration(".*BasicSummarizer \\{\\}.*").retrieve();
    Assert.assertEquals(1, summaries.size());
    checkSummary(summaries, sc1, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    summaries = c.tableOperations().summaries(table).withMatchingConfiguration(".*KeySizeSummarizer \\{maxLen=512\\}.*").retrieve();
    Assert.assertEquals(1, summaries.size());
    checkSummary(summaries, sc2, "len=14", 100_000l);
    summaries = c.tableOperations().summaries(table).withMatchingConfiguration(".*KeySizeSummarizer \\{maxLen=256\\}.*").retrieve();
    Assert.assertEquals(0, summaries.size());
    summaries = c.tableOperations().summaries(table).withMatchingConfiguration(".*BasicSummarizer \\{\\}.*").withConfiguration(sc2).retrieve();
    Assert.assertEquals(2, summaries.size());
    checkSummary(summaries, sc1, TOTAL_STAT, 100_000l, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0l);
    checkSummary(summaries, sc2, "len=14", 100_000l);
    // Ensure a bad regex fails fast.
    try {
        summaries = c.tableOperations().summaries(table).withMatchingConfiguration(".*KeySizeSummarizer {maxLen=256}.*").retrieve();
        Assert.fail("Bad regex should have caused exception");
    } catch (PatternSyntaxException e) {
    }
}

Example 15 with Summary

use of org.apache.accumulo.core.client.summary.Summary in project accumulo by apache.

the class SummaryIT method testPermissions.

@Test
public void testPermissions() throws Exception {
    final String table = getUniqueNames(1)[0];
    Connector c = getConnector();
    NewTableConfiguration ntc = new NewTableConfiguration();
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(FooCounter.class).build();
    ntc.enableSummarization(sc1);
    c.tableOperations().create(table, ntc);
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        write(bw, "bar1", "f1", "q1", "v1");
        write(bw, "bar2", "f1", "q1", "v2");
        write(bw, "foo1", "f1", "q1", "v3");
    }
    c.tableOperations().flush(table, null, null, true);
    PasswordToken passTok = new PasswordToken("letmesee");
    c.securityOperations().createLocalUser("user1", passTok);
    String instanceName = c.getInstance().getInstanceName();
    String zookeepers = c.getInstance().getZooKeepers();
    Connector c2 = new ZooKeeperInstance(instanceName, zookeepers).getConnector("user1", passTok);
    try {
        c2.tableOperations().summaries(table).retrieve();
        Assert.fail("Expected operation to fail because user does not have permssion to get summaries");
    } catch (AccumuloSecurityException ase) {
        Assert.assertEquals(SecurityErrorCode.PERMISSION_DENIED, ase.getSecurityErrorCode());
    }
    c.securityOperations().grantTablePermission("user1", table, TablePermission.GET_SUMMARIES);
    int tries = 0;
    while (tries < 10) {
        try {
            Summary summary = c2.tableOperations().summaries(table).retrieve().get(0);
            Assert.assertEquals(2, summary.getStatistics().size());
            Assert.assertEquals(2l, (long) summary.getStatistics().getOrDefault("bars", 0l));
            Assert.assertEquals(1l, (long) summary.getStatistics().getOrDefault("foos", 0l));
            break;
        } catch (AccumuloSecurityException ase) {
            UtilWaitThread.sleep(500);
            tries++;
        }
    }
}