Examples with CompactionConfig org.apache.accumulo.core.client.admin.CompactionConfig used on opensource projects

Example 51 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class SummaryIT method compactionTest.

private void compactionTest(CompactionConfig compactConfig) throws Exception {
    final String table = getUniqueNames(1)[0];
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        NewTableConfiguration ntc = new NewTableConfiguration();
        SummarizerConfiguration sc1 = SummarizerConfiguration.builder(FooCounter.class.getName()).build();
        ntc.enableSummarization(sc1);
        c.tableOperations().create(table, ntc);
        try (BatchWriter bw = c.createBatchWriter(table)) {
            write(bw, "bar1", "f1", "q1", "v1");
            write(bw, "bar2", "f1", "q1", "v2");
            write(bw, "foo1", "f1", "q1", "v3");
        }
        List<IteratorSetting> iterators = Collections.singletonList(new IteratorSetting(100, FooFilter.class));
        compactConfig = compactConfig.setFlush(true).setIterators(iterators).setWait(true);
        // this compaction should make no changes because there are less foos than bars
        c.tableOperations().compact(table, compactConfig);
        try (Scanner scanner = c.createScanner(table, Authorizations.EMPTY)) {
            Stream<Entry<Key, Value>> stream = StreamSupport.stream(scanner.spliterator(), false);
            Map<String, Long> counts = // convert to
            stream.map(e -> e.getKey().getRowData().toString()).map(// strip numbers off row
            r -> r.replaceAll("[0-9]+", "")).collect(// count different row types
            groupingBy(identity(), counting()));
            assertEquals(1L, (long) counts.getOrDefault("foo", 0L));
            assertEquals(2L, (long) counts.getOrDefault("bar", 0L));
            assertEquals(2, counts.size());
        }
        try (BatchWriter bw = c.createBatchWriter(table)) {
            write(bw, "foo2", "f1", "q1", "v4");
            write(bw, "foo3", "f1", "q1", "v5");
            write(bw, "foo4", "f1", "q1", "v6");
        }
        // this compaction should remove all foos because there are more foos than bars
        c.tableOperations().compact(table, compactConfig);
        try (Scanner scanner = c.createScanner(table, Authorizations.EMPTY)) {
            Stream<Entry<Key, Value>> stream = StreamSupport.stream(scanner.spliterator(), false);
            Map<String, Long> counts = // convert to
            stream.map(e -> e.getKey().getRowData().toString()).map(// strip numbers off row
            r -> r.replaceAll("[0-9]+", "")).collect(// count different row types
            groupingBy(identity(), counting()));
            assertEquals(0L, (long) counts.getOrDefault("foo", 0L));
            assertEquals(2L, (long) counts.getOrDefault("bar", 0L));
            assertEquals(1, counts.size());
        }
    }
}

Example 52 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class KerberosIT method testUserPrivilegesThroughGrant.

@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "path provided by test")
@Test
public void testUserPrivilegesThroughGrant() throws Exception {
    String user1 = testName.getMethodName();
    final File user1Keytab = new File(kdc.getKeytabDir(), user1 + ".keytab");
    if (user1Keytab.exists() && !user1Keytab.delete()) {
        log.warn("Unable to delete {}", user1Keytab);
    }
    // Create some new users
    kdc.createPrincipal(user1Keytab, user1);
    final String qualifiedUser1 = kdc.qualifyUser(user1);
    // Log in as user1
    UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(user1, user1Keytab.getAbsolutePath());
    log.info("Logged in as {}", user1);
    ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
        // Indirectly creates this user when we use it
        AccumuloClient client = mac.createAccumuloClient(qualifiedUser1, new KerberosToken());
        log.info("Created client as {}", qualifiedUser1);
        // The new user should have no system permissions
        for (SystemPermission perm : SystemPermission.values()) {
            assertFalse(client.securityOperations().hasSystemPermission(qualifiedUser1, perm));
        }
        return null;
    });
    ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
    ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
        AccumuloClient client = mac.createAccumuloClient(rootUser.getPrincipal(), new KerberosToken());
        client.securityOperations().grantSystemPermission(qualifiedUser1, SystemPermission.CREATE_TABLE);
        return null;
    });
    // Switch back to the original user
    ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(user1, user1Keytab.getAbsolutePath());
    ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
        AccumuloClient client = mac.createAccumuloClient(qualifiedUser1, new KerberosToken());
        // Shouldn't throw an exception since we granted the create table permission
        final String table = testName.getMethodName() + "_user_table";
        client.tableOperations().create(table);
        // Make sure we can actually use the table we made
        try (BatchWriter bw = client.createBatchWriter(table)) {
            Mutation m = new Mutation("a");
            m.put("b", "c", "d");
            bw.addMutation(m);
        }
        client.tableOperations().compact(table, new CompactionConfig().setWait(true).setFlush(true));
        return null;
    });
}

Example 53 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class KerberosIT method createTableWithDataAndCompact.

/**
 * Creates a table, adds a record to it, and then compacts the table. A simple way to make sure
 * that the system user exists (since the manager does an RPC to the tserver which will create the
 * system user if it doesn't already exist).
 */
private void createTableWithDataAndCompact(AccumuloClient client) throws TableNotFoundException, AccumuloSecurityException, AccumuloException, TableExistsException {
    final String table = testName.getMethodName() + "_table";
    client.tableOperations().create(table);
    try (BatchWriter bw = client.createBatchWriter(table)) {
        Mutation m = new Mutation("a");
        m.put("b", "c", "d");
        bw.addMutation(m);
    }
    client.tableOperations().compact(table, new CompactionConfig().setFlush(true).setWait(true));
}

Example 54 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class CompactionIT method testCompactionWithTableIterator.

@Test
public void testCompactionWithTableIterator() throws Exception {
    String table1 = this.getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        client.tableOperations().create(table1);
        try (BatchWriter bw = client.createBatchWriter(table1)) {
            for (int i = 1; i <= 4; i++) {
                Mutation m = new Mutation(Integer.toString(i));
                m.put("cf", "cq", new Value());
                bw.addMutation(m);
                bw.flush();
                client.tableOperations().flush(table1, null, null, true);
            }
        }
        IteratorSetting setting = new IteratorSetting(50, "delete", DevNull.class);
        client.tableOperations().attachIterator(table1, setting, EnumSet.of(IteratorScope.majc));
        client.tableOperations().compact(table1, new CompactionConfig().setWait(true));
        try (Scanner s = client.createScanner(table1)) {
            assertFalse(s.iterator().hasNext());
        }
    }
}

Example 55 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class CompactionIT method testPartialCompaction.

@Test
public void testPartialCompaction() throws Exception {
    String tableName = getUniqueNames(1)[0];
    try (final AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        client.tableOperations().create(tableName);
        // Insert MAX_DATA rows
        try (BatchWriter bw = client.createBatchWriter(tableName)) {
            for (int i = 0; i < MAX_DATA; i++) {
                Mutation m = new Mutation(String.format("r:%04d", i));
                m.put("", "", "" + i);
                bw.addMutation(m);
            }
        }
        client.tableOperations().flush(tableName);
        IteratorSetting iterSetting = new IteratorSetting(100, TestFilter.class);
        // make sure iterator options make it to compactor process
        iterSetting.addOption("modulus", 17 + "");
        CompactionConfig config = new CompactionConfig().setIterators(List.of(iterSetting)).setWait(true);
        client.tableOperations().compact(tableName, config);
        // Insert 2 * MAX_DATA rows
        try (BatchWriter bw = client.createBatchWriter(tableName)) {
            for (int i = MAX_DATA; i < MAX_DATA * 2; i++) {
                Mutation m = new Mutation(String.format("r:%04d", i));
                m.put("", "", "" + i);
                bw.addMutation(m);
            }
        }
        // this should create an F file
        client.tableOperations().flush(tableName);
        // run a compaction that only compacts F files
        iterSetting = new IteratorSetting(100, TestFilter.class);
        // compact F file w/ different modulus and user pmodulus option for partial compaction
        iterSetting.addOption("pmodulus", 19 + "");
        config = new CompactionConfig().setIterators(List.of(iterSetting)).setWait(true).setSelector(new PluginConfig(FSelector.class.getName()));
        client.tableOperations().compact(tableName, config);
        try (Scanner scanner = client.createScanner(tableName)) {
            int count = 0;
            for (Entry<Key, Value> entry : scanner) {
                int v = Integer.parseInt(entry.getValue().toString());
                int modulus = v < MAX_DATA ? 17 : 19;
                assertTrue(String.format("%s %s %d != 0", entry.getValue(), "%", modulus), Integer.parseInt(entry.getValue().toString()) % modulus == 0);
                count++;
            }
            // Verify
            int expectedCount = 0;
            for (int i = 0; i < MAX_DATA * 2; i++) {
                int modulus = i < MAX_DATA ? 17 : 19;
                if (i % modulus == 0) {
                    expectedCount++;
                }
            }
            assertEquals(expectedCount, count);
        }
    }
}