Examples with CompactionConfig org.apache.accumulo.core.client.admin.CompactionConfig used on opensource projects

Example 16 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class ConcurrentDeleteTableIT method testConcurrentFateOpsWithDelete.

@Test
public void testConcurrentFateOpsWithDelete() throws Exception {
    final Connector c = getConnector();
    String[] tables = getUniqueNames(2);
    TreeSet<Text> splits = createSplits();
    int numOperations = 8;
    ExecutorService es = Executors.newFixedThreadPool(numOperations);
    int count = 0;
    for (final String table : tables) {
        c.tableOperations().create(table);
        c.tableOperations().addSplits(table, splits);
        writeData(c, table);
        if (count == 1) {
            c.tableOperations().flush(table, null, null, true);
        }
        count++;
        // increment this for each test
        final CountDownLatch cdl = new CountDownLatch(numOperations);
        List<Future<?>> futures = new ArrayList<>();
        futures.add(es.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    cdl.countDown();
                    cdl.await();
                    c.tableOperations().delete(table);
                } catch (TableNotFoundException | TableOfflineException e) {
                // expected
                } catch (InterruptedException | AccumuloException | AccumuloSecurityException e) {
                    throw new RuntimeException(e);
                }
            }
        }));
        futures.add(es.submit(new DelayedTableOp(cdl) {

            @Override
            protected void doTableOp() throws Exception {
                c.tableOperations().compact(table, new CompactionConfig());
            }
        }));
        futures.add(es.submit(new DelayedTableOp(cdl) {

            @Override
            protected void doTableOp() throws Exception {
                c.tableOperations().merge(table, null, null);
            }
        }));
        futures.add(es.submit(new DelayedTableOp(cdl) {

            @Override
            protected void doTableOp() throws Exception {
                Map<String, String> m = Collections.emptyMap();
                Set<String> s = Collections.emptySet();
                c.tableOperations().clone(table, table + "_clone", true, m, s);
            }
        }));
        futures.add(es.submit(new DelayedTableOp(cdl) {

            @Override
            protected void doTableOp() throws Exception {
                c.tableOperations().deleteRows(table, null, null);
            }
        }));
        futures.add(es.submit(new DelayedTableOp(cdl) {

            @Override
            protected void doTableOp() throws Exception {
                c.tableOperations().cancelCompaction(table);
            }
        }));
        futures.add(es.submit(new DelayedTableOp(cdl) {

            @Override
            protected void doTableOp() throws Exception {
                c.tableOperations().rename(table, table + "_renamed");
            }
        }));
        futures.add(es.submit(new DelayedTableOp(cdl) {

            @Override
            protected void doTableOp() throws Exception {
                c.tableOperations().offline(table);
            }
        }));
        Assert.assertEquals(numOperations, futures.size());
        for (Future<?> future : futures) {
            future.get();
        }
        try {
            c.createScanner(table, Authorizations.EMPTY);
            Assert.fail("Expected table " + table + " to be gone.");
        } catch (TableNotFoundException tnfe) {
        // expected
        }
        FunctionalTestUtils.assertNoDanglingFateLocks(getConnector().getInstance(), getCluster());
    }
    es.shutdown();
}

Example 17 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class KerberosIT method testUserPrivilegesThroughGrant.

@Test
public void testUserPrivilegesThroughGrant() throws Exception {
    String user1 = testName.getMethodName();
    final File user1Keytab = new File(kdc.getKeytabDir(), user1 + ".keytab");
    if (user1Keytab.exists() && !user1Keytab.delete()) {
        log.warn("Unable to delete {}", user1Keytab);
    }
    // Create some new users
    kdc.createPrincipal(user1Keytab, user1);
    final String qualifiedUser1 = kdc.qualifyUser(user1);
    // Log in as user1
    UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(user1, user1Keytab.getAbsolutePath());
    log.info("Logged in as {}", user1);
    ugi.doAs(new PrivilegedExceptionAction<Void>() {

        @Override
        public Void run() throws Exception {
            // Indirectly creates this user when we use it
            Connector conn = mac.getConnector(qualifiedUser1, new KerberosToken());
            log.info("Created connector as {}", qualifiedUser1);
            // The new user should have no system permissions
            for (SystemPermission perm : SystemPermission.values()) {
                assertFalse(conn.securityOperations().hasSystemPermission(qualifiedUser1, perm));
            }
            return null;
        }
    });
    ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
    ugi.doAs(new PrivilegedExceptionAction<Void>() {

        @Override
        public Void run() throws Exception {
            Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
            conn.securityOperations().grantSystemPermission(qualifiedUser1, SystemPermission.CREATE_TABLE);
            return null;
        }
    });
    // Switch back to the original user
    ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(user1, user1Keytab.getAbsolutePath());
    ugi.doAs(new PrivilegedExceptionAction<Void>() {

        @Override
        public Void run() throws Exception {
            Connector conn = mac.getConnector(qualifiedUser1, new KerberosToken());
            // Shouldn't throw an exception since we granted the create table permission
            final String table = testName.getMethodName() + "_user_table";
            conn.tableOperations().create(table);
            // Make sure we can actually use the table we made
            BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
            Mutation m = new Mutation("a");
            m.put("b", "c", "d");
            bw.addMutation(m);
            bw.close();
            conn.tableOperations().compact(table, new CompactionConfig().setWait(true).setFlush(true));
            return null;
        }
    });
}

Example 18 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class SummaryIT method compactionTest.

@Test
public void compactionTest() throws Exception {
    final String table = getUniqueNames(1)[0];
    Connector c = getConnector();
    NewTableConfiguration ntc = new NewTableConfiguration();
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(FooCounter.class.getName()).build();
    ntc.enableSummarization(sc1);
    c.tableOperations().create(table, ntc);
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        write(bw, "bar1", "f1", "q1", "v1");
        write(bw, "bar2", "f1", "q1", "v2");
        write(bw, "foo1", "f1", "q1", "v3");
    }
    // Create a compaction config that will filter out foos if there are too many. Uses summary data to know if there are too many foos.
    CompactionStrategyConfig csc = new CompactionStrategyConfig(FooCS.class.getName());
    List<IteratorSetting> iterators = Collections.singletonList(new IteratorSetting(100, FooFilter.class));
    CompactionConfig compactConfig = new CompactionConfig().setFlush(true).setCompactionStrategy(csc).setIterators(iterators).setWait(true);
    // this compaction should make no changes because there are less foos than bars
    c.tableOperations().compact(table, compactConfig);
    try (Scanner scanner = c.createScanner(table, Authorizations.EMPTY)) {
        Stream<Entry<Key, Value>> stream = StreamSupport.stream(scanner.spliterator(), false);
        Map<String, Long> counts = // convert to row
        stream.map(e -> e.getKey().getRowData().toString()).map(// strip numbers off row
        r -> r.replaceAll("[0-9]+", "")).collect(// count different row types
        groupingBy(identity(), counting()));
        Assert.assertEquals(1l, (long) counts.getOrDefault("foo", 0l));
        Assert.assertEquals(2l, (long) counts.getOrDefault("bar", 0l));
        Assert.assertEquals(2, counts.size());
    }
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        write(bw, "foo2", "f1", "q1", "v4");
        write(bw, "foo3", "f1", "q1", "v5");
        write(bw, "foo4", "f1", "q1", "v6");
    }
    // this compaction should remove all foos because there are more foos than bars
    c.tableOperations().compact(table, compactConfig);
    try (Scanner scanner = c.createScanner(table, Authorizations.EMPTY)) {
        Stream<Entry<Key, Value>> stream = StreamSupport.stream(scanner.spliterator(), false);
        Map<String, Long> counts = // convert to row
        stream.map(e -> e.getKey().getRowData().toString()).map(// strip numbers off row
        r -> r.replaceAll("[0-9]+", "")).collect(// count different row types
        groupingBy(identity(), counting()));
        Assert.assertEquals(0l, (long) counts.getOrDefault("foo", 0l));
        Assert.assertEquals(2l, (long) counts.getOrDefault("bar", 0l));
        Assert.assertEquals(1, counts.size());
    }
}