Examples with CompactionConfig org.apache.accumulo.core.client.admin.CompactionConfig used on opensource projects

Example 26 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class KerberosIT method testUserPrivilegesForTable.

@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "path provided by test")
@Test
public void testUserPrivilegesForTable() throws Exception {
    String user1 = testName.getMethodName();
    final File user1Keytab = new File(kdc.getKeytabDir(), user1 + ".keytab");
    if (user1Keytab.exists() && !user1Keytab.delete()) {
        log.warn("Unable to delete {}", user1Keytab);
    }
    // Create some new users -- cannot contain realm
    kdc.createPrincipal(user1Keytab, user1);
    final String qualifiedUser1 = kdc.qualifyUser(user1);
    // Log in as user1
    UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(qualifiedUser1, user1Keytab.getAbsolutePath());
    log.info("Logged in as {}", user1);
    ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
        // Indirectly creates this user when we use it
        AccumuloClient client = mac.createAccumuloClient(qualifiedUser1, new KerberosToken());
        log.info("Created client as {}", qualifiedUser1);
        // The new user should have no system permissions
        for (SystemPermission perm : SystemPermission.values()) {
            assertFalse(client.securityOperations().hasSystemPermission(qualifiedUser1, perm));
        }
        return null;
    });
    final String table = testName.getMethodName() + "_user_table";
    final String viz = "viz";
    ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
    ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
        AccumuloClient client = mac.createAccumuloClient(rootUser.getPrincipal(), new KerberosToken());
        client.tableOperations().create(table);
        // Give our unprivileged user permission on the table we made for them
        client.securityOperations().grantTablePermission(qualifiedUser1, table, TablePermission.READ);
        client.securityOperations().grantTablePermission(qualifiedUser1, table, TablePermission.WRITE);
        client.securityOperations().grantTablePermission(qualifiedUser1, table, TablePermission.ALTER_TABLE);
        client.securityOperations().grantTablePermission(qualifiedUser1, table, TablePermission.DROP_TABLE);
        client.securityOperations().changeUserAuthorizations(qualifiedUser1, new Authorizations(viz));
        return null;
    });
    // Switch back to the original user
    ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(qualifiedUser1, user1Keytab.getAbsolutePath());
    ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
        AccumuloClient client = mac.createAccumuloClient(qualifiedUser1, new KerberosToken());
        // Make sure we can actually use the table we made
        // Write data
        final long ts = 1000L;
        try (BatchWriter bw = client.createBatchWriter(table)) {
            Mutation m = new Mutation("a");
            m.put("b", "c", new ColumnVisibility(viz.getBytes()), ts, "d");
            bw.addMutation(m);
        }
        // Compact
        client.tableOperations().compact(table, new CompactionConfig().setWait(true).setFlush(true));
        // Alter
        client.tableOperations().setProperty(table, Property.TABLE_BLOOM_ENABLED.getKey(), "true");
        // Read (and proper authorizations)
        try (Scanner s = client.createScanner(table, new Authorizations(viz))) {
            Iterator<Entry<Key, Value>> iter = s.iterator();
            assertTrue("No results from iterator", iter.hasNext());
            Entry<Key, Value> entry = iter.next();
            assertEquals(new Key("a", "b", "c", viz, ts), entry.getKey());
            assertEquals(new Value("d"), entry.getValue());
            assertFalse("Had more results from iterator", iter.hasNext());
            return null;
        }
    });
}

Example 27 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class CompactionIT method testConfigurer.

@Test
public void testConfigurer() throws Exception {
    String tableName = this.getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        Map<String, String> props = Map.of(Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
        NewTableConfiguration ntc = new NewTableConfiguration().setProperties(props);
        client.tableOperations().create(tableName, ntc);
        byte[] data = new byte[100000];
        Arrays.fill(data, (byte) 65);
        try (var writer = client.createBatchWriter(tableName)) {
            for (int row = 0; row < 10; row++) {
                Mutation m = new Mutation(row + "");
                m.at().family("big").qualifier("stuff").put(data);
                writer.addMutation(m);
            }
        }
        client.tableOperations().flush(tableName, null, null, true);
        // without compression, expect file to be large
        long sizes = CompactionExecutorIT.getFileSizes(client, tableName);
        assertTrue("Unexpected files sizes : " + sizes, sizes > data.length * 10 && sizes < data.length * 11);
        client.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setConfigurer(new PluginConfig(CompressionConfigurer.class.getName(), Map.of(CompressionConfigurer.LARGE_FILE_COMPRESSION_TYPE, "gz", CompressionConfigurer.LARGE_FILE_COMPRESSION_THRESHOLD, data.length + ""))));
        // after compacting with compression, expect small file
        sizes = CompactionExecutorIT.getFileSizes(client, tableName);
        assertTrue("Unexpected files sizes: data: " + data.length + ", file:" + sizes, sizes < data.length);
        client.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
        // after compacting without compression, expect big files again
        sizes = CompactionExecutorIT.getFileSizes(client, tableName);
        assertTrue("Unexpected files sizes : " + sizes, sizes > data.length * 10 && sizes < data.length * 11);
    }
}

Example 28 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class CompactionIT method testUserCompactionCancellation.

@Test
public void testUserCompactionCancellation() throws Exception {
    final String table1 = this.getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        client.tableOperations().create(table1);
        try (BatchWriter bw = client.createBatchWriter(table1)) {
            for (int i = 1; i <= MAX_DATA; i++) {
                Mutation m = new Mutation(Integer.toString(i));
                m.put("cf", "cq", new Value());
                bw.addMutation(m);
                bw.flush();
                client.tableOperations().flush(table1, null, null, true);
            }
        }
        final AtomicReference<Exception> error = new AtomicReference<>();
        final AtomicBoolean started = new AtomicBoolean(false);
        Thread t = new Thread(() -> {
            try {
                started.set(true);
                IteratorSetting setting = new IteratorSetting(50, "sleepy", SlowIterator.class);
                setting.addOption("sleepTime", "3000");
                setting.addOption("seekSleepTime", "3000");
                client.tableOperations().attachIterator(table1, setting, EnumSet.of(IteratorScope.majc));
                client.tableOperations().compact(table1, new CompactionConfig().setWait(true));
            } catch (AccumuloSecurityException | TableNotFoundException | AccumuloException e) {
                error.set(e);
            }
        });
        t.start();
        while (!started.get()) {
            Thread.sleep(1000);
        }
        client.tableOperations().cancelCompaction(table1);
        t.join();
        Exception e = error.get();
        assertNotNull(e);
        assertEquals(TableOperationsImpl.COMPACTION_CANCELED_MSG, e.getMessage());
    }
}

Example 29 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class CompactionIT method testTableDeletedDuringUserCompaction.

@Test
public void testTableDeletedDuringUserCompaction() throws Exception {
    final String table1 = this.getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        client.tableOperations().create(table1);
        try (BatchWriter bw = client.createBatchWriter(table1)) {
            for (int i = 1; i <= MAX_DATA; i++) {
                Mutation m = new Mutation(Integer.toString(i));
                m.put("cf", "cq", new Value());
                bw.addMutation(m);
                bw.flush();
                client.tableOperations().flush(table1, null, null, true);
            }
        }
        final AtomicReference<Exception> error = new AtomicReference<>();
        final AtomicBoolean started = new AtomicBoolean(false);
        Thread t = new Thread(() -> {
            try {
                started.set(true);
                IteratorSetting setting = new IteratorSetting(50, "sleepy", SlowIterator.class);
                setting.addOption("sleepTime", "3000");
                setting.addOption("seekSleepTime", "3000");
                client.tableOperations().attachIterator(table1, setting, EnumSet.of(IteratorScope.majc));
                client.tableOperations().compact(table1, new CompactionConfig().setWait(true));
            } catch (AccumuloSecurityException | TableNotFoundException | AccumuloException e) {
                error.set(e);
            }
        });
        t.start();
        while (!started.get()) {
            Thread.sleep(1000);
        }
        client.tableOperations().delete(table1);
        t.join();
        Exception e = error.get();
        assertNotNull(e);
        assertEquals(TableOperationsImpl.COMPACTION_CANCELED_MSG, e.getMessage());
    }
}

Example 30 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class CompactionIT method testBadSelector.

@Test
public void testBadSelector() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        final String tableName = getUniqueNames(1)[0];
        NewTableConfiguration tc = new NewTableConfiguration();
        // Ensure compactions don't kick off
        tc.setProperties(Map.of(Property.TABLE_MAJC_RATIO.getKey(), "10.0"));
        c.tableOperations().create(tableName, tc);
        // Create multiple RFiles
        try (BatchWriter bw = c.createBatchWriter(tableName)) {
            for (int i = 1; i <= 4; i++) {
                Mutation m = new Mutation(Integer.toString(i));
                m.put("cf", "cq", new Value());
                bw.addMutation(m);
                bw.flush();
                c.tableOperations().flush(tableName, null, null, true);
            }
        }
        List<String> files = FunctionalTestUtils.getRFilePaths(c, tableName);
        assertEquals(4, files.size());
        String subset = files.get(0).substring(files.get(0).lastIndexOf('/') + 1) + "," + files.get(3).substring(files.get(3).lastIndexOf('/') + 1);
        CompactionConfig config = new CompactionConfig().setSelector(new PluginConfig(RandomErrorThrowingSelector.class.getName(), Map.of(RandomErrorThrowingSelector.FILE_LIST_PARAM, subset))).setWait(true);
        c.tableOperations().compact(tableName, config);
        // check that the subset of files selected are compacted, but the others remain untouched
        List<String> filesAfterCompact = FunctionalTestUtils.getRFilePaths(c, tableName);
        assertFalse(filesAfterCompact.contains(files.get(0)));
        assertTrue(filesAfterCompact.contains(files.get(1)));
        assertTrue(filesAfterCompact.contains(files.get(2)));
        assertFalse(filesAfterCompact.contains(files.get(3)));
        List<String> rows = new ArrayList<>();
        c.createScanner(tableName).forEach((k, v) -> rows.add(k.getRow().toString()));
        assertEquals(List.of("1", "2", "3", "4"), rows);
    }
}