Examples with DiskUsage org.apache.accumulo.core.client.admin.DiskUsage used on opensource projects

Example 1 with DiskUsage

use of org.apache.accumulo.core.client.admin.DiskUsage in project accumulo by apache.

the class TableOperationsIT method getDiskUsage.

@Test
public void getDiskUsage() throws TableExistsException, AccumuloException, AccumuloSecurityException, TableNotFoundException, TException {
    final String[] names = getUniqueNames(2);
    String tableName = names[0];
    connector.tableOperations().create(tableName);
    // verify 0 disk usage
    List<DiskUsage> diskUsages = connector.tableOperations().getDiskUsage(Collections.singleton(tableName));
    assertEquals(1, diskUsages.size());
    assertEquals(1, diskUsages.get(0).getTables().size());
    assertEquals(Long.valueOf(0), diskUsages.get(0).getUsage());
    assertEquals(tableName, diskUsages.get(0).getTables().first());
    // add some data
    BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
    Mutation m = new Mutation("a");
    m.put("b", "c", new Value("abcde".getBytes()));
    bw.addMutation(m);
    bw.flush();
    bw.close();
    connector.tableOperations().compact(tableName, new Text("A"), new Text("z"), true, true);
    // verify we have usage
    diskUsages = connector.tableOperations().getDiskUsage(Collections.singleton(tableName));
    assertEquals(1, diskUsages.size());
    assertEquals(1, diskUsages.get(0).getTables().size());
    assertTrue(diskUsages.get(0).getUsage() > 0);
    assertEquals(tableName, diskUsages.get(0).getTables().first());
    String newTable = names[1];
    // clone table
    connector.tableOperations().clone(tableName, newTable, false, null, null);
    // verify tables are exactly the same
    Set<String> tables = new HashSet<>();
    tables.add(tableName);
    tables.add(newTable);
    diskUsages = connector.tableOperations().getDiskUsage(tables);
    assertEquals(1, diskUsages.size());
    assertEquals(2, diskUsages.get(0).getTables().size());
    assertTrue(diskUsages.get(0).getUsage() > 0);
    connector.tableOperations().compact(tableName, new Text("A"), new Text("z"), true, true);
    connector.tableOperations().compact(newTable, new Text("A"), new Text("z"), true, true);
    // verify tables have differences
    diskUsages = connector.tableOperations().getDiskUsage(tables);
    assertEquals(2, diskUsages.size());
    assertEquals(1, diskUsages.get(0).getTables().size());
    assertEquals(1, diskUsages.get(1).getTables().size());
    assertTrue(diskUsages.get(0).getUsage() > 0);
    assertTrue(diskUsages.get(1).getUsage() > 0);
    connector.tableOperations().delete(tableName);
}

Example 2 with DiskUsage

use of org.apache.accumulo.core.client.admin.DiskUsage in project accumulo by apache.

the class CloneTestIT method testDeleteClone.

@Test
public void testDeleteClone() throws Exception {
    String[] tableNames = getUniqueNames(3);
    String table1 = tableNames[0];
    String table2 = tableNames[1];
    String table3 = tableNames[2];
    Connector c = getConnector();
    AccumuloCluster cluster = getCluster();
    Assume.assumeTrue(cluster instanceof MiniAccumuloClusterImpl);
    MiniAccumuloClusterImpl mac = (MiniAccumuloClusterImpl) cluster;
    String rootPath = mac.getConfig().getDir().getAbsolutePath();
    // verify that deleting a new table removes the files
    c.tableOperations().create(table3);
    writeData(table3, c).close();
    c.tableOperations().flush(table3, null, null, true);
    // check for files
    FileSystem fs = getCluster().getFileSystem();
    String id = c.tableOperations().tableIdMap().get(table3);
    FileStatus[] status = fs.listStatus(new Path(rootPath + "/accumulo/tables/" + id));
    assertTrue(status.length > 0);
    // verify disk usage
    List<DiskUsage> diskUsage = c.tableOperations().getDiskUsage(Collections.singleton(table3));
    assertEquals(1, diskUsage.size());
    assertTrue(diskUsage.get(0).getUsage() > 100);
    // delete the table
    c.tableOperations().delete(table3);
    // verify its gone from the file system
    Path tablePath = new Path(rootPath + "/accumulo/tables/" + id);
    if (fs.exists(tablePath)) {
        status = fs.listStatus(tablePath);
        assertTrue(status == null || status.length == 0);
    }
    c.tableOperations().create(table1);
    BatchWriter bw = writeData(table1, c);
    Map<String, String> props = new HashMap<>();
    props.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "500K");
    Set<String> exclude = new HashSet<>();
    exclude.add(Property.TABLE_FILE_MAX.getKey());
    c.tableOperations().clone(table1, table2, true, props, exclude);
    Mutation m3 = new Mutation("009");
    m3.put("data", "x", "1");
    m3.put("data", "y", "2");
    bw.addMutation(m3);
    bw.close();
    // delete source table, should not affect clone
    c.tableOperations().delete(table1);
    checkData(table2, c);
    c.tableOperations().compact(table2, null, null, true, true);
    checkData(table2, c);
    c.tableOperations().delete(table2);
}

Example 3 with DiskUsage

use of org.apache.accumulo.core.client.admin.DiskUsage in project accumulo by apache.

the class DUCommand method execute.

@Override
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws IOException, TableNotFoundException, NamespaceNotFoundException {
    final SortedSet<String> tables = new TreeSet<>(Arrays.asList(cl.getArgs()));
    if (cl.hasOption(ShellOptions.tableOption)) {
        tables.add(cl.getOptionValue(ShellOptions.tableOption));
    }
    if (cl.hasOption(optNamespace.getOpt())) {
        Instance instance = shellState.getInstance();
        Namespace.ID namespaceId = Namespaces.getNamespaceId(instance, cl.getOptionValue(optNamespace.getOpt()));
        tables.addAll(Namespaces.getTableNames(instance, namespaceId));
    }
    boolean prettyPrint = cl.hasOption(optHumanReadble.getOpt()) ? true : false;
    // Add any patterns
    if (cl.hasOption(optTablePattern.getOpt())) {
        for (String table : shellState.getConnector().tableOperations().list()) {
            if (table.matches(cl.getOptionValue(optTablePattern.getOpt()))) {
                tables.add(table);
            }
        }
    }
    // If we didn't get any tables, and we have a table selected, add the current table
    if (tables.isEmpty() && !shellState.getTableName().isEmpty()) {
        tables.add(shellState.getTableName());
    }
    // sanity check...make sure the user-specified tables exist
    for (String tableName : tables) {
        if (!shellState.getConnector().tableOperations().exists(tableName)) {
            throw new TableNotFoundException(tableName, tableName, "specified table that doesn't exist");
        }
    }
    try {
        String valueFormat = prettyPrint ? "%9s" : "%,24d";
        for (DiskUsage usage : shellState.getConnector().tableOperations().getDiskUsage(tables)) {
            Object value = prettyPrint ? NumUtil.bigNumberForSize(usage.getUsage()) : usage.getUsage();
            shellState.getReader().println(String.format(valueFormat + " %s", value, usage.getTables()));
        }
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
    return 0;
}

Example 4 with DiskUsage

use of org.apache.accumulo.core.client.admin.DiskUsage in project accumulo by apache.

the class MockTableOperations method getDiskUsage.

@Override
public List<DiskUsage> getDiskUsage(Set<String> tables) throws AccumuloException, AccumuloSecurityException {
    List<DiskUsage> diskUsages = new ArrayList<>();
    diskUsages.add(new DiskUsage(new TreeSet<>(tables), 0l));
    return diskUsages;
}

Example 5 with DiskUsage

use of org.apache.accumulo.core.client.admin.DiskUsage in project accumulo by apache.

the class VolumeIT method test.

@Test
public void test() throws Exception {
    // create a table
    Connector connector = getConnector();
    String tableName = getUniqueNames(1)[0];
    connector.tableOperations().create(tableName);
    SortedSet<Text> partitions = new TreeSet<>();
    // with some splits
    for (String s : "d,m,t".split(",")) partitions.add(new Text(s));
    connector.tableOperations().addSplits(tableName, partitions);
    // scribble over the splits
    BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
    String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
    for (String s : rows) {
        Mutation m = new Mutation(new Text(s));
        m.put(EMPTY, EMPTY, EMPTY_VALUE);
        bw.addMutation(m);
    }
    bw.close();
    // write the data to disk, read it back
    connector.tableOperations().flush(tableName, null, null, true);
    try (Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY)) {
        int i = 0;
        for (Entry<Key, Value> entry : scanner) {
            assertEquals(rows[i++], entry.getKey().getRow().toString());
        }
    }
    // verify the new files are written to the different volumes
    try (Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        scanner.setRange(new Range("1", "1<"));
        scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
        int fileCount = 0;
        for (Entry<Key, Value> entry : scanner) {
            boolean inV1 = entry.getKey().getColumnQualifier().toString().contains(v1.toString());
            boolean inV2 = entry.getKey().getColumnQualifier().toString().contains(v2.toString());
            assertTrue(inV1 || inV2);
            fileCount++;
        }
        assertEquals(4, fileCount);
        List<DiskUsage> diskUsage = connector.tableOperations().getDiskUsage(Collections.singleton(tableName));
        assertEquals(1, diskUsage.size());
        long usage = diskUsage.get(0).getUsage();
        log.debug("usage {}", usage);
        assertTrue(usage > 700 && usage < 800);
    }
}