Examples with ScannerOpts org.apache.accumulo.core.cli.ScannerOpts used on opensource projects

Example 21 with ScannerOpts

use of org.apache.accumulo.core.cli.ScannerOpts in project accumulo by apache.

the class CollectTabletStats method main.

public static void main(String[] args) throws Exception {
    final CollectOptions opts = new CollectOptions();
    final ScannerOpts scanOpts = new ScannerOpts();
    opts.parseArgs(CollectTabletStats.class.getName(), args, scanOpts);
    String[] columnsTmp = new String[] {};
    if (opts.columns != null)
        columnsTmp = opts.columns.split(",");
    final String[] columns = columnsTmp;
    final VolumeManager fs = VolumeManagerImpl.get();
    Instance instance = opts.getInstance();
    final ServerConfigurationFactory sconf = new ServerConfigurationFactory(instance);
    Credentials creds = new Credentials(opts.getPrincipal(), opts.getToken());
    ClientContext context = new ClientContext(instance, creds, sconf.getSystemConfiguration());
    Table.ID tableId = Tables.getTableId(instance, opts.getTableName());
    if (tableId == null) {
        log.error("Unable to find table named {}", opts.getTableName());
        System.exit(-1);
    }
    TreeMap<KeyExtent, String> tabletLocations = new TreeMap<>();
    List<KeyExtent> candidates = findTablets(context, !opts.selectFarTablets, opts.getTableName(), tabletLocations);
    if (candidates.size() < opts.numThreads) {
        System.err.println("ERROR : Unable to find " + opts.numThreads + " " + (opts.selectFarTablets ? "far" : "local") + " tablets");
        System.exit(-1);
    }
    List<KeyExtent> tabletsToTest = selectRandomTablets(opts.numThreads, candidates);
    Map<KeyExtent, List<FileRef>> tabletFiles = new HashMap<>();
    for (KeyExtent ke : tabletsToTest) {
        List<FileRef> files = getTabletFiles(context, ke);
        tabletFiles.put(ke, files);
    }
    System.out.println();
    System.out.println("run location      : " + InetAddress.getLocalHost().getHostName() + "/" + InetAddress.getLocalHost().getHostAddress());
    System.out.println("num threads       : " + opts.numThreads);
    System.out.println("table             : " + opts.getTableName());
    System.out.println("table id          : " + tableId);
    for (KeyExtent ke : tabletsToTest) {
        System.out.println("\t *** Information about tablet " + ke.getUUID() + " *** ");
        System.out.println("\t\t# files in tablet : " + tabletFiles.get(ke).size());
        System.out.println("\t\ttablet location   : " + tabletLocations.get(ke));
        reportHdfsBlockLocations(tabletFiles.get(ke));
    }
    System.out.println("%n*** RUNNING TEST ***%n");
    ExecutorService threadPool = Executors.newFixedThreadPool(opts.numThreads);
    for (int i = 0; i < opts.iterations; i++) {
        ArrayList<Test> tests = new ArrayList<>();
        for (final KeyExtent ke : tabletsToTest) {
            final List<FileRef> files = tabletFiles.get(ke);
            Test test = new Test(ke) {

                @Override
                public int runTest() throws Exception {
                    return readFiles(fs, sconf.getSystemConfiguration(), files, ke, columns);
                }
            };
            tests.add(test);
        }
        runTest("read files", tests, opts.numThreads, threadPool);
    }
    for (int i = 0; i < opts.iterations; i++) {
        ArrayList<Test> tests = new ArrayList<>();
        for (final KeyExtent ke : tabletsToTest) {
            final List<FileRef> files = tabletFiles.get(ke);
            Test test = new Test(ke) {

                @Override
                public int runTest() throws Exception {
                    return readFilesUsingIterStack(fs, sconf, files, opts.auths, ke, columns, false);
                }
            };
            tests.add(test);
        }
        runTest("read tablet files w/ system iter stack", tests, opts.numThreads, threadPool);
    }
    for (int i = 0; i < opts.iterations; i++) {
        ArrayList<Test> tests = new ArrayList<>();
        for (final KeyExtent ke : tabletsToTest) {
            final List<FileRef> files = tabletFiles.get(ke);
            Test test = new Test(ke) {

                @Override
                public int runTest() throws Exception {
                    return readFilesUsingIterStack(fs, sconf, files, opts.auths, ke, columns, true);
                }
            };
            tests.add(test);
        }
        runTest("read tablet files w/ table iter stack", tests, opts.numThreads, threadPool);
    }
    for (int i = 0; i < opts.iterations; i++) {
        ArrayList<Test> tests = new ArrayList<>();
        final Connector conn = opts.getConnector();
        for (final KeyExtent ke : tabletsToTest) {
            Test test = new Test(ke) {

                @Override
                public int runTest() throws Exception {
                    return scanTablet(conn, opts.getTableName(), opts.auths, scanOpts.scanBatchSize, ke.getPrevEndRow(), ke.getEndRow(), columns);
                }
            };
            tests.add(test);
        }
        runTest("read tablet data through accumulo", tests, opts.numThreads, threadPool);
    }
    for (final KeyExtent ke : tabletsToTest) {
        final Connector conn = opts.getConnector();
        threadPool.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    calcTabletStats(conn, opts.getTableName(), opts.auths, scanOpts.scanBatchSize, ke, columns);
                } catch (Exception e) {
                    log.error("Failed to calculate tablet stats.", e);
                }
            }
        });
    }
    threadPool.shutdown();
}

Example 22 with ScannerOpts

use of org.apache.accumulo.core.cli.ScannerOpts in project accumulo by apache.

the class CollectTabletStatsTest method paramsDefaulThreadTest.

@Test
public void paramsDefaulThreadTest() {
    String tablename = "aTable";
    String[] args = { "-t", tablename, "--iterations", "2" };
    final CollectTabletStats.CollectOptions opts = new CollectTabletStats.CollectOptions();
    final ScannerOpts scanOpts = new ScannerOpts();
    opts.parseArgs(CollectTabletStats.class.getName(), args, scanOpts);
    assertEquals("Check iterations is set, default is 3", 2, opts.iterations);
    assertEquals("Check tablename is set", 0, tablename.compareTo(opts.getTableName()));
    assertEquals("Check default numThreads", 1, opts.numThreads);
}

Example 23 with ScannerOpts

use of org.apache.accumulo.core.cli.ScannerOpts in project accumulo by apache.

the class CollectTabletStatsTest method paramsSetThreadsTest.

@Test
public void paramsSetThreadsTest() {
    String tablename = "aTable";
    String[] args = { "-t", tablename, "--iterations", "2", "--numThreads", "99" };
    final CollectTabletStats.CollectOptions opts = new CollectTabletStats.CollectOptions();
    final ScannerOpts scanOpts = new ScannerOpts();
    opts.parseArgs(CollectTabletStats.class.getName(), args, scanOpts);
    assertEquals("Check iterations is set, default is 3", 2, opts.iterations);
    assertEquals("Check tablename is set", 0, tablename.compareTo(opts.getTableName()));
    assertEquals("Check numThreads is set", 99, opts.numThreads);
    System.out.println(opts.columns);
}

Example 24 with ScannerOpts

use of org.apache.accumulo.core.cli.ScannerOpts in project accumulo by apache.

the class TraceDump method main.

public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    ScannerOpts scanOpts = new ScannerOpts();
    opts.parseArgs(TraceDump.class.getName(), args, scanOpts);
    int code = 0;
    if (opts.list) {
        code = listSpans(opts, scanOpts);
    }
    if (code == 0 && opts.dump) {
        code = dumpTrace(opts, scanOpts);
    }
    System.exit(code);
}

Example 25 with ScannerOpts

use of org.apache.accumulo.core.cli.ScannerOpts in project accumulo by apache.

the class SplitIT method tabletShouldSplit.

@Test
public void tabletShouldSplit() throws Exception {
    Connector c = getConnector();
    String table = getUniqueNames(1)[0];
    c.tableOperations().create(table);
    c.tableOperations().setProperty(table, Property.TABLE_SPLIT_THRESHOLD.getKey(), "256K");
    c.tableOperations().setProperty(table, Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1K");
    TestIngest.Opts opts = new TestIngest.Opts();
    VerifyIngest.Opts vopts = new VerifyIngest.Opts();
    opts.rows = 100000;
    opts.setTableName(table);
    ClientConfiguration clientConfig = cluster.getClientConfig();
    if (clientConfig.hasSasl()) {
        opts.updateKerberosCredentials(clientConfig);
        vopts.updateKerberosCredentials(clientConfig);
    } else {
        opts.setPrincipal(getAdminPrincipal());
        vopts.setPrincipal(getAdminPrincipal());
    }
    TestIngest.ingest(c, opts, new BatchWriterOpts());
    vopts.rows = opts.rows;
    vopts.setTableName(table);
    VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
    while (c.tableOperations().listSplits(table).size() < 10) {
        sleepUninterruptibly(15, TimeUnit.SECONDS);
    }
    Table.ID id = Table.ID.of(c.tableOperations().tableIdMap().get(table));
    try (Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        KeyExtent extent = new KeyExtent(id, null, null);
        s.setRange(extent.toMetadataRange());
        MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(s);
        int count = 0;
        int shortened = 0;
        for (Entry<Key, Value> entry : s) {
            extent = new KeyExtent(entry.getKey().getRow(), entry.getValue());
            if (extent.getEndRow() != null && extent.getEndRow().toString().length() < 14)
                shortened++;
            count++;
        }
        assertTrue("Shortened should be greater than zero: " + shortened, shortened > 0);
        assertTrue("Count should be cgreater than 10: " + count, count > 10);
    }
    String[] args;
    if (clientConfig.hasSasl()) {
        ClusterUser rootUser = getAdminUser();
        args = new String[] { "-i", cluster.getInstanceName(), "-u", rootUser.getPrincipal(), "--keytab", rootUser.getKeytab().getAbsolutePath(), "-z", cluster.getZooKeepers() };
    } else {
        PasswordToken token = (PasswordToken) getAdminToken();
        args = new String[] { "-i", cluster.getInstanceName(), "-u", "root", "-p", new String(token.getPassword(), UTF_8), "-z", cluster.getZooKeepers() };
    }
    assertEquals(0, getCluster().getClusterControl().exec(CheckForMetadataProblems.class, args));
}