Examples with BatchWriterOpts org.apache.accumulo.core.cli.BatchWriterOpts used on opensource projects

Example 16 with BatchWriterOpts

use of org.apache.accumulo.core.cli.BatchWriterOpts in project accumulo by apache.

the class RemoveEntriesForMissingFiles method main.

public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    ScannerOpts scanOpts = new ScannerOpts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(RemoveEntriesForMissingFiles.class.getName(), args, scanOpts, bwOpts);
    checkAllTables(new ClientContext(opts.getInstance(), new Credentials(opts.getPrincipal(), opts.getToken()), ClientConfiguration.loadDefault()), opts.fix);
}

Example 17 with BatchWriterOpts

use of org.apache.accumulo.core.cli.BatchWriterOpts in project accumulo by apache.

the class DynamicThreadPoolsIT method test.

@Test
public void test() throws Exception {
    final String[] tables = getUniqueNames(15);
    String firstTable = tables[0];
    Connector c = getConnector();
    c.instanceOperations().setProperty(Property.TSERV_MAJC_MAXCONCURRENT.getKey(), "5");
    TestIngest.Opts opts = new TestIngest.Opts();
    opts.rows = 500 * 1000;
    opts.createTable = true;
    opts.setTableName(firstTable);
    ClientConfiguration clientConf = cluster.getClientConfig();
    if (clientConf.hasSasl()) {
        opts.updateKerberosCredentials(clientConf);
    } else {
        opts.setPrincipal(getAdminPrincipal());
    }
    TestIngest.ingest(c, opts, new BatchWriterOpts());
    c.tableOperations().flush(firstTable, null, null, true);
    for (int i = 1; i < tables.length; i++) c.tableOperations().clone(firstTable, tables[i], true, null, null);
    // time between checks of the thread pool sizes
    sleepUninterruptibly(11, TimeUnit.SECONDS);
    Credentials creds = new Credentials(getAdminPrincipal(), getAdminToken());
    for (int i = 1; i < tables.length; i++) c.tableOperations().compact(tables[i], null, null, true, false);
    for (int i = 0; i < 30; i++) {
        int count = 0;
        MasterClientService.Iface client = null;
        MasterMonitorInfo stats = null;
        while (true) {
            try {
                client = MasterClient.getConnectionWithRetry(new ClientContext(c.getInstance(), creds, clientConf));
                stats = client.getMasterStats(Tracer.traceInfo(), creds.toThrift(c.getInstance()));
                break;
            } catch (ThriftNotActiveServiceException e) {
                // Let it loop, fetching a new location
                sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            } finally {
                if (client != null)
                    MasterClient.close(client);
            }
        }
        for (TabletServerStatus server : stats.tServerInfo) {
            for (TableInfo table : server.tableMap.values()) {
                count += table.majors.running;
            }
        }
        System.out.println("count " + count);
        if (count > 3)
            return;
        sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    }
    fail("Could not observe higher number of threads after changing the config");
}

Example 18 with BatchWriterOpts

use of org.apache.accumulo.core.cli.BatchWriterOpts in project accumulo by apache.

the class FunctionalTestUtils method createRFiles.

public static void createRFiles(final Connector c, final FileSystem fs, String path, int rows, int splits, int threads) throws Exception {
    fs.delete(new Path(path), true);
    ExecutorService threadPool = Executors.newFixedThreadPool(threads);
    final AtomicBoolean fail = new AtomicBoolean(false);
    for (int i = 0; i < rows; i += rows / splits) {
        final TestIngest.Opts opts = new TestIngest.Opts();
        opts.outputFile = String.format("%s/mf%s", path, i);
        opts.random = 56;
        opts.timestamp = 1;
        opts.dataSize = 50;
        opts.rows = rows / splits;
        opts.startRow = i;
        opts.cols = 1;
        threadPool.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    TestIngest.ingest(c, fs, opts, new BatchWriterOpts());
                } catch (Exception e) {
                    fail.set(true);
                }
            }
        });
    }
    threadPool.shutdown();
    threadPool.awaitTermination(1, TimeUnit.HOURS);
    assertFalse(fail.get());
}

Example 19 with BatchWriterOpts

use of org.apache.accumulo.core.cli.BatchWriterOpts in project accumulo by apache.

the class GarbageCollectorIT method gcTest.

@Test
public void gcTest() throws Exception {
    killMacGc();
    Connector c = getConnector();
    c.tableOperations().create("test_ingest");
    c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "5K");
    TestIngest.Opts opts = new TestIngest.Opts();
    VerifyIngest.Opts vopts = new VerifyIngest.Opts();
    vopts.rows = opts.rows = 10000;
    vopts.cols = opts.cols = 1;
    opts.setPrincipal("root");
    vopts.setPrincipal("root");
    TestIngest.ingest(c, cluster.getFileSystem(), opts, new BatchWriterOpts());
    c.tableOperations().compact("test_ingest", null, null, true, true);
    int before = countFiles();
    while (true) {
        sleepUninterruptibly(1, TimeUnit.SECONDS);
        int more = countFiles();
        if (more <= before)
            break;
        before = more;
    }
    // restart GC
    getCluster().start();
    sleepUninterruptibly(15, TimeUnit.SECONDS);
    int after = countFiles();
    VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
    assertTrue(after < before);
}

Example 20 with BatchWriterOpts

use of org.apache.accumulo.core.cli.BatchWriterOpts in project accumulo by apache.

the class GarbageCollectorIT method gcLotsOfCandidatesIT.

@Test
public void gcLotsOfCandidatesIT() throws Exception {
    killMacGc();
    log.info("Filling metadata table with bogus delete flags");
    Connector c = getConnector();
    addEntries(c, new BatchWriterOpts());
    cluster.getConfig().setDefaultMemory(10, MemoryUnit.MEGABYTE);
    Process gc = cluster.exec(SimpleGarbageCollector.class);
    sleepUninterruptibly(20, TimeUnit.SECONDS);
    String output = "";
    while (!output.contains("delete candidates has exceeded")) {
        byte[] buffer = new byte[10 * 1024];
        try {
            int n = gc.getInputStream().read(buffer);
            output = new String(buffer, 0, n, UTF_8);
        } catch (IOException ex) {
            break;
        }
    }
    gc.destroy();
    assertTrue(output.contains("delete candidates has exceeded"));
}