Examples with BatchWriterOpts org.apache.accumulo.core.cli.BatchWriterOpts used on opensource projects

Example 1 with BatchWriterOpts

use of org.apache.accumulo.core.cli.BatchWriterOpts in project accumulo by apache.

the class TestBinaryRows method main.

public static void main(String[] args) {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    ScannerOpts scanOpts = new ScannerOpts();
    opts.parseArgs(TestBinaryRows.class.getName(), args, scanOpts, bwOpts);
    try {
        runTest(opts.getConnector(), opts, bwOpts, scanOpts);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 2 with BatchWriterOpts

use of org.apache.accumulo.core.cli.BatchWriterOpts in project accumulo by apache.

the class TestIngest method main.

public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(TestIngest.class.getName(), args, bwOpts);
    String name = TestIngest.class.getSimpleName();
    DistributedTrace.enable(name);
    try {
        opts.startTracing(name);
        if (opts.debug)
            Logger.getLogger(TabletServerBatchWriter.class.getName()).setLevel(Level.TRACE);
        // test batch update
        ingest(opts.getConnector(), opts, bwOpts);
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        opts.stopTracing();
        DistributedTrace.disable();
    }
}

Example 3 with BatchWriterOpts

use of org.apache.accumulo.core.cli.BatchWriterOpts in project accumulo by apache.

the class TestRandomDeletes method main.

public static void main(String[] args) {
    ClientOnDefaultTable opts = new ClientOnDefaultTable("test_ingest");
    ScannerOpts scanOpts = new ScannerOpts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(TestRandomDeletes.class.getName(), args, scanOpts, bwOpts);
    log.info("starting random delete test");
    try {
        long deleted = 0;
        String tableName = opts.getTableName();
        TreeSet<RowColumn> doomed = scanAll(opts, scanOpts, tableName);
        log.info("Got {} rows", doomed.size());
        long startTime = System.currentTimeMillis();
        while (true) {
            long half = scrambleDeleteHalfAndCheck(opts, scanOpts, bwOpts, tableName, doomed);
            deleted += half;
            if (half == 0)
                break;
        }
        long stopTime = System.currentTimeMillis();
        long elapsed = (stopTime - startTime) / 1000;
        log.info("deleted {} values in {} seconds", deleted, elapsed);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 4 with BatchWriterOpts

use of org.apache.accumulo.core.cli.BatchWriterOpts in project accumulo by apache.

the class ChaoticBalancerIT method test.

@Test
public void test() throws Exception {
    Connector c = getConnector();
    String[] names = getUniqueNames(1);
    String tableName = names[0];
    NewTableConfiguration ntc = new NewTableConfiguration();
    ntc.setProperties(Stream.of(new Pair<>(Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K"), new Pair<>(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1K")).collect(Collectors.toMap(k -> k.getFirst(), v -> v.getSecond())));
    c.tableOperations().create(tableName, ntc);
    TestIngest.Opts opts = new TestIngest.Opts();
    VerifyIngest.Opts vopts = new VerifyIngest.Opts();
    vopts.rows = opts.rows = 20000;
    opts.setTableName(tableName);
    vopts.setTableName(tableName);
    ClientConfiguration clientConfig = getCluster().getClientConfig();
    if (clientConfig.hasSasl()) {
        opts.updateKerberosCredentials(clientConfig);
        vopts.updateKerberosCredentials(clientConfig);
    } else {
        opts.setPrincipal(getAdminPrincipal());
        vopts.setPrincipal(getAdminPrincipal());
    }
    TestIngest.ingest(c, opts, new BatchWriterOpts());
    c.tableOperations().flush(tableName, null, null, true);
    VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
}

Example 5 with BatchWriterOpts

use of org.apache.accumulo.core.cli.BatchWriterOpts in project accumulo by apache.

the class BalanceInPresenceOfOfflineTableIT method test.

@Test
public void test() throws Exception {
    log.info("Test that balancing is not stopped by an offline table with outstanding migrations.");
    log.debug("starting test ingestion");
    TestIngest.Opts opts = new TestIngest.Opts();
    VerifyIngest.Opts vopts = new VerifyIngest.Opts();
    ClientConfiguration conf = cluster.getClientConfig();
    if (conf.hasSasl()) {
        opts.updateKerberosCredentials(cluster.getClientConfig());
        vopts.updateKerberosCredentials(cluster.getClientConfig());
    } else {
        opts.setPrincipal("root");
        vopts.setPrincipal("root");
    }
    vopts.rows = opts.rows = 200000;
    opts.setTableName(TEST_TABLE);
    TestIngest.ingest(connector, opts, new BatchWriterOpts());
    connector.tableOperations().flush(TEST_TABLE, null, null, true);
    vopts.setTableName(TEST_TABLE);
    VerifyIngest.verifyIngest(connector, vopts, new ScannerOpts());
    log.debug("waiting for balancing, up to ~5 minutes to allow for migration cleanup.");
    final long startTime = System.currentTimeMillis();
    long currentWait = 10 * 1000;
    boolean balancingWorked = false;
    Credentials creds = new Credentials(getAdminPrincipal(), getAdminToken());
    while (!balancingWorked && (System.currentTimeMillis() - startTime) < ((5 * 60 + 15) * 1000)) {
        Thread.sleep(currentWait);
        currentWait *= 2;
        log.debug("fetch the list of tablets assigned to each tserver.");
        MasterClientService.Iface client = null;
        MasterMonitorInfo stats = null;
        Instance instance = new ZooKeeperInstance(cluster.getClientConfig());
        while (true) {
            try {
                client = MasterClient.getConnectionWithRetry(new ClientContext(instance, creds, cluster.getClientConfig()));
                stats = client.getMasterStats(Tracer.traceInfo(), creds.toThrift(instance));
                break;
            } catch (ThriftSecurityException exception) {
                throw new AccumuloSecurityException(exception);
            } catch (ThriftNotActiveServiceException e) {
                // Let it loop, fetching a new location
                log.debug("Contacted a Master which is no longer active, retrying");
                sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            } catch (TException exception) {
                throw new AccumuloException(exception);
            } finally {
                if (client != null) {
                    MasterClient.close(client);
                }
            }
        }
        if (stats.getTServerInfoSize() < 2) {
            log.debug("we need >= 2 servers. sleeping for {}ms", currentWait);
            continue;
        }
        if (stats.getUnassignedTablets() != 0) {
            log.debug("We shouldn't have unassigned tablets. sleeping for {}ms", currentWait);
            continue;
        }
        long[] tabletsPerServer = new long[stats.getTServerInfoSize()];
        Arrays.fill(tabletsPerServer, 0l);
        for (int i = 0; i < stats.getTServerInfoSize(); i++) {
            for (Map.Entry<String, TableInfo> entry : stats.getTServerInfo().get(i).getTableMap().entrySet()) {
                tabletsPerServer[i] += entry.getValue().getTablets();
            }
        }
        if (tabletsPerServer[0] <= 10) {
            log.debug("We should have > 10 tablets. sleeping for {}ms", currentWait);
            continue;
        }
        long min = NumberUtils.min(tabletsPerServer), max = NumberUtils.max(tabletsPerServer);
        log.debug("Min={}, Max={}", min, max);
        if ((min / ((double) max)) < 0.5) {
            log.debug("ratio of min to max tablets per server should be roughly even. sleeping for {}ms", currentWait);
            continue;
        }
        balancingWorked = true;
    }
    Assert.assertTrue("did not properly balance", balancingWorked);
}