Examples with ScannerOpts org.apache.accumulo.core.cli.ScannerOpts used on opensource projects

Example 1 with ScannerOpts

use of org.apache.accumulo.core.cli.ScannerOpts in project accumulo by apache.

the class TestBinaryRows method main.

public static void main(String[] args) {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    ScannerOpts scanOpts = new ScannerOpts();
    opts.parseArgs(TestBinaryRows.class.getName(), args, scanOpts, bwOpts);
    try {
        runTest(opts.getConnector(), opts, bwOpts, scanOpts);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 2 with ScannerOpts

use of org.apache.accumulo.core.cli.ScannerOpts in project accumulo by apache.

the class TestRandomDeletes method main.

public static void main(String[] args) {
    ClientOnDefaultTable opts = new ClientOnDefaultTable("test_ingest");
    ScannerOpts scanOpts = new ScannerOpts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(TestRandomDeletes.class.getName(), args, scanOpts, bwOpts);
    log.info("starting random delete test");
    try {
        long deleted = 0;
        String tableName = opts.getTableName();
        TreeSet<RowColumn> doomed = scanAll(opts, scanOpts, tableName);
        log.info("Got {} rows", doomed.size());
        long startTime = System.currentTimeMillis();
        while (true) {
            long half = scrambleDeleteHalfAndCheck(opts, scanOpts, bwOpts, tableName, doomed);
            deleted += half;
            if (half == 0)
                break;
        }
        long stopTime = System.currentTimeMillis();
        long elapsed = (stopTime - startTime) / 1000;
        log.info("deleted {} values in {} seconds", deleted, elapsed);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 3 with ScannerOpts

use of org.apache.accumulo.core.cli.ScannerOpts in project accumulo by apache.

the class ChaoticBalancerIT method test.

@Test
public void test() throws Exception {
    Connector c = getConnector();
    String[] names = getUniqueNames(1);
    String tableName = names[0];
    NewTableConfiguration ntc = new NewTableConfiguration();
    ntc.setProperties(Stream.of(new Pair<>(Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K"), new Pair<>(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1K")).collect(Collectors.toMap(k -> k.getFirst(), v -> v.getSecond())));
    c.tableOperations().create(tableName, ntc);
    TestIngest.Opts opts = new TestIngest.Opts();
    VerifyIngest.Opts vopts = new VerifyIngest.Opts();
    vopts.rows = opts.rows = 20000;
    opts.setTableName(tableName);
    vopts.setTableName(tableName);
    ClientConfiguration clientConfig = getCluster().getClientConfig();
    if (clientConfig.hasSasl()) {
        opts.updateKerberosCredentials(clientConfig);
        vopts.updateKerberosCredentials(clientConfig);
    } else {
        opts.setPrincipal(getAdminPrincipal());
        vopts.setPrincipal(getAdminPrincipal());
    }
    TestIngest.ingest(c, opts, new BatchWriterOpts());
    c.tableOperations().flush(tableName, null, null, true);
    VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
}

Example 4 with ScannerOpts

use of org.apache.accumulo.core.cli.ScannerOpts in project accumulo by apache.

the class BalanceInPresenceOfOfflineTableIT method test.

@Test
public void test() throws Exception {
    log.info("Test that balancing is not stopped by an offline table with outstanding migrations.");
    log.debug("starting test ingestion");
    TestIngest.Opts opts = new TestIngest.Opts();
    VerifyIngest.Opts vopts = new VerifyIngest.Opts();
    ClientConfiguration conf = cluster.getClientConfig();
    if (conf.hasSasl()) {
        opts.updateKerberosCredentials(cluster.getClientConfig());
        vopts.updateKerberosCredentials(cluster.getClientConfig());
    } else {
        opts.setPrincipal("root");
        vopts.setPrincipal("root");
    }
    vopts.rows = opts.rows = 200000;
    opts.setTableName(TEST_TABLE);
    TestIngest.ingest(connector, opts, new BatchWriterOpts());
    connector.tableOperations().flush(TEST_TABLE, null, null, true);
    vopts.setTableName(TEST_TABLE);
    VerifyIngest.verifyIngest(connector, vopts, new ScannerOpts());
    log.debug("waiting for balancing, up to ~5 minutes to allow for migration cleanup.");
    final long startTime = System.currentTimeMillis();
    long currentWait = 10 * 1000;
    boolean balancingWorked = false;
    Credentials creds = new Credentials(getAdminPrincipal(), getAdminToken());
    while (!balancingWorked && (System.currentTimeMillis() - startTime) < ((5 * 60 + 15) * 1000)) {
        Thread.sleep(currentWait);
        currentWait *= 2;
        log.debug("fetch the list of tablets assigned to each tserver.");
        MasterClientService.Iface client = null;
        MasterMonitorInfo stats = null;
        Instance instance = new ZooKeeperInstance(cluster.getClientConfig());
        while (true) {
            try {
                client = MasterClient.getConnectionWithRetry(new ClientContext(instance, creds, cluster.getClientConfig()));
                stats = client.getMasterStats(Tracer.traceInfo(), creds.toThrift(instance));
                break;
            } catch (ThriftSecurityException exception) {
                throw new AccumuloSecurityException(exception);
            } catch (ThriftNotActiveServiceException e) {
                // Let it loop, fetching a new location
                log.debug("Contacted a Master which is no longer active, retrying");
                sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            } catch (TException exception) {
                throw new AccumuloException(exception);
            } finally {
                if (client != null) {
                    MasterClient.close(client);
                }
            }
        }
        if (stats.getTServerInfoSize() < 2) {
            log.debug("we need >= 2 servers. sleeping for {}ms", currentWait);
            continue;
        }
        if (stats.getUnassignedTablets() != 0) {
            log.debug("We shouldn't have unassigned tablets. sleeping for {}ms", currentWait);
            continue;
        }
        long[] tabletsPerServer = new long[stats.getTServerInfoSize()];
        Arrays.fill(tabletsPerServer, 0l);
        for (int i = 0; i < stats.getTServerInfoSize(); i++) {
            for (Map.Entry<String, TableInfo> entry : stats.getTServerInfo().get(i).getTableMap().entrySet()) {
                tabletsPerServer[i] += entry.getValue().getTablets();
            }
        }
        if (tabletsPerServer[0] <= 10) {
            log.debug("We should have > 10 tablets. sleeping for {}ms", currentWait);
            continue;
        }
        long min = NumberUtils.min(tabletsPerServer), max = NumberUtils.max(tabletsPerServer);
        log.debug("Min={}, Max={}", min, max);
        if ((min / ((double) max)) < 0.5) {
            log.debug("ratio of min to max tablets per server should be roughly even. sleeping for {}ms", currentWait);
            continue;
        }
        balancingWorked = true;
    }
    Assert.assertTrue("did not properly balance", balancingWorked);
}

Example 5 with ScannerOpts

use of org.apache.accumulo.core.cli.ScannerOpts in project accumulo by apache.

the class HalfDeadTServerIT method test.

public String test(int seconds, boolean expectTserverDied) throws Exception {
    if (!makeDiskFailureLibrary())
        return null;
    Connector c = getConnector();
    assertEquals(1, c.instanceOperations().getTabletServers().size());
    // create our own tablet server with the special test library
    String javaHome = System.getProperty("java.home");
    String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
    String classpath = System.getProperty("java.class.path");
    classpath = new File(cluster.getConfig().getDir(), "conf") + File.pathSeparator + classpath;
    String className = TabletServer.class.getName();
    ArrayList<String> argList = new ArrayList<>();
    argList.addAll(Arrays.asList(javaBin, "-cp", classpath));
    argList.addAll(Arrays.asList(Main.class.getName(), className));
    ProcessBuilder builder = new ProcessBuilder(argList);
    Map<String, String> env = builder.environment();
    env.put("ACCUMULO_HOME", cluster.getConfig().getDir().getAbsolutePath());
    env.put("ACCUMULO_LOG_DIR", cluster.getConfig().getLogDir().getAbsolutePath());
    String trickFilename = cluster.getConfig().getLogDir().getAbsolutePath() + "/TRICK_FILE";
    env.put("TRICK_FILE", trickFilename);
    String libPath = System.getProperty("user.dir") + "/target/fake_disk_failure.so";
    env.put("LD_PRELOAD", libPath);
    env.put("DYLD_INSERT_LIBRARIES", libPath);
    env.put("DYLD_FORCE_FLAT_NAMESPACE", "true");
    Process ingest = null;
    Process tserver = builder.start();
    DumpOutput t = new DumpOutput(tserver.getInputStream());
    try {
        t.start();
        sleepUninterruptibly(1, TimeUnit.SECONDS);
        // don't need the regular tablet server
        cluster.killProcess(ServerType.TABLET_SERVER, cluster.getProcesses().get(ServerType.TABLET_SERVER).iterator().next());
        sleepUninterruptibly(1, TimeUnit.SECONDS);
        c.tableOperations().create("test_ingest");
        assertEquals(1, c.instanceOperations().getTabletServers().size());
        int rows = 100 * 1000;
        ingest = cluster.exec(TestIngest.class, "-u", "root", "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers(), "-p", ROOT_PASSWORD, "--rows", rows + "");
        sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
        // block I/O with some side-channel trickiness
        File trickFile = new File(trickFilename);
        try {
            assertTrue(trickFile.createNewFile());
            sleepUninterruptibly(seconds, TimeUnit.SECONDS);
        } finally {
            if (!trickFile.delete()) {
                log.error("Couldn't delete {}", trickFile);
            }
        }
        if (seconds <= 10) {
            assertEquals(0, ingest.waitFor());
            VerifyIngest.Opts vopts = new VerifyIngest.Opts();
            vopts.rows = rows;
            vopts.setPrincipal("root");
            VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
        } else {
            sleepUninterruptibly(5, TimeUnit.SECONDS);
            tserver.waitFor();
            t.join();
            tserver = null;
        }
        // verify the process was blocked
        String results = t.toString();
        assertTrue(results.contains("sleeping\nsleeping\nsleeping\n"));
        return results;
    } finally {
        if (ingest != null) {
            ingest.destroy();
            ingest.waitFor();
        }
        if (tserver != null) {
            try {
                if (expectTserverDied) {
                    try {
                        tserver.exitValue();
                    } catch (IllegalThreadStateException e) {
                        fail("Expected TServer to kill itself, but it is still running");
                    }
                }
            } finally {
                tserver.destroy();
                tserver.waitFor();
                t.join();
            }
        }
    }
}