Examples with VerifyParams org.apache.accumulo.test.VerifyIngest.VerifyParams used on opensource projects

Example 1 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class BalanceInPresenceOfOfflineTableIT method test.

@Test
public void test() throws Exception {
    log.info("Test that balancing is not stopped by an offline table with outstanding migrations.");
    log.debug("starting test ingestion");
    VerifyParams params = new VerifyParams(getClientProps(), TEST_TABLE, 200_000);
    TestIngest.ingest(accumuloClient, params);
    accumuloClient.tableOperations().flush(TEST_TABLE, null, null, true);
    VerifyIngest.verifyIngest(accumuloClient, params);
    log.debug("waiting for balancing, up to ~5 minutes to allow for migration cleanup.");
    final long startTime = System.currentTimeMillis();
    long currentWait = 10_000;
    boolean balancingWorked = false;
    Credentials creds = new Credentials(getAdminPrincipal(), getAdminToken());
    while (!balancingWorked && (System.currentTimeMillis() - startTime) < ((5 * 60 + 15) * 1000)) {
        Thread.sleep(currentWait);
        currentWait *= 2;
        log.debug("fetch the list of tablets assigned to each tserver.");
        ManagerClientService.Iface client = null;
        ManagerMonitorInfo stats;
        while (true) {
            try {
                client = ManagerClient.getConnectionWithRetry((ClientContext) accumuloClient);
                stats = client.getManagerStats(TraceUtil.traceInfo(), creds.toThrift(accumuloClient.instanceOperations().getInstanceId()));
                break;
            } catch (ThriftSecurityException exception) {
                throw new AccumuloSecurityException(exception);
            } catch (ThriftNotActiveServiceException e) {
                // Let it loop, fetching a new location
                log.debug("Contacted a Manager which is no longer active, retrying");
                sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            } catch (TException exception) {
                throw new AccumuloException(exception);
            } finally {
                if (client != null) {
                    ManagerClient.close(client, (ClientContext) accumuloClient);
                }
            }
        }
        if (stats.getTServerInfoSize() < 2) {
            log.debug("we need >= 2 servers. sleeping for {}ms", currentWait);
            continue;
        }
        if (stats.getUnassignedTablets() != 0) {
            log.debug("We shouldn't have unassigned tablets. sleeping for {}ms", currentWait);
            continue;
        }
        long[] tabletsPerServer = new long[stats.getTServerInfoSize()];
        Arrays.fill(tabletsPerServer, 0L);
        for (int i = 0; i < stats.getTServerInfoSize(); i++) {
            for (Map.Entry<String, TableInfo> entry : stats.getTServerInfo().get(i).getTableMap().entrySet()) {
                tabletsPerServer[i] += entry.getValue().getTablets();
            }
        }
        if (tabletsPerServer[0] <= 10) {
            log.debug("We should have > 10 tablets. sleeping for {}ms", currentWait);
            continue;
        }
        long min = NumberUtils.min(tabletsPerServer), max = NumberUtils.max(tabletsPerServer);
        log.debug("Min={}, Max={}", min, max);
        if ((min / ((double) max)) < 0.5) {
            log.debug("ratio of min to max tablets per server should be roughly even. sleeping for {}ms", currentWait);
            continue;
        }
        balancingWorked = true;
    }
    assertTrue("did not properly balance", balancingWorked);
}

Example 2 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class WriteAheadLogIT method testWAL.

public static void testWAL(AccumuloClient c, String tableName) throws Exception {
    c.tableOperations().create(tableName);
    c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "750K");
    VerifyParams params = new VerifyParams(getClientProps(), tableName);
    TestIngest.ingest(c, params);
    VerifyIngest.verifyIngest(c, params);
    getCluster().getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
    getCluster().getClusterControl().startAllServers(ServerType.TABLET_SERVER);
    VerifyIngest.verifyIngest(c, params);
}

Example 3 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class WriteLotsIT method writeLots.

@Test
public void writeLots() throws Exception {
    BatchWriterConfig bwConfig = new BatchWriterConfig();
    bwConfig.setMaxMemory(1024L * 1024);
    bwConfig.setMaxWriteThreads(2);
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).batchWriterConfig(bwConfig).build()) {
        final String tableName = getUniqueNames(1)[0];
        c.tableOperations().create(tableName);
        final AtomicReference<Exception> ref = new AtomicReference<>();
        final int THREADS = 5;
        ThreadPoolExecutor tpe = new ThreadPoolExecutor(0, THREADS, 0, TimeUnit.SECONDS, new ArrayBlockingQueue<>(THREADS));
        for (int i = 0; i < THREADS; i++) {
            final int index = i;
            Runnable r = () -> {
                try {
                    IngestParams ingestParams = new IngestParams(getClientProps(), tableName, 10_000);
                    ingestParams.startRow = index * 10000;
                    TestIngest.ingest(c, ingestParams);
                } catch (Exception ex) {
                    ref.set(ex);
                }
            };
            tpe.execute(r);
        }
        tpe.shutdown();
        tpe.awaitTermination(90, TimeUnit.SECONDS);
        if (ref.get() != null) {
            throw ref.get();
        }
        VerifyParams params = new VerifyParams(getClientProps(), tableName, 10_000 * THREADS);
        VerifyIngest.verifyIngest(c, params);
    }
}

Example 4 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class TableIT method test.

@Test
public void test() throws Exception {
    assumeTrue(getClusterType() == ClusterType.MINI);
    AccumuloCluster cluster = getCluster();
    MiniAccumuloClusterImpl mac = (MiniAccumuloClusterImpl) cluster;
    String rootPath = mac.getConfig().getDir().getAbsolutePath();
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        TableOperations to = c.tableOperations();
        String tableName = getUniqueNames(1)[0];
        to.create(tableName);
        VerifyParams params = new VerifyParams(getClientProps(), tableName);
        TestIngest.ingest(c, params);
        to.flush(tableName, null, null, true);
        VerifyIngest.verifyIngest(c, params);
        TableId id = TableId.of(to.tableIdMap().get(tableName));
        try (Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            s.setRange(new KeyExtent(id, null, null).toMetaRange());
            s.fetchColumnFamily(DataFileColumnFamily.NAME);
            assertTrue(Iterators.size(s.iterator()) > 0);
            FileSystem fs = getCluster().getFileSystem();
            assertTrue(fs.listStatus(new Path(rootPath + "/accumulo/tables/" + id)).length > 0);
            to.delete(tableName);
            assertEquals(0, Iterators.size(s.iterator()));
            try {
                assertEquals(0, fs.listStatus(new Path(rootPath + "/accumulo/tables/" + id)).length);
            } catch (FileNotFoundException ex) {
            // that's fine, too
            }
            assertNull(to.tableIdMap().get(tableName));
            to.create(tableName);
            TestIngest.ingest(c, params);
            VerifyIngest.verifyIngest(c, params);
            to.delete(tableName);
        }
    }
}

Example 5 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class RenameIT method renameTest.

@Test
public void renameTest() throws Exception {
    String[] tableNames = getUniqueNames(2);
    String name1 = tableNames[0];
    String name2 = tableNames[1];
    VerifyParams params = new VerifyParams(cluster.getClientProperties(), name1);
    params.createTable = true;
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        TestIngest.ingest(c, params);
        c.tableOperations().rename(name1, name2);
        TestIngest.ingest(c, params);
        params.tableName = name2;
        VerifyIngest.verifyIngest(c, params);
        c.tableOperations().delete(name1);
        c.tableOperations().rename(name2, name1);
        params.tableName = name1;
        VerifyIngest.verifyIngest(c, params);
        FunctionalTestUtils.assertNoDanglingFateLocks((ClientContext) c, getCluster());
    }
}