Examples with VerifyParams org.apache.accumulo.test.VerifyIngest.VerifyParams used on opensource projects

Example 16 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class BulkIT method runTest.

static void runTest(AccumuloClient c, ClientInfo info, FileSystem fs, Path basePath, String tableName, String filePrefix, String dirSuffix, boolean useOld) throws Exception {
    c.tableOperations().create(tableName);
    Path base = new Path(basePath, "testBulkFail_" + dirSuffix);
    fs.delete(base, true);
    fs.mkdirs(base);
    fs.deleteOnExit(base);
    Path bulkFailures = new Path(base, "failures");
    fs.deleteOnExit(bulkFailures);
    Path files = new Path(base, "files");
    fs.deleteOnExit(files);
    fs.mkdirs(bulkFailures);
    fs.mkdirs(files);
    IngestParams params = new IngestParams(info.getProperties(), tableName, N);
    params.timestamp = 1;
    params.random = 56;
    params.cols = 1;
    String fileFormat = filePrefix + "rf%02d";
    for (int i = 0; i < COUNT; i++) {
        params.outputFile = new Path(files, String.format(fileFormat, i)).toString();
        params.startRow = N * i;
        TestIngest.ingest(c, fs, params);
    }
    params.outputFile = new Path(files, String.format(fileFormat, N)).toString();
    params.startRow = N;
    params.rows = 1;
    // create an rfile with one entry, there was a bug with this:
    TestIngest.ingest(c, fs, params);
    bulkLoad(c, tableName, bulkFailures, files, useOld);
    VerifyParams verifyParams = new VerifyParams(info.getProperties(), tableName, N);
    verifyParams.random = 56;
    for (int i = 0; i < COUNT; i++) {
        verifyParams.startRow = i * N;
        VerifyIngest.verifyIngest(c, verifyParams);
    }
    verifyParams.startRow = N;
    verifyParams.rows = 1;
    VerifyIngest.verifyIngest(c, verifyParams);
}

Example 17 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class ManagerFailoverIT method test.

@Test
public void test() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        String[] names = getUniqueNames(2);
        c.tableOperations().create(names[0]);
        VerifyParams params = new VerifyParams(getClientProps(), names[0]);
        TestIngest.ingest(c, params);
        ClusterControl control = cluster.getClusterControl();
        control.stopAllServers(ServerType.MANAGER);
        // start up a new one
        control.startAllServers(ServerType.MANAGER);
        // talk to it
        c.tableOperations().rename(names[0], names[1]);
        params.tableName = names[1];
        VerifyIngest.verifyIngest(c, params);
    }
}

Example 18 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class GarbageCollectorIT method gcTest.

@Test
public void gcTest() throws Exception {
    killMacGc();
    final String table = "test_ingest";
    try (AccumuloClient c = Accumulo.newClient().from(getClientProperties()).build()) {
        c.tableOperations().create(table);
        c.tableOperations().setProperty(table, Property.TABLE_SPLIT_THRESHOLD.getKey(), "5K");
        VerifyParams params = new VerifyParams(getClientProperties(), table, 10_000);
        params.cols = 1;
        TestIngest.ingest(c, cluster.getFileSystem(), params);
        c.tableOperations().compact(table, null, null, true, true);
        int before = countFiles();
        while (true) {
            sleepUninterruptibly(1, TimeUnit.SECONDS);
            int more = countFiles();
            if (more <= before)
                break;
            before = more;
        }
        // restart GC
        getCluster().start();
        sleepUninterruptibly(15, TimeUnit.SECONDS);
        int after = countFiles();
        VerifyIngest.verifyIngest(c, params);
        assertTrue(after < before);
    }
}

Example 19 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class BulkSplitOptimizationIT method testBulkSplitOptimization.

@Test
public void testBulkSplitOptimization() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        final String tableName = getUniqueNames(1)[0];
        c.tableOperations().create(tableName);
        c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "1000");
        c.tableOperations().setProperty(tableName, Property.TABLE_FILE_MAX.getKey(), "1000");
        c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "1G");
        FileSystem fs = cluster.getFileSystem();
        Path testDir = new Path(cluster.getTemporaryPath(), "testmf");
        fs.deleteOnExit(testDir);
        FunctionalTestUtils.createRFiles(c, fs, testDir.toString(), ROWS, SPLITS, 8);
        FileStatus[] stats = fs.listStatus(testDir);
        System.out.println("Number of generated files: " + stats.length);
        c.tableOperations().importDirectory(testDir.toString()).to(tableName).load();
        FunctionalTestUtils.checkSplits(c, tableName, 0, 0);
        FunctionalTestUtils.checkRFiles(c, tableName, 1, 1, 100, 100);
        // initiate splits
        c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "100K");
        sleepUninterruptibly(2, TimeUnit.SECONDS);
        // wait until over split threshold -- should be 78 splits
        while (c.tableOperations().listSplits(tableName).size() < 75) {
            sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
        }
        FunctionalTestUtils.checkSplits(c, tableName, 50, 100);
        VerifyParams params = new VerifyParams(getClientProps(), tableName, ROWS);
        params.timestamp = 1;
        params.dataSize = 50;
        params.random = 56;
        params.startRow = 0;
        params.cols = 1;
        VerifyIngest.verifyIngest(c, params);
        // ensure each tablet does not have all map files, should be ~2.5 files per tablet
        FunctionalTestUtils.checkRFiles(c, tableName, 50, 100, 1, 4);
    }
}

Example 20 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class DeleteIT method deleteTest.

public static void deleteTest(AccumuloClient c, AccumuloCluster cluster, String tableName) throws Exception {
    VerifyParams params = new VerifyParams(getClientProps(), tableName, 1000);
    params.cols = 1;
    params.random = 56;
    TestIngest.ingest(c, params);
    assertEquals(0, cluster.getClusterControl().exec(TestRandomDeletes.class, new String[] { "-c", cluster.getClientPropsPath(), "--table", tableName }));
    TestIngest.ingest(c, params);
    VerifyIngest.verifyIngest(c, params);
}