Examples with IngestParams org.apache.accumulo.test.TestIngest.IngestParams used on opensource projects

Example 1 with IngestParams

use of org.apache.accumulo.test.TestIngest.IngestParams in project accumulo by apache.

the class WriteLotsIT method writeLots.

@Test
public void writeLots() throws Exception {
    BatchWriterConfig bwConfig = new BatchWriterConfig();
    bwConfig.setMaxMemory(1024L * 1024);
    bwConfig.setMaxWriteThreads(2);
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).batchWriterConfig(bwConfig).build()) {
        final String tableName = getUniqueNames(1)[0];
        c.tableOperations().create(tableName);
        final AtomicReference<Exception> ref = new AtomicReference<>();
        final int THREADS = 5;
        ThreadPoolExecutor tpe = new ThreadPoolExecutor(0, THREADS, 0, TimeUnit.SECONDS, new ArrayBlockingQueue<>(THREADS));
        for (int i = 0; i < THREADS; i++) {
            final int index = i;
            Runnable r = () -> {
                try {
                    IngestParams ingestParams = new IngestParams(getClientProps(), tableName, 10_000);
                    ingestParams.startRow = index * 10000;
                    TestIngest.ingest(c, ingestParams);
                } catch (Exception ex) {
                    ref.set(ex);
                }
            };
            tpe.execute(r);
        }
        tpe.shutdown();
        tpe.awaitTermination(90, TimeUnit.SECONDS);
        if (ref.get() != null) {
            throw ref.get();
        }
        VerifyParams params = new VerifyParams(getClientProps(), tableName, 10_000 * THREADS);
        VerifyIngest.verifyIngest(c, params);
    }
}

Example 2 with IngestParams

use of org.apache.accumulo.test.TestIngest.IngestParams in project accumulo by apache.

the class RestartIT method killedTabletServerDuringShutdown.

@Test
public void killedTabletServerDuringShutdown() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        String tableName = getUniqueNames(1)[0];
        c.tableOperations().create(tableName);
        IngestParams params = new IngestParams(getClientProps(), tableName, 10_000);
        TestIngest.ingest(c, params);
        try {
            getCluster().getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
            getCluster().getClusterControl().adminStopAll();
        } finally {
            // make sure processes are cleaned up before we restart clean for other tests
            getCluster().stop();
            getCluster().start();
        }
    }
}

Example 3 with IngestParams

use of org.apache.accumulo.test.TestIngest.IngestParams in project accumulo by apache.

the class FateStarvationIT method run.

@Test
public void run() throws Exception {
    String tableName = getUniqueNames(1)[0];
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        var ntc = new NewTableConfiguration().withSplits(TestIngest.getSplitPoints(0, 100000, 50));
        c.tableOperations().create(tableName, ntc);
        IngestParams params = new IngestParams(getClientProps(), tableName, 100_000);
        params.random = 89;
        params.timestamp = 7;
        params.dataSize = 50;
        params.cols = 1;
        TestIngest.ingest(c, params);
        c.tableOperations().flush(tableName, null, null, true);
        List<Text> splits = new ArrayList<>(TestIngest.getSplitPoints(0, 100000, 67));
        for (int i = 0; i < 100; i++) {
            int idx1 = random.nextInt(splits.size() - 1);
            int idx2 = random.nextInt(splits.size() - (idx1 + 1)) + idx1 + 1;
            c.tableOperations().compact(tableName, splits.get(idx1), splits.get(idx2), false, false);
        }
        c.tableOperations().offline(tableName);
        FunctionalTestUtils.assertNoDanglingFateLocks((ClientContext) c, getCluster());
    }
}

Example 4 with IngestParams

use of org.apache.accumulo.test.TestIngest.IngestParams in project accumulo by apache.

the class MaxOpenIT method run.

@Test
public void run() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        final String tableName = getUniqueNames(1)[0];
        HashMap<String, String> props = new HashMap<>();
        props.put(Property.TABLE_MAJC_RATIO.getKey(), "10");
        NewTableConfiguration ntc = new NewTableConfiguration().setProperties(props).withSplits(TestIngest.getSplitPoints(0, NUM_TO_INGEST, NUM_TABLETS));
        c.tableOperations().create(tableName, ntc);
        // the following loop should create three tablets in each map file
        for (int i = 0; i < 3; i++) {
            IngestParams params = new IngestParams(getClientProps(), tableName, NUM_TO_INGEST);
            params.timestamp = i;
            params.dataSize = 50;
            params.rows = NUM_TO_INGEST;
            params.cols = 1;
            params.random = i;
            TestIngest.ingest(c, params);
            c.tableOperations().flush(tableName, null, null, true);
            FunctionalTestUtils.checkRFiles(c, tableName, NUM_TABLETS, NUM_TABLETS, i + 1, i + 1);
        }
        List<Range> ranges = new ArrayList<>(NUM_TO_INGEST);
        for (int i = 0; i < NUM_TO_INGEST; i++) {
            ranges.add(new Range(TestIngest.generateRow(i, 0)));
        }
        long time1 = batchScan(c, tableName, ranges, 1);
        // run it again, now that stuff is cached on the client and sever
        time1 = batchScan(c, tableName, ranges, 1);
        long time2 = batchScan(c, tableName, ranges, NUM_TABLETS);
        System.out.printf("Single thread scan time   %6.2f %n", time1 / 1000.0);
        System.out.printf("Multiple thread scan time %6.2f %n", time2 / 1000.0);
    }
}

Example 5 with IngestParams

use of org.apache.accumulo.test.TestIngest.IngestParams in project accumulo by apache.

the class ReadWriteIT method ingest.

public static void ingest(AccumuloClient accumuloClient, ClientInfo info, int rows, int cols, int width, int offset, String colf, String tableName) throws Exception {
    IngestParams params = new IngestParams(info.getProperties(), tableName, rows);
    params.cols = cols;
    params.dataSize = width;
    params.startRow = offset;
    params.columnFamily = colf;
    params.createTable = true;
    TestIngest.ingest(accumuloClient, params);
}