Examples with VerifyParams org.apache.accumulo.test.VerifyIngest.VerifyParams used on opensource projects

Example 11 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class SplitIT method tabletShouldSplit.

@Test
public void tabletShouldSplit() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        String table = getUniqueNames(1)[0];
        Map<String, String> props = new HashMap<>();
        props.put(Property.TABLE_SPLIT_THRESHOLD.getKey(), "256K");
        props.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1K");
        c.tableOperations().create(table, new NewTableConfiguration().setProperties(props));
        VerifyParams params = new VerifyParams(getClientProps(), table, 100_000);
        TestIngest.ingest(c, params);
        VerifyIngest.verifyIngest(c, params);
        while (c.tableOperations().listSplits(table).size() < 10) {
            sleepUninterruptibly(15, TimeUnit.SECONDS);
        }
        TableId id = TableId.of(c.tableOperations().tableIdMap().get(table));
        try (Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            KeyExtent extent = new KeyExtent(id, null, null);
            s.setRange(extent.toMetaRange());
            TabletColumnFamily.PREV_ROW_COLUMN.fetch(s);
            int count = 0;
            int shortened = 0;
            for (Entry<Key, Value> entry : s) {
                extent = KeyExtent.fromMetaPrevRow(entry);
                if (extent.endRow() != null && extent.endRow().toString().length() < 14)
                    shortened++;
                count++;
            }
            assertTrue("Shortened should be greater than zero: " + shortened, shortened > 0);
            assertTrue("Count should be cgreater than 10: " + count, count > 10);
        }
        assertEquals(0, getCluster().getClusterControl().exec(CheckForMetadataProblems.class, new String[] { "-c", cluster.getClientPropsPath() }));
    }
}

Example 12 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class RestartIT method restartManagerSplit.

@Test
public void restartManagerSplit() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        final String tableName = getUniqueNames(1)[0];
        final ClusterControl control = getCluster().getClusterControl();
        c.tableOperations().create(tableName);
        c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "5K");
        VerifyParams params = new VerifyParams(getClientProps(), tableName, 10_000);
        Future<Integer> ret = svc.submit(() -> {
            try {
                return control.exec(TestIngest.class, new String[] { "-c", cluster.getClientPropsPath(), "--rows", "" + params.rows, "--table", tableName });
            } catch (Exception e) {
                log.error("Error running TestIngest", e);
                return -1;
            }
        });
        control.stopAllServers(ServerType.MANAGER);
        ClientInfo info = ClientInfo.from(c.properties());
        ZooReader zreader = new ZooReader(info.getZooKeepers(), info.getZooKeepersSessionTimeOut());
        ZooCache zcache = new ZooCache(zreader, null);
        var zLockPath = ServiceLock.path(ZooUtil.getRoot(c.instanceOperations().getInstanceId()) + Constants.ZMANAGER_LOCK);
        byte[] managerLockData;
        do {
            managerLockData = ServiceLock.getLockData(zcache, zLockPath, null);
            if (managerLockData != null) {
                log.info("Manager lock is still held");
                Thread.sleep(1000);
            }
        } while (managerLockData != null);
        cluster.start();
        assertEquals(0, ret.get().intValue());
        VerifyIngest.verifyIngest(c, params);
    }
}

Example 13 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class RestartIT method restartManagerRecovery.

@Test
public void restartManagerRecovery() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        String tableName = getUniqueNames(1)[0];
        c.tableOperations().create(tableName);
        VerifyParams params = new VerifyParams(getClientProps(), tableName, 10_000);
        TestIngest.ingest(c, params);
        ClusterControl control = getCluster().getClusterControl();
        // TODO implement a kill all too?
        // cluster.stop() would also stop ZooKeeper
        control.stopAllServers(ServerType.MANAGER);
        control.stopAllServers(ServerType.TABLET_SERVER);
        control.stopAllServers(ServerType.GARBAGE_COLLECTOR);
        control.stopAllServers(ServerType.MONITOR);
        ClientInfo info = ClientInfo.from(c.properties());
        ZooReader zreader = new ZooReader(info.getZooKeepers(), info.getZooKeepersSessionTimeOut());
        ZooCache zcache = new ZooCache(zreader, null);
        var zLockPath = ServiceLock.path(ZooUtil.getRoot(c.instanceOperations().getInstanceId()) + Constants.ZMANAGER_LOCK);
        byte[] managerLockData;
        do {
            managerLockData = ServiceLock.getLockData(zcache, zLockPath, null);
            if (managerLockData != null) {
                log.info("Manager lock is still held");
                Thread.sleep(1000);
            }
        } while (managerLockData != null);
        cluster.start();
        sleepUninterruptibly(5, TimeUnit.MILLISECONDS);
        control.stopAllServers(ServerType.MANAGER);
        managerLockData = new byte[0];
        do {
            managerLockData = ServiceLock.getLockData(zcache, zLockPath, null);
            if (managerLockData != null) {
                log.info("Manager lock is still held");
                Thread.sleep(1000);
            }
        } while (managerLockData != null);
        cluster.start();
        VerifyIngest.verifyIngest(c, params);
    }
}

Example 14 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class RestartStressIT method test.

@Test
public void test() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        final String tableName = getUniqueNames(1)[0];
        c.tableOperations().create(tableName);
        c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "500K");
        final ClusterControl control = getCluster().getClusterControl();
        VerifyParams params = new VerifyParams(getClientProps(), tableName, 10_000);
        Future<Integer> retCode = svc.submit(() -> {
            try {
                return control.exec(TestIngest.class, new String[] { "-c", cluster.getClientPropsPath(), "--rows", "" + params.rows, "--table", tableName });
            } catch (Exception e) {
                log.error("Error running TestIngest", e);
                return -1;
            }
        });
        for (int i = 0; i < 2; i++) {
            sleepUninterruptibly(10, TimeUnit.SECONDS);
            control.stopAllServers(ServerType.TABLET_SERVER);
            control.startAllServers(ServerType.TABLET_SERVER);
        }
        assertEquals(0, retCode.get().intValue());
        VerifyIngest.verifyIngest(c, params);
    }
}

Example 15 with VerifyParams

use of org.apache.accumulo.test.VerifyIngest.VerifyParams in project accumulo by apache.

the class CompactionIT method testSuccessfulCompaction.

@Test
public void testSuccessfulCompaction() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        final String tableName = getUniqueNames(1)[0];
        c.tableOperations().create(tableName);
        c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "1.0");
        FileSystem fs = getFileSystem();
        Path root = new Path(cluster.getTemporaryPath(), getClass().getName());
        fs.deleteOnExit(root);
        Path testrf = new Path(root, "testrf");
        fs.deleteOnExit(testrf);
        FunctionalTestUtils.createRFiles(c, fs, testrf.toString(), 500000, 59, 4);
        c.tableOperations().importDirectory(testrf.toString()).to(tableName).load();
        int beforeCount = countFiles(c);
        final AtomicBoolean fail = new AtomicBoolean(false);
        final int THREADS = 5;
        for (int count = 0; count < THREADS; count++) {
            ExecutorService executor = Executors.newFixedThreadPool(THREADS);
            final int span = 500000 / 59;
            for (int i = 0; i < 500000; i += 500000 / 59) {
                final int finalI = i;
                Runnable r = () -> {
                    try {
                        VerifyParams params = new VerifyParams(getClientProps(), tableName, span);
                        params.startRow = finalI;
                        params.random = 56;
                        params.dataSize = 50;
                        params.cols = 1;
                        VerifyIngest.verifyIngest(c, params);
                    } catch (Exception ex) {
                        log.warn("Got exception verifying data", ex);
                        fail.set(true);
                    }
                };
                executor.execute(r);
            }
            executor.shutdown();
            executor.awaitTermination(defaultTimeoutSeconds(), TimeUnit.SECONDS);
            assertFalse("Failed to successfully run all threads, Check the test output for error", fail.get());
        }
        int finalCount = countFiles(c);
        assertTrue(finalCount < beforeCount);
        try {
            getClusterControl().adminStopAll();
        } finally {
            // Make sure the internal state in the cluster is reset (e.g. processes in MAC)
            getCluster().stop();
            if (getClusterType() == ClusterType.STANDALONE) {
                // Then restart things for the next test if it's a standalone
                getCluster().start();
            }
        }
    }
}