Examples with ActiveScan org.apache.accumulo.core.client.admin.ActiveScan used on opensource projects

Example 1 with ActiveScan

use of org.apache.accumulo.core.client.admin.ActiveScan in project accumulo by apache.

the class ScanIdIT method testScanId.

/**
 * @throws Exception
 *           any exception is a test failure.
 */
@Test
public void testScanId() throws Exception {
    final String tableName = getUniqueNames(1)[0];
    Connector conn = getConnector();
    conn.tableOperations().create(tableName);
    addSplits(conn, tableName);
    log.info("Splits added");
    generateSampleData(conn, tableName);
    log.info("Generated data for {}", tableName);
    attachSlowIterator(conn, tableName);
    CountDownLatch latch = new CountDownLatch(NUM_SCANNERS);
    for (int scannerIndex = 0; scannerIndex < NUM_SCANNERS; scannerIndex++) {
        ScannerThread st = new ScannerThread(conn, scannerIndex, tableName, latch);
        pool.submit(st);
    }
    // wait for scanners to report a result.
    while (testInProgress.get()) {
        if (resultsByWorker.size() < NUM_SCANNERS) {
            log.trace("Results reported {}", resultsByWorker.size());
            sleepUninterruptibly(750, TimeUnit.MILLISECONDS);
        } else {
            // each worker has reported at least one result.
            testInProgress.set(false);
            log.debug("Final result count {}", resultsByWorker.size());
            // delay to allow scanners to react to end of test and cleanly close.
            sleepUninterruptibly(1, TimeUnit.SECONDS);
        }
    }
    // all scanner have reported at least 1 result, so check for unique scan ids.
    Set<Long> scanIds = new HashSet<>();
    List<String> tservers = conn.instanceOperations().getTabletServers();
    log.debug("tablet servers {}", tservers.toString());
    for (String tserver : tservers) {
        List<ActiveScan> activeScans = null;
        for (int i = 0; i < 10; i++) {
            try {
                activeScans = conn.instanceOperations().getActiveScans(tserver);
                break;
            } catch (AccumuloException e) {
                if (e.getCause() instanceof TableNotFoundException) {
                    log.debug("Got TableNotFoundException, will retry");
                    Thread.sleep(200);
                    continue;
                }
                throw e;
            }
        }
        assertNotNull("Repeatedly got exception trying to active scans", activeScans);
        log.debug("TServer {} has {} active scans", tserver, activeScans.size());
        for (ActiveScan scan : activeScans) {
            log.debug("Tserver {} scan id {}", tserver, scan.getScanid());
            scanIds.add(scan.getScanid());
        }
    }
    assertTrue("Expected at least " + NUM_SCANNERS + " scanIds, but saw " + scanIds.size(), NUM_SCANNERS <= scanIds.size());
}

Example 2 with ActiveScan

use of org.apache.accumulo.core.client.admin.ActiveScan in project accumulo by apache.

the class SessionBlockVerifyIT method run.

@Test
public void run() throws Exception {
    Connector c = getConnector();
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
    for (int i = 0; i < 1000; i++) {
        Mutation m = new Mutation(new Text(String.format("%08d", i)));
        for (int j = 0; j < 3; j++) m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(UTF_8)));
        bw.addMutation(m);
    }
    bw.close();
    try (Scanner scanner = c.createScanner(tableName, new Authorizations())) {
        scanner.setReadaheadThreshold(20000);
        scanner.setRange(new Range(String.format("%08d", 0), String.format("%08d", 1000)));
        // test by making a slow iterator and then a couple of fast ones.
        // when then checking we shouldn't have any running except the slow iterator
        IteratorSetting setting = new IteratorSetting(21, SlowIterator.class);
        SlowIterator.setSeekSleepTime(setting, Long.MAX_VALUE);
        SlowIterator.setSleepTime(setting, Long.MAX_VALUE);
        scanner.addScanIterator(setting);
        final Iterator<Entry<Key, Value>> slow = scanner.iterator();
        final List<Future<Boolean>> callables = new ArrayList<>();
        final CountDownLatch latch = new CountDownLatch(10);
        for (int i = 0; i < 10; i++) {
            Future<Boolean> callable = service.submit(new Callable<Boolean>() {

                public Boolean call() {
                    latch.countDown();
                    while (slow.hasNext()) {
                        slow.next();
                    }
                    return slow.hasNext();
                }
            });
            callables.add(callable);
        }
        latch.await();
        log.info("Starting SessionBlockVerifyIT");
        // let's add more for good measure.
        for (int i = 0; i < 2; i++) {
            try (Scanner scanner2 = c.createScanner(tableName, new Authorizations())) {
                scanner2.setRange(new Range(String.format("%08d", 0), String.format("%08d", 1000)));
                scanner2.setBatchSize(1);
                Iterator<Entry<Key, Value>> iter = scanner2.iterator();
                // call super's verify mechanism
                verify(iter, 0, 1000);
            }
        }
        int sessionsFound = 0;
        // we have configured 1 tserver, so we can grab the one and only
        String tserver = Iterables.getOnlyElement(c.instanceOperations().getTabletServers());
        final List<ActiveScan> scans = c.instanceOperations().getActiveScans(tserver);
        for (ActiveScan scan : scans) {
            if (tableName.equals(scan.getTable()) && scan.getSsiList().size() > 0) {
                assertEquals("Not the expected iterator", 1, scan.getSsiList().size());
                assertTrue("Not the expected iterator", scan.getSsiList().iterator().next().contains("SlowIterator"));
                sessionsFound++;
            }
        }
        /**
         * The message below indicates the problem that we experience within ACCUMULO-3509. The issue manifests as a blockage in the Scanner synchronization that
         * prevent us from making the close call against it. Since the close blocks until a read is finished, we ultimately have a block within the sweep of
         * SessionManager. As a result never reap subsequent idle sessions AND we will orphan the sessionsToCleanup in the sweep, leading to an inaccurate count
         * within sessionsFound.
         */
        assertEquals("Must have ten sessions. Failure indicates a synchronization block within the sweep mechanism", 10, sessionsFound);
        for (Future<Boolean> callable : callables) {
            callable.cancel(true);
        }
    }
    service.shutdown();
}

Example 3 with ActiveScan

use of org.apache.accumulo.core.client.admin.ActiveScan in project accumulo by apache.

the class ActiveScanIterator method readNext.

private void readNext() {
    final List<String> scans = new ArrayList<>();
    while (tsIter.hasNext()) {
        final String tserver = tsIter.next();
        try {
            final List<ActiveScan> asl = instanceOps.getActiveScans(tserver);
            for (ActiveScan as : asl) {
                scans.add(String.format("%21s |%21s |%9s |%9s |%7s |%6s |%8s |%8s |%10s |%20s |%10s |%20s |%10s | %s", tserver, as.getClient(), Duration.format(as.getAge(), "", "-"), Duration.format(as.getLastContactTime(), "", "-"), as.getState(), as.getType(), as.getUser(), as.getTable(), as.getColumns(), as.getAuthorizations(), (as.getType() == ScanType.SINGLE ? as.getTablet() : "N/A"), as.getScanid(), as.getSsiList(), as.getSsio()));
            }
        } catch (Exception e) {
            scans.add(tserver + " ERROR " + e.getMessage());
        }
        if (scans.size() > 0) {
            break;
        }
    }
    scansIter = scans.iterator();
}

Example 4 with ActiveScan

use of org.apache.accumulo.core.client.admin.ActiveScan in project accumulo by apache.

the class ProxyServer method getActiveScans.

@Override
public List<org.apache.accumulo.proxy.thrift.ActiveScan> getActiveScans(ByteBuffer login, String tserver) throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException {
    List<org.apache.accumulo.proxy.thrift.ActiveScan> result = new ArrayList<>();
    try {
        List<ActiveScan> activeScans = getConnector(login).instanceOperations().getActiveScans(tserver);
        for (ActiveScan scan : activeScans) {
            org.apache.accumulo.proxy.thrift.ActiveScan pscan = new org.apache.accumulo.proxy.thrift.ActiveScan();
            pscan.client = scan.getClient();
            pscan.user = scan.getUser();
            pscan.table = scan.getTable();
            pscan.age = scan.getAge();
            pscan.idleTime = scan.getIdleTime();
            pscan.type = ScanType.valueOf(scan.getType().toString());
            pscan.state = ScanState.valueOf(scan.getState().toString());
            TabletId e = scan.getTablet();
            pscan.extent = new org.apache.accumulo.proxy.thrift.KeyExtent(e.getTableId().toString(), TextUtil.getByteBuffer(e.getEndRow()), TextUtil.getByteBuffer(e.getPrevEndRow()));
            pscan.columns = new ArrayList<>();
            if (scan.getColumns() != null) {
                for (Column c : scan.getColumns()) {
                    org.apache.accumulo.proxy.thrift.Column column = new org.apache.accumulo.proxy.thrift.Column();
                    column.setColFamily(c.getColumnFamily());
                    column.setColQualifier(c.getColumnQualifier());
                    column.setColVisibility(c.getColumnVisibility());
                    pscan.columns.add(column);
                }
            }
            pscan.iterators = new ArrayList<>();
            for (String iteratorString : scan.getSsiList()) {
                String[] parts = iteratorString.split("[=,]");
                if (parts.length == 3) {
                    String name = parts[0];
                    int priority = Integer.parseInt(parts[1]);
                    String classname = parts[2];
                    org.apache.accumulo.proxy.thrift.IteratorSetting settings = new org.apache.accumulo.proxy.thrift.IteratorSetting(priority, name, classname, scan.getSsio().get(name));
                    pscan.iterators.add(settings);
                }
            }
            pscan.authorizations = new ArrayList<>();
            if (scan.getAuthorizations() != null) {
                for (byte[] a : scan.getAuthorizations()) {
                    pscan.authorizations.add(ByteBuffer.wrap(a));
                }
            }
            result.add(pscan);
        }
        return result;
    } catch (Exception e) {
        handleException(e);
        return null;
    }
}

Example 5 with ActiveScan

use of org.apache.accumulo.core.client.admin.ActiveScan in project accumulo by apache.

the class InterruptibleScannersIT method test.

@Test
public void test() throws Exception {
    // make a table
    final String tableName = getUniqueNames(1)[0];
    final Connector conn = getConnector();
    conn.tableOperations().create(tableName);
    // make the world's slowest scanner
    try (Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY)) {
        final IteratorSetting cfg = new IteratorSetting(100, SlowIterator.class);
        // Wait long enough to be sure we can catch it, but not indefinitely.
        SlowIterator.setSeekSleepTime(cfg, 60 * 1000);
        scanner.addScanIterator(cfg);
        // create a thread to interrupt the slow scan
        final Thread scanThread = Thread.currentThread();
        Thread thread = new Thread() {

            @Override
            public void run() {
                try {
                    // ensure the scan is running: not perfect, the metadata tables could be scanned, too.
                    String tserver = conn.instanceOperations().getTabletServers().iterator().next();
                    do {
                        ArrayList<ActiveScan> scans = new ArrayList<>(conn.instanceOperations().getActiveScans(tserver));
                        Iterator<ActiveScan> iter = scans.iterator();
                        while (iter.hasNext()) {
                            ActiveScan scan = iter.next();
                            // Remove scans not against our table and not owned by us
                            if (!getAdminPrincipal().equals(scan.getUser()) || !tableName.equals(scan.getTable())) {
                                iter.remove();
                            }
                        }
                        if (!scans.isEmpty()) {
                            // We found our scan
                            break;
                        }
                    } while (true);
                } catch (Exception e) {
                    e.printStackTrace();
                }
                // BAM!
                scanThread.interrupt();
            }
        };
        thread.start();
        try {
            // Use the scanner, expect problems
            Iterators.size(scanner.iterator());
            Assert.fail("Scan should not succeed");
        } catch (Exception ex) {
        } finally {
            thread.join();
        }
    }
}