Examples with SampleNotPresentException org.apache.accumulo.core.client.SampleNotPresentException used on opensource projects

Example 1 with SampleNotPresentException

use of org.apache.accumulo.core.client.SampleNotPresentException in project accumulo by apache.

the class TabletIteratorEnvironment method cloneWithSamplingEnabled.

@Override
public IteratorEnvironment cloneWithSamplingEnabled() {
    if (!scope.equals(IteratorScope.scan)) {
        throw new UnsupportedOperationException();
    }
    SamplerConfigurationImpl sci = SamplerConfigurationImpl.newSamplerConfig(config);
    if (sci == null) {
        throw new SampleNotPresentException();
    }
    TabletIteratorEnvironment te = new TabletIteratorEnvironment(scope, config, trm, files, authorizations, sci, topLevelIterators);
    return te;
}

Example 2 with SampleNotPresentException

use of org.apache.accumulo.core.client.SampleNotPresentException in project accumulo by apache.

the class TabletServerBatchReaderIterator method doLookup.

static void doLookup(ClientContext context, String server, Map<KeyExtent, List<Range>> requested, Map<KeyExtent, List<Range>> failures, Map<KeyExtent, List<Range>> unscanned, ResultReceiver receiver, List<Column> columns, ScannerOptions options, Authorizations authorizations, TimeoutTracker timeoutTracker) throws IOException, AccumuloSecurityException, AccumuloServerException {
    if (requested.size() == 0) {
        return;
    }
    // copy requested to unscanned map. we will remove ranges as they are scanned in trackScanning()
    for (Entry<KeyExtent, List<Range>> entry : requested.entrySet()) {
        ArrayList<Range> ranges = new ArrayList<>();
        for (Range range : entry.getValue()) {
            ranges.add(new Range(range));
        }
        unscanned.put(new KeyExtent(entry.getKey()), ranges);
    }
    timeoutTracker.startingScan();
    TTransport transport = null;
    try {
        final HostAndPort parsedServer = HostAndPort.fromString(server);
        final TabletClientService.Client client;
        if (timeoutTracker.getTimeOut() < context.getClientTimeoutInMillis())
            client = ThriftUtil.getTServerClient(parsedServer, context, timeoutTracker.getTimeOut());
        else
            client = ThriftUtil.getTServerClient(parsedServer, context);
        try {
            OpTimer timer = null;
            if (log.isTraceEnabled()) {
                log.trace("tid={} Starting multi scan, tserver={}  #tablets={}  #ranges={} ssil={} ssio={}", Thread.currentThread().getId(), server, requested.size(), sumSizes(requested.values()), options.serverSideIteratorList, options.serverSideIteratorOptions);
                timer = new OpTimer().start();
            }
            TabletType ttype = TabletType.type(requested.keySet());
            boolean waitForWrites = !ThriftScanner.serversWaitedForWrites.get(ttype).contains(server);
            Map<TKeyExtent, List<TRange>> thriftTabletRanges = Translator.translate(requested, Translators.KET, new Translator.ListTranslator<>(Translators.RT));
            InitialMultiScan imsr = client.startMultiScan(Tracer.traceInfo(), context.rpcCreds(), thriftTabletRanges, Translator.translate(columns, Translators.CT), options.serverSideIteratorList, options.serverSideIteratorOptions, ByteBufferUtil.toByteBuffers(authorizations.getAuthorizations()), waitForWrites, SamplerConfigurationImpl.toThrift(options.getSamplerConfiguration()), options.batchTimeOut, options.classLoaderContext);
            if (waitForWrites)
                ThriftScanner.serversWaitedForWrites.get(ttype).add(server.toString());
            MultiScanResult scanResult = imsr.result;
            if (timer != null) {
                timer.stop();
                log.trace("tid={} Got 1st multi scan results, #results={} {} in {}", Thread.currentThread().getId(), scanResult.results.size(), (scanResult.more ? "scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
            }
            ArrayList<Entry<Key, Value>> entries = new ArrayList<>(scanResult.results.size());
            for (TKeyValue kv : scanResult.results) {
                entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
            }
            if (entries.size() > 0)
                receiver.receive(entries);
            if (entries.size() > 0 || scanResult.fullScans.size() > 0)
                timeoutTracker.madeProgress();
            trackScanning(failures, unscanned, scanResult);
            AtomicLong nextOpid = new AtomicLong();
            while (scanResult.more) {
                timeoutTracker.check();
                if (timer != null) {
                    log.trace("tid={} oid={} Continuing multi scan, scanid={}", Thread.currentThread().getId(), nextOpid.get(), imsr.scanID);
                    timer.reset().start();
                }
                scanResult = client.continueMultiScan(Tracer.traceInfo(), imsr.scanID);
                if (timer != null) {
                    timer.stop();
                    log.trace("tid={} oid={} Got more multi scan results, #results={} {} in {}", Thread.currentThread().getId(), nextOpid.getAndIncrement(), scanResult.results.size(), (scanResult.more ? " scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
                }
                entries = new ArrayList<>(scanResult.results.size());
                for (TKeyValue kv : scanResult.results) {
                    entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
                }
                if (entries.size() > 0)
                    receiver.receive(entries);
                if (entries.size() > 0 || scanResult.fullScans.size() > 0)
                    timeoutTracker.madeProgress();
                trackScanning(failures, unscanned, scanResult);
            }
            client.closeMultiScan(Tracer.traceInfo(), imsr.scanID);
        } finally {
            ThriftUtil.returnClient(client);
        }
    } catch (TTransportException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage());
        timeoutTracker.errorOccured(e);
        throw new IOException(e);
    } catch (ThriftSecurityException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new AccumuloSecurityException(e.user, e.code, e);
    } catch (TApplicationException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new AccumuloServerException(server, e);
    } catch (NoSuchScanIDException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new IOException(e);
    } catch (TSampleNotPresentException e) {
        log.debug("Server : " + server + " msg : " + e.getMessage(), e);
        String tableInfo = "?";
        if (e.getExtent() != null) {
            Table.ID tableId = new KeyExtent(e.getExtent()).getTableId();
            tableInfo = Tables.getPrintableTableInfoFromId(context.getInstance(), tableId);
        }
        String message = "Table " + tableInfo + " does not have sampling configured or built";
        throw new SampleNotPresentException(message, e);
    } catch (TException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        timeoutTracker.errorOccured(e);
        throw new IOException(e);
    } finally {
        ThriftTransportPool.getInstance().returnTransport(transport);
    }
}

Example 3 with SampleNotPresentException

use of org.apache.accumulo.core.client.SampleNotPresentException in project accumulo by apache.

the class OfflineIterator method createIterator.

private SortedKeyValueIterator<Key, Value> createIterator(KeyExtent extent, List<String> absFiles) throws TableNotFoundException, AccumuloException, IOException {
    // TODO share code w/ tablet - ACCUMULO-1303
    // possible race condition here, if table is renamed
    String tableName = Tables.getTableName(conn.getInstance(), tableId);
    AccumuloConfiguration acuTableConf = new ConfigurationCopy(conn.tableOperations().getProperties(tableName));
    Configuration conf = CachedConfiguration.getInstance();
    for (SortedKeyValueIterator<Key, Value> reader : readers) {
        ((FileSKVIterator) reader).close();
    }
    readers.clear();
    SamplerConfiguration scannerSamplerConfig = options.getSamplerConfiguration();
    SamplerConfigurationImpl scannerSamplerConfigImpl = scannerSamplerConfig == null ? null : new SamplerConfigurationImpl(scannerSamplerConfig);
    SamplerConfigurationImpl samplerConfImpl = SamplerConfigurationImpl.newSamplerConfig(acuTableConf);
    if (scannerSamplerConfigImpl != null && ((samplerConfImpl != null && !scannerSamplerConfigImpl.equals(samplerConfImpl)) || samplerConfImpl == null)) {
        throw new SampleNotPresentException();
    }
    // TODO need to close files - ACCUMULO-1303
    for (String file : absFiles) {
        FileSystem fs = VolumeConfiguration.getVolume(file, conf, config).getFileSystem();
        FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder().forFile(file, fs, conf).withTableConfiguration(acuTableConf).build();
        if (scannerSamplerConfigImpl != null) {
            reader = reader.getSample(scannerSamplerConfigImpl);
            if (reader == null)
                throw new SampleNotPresentException();
        }
        readers.add(reader);
    }
    MultiIterator multiIter = new MultiIterator(readers, extent);
    OfflineIteratorEnvironment iterEnv = new OfflineIteratorEnvironment(authorizations, acuTableConf, false, samplerConfImpl == null ? null : samplerConfImpl.toSamplerConfiguration());
    byte[] defaultSecurityLabel;
    ColumnVisibility cv = new ColumnVisibility(acuTableConf.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
    defaultSecurityLabel = cv.getExpression();
    SortedKeyValueIterator<Key, Value> visFilter = IteratorUtil.setupSystemScanIterators(multiIter, new HashSet<>(options.fetchedColumns), authorizations, defaultSecurityLabel);
    return iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(IteratorScope.scan, visFilter, extent, acuTableConf, options.serverSideIteratorList, options.serverSideIteratorOptions, iterEnv, false));
}

Example 4 with SampleNotPresentException

use of org.apache.accumulo.core.client.SampleNotPresentException in project accumulo by apache.

the class SampleIT method assertSampleNotPresent.

private void assertSampleNotPresent(SamplerConfiguration sc, ScannerBase... scanners) {
    for (ScannerBase scanner : scanners) {
        SamplerConfiguration csc = scanner.getSamplerConfiguration();
        scanner.setSamplerConfiguration(sc);
        try {
            for (Entry<Key, Value> entry : scanner) {
                entry.getKey();
            }
            Assert.fail("Expected SampleNotPresentException, but it did not happen : " + scanner.getClass().getSimpleName());
        } catch (SampleNotPresentException e) {
        }
        scanner.clearSamplerConfiguration();
        for (Entry<Key, Value> entry : scanner) {
            entry.getKey();
        }
        if (csc == null) {
            scanner.clearSamplerConfiguration();
        } else {
            scanner.setSamplerConfiguration(csc);
        }
    }
}

Example 5 with SampleNotPresentException

use of org.apache.accumulo.core.client.SampleNotPresentException in project accumulo by apache.

the class SampleIT method testIterator.

@Test
public void testIterator() throws Exception {
    Connector conn = getConnector();
    String tableName = getUniqueNames(1)[0];
    String clone = tableName + "_clone";
    conn.tableOperations().create(tableName, new NewTableConfiguration().enableSampling(SC1));
    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
    TreeMap<Key, Value> expected = new TreeMap<>();
    writeData(bw, SC1, expected);
    ArrayList<Key> keys = new ArrayList<>(expected.keySet());
    Range range1 = new Range(keys.get(6), true, keys.get(11), true);
    Scanner scanner = null;
    Scanner isoScanner = null;
    ClientSideIteratorScanner csiScanner = null;
    BatchScanner bScanner = null;
    Scanner oScanner = null;
    try {
        scanner = conn.createScanner(tableName, Authorizations.EMPTY);
        isoScanner = new IsolatedScanner(conn.createScanner(tableName, Authorizations.EMPTY));
        csiScanner = new ClientSideIteratorScanner(conn.createScanner(tableName, Authorizations.EMPTY));
        bScanner = conn.createBatchScanner(tableName, Authorizations.EMPTY, 2);
        csiScanner.setIteratorSamplerConfiguration(SC1);
        List<? extends ScannerBase> scanners = Arrays.asList(scanner, isoScanner, bScanner, csiScanner);
        for (ScannerBase s : scanners) {
            s.addScanIterator(new IteratorSetting(100, IteratorThatUsesSample.class));
        }
        // the iterator should see less than 10 entries in sample data, and return data
        setRange(range1, scanners);
        for (ScannerBase s : scanners) {
            Assert.assertEquals(2954, countEntries(s));
        }
        Range range2 = new Range(keys.get(5), true, keys.get(18), true);
        setRange(range2, scanners);
        // the iterator should see more than 10 entries in sample data, and return no data
        for (ScannerBase s : scanners) {
            Assert.assertEquals(0, countEntries(s));
        }
        // flush an rerun same test against files
        conn.tableOperations().flush(tableName, null, null, true);
        oScanner = newOfflineScanner(conn, tableName, clone, null);
        oScanner.addScanIterator(new IteratorSetting(100, IteratorThatUsesSample.class));
        scanners = Arrays.asList(scanner, isoScanner, bScanner, csiScanner, oScanner);
        setRange(range1, scanners);
        for (ScannerBase s : scanners) {
            Assert.assertEquals(2954, countEntries(s));
        }
        setRange(range2, scanners);
        for (ScannerBase s : scanners) {
            Assert.assertEquals(0, countEntries(s));
        }
        updateSamplingConfig(conn, tableName, SC2);
        csiScanner.setIteratorSamplerConfiguration(SC2);
        oScanner = newOfflineScanner(conn, tableName, clone, null);
        oScanner.addScanIterator(new IteratorSetting(100, IteratorThatUsesSample.class));
        scanners = Arrays.asList(scanner, isoScanner, bScanner, csiScanner, oScanner);
        for (ScannerBase s : scanners) {
            try {
                countEntries(s);
                Assert.fail("Expected SampleNotPresentException, but it did not happen : " + s.getClass().getSimpleName());
            } catch (SampleNotPresentException e) {
            }
        }
    } finally {
        if (scanner != null) {
            scanner.close();
        }
        if (bScanner != null) {
            bScanner.close();
        }
        if (isoScanner != null) {
            isoScanner.close();
        }
        if (csiScanner != null) {
            csiScanner.close();
        }
        if (oScanner != null) {
            oScanner.close();
        }
    }
}