Examples with InterruptibleIterator org.apache.accumulo.core.iteratorsImpl.system.InterruptibleIterator used on opensource projects

Example 1 with InterruptibleIterator

use of org.apache.accumulo.core.iteratorsImpl.system.InterruptibleIterator in project accumulo by apache.

the class InMemoryMap method delete.

public void delete(long waitTime) {
    synchronized (this) {
        if (deleted)
            throw new IllegalStateException("Double delete");
        deleted = true;
    }
    long t1 = System.currentTimeMillis();
    while (!activeIters.isEmpty() && System.currentTimeMillis() - t1 < waitTime) {
        sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
    }
    if (!activeIters.isEmpty()) {
        // dump memmap exactly as is to a tmp file on disk, and switch scans to that temp file
        try {
            Configuration conf = context.getHadoopConf();
            FileSystem fs = FileSystem.getLocal(conf);
            String tmpFile = memDumpDir + "/memDump" + UUID.randomUUID() + "." + RFile.EXTENSION;
            Configuration newConf = new Configuration(conf);
            newConf.setInt("io.seqfile.compress.blocksize", 100000);
            AccumuloConfiguration aconf = context.getConfiguration();
            if (getOrCreateSampler() != null) {
                aconf = createSampleConfig(aconf);
            }
            FileSKVWriter out = new RFileOperations().newWriterBuilder().forFile(tmpFile, fs, newConf, context.getCryptoService()).withTableConfiguration(aconf).build();
            InterruptibleIterator iter = map.skvIterator(null);
            HashSet<ByteSequence> allfams = new HashSet<>();
            for (Entry<String, Set<ByteSequence>> entry : lggroups.entrySet()) {
                allfams.addAll(entry.getValue());
                out.startNewLocalityGroup(entry.getKey(), entry.getValue());
                iter.seek(new Range(), entry.getValue(), true);
                dumpLocalityGroup(out, iter);
            }
            out.startDefaultLocalityGroup();
            iter.seek(new Range(), allfams, false);
            dumpLocalityGroup(out, iter);
            out.close();
            log.debug("Created mem dump file {}", tmpFile);
            memDumpFile = tmpFile;
            synchronized (activeIters) {
                for (MemoryIterator mi : activeIters) {
                    mi.switchNow();
                }
            }
            // rely on unix behavior that file will be deleted when last
            // reader closes it
            fs.delete(new Path(memDumpFile), true);
        } catch (IOException ioe) {
            log.error("Failed to create mem dump file", ioe);
            while (!activeIters.isEmpty()) {
                sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            }
        }
    }
    SimpleMap tmpMap = map;
    synchronized (this) {
        map = null;
    }
    tmpMap.delete();
}

Example 2 with InterruptibleIterator

use of org.apache.accumulo.core.iteratorsImpl.system.InterruptibleIterator in project accumulo by apache.

the class ScanDataSource method createIterator.

private SortedKeyValueIterator<Key, Value> createIterator() throws IOException {
    Map<TabletFile, DataFileValue> files;
    SamplerConfigurationImpl samplerConfig = scanParams.getSamplerConfigurationImpl();
    synchronized (tablet) {
        if (memIters != null)
            throw new IllegalStateException("Tried to create new scan iterator w/o releasing memory");
        if (tablet.isClosed())
            throw new TabletClosedException();
        if (interruptFlag.get())
            throw new IterationInterruptedException(tablet.getExtent() + " " + interruptFlag.hashCode());
        // only acquire the file manager when we know the tablet is open
        if (fileManager == null) {
            fileManager = tablet.getTabletResources().newScanFileManager(scanParams.getScanDispatch());
            tablet.getTabletServer().getScanMetrics().incrementOpenFiles(fileManager.getNumOpenFiles());
            tablet.addActiveScans(this);
        }
        if (fileManager.getNumOpenFiles() != 0)
            throw new IllegalStateException("Tried to create new scan iterator w/o releasing files");
        // set this before trying to get iterators in case
        // getIterators() throws an exception
        expectedDeletionCount = tablet.getDataSourceDeletions();
        memIters = tablet.getTabletMemory().getIterators(samplerConfig);
        Pair<Long, Map<TabletFile, DataFileValue>> reservation = tablet.getDatafileManager().reserveFilesForScan();
        fileReservationId = reservation.getFirst();
        files = reservation.getSecond();
    }
    Collection<InterruptibleIterator> mapfiles = fileManager.openFiles(files, scanParams.isIsolated(), samplerConfig);
    for (SortedKeyValueIterator<Key, Value> skvi : Iterables.concat(mapfiles, memIters)) ((InterruptibleIterator) skvi).setInterruptFlag(interruptFlag);
    List<SortedKeyValueIterator<Key, Value>> iters = new ArrayList<>(mapfiles.size() + memIters.size());
    iters.addAll(mapfiles);
    iters.addAll(memIters);
    MultiIterator multiIter = new MultiIterator(iters, tablet.getExtent());
    TabletIteratorEnvironment iterEnv = new TabletIteratorEnvironment(tablet.getTabletServer().getContext(), IteratorScope.scan, tablet.getTableConfiguration(), tablet.getExtent().tableId(), fileManager, files, scanParams.getAuthorizations(), samplerConfig, new ArrayList<>());
    statsIterator = new StatsIterator(multiIter, TabletServer.seekCount, tablet.getScannedCounter());
    SortedKeyValueIterator<Key, Value> visFilter = SystemIteratorUtil.setupSystemScanIterators(statsIterator, scanParams.getColumnSet(), scanParams.getAuthorizations(), defaultLabels, tablet.getTableConfiguration());
    if (loadIters) {
        List<IterInfo> iterInfos;
        Map<String, Map<String, String>> iterOpts;
        ParsedIteratorConfig pic = tablet.getTableConfiguration().getParsedIteratorConfig(IteratorScope.scan);
        if (scanParams.getSsiList().isEmpty() && scanParams.getSsio().isEmpty()) {
            // No scan time iterator options were set, so can just use the pre-parsed table iterator
            // options.
            iterInfos = pic.getIterInfo();
            iterOpts = pic.getOpts();
        } else {
            // Scan time iterator options were set, so need to merge those with pre-parsed table
            // iterator options.
            iterOpts = new HashMap<>(pic.getOpts().size() + scanParams.getSsio().size());
            iterInfos = new ArrayList<>(pic.getIterInfo().size() + scanParams.getSsiList().size());
            IterConfigUtil.mergeIteratorConfig(iterInfos, iterOpts, pic.getIterInfo(), pic.getOpts(), scanParams.getSsiList(), scanParams.getSsio());
        }
        String context;
        if (scanParams.getClassLoaderContext() != null) {
            log.trace("Loading iterators for scan with scan context: {}", scanParams.getClassLoaderContext());
            context = scanParams.getClassLoaderContext();
        } else {
            context = pic.getServiceEnv();
            if (context != null) {
                log.trace("Loading iterators for scan with table context: {}", scanParams.getClassLoaderContext());
            } else {
                log.trace("Loading iterators for scan");
            }
        }
        IterLoad il = new IterLoad().iters(iterInfos).iterOpts(iterOpts).iterEnv(iterEnv).useAccumuloClassLoader(true).context(context);
        return iterEnv.getTopLevelIterator(IterConfigUtil.loadIterators(visFilter, il));
    } else {
        return visFilter;
    }
}