Examples with IterInfo org.apache.accumulo.core.dataImpl.thrift.IterInfo used on opensource projects

Example 6 with IterInfo

use of org.apache.accumulo.core.dataImpl.thrift.IterInfo in project accumulo by apache.

the class IterConfigUtil method convertItersAndLoad.

/**
 * Convert the list of iterators to IterInfo objects and then load the stack.
 */
public static SortedKeyValueIterator<Key, Value> convertItersAndLoad(IteratorScope scope, SortedKeyValueIterator<Key, Value> source, AccumuloConfiguration conf, List<IteratorSetting> iterators, IteratorEnvironment env) throws IOException {
    List<IterInfo> ssiList = new ArrayList<>();
    Map<String, Map<String, String>> ssio = new HashMap<>();
    for (IteratorSetting is : iterators) {
        ssiList.add(new IterInfo(is.getPriority(), is.getIteratorClass(), is.getName()));
        ssio.put(is.getName(), is.getOptions());
    }
    IterLoad il = loadIterConf(scope, ssiList, ssio, conf);
    il = il.iterEnv(env).useAccumuloClassLoader(true).context(ClassLoaderUtil.tableContext(conf));
    return loadIterators(source, il);
}

Example 7 with IterInfo

use of org.apache.accumulo.core.dataImpl.thrift.IterInfo in project accumulo by apache.

the class IterConfigUtil method parseIterConf.

public static List<IterInfo> parseIterConf(IteratorScope scope, List<IterInfo> iters, Map<String, Map<String, String>> allOptions, AccumuloConfiguration conf) {
    Map<String, String> properties = conf.getAllPropertiesWithPrefix(getProperty(scope));
    ArrayList<IterInfo> iterators = new ArrayList<>(iters);
    final Property scopeProperty = getProperty(scope);
    final String scopePropertyKey = scopeProperty.getKey();
    for (Entry<String, String> entry : properties.entrySet()) {
        String suffix = entry.getKey().substring(scopePropertyKey.length());
        String[] suffixSplit = suffix.split("\\.", 3);
        if (suffixSplit.length == 1) {
            String[] sa = entry.getValue().split(",");
            int prio = Integer.parseInt(sa[0]);
            String className = sa[1];
            iterators.add(new IterInfo(prio, className, suffixSplit[0]));
        } else if (suffixSplit.length == 3 && suffixSplit[1].equals("opt")) {
            String iterName = suffixSplit[0];
            String optName = suffixSplit[2];
            allOptions.computeIfAbsent(iterName, k -> new HashMap<>()).put(optName, entry.getValue());
        } else {
            throw new IllegalArgumentException("Invalid iterator format: " + entry.getKey());
        }
    }
    iterators.sort(ITER_INFO_COMPARATOR);
    return iterators;
}

Example 8 with IterInfo

use of org.apache.accumulo.core.dataImpl.thrift.IterInfo in project accumulo by apache.

the class IterConfigUtil method loadIterators.

/**
 * Load a stack of iterators provided in the IterLoad, starting with source.
 */
public static SortedKeyValueIterator<Key, Value> loadIterators(SortedKeyValueIterator<Key, Value> source, IterLoad iterLoad) throws IOException {
    SortedKeyValueIterator<Key, Value> prev = source;
    try {
        for (IterInfo iterInfo : iterLoad.iters) {
            Class<SortedKeyValueIterator<Key, Value>> clazz = null;
            log.trace("Attempting to load iterator class {}", iterInfo.className);
            if (iterLoad.classCache != null) {
                clazz = iterLoad.classCache.get(iterInfo.className);
                if (clazz == null) {
                    clazz = loadClass(iterLoad.useAccumuloClassLoader, iterLoad.context, iterInfo);
                    iterLoad.classCache.put(iterInfo.className, clazz);
                }
            } else {
                clazz = loadClass(iterLoad.useAccumuloClassLoader, iterLoad.context, iterInfo);
            }
            SortedKeyValueIterator<Key, Value> skvi = clazz.getDeclaredConstructor().newInstance();
            Map<String, String> options = iterLoad.iterOpts.get(iterInfo.iterName);
            if (options == null)
                options = Collections.emptyMap();
            skvi.init(prev, options, iterLoad.iteratorEnvironment);
            prev = skvi;
        }
    } catch (ReflectiveOperationException e) {
        log.error(e.toString());
        throw new RuntimeException(e);
    }
    return prev;
}

Example 9 with IterInfo

use of org.apache.accumulo.core.dataImpl.thrift.IterInfo in project accumulo by apache.

the class ConditionCheckerContext method buildIterator.

SortedKeyValueIterator<Key, Value> buildIterator(SortedKeyValueIterator<Key, Value> systemIter, TCondition tc) throws IOException {
    ArrayByteSequence key = new ArrayByteSequence(tc.iterators);
    MergedIterConfig mic = mergedIterCache.get(key);
    if (mic == null) {
        IterConfig ic = compressedIters.decompress(tc.iterators);
        List<IterInfo> mergedIters = new ArrayList<>(tableIters.size() + ic.ssiList.size());
        Map<String, Map<String, String>> mergedItersOpts = new HashMap<>(tableIterOpts.size() + ic.ssio.size());
        IterConfigUtil.mergeIteratorConfig(mergedIters, mergedItersOpts, tableIters, tableIterOpts, ic.ssiList, ic.ssio);
        mic = new MergedIterConfig(mergedIters, mergedItersOpts);
        mergedIterCache.put(key, mic);
    }
    IterLoad iterLoad = new IterLoad().iters(mic.mergedIters).iterOpts(mic.mergedItersOpts).iterEnv(tie).useAccumuloClassLoader(true).context(context).classCache(classCache);
    return IterConfigUtil.loadIterators(systemIter, iterLoad);
}

Example 10 with IterInfo

use of org.apache.accumulo.core.dataImpl.thrift.IterInfo in project accumulo by apache.

the class ScanDataSource method createIterator.

private SortedKeyValueIterator<Key, Value> createIterator() throws IOException {
    Map<TabletFile, DataFileValue> files;
    SamplerConfigurationImpl samplerConfig = scanParams.getSamplerConfigurationImpl();
    synchronized (tablet) {
        if (memIters != null)
            throw new IllegalStateException("Tried to create new scan iterator w/o releasing memory");
        if (tablet.isClosed())
            throw new TabletClosedException();
        if (interruptFlag.get())
            throw new IterationInterruptedException(tablet.getExtent() + " " + interruptFlag.hashCode());
        // only acquire the file manager when we know the tablet is open
        if (fileManager == null) {
            fileManager = tablet.getTabletResources().newScanFileManager(scanParams.getScanDispatch());
            tablet.getTabletServer().getScanMetrics().incrementOpenFiles(fileManager.getNumOpenFiles());
            tablet.addActiveScans(this);
        }
        if (fileManager.getNumOpenFiles() != 0)
            throw new IllegalStateException("Tried to create new scan iterator w/o releasing files");
        // set this before trying to get iterators in case
        // getIterators() throws an exception
        expectedDeletionCount = tablet.getDataSourceDeletions();
        memIters = tablet.getTabletMemory().getIterators(samplerConfig);
        Pair<Long, Map<TabletFile, DataFileValue>> reservation = tablet.getDatafileManager().reserveFilesForScan();
        fileReservationId = reservation.getFirst();
        files = reservation.getSecond();
    }
    Collection<InterruptibleIterator> mapfiles = fileManager.openFiles(files, scanParams.isIsolated(), samplerConfig);
    for (SortedKeyValueIterator<Key, Value> skvi : Iterables.concat(mapfiles, memIters)) ((InterruptibleIterator) skvi).setInterruptFlag(interruptFlag);
    List<SortedKeyValueIterator<Key, Value>> iters = new ArrayList<>(mapfiles.size() + memIters.size());
    iters.addAll(mapfiles);
    iters.addAll(memIters);
    MultiIterator multiIter = new MultiIterator(iters, tablet.getExtent());
    TabletIteratorEnvironment iterEnv = new TabletIteratorEnvironment(tablet.getTabletServer().getContext(), IteratorScope.scan, tablet.getTableConfiguration(), tablet.getExtent().tableId(), fileManager, files, scanParams.getAuthorizations(), samplerConfig, new ArrayList<>());
    statsIterator = new StatsIterator(multiIter, TabletServer.seekCount, tablet.getScannedCounter());
    SortedKeyValueIterator<Key, Value> visFilter = SystemIteratorUtil.setupSystemScanIterators(statsIterator, scanParams.getColumnSet(), scanParams.getAuthorizations(), defaultLabels, tablet.getTableConfiguration());
    if (loadIters) {
        List<IterInfo> iterInfos;
        Map<String, Map<String, String>> iterOpts;
        ParsedIteratorConfig pic = tablet.getTableConfiguration().getParsedIteratorConfig(IteratorScope.scan);
        if (scanParams.getSsiList().isEmpty() && scanParams.getSsio().isEmpty()) {
            // No scan time iterator options were set, so can just use the pre-parsed table iterator
            // options.
            iterInfos = pic.getIterInfo();
            iterOpts = pic.getOpts();
        } else {
            // Scan time iterator options were set, so need to merge those with pre-parsed table
            // iterator options.
            iterOpts = new HashMap<>(pic.getOpts().size() + scanParams.getSsio().size());
            iterInfos = new ArrayList<>(pic.getIterInfo().size() + scanParams.getSsiList().size());
            IterConfigUtil.mergeIteratorConfig(iterInfos, iterOpts, pic.getIterInfo(), pic.getOpts(), scanParams.getSsiList(), scanParams.getSsio());
        }
        String context;
        if (scanParams.getClassLoaderContext() != null) {
            log.trace("Loading iterators for scan with scan context: {}", scanParams.getClassLoaderContext());
            context = scanParams.getClassLoaderContext();
        } else {
            context = pic.getServiceEnv();
            if (context != null) {
                log.trace("Loading iterators for scan with table context: {}", scanParams.getClassLoaderContext());
            } else {
                log.trace("Loading iterators for scan");
            }
        }
        IterLoad il = new IterLoad().iters(iterInfos).iterOpts(iterOpts).iterEnv(iterEnv).useAccumuloClassLoader(true).context(context);
        return iterEnv.getTopLevelIterator(IterConfigUtil.loadIterators(visFilter, il));
    } else {
        return visFilter;
    }
}