Examples with SourceSwitchingIterator org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator used on opensource projects

Example 1 with SourceSwitchingIterator

use of org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator in project accumulo by apache.

the class SourceSwitchingIteratorTest method test3.

@Test
public void test3() throws Exception {
    // test switching after a row
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    put(tm1, "r1", "cf1", "cq1", 5, "v1");
    put(tm1, "r1", "cf1", "cq2", 5, "v2");
    put(tm1, "r1", "cf1", "cq3", 5, "v3");
    put(tm1, "r1", "cf1", "cq4", 5, "v4");
    put(tm1, "r3", "cf1", "cq1", 5, "v5");
    put(tm1, "r3", "cf1", "cq2", 5, "v6");
    SortedMapIterator smi = new SortedMapIterator(tm1);
    TestDataSource tds = new TestDataSource(smi);
    SourceSwitchingIterator ssi = new SourceSwitchingIterator(tds, true);
    ssi.seek(new Range(), new ArrayList<>(), false);
    testAndCallNext(ssi, "r1", "cf1", "cq1", 5, "v1", true);
    TreeMap<Key, Value> tm2 = new TreeMap<>(tm1);
    // should not see this because it should not switch until
    put(tm2, "r1", "cf1", "cq5", 5, "v7");
    // the row is finished
    // should see this new row after it switches
    put(tm2, "r2", "cf1", "cq1", 5, "v8");
    // setup a new data source, but it should not switch until the current row is finished
    SortedMapIterator smi2 = new SortedMapIterator(tm2);
    tds.next = new TestDataSource(smi2);
    testAndCallNext(ssi, "r1", "cf1", "cq2", 5, "v2", true);
    testAndCallNext(ssi, "r1", "cf1", "cq3", 5, "v3", true);
    testAndCallNext(ssi, "r1", "cf1", "cq4", 5, "v4", true);
    testAndCallNext(ssi, "r2", "cf1", "cq1", 5, "v8", true);
    testAndCallNext(ssi, "r3", "cf1", "cq1", 5, "v5", true);
    testAndCallNext(ssi, "r3", "cf1", "cq2", 5, "v6", true);
}

Example 2 with SourceSwitchingIterator

use of org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator in project accumulo by apache.

the class SourceSwitchingIteratorTest method test4.

@Test
public void test4() throws Exception {
    // ensure switch is done on initial seek
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    put(tm1, "r1", "cf1", "cq1", 5, "v1");
    put(tm1, "r1", "cf1", "cq2", 5, "v2");
    SortedMapIterator smi = new SortedMapIterator(tm1);
    TestDataSource tds = new TestDataSource(smi);
    SourceSwitchingIterator ssi = new SourceSwitchingIterator(tds, false);
    TreeMap<Key, Value> tm2 = new TreeMap<>();
    put(tm2, "r1", "cf1", "cq1", 6, "v3");
    put(tm2, "r1", "cf1", "cq2", 6, "v4");
    SortedMapIterator smi2 = new SortedMapIterator(tm2);
    tds.next = new TestDataSource(smi2);
    ssi.seek(new Range(), new ArrayList<>(), false);
    testAndCallNext(ssi, "r1", "cf1", "cq1", 6, "v3", true);
    testAndCallNext(ssi, "r1", "cf1", "cq2", 6, "v4", true);
}

Example 3 with SourceSwitchingIterator

use of org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator in project accumulo by apache.

the class SourceSwitchingIteratorTest method test5.

@Test
public void test5() throws Exception {
    // ensure switchNow() works w/ deepCopy()
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    put(tm1, "r1", "cf1", "cq1", 5, "v1");
    put(tm1, "r1", "cf1", "cq2", 5, "v2");
    SortedMapIterator smi = new SortedMapIterator(tm1);
    TestDataSource tds = new TestDataSource(smi);
    SourceSwitchingIterator ssi = new SourceSwitchingIterator(tds, false);
    SortedKeyValueIterator<Key, Value> dc1 = ssi.deepCopy(null);
    TreeMap<Key, Value> tm2 = new TreeMap<>();
    put(tm2, "r1", "cf1", "cq1", 6, "v3");
    put(tm2, "r2", "cf1", "cq2", 6, "v4");
    SortedMapIterator smi2 = new SortedMapIterator(tm2);
    TestDataSource tds2 = new TestDataSource(smi2);
    tds.setNext(tds2);
    ssi.switchNow();
    ssi.seek(new Range("r1"), new ArrayList<>(), false);
    dc1.seek(new Range("r2"), new ArrayList<>(), false);
    testAndCallNext(ssi, "r1", "cf1", "cq1", 6, "v3", true);
    assertFalse(ssi.hasTop());
    testAndCallNext(dc1, "r2", "cf1", "cq2", 6, "v4", true);
    assertFalse(dc1.hasTop());
}

Example 4 with SourceSwitchingIterator

use of org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator in project accumulo by apache.

the class Tablet method lookup.

public LookupResult lookup(List<Range> ranges, List<KVEntry> results, ScanParameters scanParams, long maxResultSize, AtomicBoolean interruptFlag) throws IOException {
    if (ranges.isEmpty()) {
        return new LookupResult();
    }
    ranges = Range.mergeOverlapping(ranges);
    if (ranges.size() > 1) {
        Collections.sort(ranges);
    }
    Range tabletRange = extent.toDataRange();
    for (Range range : ranges) {
        // do a test to see if this range falls within the tablet, if it does not
        // then clip will throw an exception
        tabletRange.clip(range);
    }
    ScanDataSource dataSource = new ScanDataSource(this, scanParams, true, interruptFlag);
    LookupResult result = null;
    try {
        SortedKeyValueIterator<Key, Value> iter = new SourceSwitchingIterator(dataSource);
        lookupCount++;
        result = lookup(iter, ranges, results, scanParams, maxResultSize);
        return result;
    } catch (IOException ioe) {
        dataSource.close(true);
        throw ioe;
    } finally {
        // code in finally block because always want
        // to return mapfiles, even when exception is thrown
        dataSource.close(false);
        synchronized (this) {
            queryResultCount += results.size();
            if (result != null) {
                queryResultBytes += result.dataSize;
            }
        }
    }
}

Example 5 with SourceSwitchingIterator

use of org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator in project accumulo by apache.

the class Scanner method read.

public ScanBatch read() throws IOException, TabletClosedException {
    ScanDataSource dataSource = null;
    Batch results = null;
    try {
        try {
            scannerSemaphore.acquire();
        } catch (InterruptedException e) {
            sawException = true;
        }
        // exception was its cause
        if (sawException)
            throw new IllegalStateException("Tried to use scanner after exception occurred.");
        if (scanClosed)
            throw new IllegalStateException("Tried to use scanner after it was closed.");
        if (scanParams.isIsolated()) {
            if (isolatedDataSource == null)
                isolatedDataSource = new ScanDataSource(tablet, scanParams, true, interruptFlag);
            dataSource = isolatedDataSource;
        } else {
            dataSource = new ScanDataSource(tablet, scanParams, true, interruptFlag);
        }
        SortedKeyValueIterator<Key, Value> iter;
        if (scanParams.isIsolated()) {
            if (isolatedIter == null)
                isolatedIter = new SourceSwitchingIterator(dataSource, true);
            else
                isolatedDataSource.reattachFileManager();
            iter = isolatedIter;
        } else {
            iter = new SourceSwitchingIterator(dataSource, false);
        }
        results = tablet.nextBatch(iter, range, scanParams);
        if (results.getResults() == null) {
            range = null;
            return new ScanBatch(new ArrayList<>(), false);
        } else if (results.getContinueKey() == null) {
            return new ScanBatch(results.getResults(), false);
        } else {
            range = new Range(results.getContinueKey(), !results.isSkipContinueKey(), range.getEndKey(), range.isEndKeyInclusive());
            return new ScanBatch(results.getResults(), true);
        }
    } catch (IterationInterruptedException iie) {
        sawException = true;
        if (tablet.isClosed())
            throw new TabletClosedException(iie);
        else
            throw iie;
    } catch (IOException ioe) {
        if (ShutdownUtil.isShutdownInProgress()) {
            log.debug("IOException while shutdown in progress ", ioe);
            // assume IOException was caused by execution of HDFS
            throw new TabletClosedException(ioe);
        // shutdown hook
        }
        sawException = true;
        dataSource.close(true);
        throw ioe;
    } catch (RuntimeException re) {
        sawException = true;
        throw re;
    } finally {
        // to return mapfiles, even when exception is thrown
        if (dataSource != null && !scanParams.isIsolated()) {
            dataSource.close(false);
        } else if (dataSource != null) {
            dataSource.detachFileManager();
        }
        if (results != null && results.getResults() != null)
            tablet.updateQueryStats(results.getResults().size(), results.getNumBytes());
        scannerSemaphore.release();
    }
}