Examples with ClientSideIteratorScanner org.apache.accumulo.core.client.ClientSideIteratorScanner used on opensource projects

Example 1 with ClientSideIteratorScanner

use of org.apache.accumulo.core.client.ClientSideIteratorScanner in project accumulo by apache.

the class SampleIT method testBasic.

@Test
public void testBasic() throws Exception {
    Connector conn = getConnector();
    String tableName = getUniqueNames(1)[0];
    String clone = tableName + "_clone";
    conn.tableOperations().create(tableName, new NewTableConfiguration().enableSampling(SC1));
    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
    TreeMap<Key, Value> expected = new TreeMap<>();
    String someRow = writeData(bw, SC1, expected);
    Assert.assertEquals(20, expected.size());
    Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
    Scanner isoScanner = new IsolatedScanner(conn.createScanner(tableName, Authorizations.EMPTY));
    Scanner csiScanner = new ClientSideIteratorScanner(conn.createScanner(tableName, Authorizations.EMPTY));
    scanner.setSamplerConfiguration(SC1);
    csiScanner.setSamplerConfiguration(SC1);
    isoScanner.setSamplerConfiguration(SC1);
    isoScanner.setBatchSize(10);
    BatchScanner bScanner = conn.createBatchScanner(tableName, Authorizations.EMPTY, 2);
    bScanner.setSamplerConfiguration(SC1);
    bScanner.setRanges(Arrays.asList(new Range()));
    check(expected, scanner, bScanner, isoScanner, csiScanner);
    conn.tableOperations().flush(tableName, null, null, true);
    Scanner oScanner = newOfflineScanner(conn, tableName, clone, SC1);
    check(expected, scanner, bScanner, isoScanner, csiScanner, oScanner);
    // ensure non sample data can be scanned after scanning sample data
    for (ScannerBase sb : Arrays.asList(scanner, bScanner, isoScanner, csiScanner, oScanner)) {
        sb.clearSamplerConfiguration();
        Assert.assertEquals(20000, Iterables.size(sb));
        sb.setSamplerConfiguration(SC1);
    }
    Iterator<Key> it = expected.keySet().iterator();
    while (it.hasNext()) {
        Key k = it.next();
        if (k.getRow().toString().equals(someRow)) {
            it.remove();
        }
    }
    expected.put(new Key(someRow, "cf1", "cq1", 8), new Value("42".getBytes()));
    expected.put(new Key(someRow, "cf1", "cq3", 8), new Value("suprise".getBytes()));
    Mutation m = new Mutation(someRow);
    m.put("cf1", "cq1", 8, "42");
    m.putDelete("cf1", "cq2", 8);
    m.put("cf1", "cq3", 8, "suprise");
    bw.addMutation(m);
    bw.close();
    check(expected, scanner, bScanner, isoScanner, csiScanner);
    conn.tableOperations().flush(tableName, null, null, true);
    oScanner = newOfflineScanner(conn, tableName, clone, SC1);
    check(expected, scanner, bScanner, isoScanner, csiScanner, oScanner);
    scanner.setRange(new Range(someRow));
    isoScanner.setRange(new Range(someRow));
    csiScanner.setRange(new Range(someRow));
    oScanner.setRange(new Range(someRow));
    bScanner.setRanges(Arrays.asList(new Range(someRow)));
    expected.clear();
    expected.put(new Key(someRow, "cf1", "cq1", 8), new Value("42".getBytes()));
    expected.put(new Key(someRow, "cf1", "cq3", 8), new Value("suprise".getBytes()));
    check(expected, scanner, bScanner, isoScanner, csiScanner, oScanner);
}

Example 2 with ClientSideIteratorScanner

use of org.apache.accumulo.core.client.ClientSideIteratorScanner in project accumulo by apache.

the class SampleIT method testSampleNotPresent.

@Test
public void testSampleNotPresent() throws Exception {
    Connector conn = getConnector();
    String tableName = getUniqueNames(1)[0];
    String clone = tableName + "_clone";
    conn.tableOperations().create(tableName);
    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
    TreeMap<Key, Value> expected = new TreeMap<>();
    writeData(bw, SC1, expected);
    Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
    Scanner isoScanner = new IsolatedScanner(conn.createScanner(tableName, Authorizations.EMPTY));
    isoScanner.setBatchSize(10);
    Scanner csiScanner = new ClientSideIteratorScanner(conn.createScanner(tableName, Authorizations.EMPTY));
    BatchScanner bScanner = conn.createBatchScanner(tableName, Authorizations.EMPTY, 2);
    bScanner.setRanges(Arrays.asList(new Range()));
    // ensure sample not present exception occurs when sampling is not configured
    assertSampleNotPresent(SC1, scanner, isoScanner, bScanner, csiScanner);
    conn.tableOperations().flush(tableName, null, null, true);
    Scanner oScanner = newOfflineScanner(conn, tableName, clone, SC1);
    assertSampleNotPresent(SC1, scanner, isoScanner, bScanner, csiScanner, oScanner);
    // configure sampling, however there exist an rfile w/o sample data... so should still see sample not present exception
    updateSamplingConfig(conn, tableName, SC1);
    // create clone with new config
    oScanner = newOfflineScanner(conn, tableName, clone, SC1);
    assertSampleNotPresent(SC1, scanner, isoScanner, bScanner, csiScanner, oScanner);
    // create rfile with sample data present
    conn.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    // should be able to scan sample now
    oScanner = newOfflineScanner(conn, tableName, clone, SC1);
    setSamplerConfig(SC1, scanner, csiScanner, isoScanner, bScanner, oScanner);
    check(expected, scanner, isoScanner, bScanner, csiScanner, oScanner);
    // change sampling config
    updateSamplingConfig(conn, tableName, SC2);
    // create clone with new config
    oScanner = newOfflineScanner(conn, tableName, clone, SC2);
    // rfile should have different sample config than table, and scan should not work
    assertSampleNotPresent(SC2, scanner, isoScanner, bScanner, csiScanner, oScanner);
    // create rfile that has same sample data as table config
    conn.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    // should be able to scan sample now
    updateExpected(SC2, expected);
    oScanner = newOfflineScanner(conn, tableName, clone, SC2);
    setSamplerConfig(SC2, scanner, csiScanner, isoScanner, bScanner, oScanner);
    check(expected, scanner, isoScanner, bScanner, csiScanner, oScanner);
}

Example 3 with ClientSideIteratorScanner

use of org.apache.accumulo.core.client.ClientSideIteratorScanner in project accumulo by apache.

the class SampleIT method testIterator.

@Test
public void testIterator() throws Exception {
    Connector conn = getConnector();
    String tableName = getUniqueNames(1)[0];
    String clone = tableName + "_clone";
    conn.tableOperations().create(tableName, new NewTableConfiguration().enableSampling(SC1));
    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
    TreeMap<Key, Value> expected = new TreeMap<>();
    writeData(bw, SC1, expected);
    ArrayList<Key> keys = new ArrayList<>(expected.keySet());
    Range range1 = new Range(keys.get(6), true, keys.get(11), true);
    Scanner scanner = null;
    Scanner isoScanner = null;
    ClientSideIteratorScanner csiScanner = null;
    BatchScanner bScanner = null;
    Scanner oScanner = null;
    try {
        scanner = conn.createScanner(tableName, Authorizations.EMPTY);
        isoScanner = new IsolatedScanner(conn.createScanner(tableName, Authorizations.EMPTY));
        csiScanner = new ClientSideIteratorScanner(conn.createScanner(tableName, Authorizations.EMPTY));
        bScanner = conn.createBatchScanner(tableName, Authorizations.EMPTY, 2);
        csiScanner.setIteratorSamplerConfiguration(SC1);
        List<? extends ScannerBase> scanners = Arrays.asList(scanner, isoScanner, bScanner, csiScanner);
        for (ScannerBase s : scanners) {
            s.addScanIterator(new IteratorSetting(100, IteratorThatUsesSample.class));
        }
        // the iterator should see less than 10 entries in sample data, and return data
        setRange(range1, scanners);
        for (ScannerBase s : scanners) {
            Assert.assertEquals(2954, countEntries(s));
        }
        Range range2 = new Range(keys.get(5), true, keys.get(18), true);
        setRange(range2, scanners);
        // the iterator should see more than 10 entries in sample data, and return no data
        for (ScannerBase s : scanners) {
            Assert.assertEquals(0, countEntries(s));
        }
        // flush an rerun same test against files
        conn.tableOperations().flush(tableName, null, null, true);
        oScanner = newOfflineScanner(conn, tableName, clone, null);
        oScanner.addScanIterator(new IteratorSetting(100, IteratorThatUsesSample.class));
        scanners = Arrays.asList(scanner, isoScanner, bScanner, csiScanner, oScanner);
        setRange(range1, scanners);
        for (ScannerBase s : scanners) {
            Assert.assertEquals(2954, countEntries(s));
        }
        setRange(range2, scanners);
        for (ScannerBase s : scanners) {
            Assert.assertEquals(0, countEntries(s));
        }
        updateSamplingConfig(conn, tableName, SC2);
        csiScanner.setIteratorSamplerConfiguration(SC2);
        oScanner = newOfflineScanner(conn, tableName, clone, null);
        oScanner.addScanIterator(new IteratorSetting(100, IteratorThatUsesSample.class));
        scanners = Arrays.asList(scanner, isoScanner, bScanner, csiScanner, oScanner);
        for (ScannerBase s : scanners) {
            try {
                countEntries(s);
                Assert.fail("Expected SampleNotPresentException, but it did not happen : " + s.getClass().getSimpleName());
            } catch (SampleNotPresentException e) {
            }
        }
    } finally {
        if (scanner != null) {
            scanner.close();
        }
        if (bScanner != null) {
            bScanner.close();
        }
        if (isoScanner != null) {
            isoScanner.close();
        }
        if (csiScanner != null) {
            csiScanner.close();
        }
        if (oScanner != null) {
            oScanner.close();
        }
    }
}

Example 4 with ClientSideIteratorScanner

use of org.apache.accumulo.core.client.ClientSideIteratorScanner in project accumulo by apache.

the class ClientSideIteratorIT method testVersioning.

@Test
public void testVersioning() throws Exception {
    conn.tableOperations().create(tableName);
    conn.tableOperations().removeProperty(tableName, "table.iterator.scan.vers");
    conn.tableOperations().removeProperty(tableName, "table.iterator.majc.vers");
    conn.tableOperations().removeProperty(tableName, "table.iterator.minc.vers");
    final BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
    Mutation m = new Mutation("row1");
    m.put("colf", "colq", 1l, "value");
    m.put("colf", "colq", 2l, "value");
    bw.addMutation(m);
    bw.flush();
    m = new Mutation("row1");
    m.put("colf", "colq", 3l, "value");
    m.put("colf", "colq", 4l, "value");
    bw.addMutation(m);
    bw.flush();
    try (Scanner scanner = conn.createScanner(tableName, new Authorizations());
        ClientSideIteratorScanner csis = new ClientSideIteratorScanner(scanner)) {
        final IteratorSetting si = new IteratorSetting(10, "localvers", VersioningIterator.class);
        si.addOption("maxVersions", "2");
        csis.addScanIterator(si);
        checkResults(csis, resultSet1, PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME);
        checkResults(scanner, resultSet2, PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME);
        csis.fetchColumnFamily(new Text("colf"));
        checkResults(csis, resultSet1, PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME);
        csis.clearColumns();
        csis.fetchColumnFamily(new Text("none"));
        assertFalse(csis.iterator().hasNext());
    }
}

Example 5 with ClientSideIteratorScanner

use of org.apache.accumulo.core.client.ClientSideIteratorScanner in project accumulo by apache.

the class ClientSideIteratorIT method testIntersect.

@Test
public void testIntersect() throws Exception {
    conn.tableOperations().create(tableName);
    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
    Mutation m = new Mutation("part1");
    m.put("bar", "doc1", "value");
    m.put("bar", "doc2", "value");
    m.put("dog", "doc3", "value");
    m.put("foo", "doc2", "value");
    m.put("foo", "doc3", "value");
    bw.addMutation(m);
    m = new Mutation("part2");
    m.put("bar", "DOC1", "value");
    m.put("bar", "DOC2", "value");
    m.put("dog", "DOC3", "value");
    m.put("foo", "DOC2", "value");
    m.put("foo", "DOC3", "value");
    bw.addMutation(m);
    bw.flush();
    final IteratorSetting si = new IteratorSetting(10, tableName, IntersectingIterator.class);
    try (ClientSideIteratorScanner csis = new ClientSideIteratorScanner(conn.createScanner(tableName, new Authorizations()))) {
        IntersectingIterator.setColumnFamilies(si, new Text[] { new Text("bar"), new Text("foo") });
        csis.addScanIterator(si);
        checkResults(csis, resultSet3, PartialKey.ROW_COLFAM_COLQUAL);
    }
}