Examples with VisitorIterator com.yahoo.documentapi.VisitorIterator used on opensource projects

Example 6 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testEntireBucketSpaceCovered.

public void testEntireBucketSpaceCovered() throws ParseException {
    int db = 4;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];
    for (int i = 0; i < 3; ++i) {
        bpp[i] = iter.getNext();
    }
    for (int i = 0; i < 3; ++i) {
        // Must use non-zero progress or all pending will be optimized
        // away by the reset-logic
        iter.update(bpp[i].getSuperbucket(), new BucketId(db + 1, bpp[i].getSuperbucket().getId()));
    }
    Set<BucketId> buckets = new TreeSet<BucketId>();
    db = 7;
    for (int i = 0; i < (1 << db); ++i) {
        buckets.add(new BucketId(db, i));
    }
    iter.setDistributionBitCount(db);
    assertEquals(p.getFinishedBucketCount(), 0);
    assertEquals(p.getPendingBucketCount(), 3 << 3);
    // Ensure all buckets are visited once and only once
    while (iter.hasNext()) {
        VisitorIterator.BucketProgress bp = iter.getNext();
        assertTrue(buckets.contains(bp.getSuperbucket()));
        buckets.remove(bp.getSuperbucket());
    }
    assertTrue(buckets.isEmpty());
}

Example 7 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionLosslessReset.

// Similar to testRangeDistributionBitInitialDrop, but going from 1 to 11
// This tests that doing so may be done in an optimized way rather than
// attempting to split enough buckets to cover the entire bucket space!
public void testRangeDistributionLosslessReset() throws ParseException {
    int db = 1;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.DistributionRangeBucketSource src = (VisitorIterator.DistributionRangeBucketSource) iter.getBucketSource();
    VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[2];
    bp[0] = iter.getNext();
    bp[1] = iter.getNext();
    String serialized = p.toString();
    assertFalse(src.isLosslessResetPossible());
    iter.update(bp[1].getSuperbucket(), new BucketId());
    assertEquals(p.getActiveBucketCount(), 1);
    iter.setDistributionBitCount(11);
    assertFalse(src.isLosslessResetPossible());
    // Still at 1
    assertEquals(p.getDistributionBitCount(), 1);
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(p.getActiveBucketCount(), 1);
    // Updating the active bucket allows the reset to take place
    iter.update(new BucketId(1, 0), new BucketId());
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    // Should not be any buckets pending/active and the cursor should be
    // back at 0
    assertEquals(p.getPendingBucketCount(), 0);
    assertEquals(p.getActiveBucketCount(), 0);
    assertEquals(p.getFinishedBucketCount(), 0);
    assertEquals(p.getBucketCursor(), 0);
    assertEquals(p.getDistributionBitCount(), 11);
    bp[0] = iter.getNext();
    assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
    // Ensure resetting also works when you're importing existing
    // progress
    p = new ProgressToken(serialized);
    idFactory = new BucketIdFactory();
    iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 1, p);
    iter.setDistributionBitCount(11);
    assertEquals(p.getPendingBucketCount(), 0);
    assertEquals(p.getActiveBucketCount(), 0);
    assertEquals(p.getFinishedBucketCount(), 0);
    assertEquals(p.getBucketCursor(), 0);
    assertEquals(p.getDistributionBitCount(), 11);
    bp[0] = iter.getNext();
    assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
}

Example 8 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionBitIncrease1NoPending.

/**
 * Test increasing the distribution bits for a full bucket space range
 * source with no finished, active or pending buckets
 * @throws ParseException upon docsel parse failure (shouldn't happen)
 */
public void testRangeDistributionBitIncrease1NoPending() throws ParseException {
    int db = 2;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    // Test for empty progress token. no splitting involved
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    assertEquals(p.getTotalBucketCount(), 4);
    iter.setDistributionBitCount(db + 1);
    assertEquals(p.getTotalBucketCount(), 8);
    assertEquals(p.getDistributionBitCount(), db + 1);
    assertEquals(iter.getDistributionBitCount(), db + 1);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);
    int[] desired = new int[] { 0, 4, 2, 6, 1, 5, 3, 7 };
    for (int i = 0; i < 8; ++i) {
        VisitorIterator.BucketProgress bp = iter.getNext();
        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, desired[i]));
    }
}

Example 9 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testInconsistentState.

// Test that altering distribution bit count sets ProgressToken as
// inconsistent when there are active buckets
public void testInconsistentState() throws ParseException {
    int db = 16;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    // For this test, have 3 finished bucket, 2 pending and 1 active
    for (int i = 0; i < 3; ++i) {
        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    }
    VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];
    bpp[0] = iter.getNext();
    bpp[1] = iter.getNext();
    // Leave this hanging as active
    VisitorIterator.BucketProgress bpa = iter.getNext();
    iter.update(bpp[0].getSuperbucket(), new BucketId());
    iter.update(bpp[1].getSuperbucket(), new BucketId());
    assertFalse(p.isInconsistentState());
    iter.setDistributionBitCount(20);
    assertTrue(p.isInconsistentState());
    // Finish active, triggering the consistency fixes
    iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertFalse(p.isInconsistentState());
}

Example 10 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testNullAndSuperUpdate.

/**
 * Test that ensures doing update(superbucket, 0) simply puts the bucket back in
 * pending
 * @throws ParseException
 */
public void testNullAndSuperUpdate() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group = \"yahoo.com\"", idFactory, 16, progress);
    assertEquals(progress.getPendingBucketCount(), 1);
    VisitorIterator.BucketProgress bp = iter.getNext();
    assertEquals(bp.getProgress(), new BucketId());
    BucketId superbucket = bp.getSuperbucket();
    BucketId sub = bp.getProgress();
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
    // 0-bucket
    iter.update(superbucket, ProgressToken.NULL_BUCKET);
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    VisitorIterator.BucketProgress bp2 = iter.getNext();
    assertEquals(bp2.getSuperbucket(), superbucket);
    assertEquals(bp2.getProgress(), ProgressToken.NULL_BUCKET);
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
    // progress == super
    iter.update(superbucket, superbucket);
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    bp2 = iter.getNext();
    assertEquals(bp2.getSuperbucket(), superbucket);
    assertEquals(bp2.getProgress(), superbucket);
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
}