Examples with VisitorIterator com.yahoo.documentapi.VisitorIterator used on opensource projects

Example 1 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testIterationSingleBucketUpdate.

public void testIterationSingleBucketUpdate() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.user = 1234", idFactory, 1, progress);
    assertFalse(progress.hasActive());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getFinishedBucketCount(), 0);
    assertEquals(progress.getTotalBucketCount(), 1);
    assertFalse(iter.isDone());
    assertTrue(iter.hasNext());
    assertEquals(iter.getRemainingBucketCount(), 1);
    VisitorIterator.BucketProgress b1 = iter.getNext();
    // Upon first getNext of a superbucket, progress == 0
    assertEquals(b1.getSuperbucket(), new BucketId(32, 1234));
    assertEquals(b1.getProgress(), new BucketId());
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(iter.getRemainingBucketCount(), 1);
    // Should only be one active bucket; the one we just got
    assertEquals(progress.getActiveBucketCount(), 1);
    // No pending yet
    assertFalse(progress.hasPending());
    // Update the bucket with a sub-bucket, moving it from active to pending
    BucketId sub = new BucketId(b1.getSuperbucket().getUsedBits() + 1, b1.getSuperbucket().getId());
    iter.update(b1.getSuperbucket(), sub);
    assertFalse(progress.hasActive());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(iter.getRemainingBucketCount(), 1);
    // Get the pending bucket
    VisitorIterator.BucketProgress b2 = iter.getNext();
    assertEquals(b2.getSuperbucket(), new BucketId(32, 1234));
    assertEquals(b2.getProgress(), new BucketId(33, 1234));
    assertFalse(iter.hasNext());
    assertEquals(progress.getActiveBucketCount(), 1);
    assertFalse(progress.hasPending());
    // Now update with progress==super, signalling that the bucket is done
    iter.update(b1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertFalse(progress.hasActive());
    assertFalse(progress.hasPending());
    assertFalse(iter.hasNext());
    assertTrue(iter.isDone());
    assertTrue(progress.isFinished());
    assertEquals(progress.getFinishedBucketCount(), 1);
    assertEquals(iter.getRemainingBucketCount(), 0);
}

Example 2 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testImportProgressWithOutdatedDistribution.

public void testImportProgressWithOutdatedDistribution() throws ParseException {
    String input = "VDS bucket progress file\n" + "10\n" + "503\n" + "500\n" + "1024\n" + "28000000000000be:0\n" + "28000000000002be:0\n" + "28000000000001be:0\n";
    int db = 12;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken(input);
    assertEquals(10, p.getDistributionBitCount());
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 1, p);
    iter.setDistributionBitCount(12);
    assertEquals(iter.getDistributionBitCount(), 12);
    assertEquals(p.getDistributionBitCount(), 12);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), 12);
    assertEquals(p.getTotalBucketCount(), 1 << 12);
    assertEquals(p.getFinishedBucketCount(), 500 << 2);
    assertEquals(p.getPendingBucketCount(), 3 << 2);
    assertEquals(p.getActiveBucketCount(), 0);
    assertEquals(p.getBucketCursor(), 503 << 2);
    assertTrue(iter.hasNext());
    ProgressToken p2 = new ProgressToken(p.serialize());
    assertEquals(p2.getDistributionBitCount(), 12);
    assertEquals(p2.getTotalBucketCount(), 1 << 12);
    assertEquals(p2.getFinishedBucketCount(), 500 << 2);
    assertEquals(p2.getPendingBucketCount(), 3 << 2);
    assertEquals(p2.getActiveBucketCount(), 0);
    assertEquals(p2.getBucketCursor(), 503 << 2);
}

Example 3 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testActiveUpdate.

/**
 * Test that doing update() on a bucket several times in a row (without re-fetching
 * from getNext first) works
 * @throws ParseException
 */
public void testActiveUpdate() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group = \"yahoo.com\"", idFactory, 16, progress);
    VisitorIterator.BucketProgress bp = iter.getNext();
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
    BucketId superbucket = bp.getSuperbucket();
    int usedBits = superbucket.getUsedBits();
    iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (2L << usedBits)));
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    bp = iter.getNext();
    assertEquals(bp.getSuperbucket(), superbucket);
    assertEquals(bp.getProgress(), new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
}

Example 4 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionBitIncreaseDecrease.

// Test that splitting and merging from and to the same db count gives
// back the initial state
public void testRangeDistributionBitIncreaseDecrease() throws ParseException {
    int db = 16;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.DistributionRangeBucketSource src = (VisitorIterator.DistributionRangeBucketSource) iter.getBucketSource();
    assertTrue(src.isLosslessResetPossible());
    // "Sabotage" resetting by having at least 1 finished
    iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];
    for (int i = 0; i < 4; ++i) {
        bpp[i] = iter.getNext();
    }
    for (int i = 0; i < 4; ++i) {
        iter.update(bpp[i].getSuperbucket(), new BucketId());
    }
    assertFalse(src.isLosslessResetPossible());
    iter.setDistributionBitCount(20);
    assertEquals(p.getDistributionBitCount(), 20);
    assertEquals(p.getPendingBucketCount(), 4 << 4);
    assertFalse(iter.getBucketSource().shouldYield());
    assertEquals(p.getBucketCursor(), 5 << 4);
    iter.setDistributionBitCount(16);
    assertEquals(p.getDistributionBitCount(), 16);
    assertEquals(p.getPendingBucketCount(), 4);
    assertFalse(iter.getBucketSource().shouldYield());
    assertEquals(p.getBucketCursor(), 5);
}

Example 5 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testSingleBucketMerge.

public void testSingleBucketMerge() throws ParseException {
    int db = 2;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    // Create a range of [0, 4) superbuckets
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.BucketProgress bp = iter.getNext();
    // Put back as pending and split it
    iter.update(bp.getSuperbucket(), new BucketId());
    p.splitPendingBucket(new BucketId(db, 0));
    assertEquals(p.getPendingBucketCount(), 2);
    // Merge both back into one node. Merge from left sibling with right present
    p.mergePendingBucket(new BucketId(db + 1, 0));
    assertEquals(p.getPendingBucketCount(), 1);
    bp = iter.getNext();
    assertEquals(bp.getSuperbucket(), new BucketId(db, 0));
}