Examples with ProgressToken com.yahoo.documentapi.ProgressToken used on opensource projects

Example 6 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionBitIncreaseDecrease.

// Test that splitting and merging from and to the same db count gives
// back the initial state
public void testRangeDistributionBitIncreaseDecrease() throws ParseException {
    int db = 16;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.DistributionRangeBucketSource src = (VisitorIterator.DistributionRangeBucketSource) iter.getBucketSource();
    assertTrue(src.isLosslessResetPossible());
    // "Sabotage" resetting by having at least 1 finished
    iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];
    for (int i = 0; i < 4; ++i) {
        bpp[i] = iter.getNext();
    }
    for (int i = 0; i < 4; ++i) {
        iter.update(bpp[i].getSuperbucket(), new BucketId());
    }
    assertFalse(src.isLosslessResetPossible());
    iter.setDistributionBitCount(20);
    assertEquals(p.getDistributionBitCount(), 20);
    assertEquals(p.getPendingBucketCount(), 4 << 4);
    assertFalse(iter.getBucketSource().shouldYield());
    assertEquals(p.getBucketCursor(), 5 << 4);
    iter.setDistributionBitCount(16);
    assertEquals(p.getDistributionBitCount(), 16);
    assertEquals(p.getPendingBucketCount(), 4);
    assertFalse(iter.getBucketSource().shouldYield());
    assertEquals(p.getBucketCursor(), 5);
}

Example 7 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testBucketProgressFraction.

public void testBucketProgressFraction() {
    double epsilon = 0.00001;
    // No progress
    BucketId b_0 = new BucketId();
    // No split; only superbucket (100%)
    BucketId b_100_0 = new BucketId(16, 1234);
    // 1 split (1/2)
    BucketId b_50_1 = new BucketId(17, 1234);
    BucketId b_100_1 = new BucketId(17, 1234 | (1 << 16));
    // 2 splits (1/4)
    BucketId b_25_2 = new BucketId(18, 1234);
    BucketId b_50_2 = new BucketId(18, 1234 | (2 << 16));
    BucketId b_75_2 = new BucketId(18, 1234 | (1 << 16));
    BucketId b_100_2 = new BucketId(18, 1234 | (3 << 16));
    ProgressToken p = new ProgressToken(16);
    BucketId sb = new BucketId(16, 1234);
    assertEquals(p.progressFraction(new BucketId(32, 1234), b_0), 0.0, epsilon);
    assertEquals(p.progressFraction(sb, b_100_0), 1.0, epsilon);
    assertEquals(p.progressFraction(sb, b_50_1), 0.5, epsilon);
    assertEquals(p.progressFraction(sb, b_100_1), 1.0, epsilon);
    assertEquals(p.progressFraction(sb, b_25_2), 0.25, epsilon);
    assertEquals(p.progressFraction(sb, b_50_2), 0.5, epsilon);
    assertEquals(p.progressFraction(sb, b_75_2), 0.75, epsilon);
    assertEquals(p.progressFraction(sb, b_100_2), 1.0, epsilon);
    assertEquals(p.progressFraction(new BucketId(0x8000000000000000L), new BucketId(0xb0000fff00000000L)), 1.0, epsilon);
}

Example 8 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testSingleBucketMerge.

public void testSingleBucketMerge() throws ParseException {
    int db = 2;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    // Create a range of [0, 4) superbuckets
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.BucketProgress bp = iter.getNext();
    // Put back as pending and split it
    iter.update(bp.getSuperbucket(), new BucketId());
    p.splitPendingBucket(new BucketId(db, 0));
    assertEquals(p.getPendingBucketCount(), 2);
    // Merge both back into one node. Merge from left sibling with right present
    p.mergePendingBucket(new BucketId(db + 1, 0));
    assertEquals(p.getPendingBucketCount(), 1);
    bp = iter.getNext();
    assertEquals(bp.getSuperbucket(), new BucketId(db, 0));
}

Example 9 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testEntireBucketSpaceCovered.

public void testEntireBucketSpaceCovered() throws ParseException {
    int db = 4;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];
    for (int i = 0; i < 3; ++i) {
        bpp[i] = iter.getNext();
    }
    for (int i = 0; i < 3; ++i) {
        // Must use non-zero progress or all pending will be optimized
        // away by the reset-logic
        iter.update(bpp[i].getSuperbucket(), new BucketId(db + 1, bpp[i].getSuperbucket().getId()));
    }
    Set<BucketId> buckets = new TreeSet<BucketId>();
    db = 7;
    for (int i = 0; i < (1 << db); ++i) {
        buckets.add(new BucketId(db, i));
    }
    iter.setDistributionBitCount(db);
    assertEquals(p.getFinishedBucketCount(), 0);
    assertEquals(p.getPendingBucketCount(), 3 << 3);
    // Ensure all buckets are visited once and only once
    while (iter.hasNext()) {
        VisitorIterator.BucketProgress bp = iter.getNext();
        assertTrue(buckets.contains(bp.getSuperbucket()));
        buckets.remove(bp.getSuperbucket());
    }
    assertTrue(buckets.isEmpty());
}

Example 10 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionLosslessReset.

// Similar to testRangeDistributionBitInitialDrop, but going from 1 to 11
// This tests that doing so may be done in an optimized way rather than
// attempting to split enough buckets to cover the entire bucket space!
public void testRangeDistributionLosslessReset() throws ParseException {
    int db = 1;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.DistributionRangeBucketSource src = (VisitorIterator.DistributionRangeBucketSource) iter.getBucketSource();
    VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[2];
    bp[0] = iter.getNext();
    bp[1] = iter.getNext();
    String serialized = p.toString();
    assertFalse(src.isLosslessResetPossible());
    iter.update(bp[1].getSuperbucket(), new BucketId());
    assertEquals(p.getActiveBucketCount(), 1);
    iter.setDistributionBitCount(11);
    assertFalse(src.isLosslessResetPossible());
    // Still at 1
    assertEquals(p.getDistributionBitCount(), 1);
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(p.getActiveBucketCount(), 1);
    // Updating the active bucket allows the reset to take place
    iter.update(new BucketId(1, 0), new BucketId());
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    // Should not be any buckets pending/active and the cursor should be
    // back at 0
    assertEquals(p.getPendingBucketCount(), 0);
    assertEquals(p.getActiveBucketCount(), 0);
    assertEquals(p.getFinishedBucketCount(), 0);
    assertEquals(p.getBucketCursor(), 0);
    assertEquals(p.getDistributionBitCount(), 11);
    bp[0] = iter.getNext();
    assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
    // Ensure resetting also works when you're importing existing
    // progress
    p = new ProgressToken(serialized);
    idFactory = new BucketIdFactory();
    iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 1, p);
    iter.setDistributionBitCount(11);
    assertEquals(p.getPendingBucketCount(), 0);
    assertEquals(p.getActiveBucketCount(), 0);
    assertEquals(p.getFinishedBucketCount(), 0);
    assertEquals(p.getBucketCursor(), 0);
    assertEquals(p.getDistributionBitCount(), 11);
    bp[0] = iter.getNext();
    assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
}