Examples with ProgressToken com.yahoo.documentapi.ProgressToken used on opensource projects

Example 31 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionBitIncrease1AllBucketStates.

public void testRangeDistributionBitIncrease1AllBucketStates() throws ParseException {
    int db = 3;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    // For this test, have 1 finished bucket, 3 pending and 0 active (we
    // want to have the splitting to be triggered immediately)
    VisitorIterator.BucketProgress bp = iter.getNext();
    iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];
    bpp[0] = iter.getNext();
    bpp[1] = iter.getNext();
    bpp[2] = iter.getNext();
    iter.update(bpp[0].getSuperbucket(), new BucketId());
    iter.update(bpp[1].getSuperbucket(), new BucketId());
    iter.update(bpp[2].getSuperbucket(), new BucketId());
    assertEquals(p.getFinishedBucketCount(), 1);
    assertEquals(p.getPendingBucketCount(), 3);
    assertEquals(p.getActiveBucketCount(), 0);
    iter.setDistributionBitCount(db + 1);
    assertEquals(p.getTotalBucketCount(), 16);
    assertEquals(p.getFinishedBucketCount(), 2);
    assertEquals(p.getPendingBucketCount(), 6);
    assertEquals(p.getActiveBucketCount(), 0);
    assertEquals(p.getDistributionBitCount(), db + 1);
    assertEquals(iter.getDistributionBitCount(), db + 1);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);
    // Bucket 3:0x4 -> 4:0x4 & 4:0xC
    assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x04));
    assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0C));
    // Bucket 3:0x2 -> 4:0x2 & 4:0xA
    assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x02));
    assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0A));
    // Bucket 3:0x6 -> 4:0x6 & 4:0xE
    assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x06));
    assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0E));
    assertEquals(p.getPendingBucketCount(), 0);
    // Bucket source should now begin returning from bucket 4:0x1
    assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x01));
    assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x09));
    assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x05));
    assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0D));
    assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x03));
// Assume correct from here on
}

Example 32 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testImportInconsistentProgressIncrease.

public void testImportInconsistentProgressIncrease() throws ParseException {
    // Bucket progress "file" that upon time of changing from 4 to 7
    // distribution bits and writing the progress had an active bucket
    String input = "VDS bucket progress file\n" + "7\n" + "32\n" + "24\n" + "128\n" + "100000000000000c:0\n";
    // Now we're at 8 distribution bits
    int db = 8;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken(input);
    assertEquals(7, p.getDistributionBitCount());
    assertEquals(p.getTotalBucketCount(), 1 << 7);
    assertEquals(p.getFinishedBucketCount(), 24);
    // Not yet corrected
    assertEquals(p.getPendingBucketCount(), 1);
    assertEquals(p.getActiveBucketCount(), 0);
    assertEquals(32, p.getBucketCursor());
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 1, p);
    // Now the range handler should have corrected the progress
    // (but not messed with the distribution bit count)
    assertEquals(7, p.getDistributionBitCount());
    assertEquals(p.getPendingBucketCount(), 1 << 3);
    assertEquals(p.getActiveBucketCount(), 0);
    assertEquals(24 + (1 << 3), p.getBucketCursor());
    iter.setDistributionBitCount(8);
    assertEquals(iter.getDistributionBitCount(), 8);
    assertEquals(p.getDistributionBitCount(), 8);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), 8);
    assertEquals(p.getTotalBucketCount(), 1 << 8);
    assertEquals(p.getFinishedBucketCount(), 24 << 1);
    // Split 4 -> 7 bits, then 7 -> 8 bits
    assertEquals(p.getPendingBucketCount(), 1 << 4);
    assertEquals(p.getActiveBucketCount(), 0);
    assertEquals(p.getBucketCursor(), 24 * 2 + (1 << 4));
    assertTrue(iter.hasNext());
}

Example 33 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testBinaryProgressSerialization.

public void testBinaryProgressSerialization() {
    String input = "VDS bucket progress file (48.828125% completed)\n" + "10\n" + "503\n" + "500\n" + "1024\n" + "28000000000000be:0\n" + "28000000000002be:0\n" + "28000000000001be:0\n";
    ProgressToken p = new ProgressToken(input);
    byte[] buf = p.serialize();
    ProgressToken p2 = new ProgressToken(buf);
    assertEquals(input, p2.toString());
}

Example 34 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testSingleBucketSplits.

public void testSingleBucketSplits() throws ParseException {
    int db = 2;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    // Create a range of [0, 4) superbuckets
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.BucketProgress bp = iter.getNext();
    assertEquals(bp.getSuperbucket(), new BucketId(db, 0));
    // Put back as pending
    iter.update(bp.getSuperbucket(), new BucketId());
    assertEquals(p.getPendingBucketCount(), 1);
    p.splitPendingBucket(new BucketId(db, 0));
    assertEquals(p.getPendingBucketCount(), 2);
    bp = iter.getNext();
    // left split
    assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 0));
    assertEquals(bp.getProgress(), new BucketId(0));
    bp = iter.getNext();
    // right split
    assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 4));
    assertEquals(bp.getProgress(), new BucketId(0));
    bp = iter.getNext();
    assertEquals(bp.getSuperbucket(), new BucketId(db, 2));
    // Put back as pending, with a progress of 10010. This implies splitting
    // the bucket should set both splits with a progress to 10010
    iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x12));
    assertEquals(p.getPendingBucketCount(), 1);
    p.splitPendingBucket(new BucketId(db, 2));
    assertEquals(p.getPendingBucketCount(), 2);
    bp = iter.getNext();
    // left split
    assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 2));
    assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));
    bp = iter.getNext();
    // right split
    assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 6));
    assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));
    bp = iter.getNext();
    // Put back as pending with a progress of 10101. This implies splitting the
    // bucket should _discard_ left and set right's progress to 10101.
    // Update: no it shouldn't, we now split with equal progress without
    // discarding
    assertEquals(bp.getSuperbucket(), new BucketId(db, 1));
    iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x15));
    assertEquals(p.getPendingBucketCount(), 1);
    p.splitPendingBucket(new BucketId(db, 1));
    assertEquals(p.getPendingBucketCount(), 2);
    bp = iter.getNext();
    assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 1));
    assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));
    bp = iter.getNext();
    // right split
    assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 5));
    assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));
}

Example 35 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionBitInitialDrop.

// Test a drop from 31->11 bits upon iteration start
public void testRangeDistributionBitInitialDrop() throws ParseException {
    int db = 31;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[3];
    bp[0] = iter.getNext();
    bp[1] = iter.getNext();
    bp[2] = iter.getNext();
    iter.update(bp[2].getSuperbucket(), new BucketId());
    iter.update(bp[1].getSuperbucket(), new BucketId());
    assertEquals(p.getActiveBucketCount(), 1);
    iter.setDistributionBitCount(11);
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(p.getActiveBucketCount(), 1);
    // Updating the active bucket allows the merging to take place
    iter.update(new BucketId(31, 0), new BucketId());
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    // All pending buckets should have been merged down to just 1 now
    // Update: now rather gets reset
    assertEquals(p.getPendingBucketCount(), 0);
    assertEquals(p.getActiveBucketCount(), 0);
    assertEquals(p.getFinishedBucketCount(), 0);
    assertEquals(p.getBucketCursor(), 0);
    bp[0] = iter.getNext();
    assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
}