use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testRangeDistributionBitIncrease1AllBucketStates.
public void testRangeDistributionBitIncrease1AllBucketStates() throws ParseException {
int db = 3;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
// For this test, have 1 finished bucket, 3 pending and 0 active (we
// want to have the splitting to be triggered immediately)
VisitorIterator.BucketProgress bp = iter.getNext();
iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];
bpp[0] = iter.getNext();
bpp[1] = iter.getNext();
bpp[2] = iter.getNext();
iter.update(bpp[0].getSuperbucket(), new BucketId());
iter.update(bpp[1].getSuperbucket(), new BucketId());
iter.update(bpp[2].getSuperbucket(), new BucketId());
assertEquals(p.getFinishedBucketCount(), 1);
assertEquals(p.getPendingBucketCount(), 3);
assertEquals(p.getActiveBucketCount(), 0);
iter.setDistributionBitCount(db + 1);
assertEquals(p.getTotalBucketCount(), 16);
assertEquals(p.getFinishedBucketCount(), 2);
assertEquals(p.getPendingBucketCount(), 6);
assertEquals(p.getActiveBucketCount(), 0);
assertEquals(p.getDistributionBitCount(), db + 1);
assertEquals(iter.getDistributionBitCount(), db + 1);
assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);
// Bucket 3:0x4 -> 4:0x4 & 4:0xC
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x04));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0C));
// Bucket 3:0x2 -> 4:0x2 & 4:0xA
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x02));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0A));
// Bucket 3:0x6 -> 4:0x6 & 4:0xE
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x06));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0E));
assertEquals(p.getPendingBucketCount(), 0);
// Bucket source should now begin returning from bucket 4:0x1
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x01));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x09));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x05));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0D));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x03));
// Assume correct from here on
}
use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testImportInconsistentProgressIncrease.
public void testImportInconsistentProgressIncrease() throws ParseException {
// Bucket progress "file" that upon time of changing from 4 to 7
// distribution bits and writing the progress had an active bucket
String input = "VDS bucket progress file\n" + "7\n" + "32\n" + "24\n" + "128\n" + "100000000000000c:0\n";
// Now we're at 8 distribution bits
int db = 8;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken(input);
assertEquals(7, p.getDistributionBitCount());
assertEquals(p.getTotalBucketCount(), 1 << 7);
assertEquals(p.getFinishedBucketCount(), 24);
// Not yet corrected
assertEquals(p.getPendingBucketCount(), 1);
assertEquals(p.getActiveBucketCount(), 0);
assertEquals(32, p.getBucketCursor());
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 1, p);
// Now the range handler should have corrected the progress
// (but not messed with the distribution bit count)
assertEquals(7, p.getDistributionBitCount());
assertEquals(p.getPendingBucketCount(), 1 << 3);
assertEquals(p.getActiveBucketCount(), 0);
assertEquals(24 + (1 << 3), p.getBucketCursor());
iter.setDistributionBitCount(8);
assertEquals(iter.getDistributionBitCount(), 8);
assertEquals(p.getDistributionBitCount(), 8);
assertEquals(iter.getBucketSource().getDistributionBitCount(), 8);
assertEquals(p.getTotalBucketCount(), 1 << 8);
assertEquals(p.getFinishedBucketCount(), 24 << 1);
// Split 4 -> 7 bits, then 7 -> 8 bits
assertEquals(p.getPendingBucketCount(), 1 << 4);
assertEquals(p.getActiveBucketCount(), 0);
assertEquals(p.getBucketCursor(), 24 * 2 + (1 << 4));
assertTrue(iter.hasNext());
}
use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testSingleBucketSplits.
public void testSingleBucketSplits() throws ParseException {
int db = 2;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
// Create a range of [0, 4) superbuckets
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
VisitorIterator.BucketProgress bp = iter.getNext();
assertEquals(bp.getSuperbucket(), new BucketId(db, 0));
// Put back as pending
iter.update(bp.getSuperbucket(), new BucketId());
assertEquals(p.getPendingBucketCount(), 1);
p.splitPendingBucket(new BucketId(db, 0));
assertEquals(p.getPendingBucketCount(), 2);
bp = iter.getNext();
// left split
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 0));
assertEquals(bp.getProgress(), new BucketId(0));
bp = iter.getNext();
// right split
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 4));
assertEquals(bp.getProgress(), new BucketId(0));
bp = iter.getNext();
assertEquals(bp.getSuperbucket(), new BucketId(db, 2));
// Put back as pending, with a progress of 10010. This implies splitting
// the bucket should set both splits with a progress to 10010
iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x12));
assertEquals(p.getPendingBucketCount(), 1);
p.splitPendingBucket(new BucketId(db, 2));
assertEquals(p.getPendingBucketCount(), 2);
bp = iter.getNext();
// left split
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 2));
assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));
bp = iter.getNext();
// right split
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 6));
assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));
bp = iter.getNext();
// Put back as pending with a progress of 10101. This implies splitting the
// bucket should _discard_ left and set right's progress to 10101.
// Update: no it shouldn't, we now split with equal progress without
// discarding
assertEquals(bp.getSuperbucket(), new BucketId(db, 1));
iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x15));
assertEquals(p.getPendingBucketCount(), 1);
p.splitPendingBucket(new BucketId(db, 1));
assertEquals(p.getPendingBucketCount(), 2);
bp = iter.getNext();
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 1));
assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));
bp = iter.getNext();
// right split
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 5));
assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));
}
use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testRangeDistributionBitInitialDrop.
// Test a drop from 31->11 bits upon iteration start
public void testRangeDistributionBitInitialDrop() throws ParseException {
int db = 31;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[3];
bp[0] = iter.getNext();
bp[1] = iter.getNext();
bp[2] = iter.getNext();
iter.update(bp[2].getSuperbucket(), new BucketId());
iter.update(bp[1].getSuperbucket(), new BucketId());
assertEquals(p.getActiveBucketCount(), 1);
iter.setDistributionBitCount(11);
assertFalse(iter.hasNext());
assertFalse(iter.isDone());
assertEquals(p.getActiveBucketCount(), 1);
// Updating the active bucket allows the merging to take place
iter.update(new BucketId(31, 0), new BucketId());
assertTrue(iter.hasNext());
assertFalse(iter.isDone());
// All pending buckets should have been merged down to just 1 now
// Update: now rather gets reset
assertEquals(p.getPendingBucketCount(), 0);
assertEquals(p.getActiveBucketCount(), 0);
assertEquals(p.getFinishedBucketCount(), 0);
assertEquals(p.getBucketCursor(), 0);
bp[0] = iter.getNext();
assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
}
use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class BatchDocumentUpdateMessage method setBucketId.
void setBucketId(IdString id) {
BucketIdFactory factory = new BucketIdFactory();
bucketId = factory.getBucketId(new DocumentId(id));
}
Aggregations