use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testIsBucketFinished.
public void testIsBucketFinished() throws ParseException {
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 4, p);
assertFalse(p.isBucketFinished(new BucketId(32, 0)));
// Finish superbucket 0x0000
iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
assertTrue(p.isBucketFinished(new BucketId(32, 0)));
// Cursor is 1, but bucket 0x1000 not yet returned
assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));
VisitorIterator.BucketProgress bp = iter.getNext();
// Cursor 2, 0x1000 returned but is contained in state, so not finished
assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));
iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
assertTrue(p.isBucketFinished(new BucketId(32, 1 << 3)));
// Only superbucket part is used
// ...0000
assertTrue(p.isBucketFinished(new BucketId(32, 0x12345670)));
// ...1000
assertTrue(p.isBucketFinished(new BucketId(32, 0x12345678)));
// ...0001
assertFalse(p.isBucketFinished(new BucketId(32, 0x12345671)));
// ...1001
assertFalse(p.isBucketFinished(new BucketId(32, 0x12345679)));
}
use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testRangeDistributionBitDecrease1.
public void testRangeDistributionBitDecrease1() throws ParseException {
int db = 16;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
VisitorIterator.DistributionRangeBucketSource src = (VisitorIterator.DistributionRangeBucketSource) iter.getBucketSource();
assertTrue(src.isLosslessResetPossible());
// active buckets have been updated, 3 merges should be triggered
for (int i = 0; i < 3; ++i) {
iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
}
assertFalse(src.isLosslessResetPossible());
VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[6];
for (int i = 0; i < 6; ++i) {
bpp[i] = iter.getNext();
}
// Leave this hanging as active
VisitorIterator.BucketProgress bpa = iter.getNext();
for (int i = 0; i < 6; ++i) {
iter.update(bpp[i].getSuperbucket(), new BucketId());
}
assertEquals(p.getBucketCursor(), 10);
iter.setDistributionBitCount(db - 1);
assertEquals(iter.getDistributionBitCount(), db - 1);
assertEquals(p.getDistributionBitCount(), db);
assertEquals(iter.getBucketSource().getDistributionBitCount(), db - 1);
// The iterator is waiting patiently for all active buckets to be updated,
// at which point it will performed the merging and actually updating the
// progress token's distribution bit count
assertTrue(iter.getBucketSource().shouldYield());
assertFalse(iter.hasNext());
assertFalse(iter.isDone());
assertEquals(p.getActiveBucketCount(), 1);
iter.update(bpa.getSuperbucket(), new BucketId());
assertEquals(p.getDistributionBitCount(), db - 1);
assertEquals(p.getActiveBucketCount(), 0);
// 3 merges, P PP PP PP -> P P P P
assertEquals(p.getPendingBucketCount(), 4);
assertEquals(p.getFinishedBucketCount(), 1);
assertEquals(p.getBucketCursor(), 5);
}
use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testRangeDistributionBitChangeWithoutDone.
// Test that intermittent changes in distribution are handled properly, e.g.
// changing from 11 -> 9 with X active and then before all those are flushed,
// the distribution goes up to 12
public void testRangeDistributionBitChangeWithoutDone() throws ParseException {
int db = 11;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
VisitorIterator.DistributionRangeBucketSource src = (VisitorIterator.DistributionRangeBucketSource) iter.getBucketSource();
VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];
for (int i = 0; i < 4; ++i) {
bpp[i] = iter.getNext();
}
for (int i = 0; i < 2; ++i) {
iter.update(bpp[i].getSuperbucket(), new BucketId());
}
assertFalse(src.isLosslessResetPossible());
// Now 2 pending, 2 active
iter.setDistributionBitCount(9);
assertEquals(p.getDistributionBitCount(), 11);
assertEquals(p.getActiveBucketCount(), 2);
assertEquals(p.getPendingBucketCount(), 2);
assertTrue(iter.getBucketSource().shouldYield());
// Update as pending, still with old count since there's 1 more active
// with bpp[2]. Have progress so that lossless reset isn't possible
iter.update(bpp[3].getSuperbucket(), new BucketId(15, bpp[3].getSuperbucket().getId()));
iter.setDistributionBitCount(12);
assertEquals(p.getActiveBucketCount(), 1);
assertEquals(p.getPendingBucketCount(), 3);
assertTrue(iter.getBucketSource().shouldYield());
// Serialize before token is updated to 12 bits
String serialized = p.toString();
iter.update(bpp[2].getSuperbucket(), ProgressToken.FINISHED_BUCKET);
assertEquals(p.getActiveBucketCount(), 0);
// All active buckets are at db=11, so they should be split once each
assertEquals(p.getPendingBucketCount(), 3 * 2);
assertFalse(iter.getBucketSource().shouldYield());
assertEquals(p.getFinishedBucketCount(), 2);
// Ensure we get a consistent progress token imported
ProgressToken p2 = new ProgressToken(serialized);
// Not yet updated
assertEquals(p2.getDistributionBitCount(), 11);
BucketIdFactory idFactory2 = new BucketIdFactory();
VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory2, 1, p2);
// Not yet updated, since we don't trust the initial BucketIdFactory
assertEquals(iter2.getDistributionBitCount(), 11);
assertEquals(p2.getDistributionBitCount(), 11);
iter2.setDistributionBitCount(12);
// Now it has been updated
assertEquals(p2.getDistributionBitCount(), 12);
assertEquals(p2.getPendingBucketCount(), 8);
assertEquals(p2.getBucketCursor(), 8);
assertEquals(p2.getFinishedBucketCount(), 0);
}
use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testRangeDistributionBitIncrease1AllBucketStates.
public void testRangeDistributionBitIncrease1AllBucketStates() throws ParseException {
int db = 3;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
// For this test, have 1 finished bucket, 3 pending and 0 active (we
// want to have the splitting to be triggered immediately)
VisitorIterator.BucketProgress bp = iter.getNext();
iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];
bpp[0] = iter.getNext();
bpp[1] = iter.getNext();
bpp[2] = iter.getNext();
iter.update(bpp[0].getSuperbucket(), new BucketId());
iter.update(bpp[1].getSuperbucket(), new BucketId());
iter.update(bpp[2].getSuperbucket(), new BucketId());
assertEquals(p.getFinishedBucketCount(), 1);
assertEquals(p.getPendingBucketCount(), 3);
assertEquals(p.getActiveBucketCount(), 0);
iter.setDistributionBitCount(db + 1);
assertEquals(p.getTotalBucketCount(), 16);
assertEquals(p.getFinishedBucketCount(), 2);
assertEquals(p.getPendingBucketCount(), 6);
assertEquals(p.getActiveBucketCount(), 0);
assertEquals(p.getDistributionBitCount(), db + 1);
assertEquals(iter.getDistributionBitCount(), db + 1);
assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);
// Bucket 3:0x4 -> 4:0x4 & 4:0xC
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x04));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0C));
// Bucket 3:0x2 -> 4:0x2 & 4:0xA
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x02));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0A));
// Bucket 3:0x6 -> 4:0x6 & 4:0xE
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x06));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0E));
assertEquals(p.getPendingBucketCount(), 0);
// Bucket source should now begin returning from bucket 4:0x1
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x01));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x09));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x05));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0D));
assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x03));
// Assume correct from here on
}
use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testSingleBucketSplits.
public void testSingleBucketSplits() throws ParseException {
int db = 2;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
// Create a range of [0, 4) superbuckets
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
VisitorIterator.BucketProgress bp = iter.getNext();
assertEquals(bp.getSuperbucket(), new BucketId(db, 0));
// Put back as pending
iter.update(bp.getSuperbucket(), new BucketId());
assertEquals(p.getPendingBucketCount(), 1);
p.splitPendingBucket(new BucketId(db, 0));
assertEquals(p.getPendingBucketCount(), 2);
bp = iter.getNext();
// left split
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 0));
assertEquals(bp.getProgress(), new BucketId(0));
bp = iter.getNext();
// right split
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 4));
assertEquals(bp.getProgress(), new BucketId(0));
bp = iter.getNext();
assertEquals(bp.getSuperbucket(), new BucketId(db, 2));
// Put back as pending, with a progress of 10010. This implies splitting
// the bucket should set both splits with a progress to 10010
iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x12));
assertEquals(p.getPendingBucketCount(), 1);
p.splitPendingBucket(new BucketId(db, 2));
assertEquals(p.getPendingBucketCount(), 2);
bp = iter.getNext();
// left split
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 2));
assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));
bp = iter.getNext();
// right split
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 6));
assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));
bp = iter.getNext();
// Put back as pending with a progress of 10101. This implies splitting the
// bucket should _discard_ left and set right's progress to 10101.
// Update: no it shouldn't, we now split with equal progress without
// discarding
assertEquals(bp.getSuperbucket(), new BucketId(db, 1));
iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x15));
assertEquals(p.getPendingBucketCount(), 1);
p.splitPendingBucket(new BucketId(db, 1));
assertEquals(p.getPendingBucketCount(), 2);
bp = iter.getNext();
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 1));
assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));
bp = iter.getNext();
// right split
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 5));
assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));
}
Aggregations