use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testBucketProgressFraction.
public void testBucketProgressFraction() {
double epsilon = 0.00001;
// No progress
BucketId b_0 = new BucketId();
// No split; only superbucket (100%)
BucketId b_100_0 = new BucketId(16, 1234);
// 1 split (1/2)
BucketId b_50_1 = new BucketId(17, 1234);
BucketId b_100_1 = new BucketId(17, 1234 | (1 << 16));
// 2 splits (1/4)
BucketId b_25_2 = new BucketId(18, 1234);
BucketId b_50_2 = new BucketId(18, 1234 | (2 << 16));
BucketId b_75_2 = new BucketId(18, 1234 | (1 << 16));
BucketId b_100_2 = new BucketId(18, 1234 | (3 << 16));
ProgressToken p = new ProgressToken(16);
BucketId sb = new BucketId(16, 1234);
assertEquals(p.progressFraction(new BucketId(32, 1234), b_0), 0.0, epsilon);
assertEquals(p.progressFraction(sb, b_100_0), 1.0, epsilon);
assertEquals(p.progressFraction(sb, b_50_1), 0.5, epsilon);
assertEquals(p.progressFraction(sb, b_100_1), 1.0, epsilon);
assertEquals(p.progressFraction(sb, b_25_2), 0.25, epsilon);
assertEquals(p.progressFraction(sb, b_50_2), 0.5, epsilon);
assertEquals(p.progressFraction(sb, b_75_2), 0.75, epsilon);
assertEquals(p.progressFraction(sb, b_100_2), 1.0, epsilon);
assertEquals(p.progressFraction(new BucketId(0x8000000000000000L), new BucketId(0xb0000fff00000000L)), 1.0, epsilon);
}
use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testSingleBucketMerge.
public void testSingleBucketMerge() throws ParseException {
int db = 2;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
// Create a range of [0, 4) superbuckets
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
VisitorIterator.BucketProgress bp = iter.getNext();
// Put back as pending and split it
iter.update(bp.getSuperbucket(), new BucketId());
p.splitPendingBucket(new BucketId(db, 0));
assertEquals(p.getPendingBucketCount(), 2);
// Merge both back into one node. Merge from left sibling with right present
p.mergePendingBucket(new BucketId(db + 1, 0));
assertEquals(p.getPendingBucketCount(), 1);
bp = iter.getNext();
assertEquals(bp.getSuperbucket(), new BucketId(db, 0));
}
use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testEntireBucketSpaceCovered.
public void testEntireBucketSpaceCovered() throws ParseException {
int db = 4;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];
for (int i = 0; i < 3; ++i) {
bpp[i] = iter.getNext();
}
for (int i = 0; i < 3; ++i) {
// Must use non-zero progress or all pending will be optimized
// away by the reset-logic
iter.update(bpp[i].getSuperbucket(), new BucketId(db + 1, bpp[i].getSuperbucket().getId()));
}
Set<BucketId> buckets = new TreeSet<BucketId>();
db = 7;
for (int i = 0; i < (1 << db); ++i) {
buckets.add(new BucketId(db, i));
}
iter.setDistributionBitCount(db);
assertEquals(p.getFinishedBucketCount(), 0);
assertEquals(p.getPendingBucketCount(), 3 << 3);
// Ensure all buckets are visited once and only once
while (iter.hasNext()) {
VisitorIterator.BucketProgress bp = iter.getNext();
assertTrue(buckets.contains(bp.getSuperbucket()));
buckets.remove(bp.getSuperbucket());
}
assertTrue(buckets.isEmpty());
}
use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testRangeDistributionLosslessReset.
// Similar to testRangeDistributionBitInitialDrop, but going from 1 to 11
// This tests that doing so may be done in an optimized way rather than
// attempting to split enough buckets to cover the entire bucket space!
public void testRangeDistributionLosslessReset() throws ParseException {
int db = 1;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
VisitorIterator.DistributionRangeBucketSource src = (VisitorIterator.DistributionRangeBucketSource) iter.getBucketSource();
VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[2];
bp[0] = iter.getNext();
bp[1] = iter.getNext();
String serialized = p.toString();
assertFalse(src.isLosslessResetPossible());
iter.update(bp[1].getSuperbucket(), new BucketId());
assertEquals(p.getActiveBucketCount(), 1);
iter.setDistributionBitCount(11);
assertFalse(src.isLosslessResetPossible());
// Still at 1
assertEquals(p.getDistributionBitCount(), 1);
assertFalse(iter.hasNext());
assertFalse(iter.isDone());
assertEquals(p.getActiveBucketCount(), 1);
// Updating the active bucket allows the reset to take place
iter.update(new BucketId(1, 0), new BucketId());
assertTrue(iter.hasNext());
assertFalse(iter.isDone());
// Should not be any buckets pending/active and the cursor should be
// back at 0
assertEquals(p.getPendingBucketCount(), 0);
assertEquals(p.getActiveBucketCount(), 0);
assertEquals(p.getFinishedBucketCount(), 0);
assertEquals(p.getBucketCursor(), 0);
assertEquals(p.getDistributionBitCount(), 11);
bp[0] = iter.getNext();
assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
// Ensure resetting also works when you're importing existing
// progress
p = new ProgressToken(serialized);
idFactory = new BucketIdFactory();
iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 1, p);
iter.setDistributionBitCount(11);
assertEquals(p.getPendingBucketCount(), 0);
assertEquals(p.getActiveBucketCount(), 0);
assertEquals(p.getFinishedBucketCount(), 0);
assertEquals(p.getBucketCursor(), 0);
assertEquals(p.getDistributionBitCount(), 11);
bp[0] = iter.getNext();
assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
}
use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testRangeDistributionBitIncrease1NoPending.
/**
* Test increasing the distribution bits for a full bucket space range
* source with no finished, active or pending buckets
* @throws ParseException upon docsel parse failure (shouldn't happen)
*/
public void testRangeDistributionBitIncrease1NoPending() throws ParseException {
int db = 2;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
// Test for empty progress token. no splitting involved
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
assertEquals(p.getTotalBucketCount(), 4);
iter.setDistributionBitCount(db + 1);
assertEquals(p.getTotalBucketCount(), 8);
assertEquals(p.getDistributionBitCount(), db + 1);
assertEquals(iter.getDistributionBitCount(), db + 1);
assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);
int[] desired = new int[] { 0, 4, 2, 6, 1, 5, 3, 7 };
for (int i = 0; i < 8; ++i) {
VisitorIterator.BucketProgress bp = iter.getNext();
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, desired[i]));
}
}
Aggregations