use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testSingleBucketMerge.
public void testSingleBucketMerge() throws ParseException {
int db = 2;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
// Create a range of [0, 4) superbuckets
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
VisitorIterator.BucketProgress bp = iter.getNext();
// Put back as pending and split it
iter.update(bp.getSuperbucket(), new BucketId());
p.splitPendingBucket(new BucketId(db, 0));
assertEquals(p.getPendingBucketCount(), 2);
// Merge both back into one node. Merge from left sibling with right present
p.mergePendingBucket(new BucketId(db + 1, 0));
assertEquals(p.getPendingBucketCount(), 1);
bp = iter.getNext();
assertEquals(bp.getSuperbucket(), new BucketId(db, 0));
}
use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testEntireBucketSpaceCovered.
public void testEntireBucketSpaceCovered() throws ParseException {
int db = 4;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];
for (int i = 0; i < 3; ++i) {
bpp[i] = iter.getNext();
}
for (int i = 0; i < 3; ++i) {
// Must use non-zero progress or all pending will be optimized
// away by the reset-logic
iter.update(bpp[i].getSuperbucket(), new BucketId(db + 1, bpp[i].getSuperbucket().getId()));
}
Set<BucketId> buckets = new TreeSet<BucketId>();
db = 7;
for (int i = 0; i < (1 << db); ++i) {
buckets.add(new BucketId(db, i));
}
iter.setDistributionBitCount(db);
assertEquals(p.getFinishedBucketCount(), 0);
assertEquals(p.getPendingBucketCount(), 3 << 3);
// Ensure all buckets are visited once and only once
while (iter.hasNext()) {
VisitorIterator.BucketProgress bp = iter.getNext();
assertTrue(buckets.contains(bp.getSuperbucket()));
buckets.remove(bp.getSuperbucket());
}
assertTrue(buckets.isEmpty());
}
use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testRangeDistributionLosslessReset.
// Similar to testRangeDistributionBitInitialDrop, but going from 1 to 11
// This tests that doing so may be done in an optimized way rather than
// attempting to split enough buckets to cover the entire bucket space!
public void testRangeDistributionLosslessReset() throws ParseException {
int db = 1;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
VisitorIterator.DistributionRangeBucketSource src = (VisitorIterator.DistributionRangeBucketSource) iter.getBucketSource();
VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[2];
bp[0] = iter.getNext();
bp[1] = iter.getNext();
String serialized = p.toString();
assertFalse(src.isLosslessResetPossible());
iter.update(bp[1].getSuperbucket(), new BucketId());
assertEquals(p.getActiveBucketCount(), 1);
iter.setDistributionBitCount(11);
assertFalse(src.isLosslessResetPossible());
// Still at 1
assertEquals(p.getDistributionBitCount(), 1);
assertFalse(iter.hasNext());
assertFalse(iter.isDone());
assertEquals(p.getActiveBucketCount(), 1);
// Updating the active bucket allows the reset to take place
iter.update(new BucketId(1, 0), new BucketId());
assertTrue(iter.hasNext());
assertFalse(iter.isDone());
// Should not be any buckets pending/active and the cursor should be
// back at 0
assertEquals(p.getPendingBucketCount(), 0);
assertEquals(p.getActiveBucketCount(), 0);
assertEquals(p.getFinishedBucketCount(), 0);
assertEquals(p.getBucketCursor(), 0);
assertEquals(p.getDistributionBitCount(), 11);
bp[0] = iter.getNext();
assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
// Ensure resetting also works when you're importing existing
// progress
p = new ProgressToken(serialized);
idFactory = new BucketIdFactory();
iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 1, p);
iter.setDistributionBitCount(11);
assertEquals(p.getPendingBucketCount(), 0);
assertEquals(p.getActiveBucketCount(), 0);
assertEquals(p.getFinishedBucketCount(), 0);
assertEquals(p.getBucketCursor(), 0);
assertEquals(p.getDistributionBitCount(), 11);
bp[0] = iter.getNext();
assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
}
use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testRangeDistributionBitIncrease1NoPending.
/**
* Test increasing the distribution bits for a full bucket space range
* source with no finished, active or pending buckets
* @throws ParseException upon docsel parse failure (shouldn't happen)
*/
public void testRangeDistributionBitIncrease1NoPending() throws ParseException {
int db = 2;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
// Test for empty progress token. no splitting involved
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
assertEquals(p.getTotalBucketCount(), 4);
iter.setDistributionBitCount(db + 1);
assertEquals(p.getTotalBucketCount(), 8);
assertEquals(p.getDistributionBitCount(), db + 1);
assertEquals(iter.getDistributionBitCount(), db + 1);
assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);
int[] desired = new int[] { 0, 4, 2, 6, 1, 5, 3, 7 };
for (int i = 0; i < 8; ++i) {
VisitorIterator.BucketProgress bp = iter.getNext();
assertEquals(bp.getSuperbucket(), new BucketId(db + 1, desired[i]));
}
}
use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testInconsistentState.
// Test that altering distribution bit count sets ProgressToken as
// inconsistent when there are active buckets
public void testInconsistentState() throws ParseException {
int db = 16;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
// For this test, have 3 finished bucket, 2 pending and 1 active
for (int i = 0; i < 3; ++i) {
iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
}
VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];
bpp[0] = iter.getNext();
bpp[1] = iter.getNext();
// Leave this hanging as active
VisitorIterator.BucketProgress bpa = iter.getNext();
iter.update(bpp[0].getSuperbucket(), new BucketId());
iter.update(bpp[1].getSuperbucket(), new BucketId());
assertFalse(p.isInconsistentState());
iter.setDistributionBitCount(20);
assertTrue(p.isInconsistentState());
// Finish active, triggering the consistency fixes
iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
assertFalse(p.isInconsistentState());
}
Aggregations