use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testExplicitDistributionBitIncrease.
public void testExplicitDistributionBitIncrease() throws ParseException {
int distBits = 12;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p);
assertEquals(iter.getDistributionBitCount(), distBits);
assertEquals(p.getDistributionBitCount(), distBits);
assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits);
iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
iter.setDistributionBitCount(16);
assertEquals(iter.getDistributionBitCount(), 16);
assertEquals(p.getDistributionBitCount(), 16);
assertEquals(iter.getBucketSource().getDistributionBitCount(), 16);
// Changing dist bits for explicit source should change nothing
assertEquals(p.getPendingBucketCount(), 2);
assertEquals(p.getFinishedBucketCount(), 1);
assertEquals(p.getTotalBucketCount(), 3);
}
use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testRangeDistributionIncreaseMultipleBits.
public void testRangeDistributionIncreaseMultipleBits() throws ParseException {
int db = 16;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
// For this test, have 3 finished bucket, 2 pending and 1 active
for (int i = 0; i < 3; ++i) {
iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
}
VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];
bpp[0] = iter.getNext();
bpp[1] = iter.getNext();
// Leave this hanging as active
VisitorIterator.BucketProgress bpa = iter.getNext();
iter.update(bpp[0].getSuperbucket(), new BucketId());
iter.update(bpp[1].getSuperbucket(), new BucketId());
iter.setDistributionBitCount(20);
// ProgressToken doesn't change yet, since it had active buckets
assertEquals(p.getDistributionBitCount(), 16);
assertEquals(iter.getDistributionBitCount(), 20);
assertEquals(iter.getBucketSource().getDistributionBitCount(), 20);
assertFalse(iter.hasNext());
assertFalse(iter.isDone());
assertTrue(iter.getBucketSource().shouldYield());
assertEquals(p.getPendingBucketCount(), 2);
assertEquals(p.getActiveBucketCount(), 1);
// Finish active, triggering the consistency fixes
iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
assertEquals(p.getDistributionBitCount(), 20);
assertEquals(p.getPendingBucketCount(), 32);
assertEquals(p.getActiveBucketCount(), 0);
// Each bucket with db:16 becomes equal to 16 buckets with db:20, so
// the bucket space position must be 16 * 6 = 96
assertEquals(p.getBucketCursor(), 96);
// Each finished bucket also covers less ground, so count is upped
// accordingly
assertEquals(p.getFinishedBucketCount(), 16 * 4);
// when using 20 distribution bits
for (int i = 0; i < 32; ++i) {
long testKey = ProgressToken.makeNthBucketKey(i + 48, 20);
VisitorIterator.BucketProgress bp = iter.getNext();
assertEquals(bp.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(testKey)));
iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
}
assertEquals(p.getPendingBucketCount(), 0);
assertEquals(p.getFinishedBucketCount(), 16 * 6);
// Bucket source should now begin returning from bucket 20:0x6000
assertEquals(iter.getNext().getSuperbucket(), new BucketId(20, 0x6000));
}
use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testProgressSerializationExplicit.
public void testProgressSerializationExplicit() throws ParseException {
int distBits = 16;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken progress = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, progress);
assertEquals(progress.getDistributionBitCount(), distBits);
assertTrue(iter.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);
assertEquals(progress.getFinishedBucketCount(), 0);
assertEquals(progress.getTotalBucketCount(), 3);
assertEquals(progress.getPendingBucketCount(), 3);
VisitorIterator.BucketProgress bp1 = iter.getNext();
VisitorIterator.BucketProgress bp2 = iter.getNext();
assertEquals(progress.getPendingBucketCount(), 1);
assertEquals(progress.getActiveBucketCount(), 2);
// Buckets are ordered by their reverse bucket id key
assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));
assertEquals(bp1.getProgress(), new BucketId());
// Put bucket 1234 back into pending
iter.update(bp1.getSuperbucket(), new BucketId(36, 1234));
assertEquals(progress.getPendingBucketCount(), 2);
assertEquals(bp2.getSuperbucket(), new BucketId(32, 8009));
assertEquals(bp2.getProgress(), new BucketId());
{
StringBuilder desired = new StringBuilder();
desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n");
desired.append(distBits);
desired.append('\n');
desired.append(0);
desired.append('\n');
desired.append(0);
desired.append('\n');
desired.append(3);
desired.append('\n');
// Pending/active buckets are written in an increasing (key, not
// bucket-id!) order
desired.append(Long.toHexString(new BucketId(32, 1234).getRawId()));
desired.append(':');
desired.append(Long.toHexString(new BucketId(36, 1234).getRawId()));
desired.append('\n');
desired.append(Long.toHexString(new BucketId(32, 8009).getRawId()));
desired.append(":0\n");
desired.append(Long.toHexString(new BucketId(32, 6789).getRawId()));
desired.append(":0\n");
assertEquals(desired.toString(), progress.toString());
ProgressToken prog2 = new ProgressToken(progress.toString());
assertEquals(prog2.getDistributionBitCount(), distBits);
assertEquals(prog2.getTotalBucketCount(), 3);
assertEquals(prog2.getFinishedBucketCount(), 0);
VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection("id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, prog2);
assertEquals(prog2.getPendingBucketCount(), 3);
assertFalse(prog2.hasActive());
assertTrue(iter2.hasNext());
assertFalse(iter2.isDone());
assertTrue(iter2.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);
assertFalse(iter2.getBucketSource().hasNext());
VisitorIterator.BucketProgress bp = iter2.getNext();
assertEquals(bp.getSuperbucket(), new BucketId(32, 1234));
assertEquals(bp.getProgress(), new BucketId(36, 1234));
assertEquals(prog2.getPendingBucketCount(), 2);
assertTrue(iter2.hasNext());
assertFalse(iter2.isDone());
bp = iter2.getNext();
assertEquals(bp.getSuperbucket(), new BucketId(32, 8009));
assertEquals(bp.getProgress(), new BucketId());
assertEquals(prog2.getPendingBucketCount(), 1);
assertTrue(iter2.hasNext());
assertFalse(iter2.isDone());
bp = iter2.getNext();
assertEquals(prog2.getPendingBucketCount(), 0);
assertEquals(bp.getSuperbucket(), new BucketId(32, 6789));
assertEquals(bp.getProgress(), new BucketId());
assertFalse(iter2.hasNext());
// Active buckets
assertFalse(iter2.isDone());
assertEquals(prog2.getActiveBucketCount(), 3);
}
// Finish off all active buckets
assertTrue(iter.hasNext());
assertFalse(iter.isDone());
bp1 = iter.getNext();
assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));
assertEquals(bp1.getProgress(), new BucketId(36, 1234));
iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
assertTrue(iter.hasNext());
assertFalse(iter.isDone());
bp1 = iter.getNext();
assertEquals(bp1.getSuperbucket(), new BucketId(32, 6789));
assertEquals(bp1.getProgress(), new BucketId());
// Just to make sure Java serializes the long properly
assertEquals(progress.toString(), "VDS bucket progress file (" + progress.percentFinished() + "% completed)\n" + "16\n" + "0\n" + "1\n" + "3\n" + "8000000000001f49:0\n" + "8000000000001a85:0\n");
iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
// At this point, we've got one active but no pending, so hasNext == false,
// but isDone is also == false
assertFalse(iter.hasNext());
assertFalse(iter.isDone());
assertEquals(progress.getPendingBucketCount(), 0);
assertEquals(progress.getActiveBucketCount(), 1);
iter.update(bp2.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
assertFalse(iter.hasNext());
assertTrue(iter.isDone());
assertTrue(progress.isFinished());
assertEquals(progress.getActiveBucketCount(), 0);
{
StringBuilder finished = new StringBuilder();
finished.append("VDS bucket progress file (100.0% completed)\n");
finished.append(distBits);
finished.append('\n');
// Cursor (not used by explicit)
finished.append(0);
finished.append('\n');
// Finished
finished.append(3);
finished.append('\n');
// Total
finished.append(3);
finished.append('\n');
assertEquals(finished.toString(), progress.toString());
}
}
use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testImportInconsistentProgressDecrease.
public void testImportInconsistentProgressDecrease() throws ParseException {
// Bucket progress "file" that upon time of changing from 4 to 7
// distribution bits and writing the progress had an active bucket
String input = "VDS bucket progress file\n" + "7\n" + "32\n" + "24\n" + "128\n" + "100000000000000c:0\n";
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken(input);
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 1, p);
assertEquals(iter.getDistributionBitCount(), 7);
// Now we're at 6 distribution bits
iter.setDistributionBitCount(6);
assertEquals(iter.getDistributionBitCount(), 6);
assertEquals(p.getDistributionBitCount(), 6);
assertEquals(iter.getBucketSource().getDistributionBitCount(), 6);
assertEquals(p.getTotalBucketCount(), 1 << 6);
assertEquals(p.getFinishedBucketCount(), 24 >> 1);
// Split 4 -> 7 bits, merge 7 -> 6 bits
assertEquals(p.getPendingBucketCount(), 1 << 2);
assertEquals(p.getActiveBucketCount(), 0);
assertEquals(p.getBucketCursor(), 24 / 2 + (1 << 2));
assertTrue(iter.hasNext());
}
use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testExplicitDistributionBitDecrease.
public void testExplicitDistributionBitDecrease() throws ParseException {
int distBits = 20;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p);
assertEquals(iter.getDistributionBitCount(), distBits);
assertEquals(p.getDistributionBitCount(), distBits);
assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits);
iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
iter.setDistributionBitCount(16);
assertEquals(iter.getDistributionBitCount(), 16);
assertEquals(p.getDistributionBitCount(), 16);
assertEquals(iter.getBucketSource().getDistributionBitCount(), 16);
// Changing dist bits for explicit source should change nothing
assertEquals(p.getPendingBucketCount(), 2);
assertEquals(p.getFinishedBucketCount(), 1);
assertEquals(p.getTotalBucketCount(), 3);
}
Aggregations