use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testInconsistentState.
// Test that altering distribution bit count sets ProgressToken as
// inconsistent when there are active buckets
public void testInconsistentState() throws ParseException {
int db = 16;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken p = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
// For this test, have 3 finished bucket, 2 pending and 1 active
for (int i = 0; i < 3; ++i) {
iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
}
VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];
bpp[0] = iter.getNext();
bpp[1] = iter.getNext();
// Leave this hanging as active
VisitorIterator.BucketProgress bpa = iter.getNext();
iter.update(bpp[0].getSuperbucket(), new BucketId());
iter.update(bpp[1].getSuperbucket(), new BucketId());
assertFalse(p.isInconsistentState());
iter.setDistributionBitCount(20);
assertTrue(p.isInconsistentState());
// Finish active, triggering the consistency fixes
iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
assertFalse(p.isInconsistentState());
}
use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testNullAndSuperUpdate.
/**
* Test that ensures doing update(superbucket, 0) simply puts the bucket back in
* pending
* @throws ParseException
*/
public void testNullAndSuperUpdate() throws ParseException {
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken progress = new ProgressToken();
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group = \"yahoo.com\"", idFactory, 16, progress);
assertEquals(progress.getPendingBucketCount(), 1);
VisitorIterator.BucketProgress bp = iter.getNext();
assertEquals(bp.getProgress(), new BucketId());
BucketId superbucket = bp.getSuperbucket();
BucketId sub = bp.getProgress();
assertFalse(iter.hasNext());
assertFalse(iter.isDone());
assertEquals(progress.getPendingBucketCount(), 0);
assertEquals(progress.getActiveBucketCount(), 1);
// 0-bucket
iter.update(superbucket, ProgressToken.NULL_BUCKET);
assertTrue(iter.hasNext());
assertFalse(iter.isDone());
assertEquals(progress.getPendingBucketCount(), 1);
assertEquals(progress.getActiveBucketCount(), 0);
VisitorIterator.BucketProgress bp2 = iter.getNext();
assertEquals(bp2.getSuperbucket(), superbucket);
assertEquals(bp2.getProgress(), ProgressToken.NULL_BUCKET);
assertEquals(progress.getPendingBucketCount(), 0);
assertEquals(progress.getActiveBucketCount(), 1);
// progress == super
iter.update(superbucket, superbucket);
assertTrue(iter.hasNext());
assertFalse(iter.isDone());
assertEquals(progress.getPendingBucketCount(), 1);
assertEquals(progress.getActiveBucketCount(), 0);
bp2 = iter.getNext();
assertEquals(bp2.getSuperbucket(), superbucket);
assertEquals(bp2.getProgress(), superbucket);
assertEquals(progress.getPendingBucketCount(), 0);
assertEquals(progress.getActiveBucketCount(), 1);
}
use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class VisitorIteratorTestCase method testProgressEstimation.
public void testProgressEstimation() throws ParseException {
int distBits = 4;
BucketIdFactory idFactory = new BucketIdFactory();
ProgressToken progress = new ProgressToken();
// Create a range of [0, 16) superbuckets
VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, distBits, progress);
assertEquals(progress.getDistributionBitCount(), 4);
double epsilon = 0.00001;
assertEquals(progress.percentFinished(), 0, epsilon);
VisitorIterator.BucketProgress bp = iter.getNext();
// Finish first superbucket (6.25% total)
iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
assertEquals(progress.percentFinished(), 6.25, epsilon);
assertEquals(progress.getFinishedBucketCount(), 1);
bp = iter.getNext();
VisitorIterator.BucketProgress bp3 = iter.getNext();
VisitorIterator.BucketProgress bp4 = iter.getNext();
// Finish second (12.5% total)
iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
assertEquals(progress.percentFinished(), 12.5, epsilon);
assertEquals(progress.getFinishedBucketCount(), 2);
// Finish third bucket 75% through (17.1875% total)
iter.update(bp3.getSuperbucket(), new BucketId(distBits + 2, bp3.getSuperbucket().getId() | (1 << distBits)));
assertEquals(progress.percentFinished(), 17.1875, epsilon);
assertEquals(progress.getFinishedBucketCount(), 2);
// Finish fourth bucket 25% through (18.75% total)
iter.update(bp4.getSuperbucket(), new BucketId(distBits + 2, bp4.getSuperbucket().getId()));
assertEquals(progress.percentFinished(), 18.75, epsilon);
assertEquals(progress.getFinishedBucketCount(), 2);
// Finish all buckets
iter.update(bp4.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
iter.update(bp3.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
assertEquals(progress.percentFinished(), 25, epsilon);
assertEquals(progress.getFinishedBucketCount(), 4);
while (iter.hasNext()) {
bp = iter.getNext();
iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
}
assertEquals(progress.getFinishedBucketCount(), 16);
assertEquals(progress.percentFinished(), 100, epsilon);
}
use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class DistributionTestCase method testSplitBeyondSplitBitDoesntAffectDistribution.
@Test
public void testSplitBeyondSplitBitDoesntAffectDistribution() throws Exception {
Random randomized = new Random(7123161);
long val = randomized.nextLong();
test = new DistributionTestFactory("abovesplitbit");
for (int i = 16; i <= 58; ++i) {
test.recordResult(new BucketId(i, val)).assertNodeUsed(2);
}
}
use of com.yahoo.document.BucketId in project vespa by vespa-engine.
the class DistributionTestCase method testDistributorNoGroupTakeover.
@Test
public void testDistributorNoGroupTakeover() throws Exception {
test = new DistributionTestFactory("hierarchical-grouping-distributor-notakeover").setDistribution(buildHierarchicalConfig(6, 3, 1, "1|2|*", 3).distributor_auto_ownership_transfer_on_whole_group_down(false)).setNodeType(NodeType.DISTRIBUTOR).setClusterState(new ClusterState("distributor:2 storage:9"));
int[] counts = new int[10];
int noneExisting = 0;
for (BucketId bucket : getTestBuckets()) {
DistributionTestFactory.Test t = test.recordResult(bucket);
List<Integer> nodes = t.getNodes();
if (nodes.isEmpty()) {
++noneExisting;
t.assertFailure(DistributionTestFactory.Failure.NO_DISTRIBUTORS_AVAILABLE);
} else {
t.assertNodeCount(1);
for (int i : nodes) {
++counts[i];
}
}
}
for (int i = 2; i < 10; ++i) {
assertEquals(0, counts[i]);
}
for (int i = 0; i < 2; ++i) {
assertTrue(counts[i] > 0);
}
assertEquals(15, noneExisting);
}
Aggregations