Examples with VisitorIterator com.yahoo.documentapi.VisitorIterator used on opensource projects

Example 11 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testProgressEstimation.

public void testProgressEstimation() throws ParseException {
    int distBits = 4;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    // Create a range of [0, 16) superbuckets
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, distBits, progress);
    assertEquals(progress.getDistributionBitCount(), 4);
    double epsilon = 0.00001;
    assertEquals(progress.percentFinished(), 0, epsilon);
    VisitorIterator.BucketProgress bp = iter.getNext();
    // Finish first superbucket (6.25% total)
    iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertEquals(progress.percentFinished(), 6.25, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 1);
    bp = iter.getNext();
    VisitorIterator.BucketProgress bp3 = iter.getNext();
    VisitorIterator.BucketProgress bp4 = iter.getNext();
    // Finish second (12.5% total)
    iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertEquals(progress.percentFinished(), 12.5, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 2);
    // Finish third bucket 75% through (17.1875% total)
    iter.update(bp3.getSuperbucket(), new BucketId(distBits + 2, bp3.getSuperbucket().getId() | (1 << distBits)));
    assertEquals(progress.percentFinished(), 17.1875, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 2);
    // Finish fourth bucket 25% through (18.75% total)
    iter.update(bp4.getSuperbucket(), new BucketId(distBits + 2, bp4.getSuperbucket().getId()));
    assertEquals(progress.percentFinished(), 18.75, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 2);
    // Finish all buckets
    iter.update(bp4.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    iter.update(bp3.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertEquals(progress.percentFinished(), 25, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 4);
    while (iter.hasNext()) {
        bp = iter.getNext();
        iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    }
    assertEquals(progress.getFinishedBucketCount(), 16);
    assertEquals(progress.percentFinished(), 100, epsilon);
}

Example 12 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testExplicitDistributionImportNoTruncation.

public void testExplicitDistributionImportNoTruncation() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, 20, p);
    assertEquals(20, iter.getDistributionBitCount());
    assertEquals(20, p.getDistributionBitCount());
    assertEquals(20, iter.getBucketSource().getDistributionBitCount());
    iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    // Make sure no truncation is done on import
    String serialized = p.toString();
    ProgressToken p2 = new ProgressToken(serialized);
    BucketIdFactory idFactory2 = new BucketIdFactory();
    VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection("id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory2, 1, p2);
    assertEquals(20, iter2.getDistributionBitCount());
    assertEquals(20, p2.getDistributionBitCount());
    assertEquals(20, iter2.getBucketSource().getDistributionBitCount());
    assertEquals(2, p2.getPendingBucketCount());
    assertEquals(1, p2.getFinishedBucketCount());
    assertEquals(3, p2.getTotalBucketCount());
}

Example 13 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testExplicitDistributionBitIncrease.

public void testExplicitDistributionBitIncrease() throws ParseException {
    int distBits = 12;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p);
    assertEquals(iter.getDistributionBitCount(), distBits);
    assertEquals(p.getDistributionBitCount(), distBits);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits);
    iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    iter.setDistributionBitCount(16);
    assertEquals(iter.getDistributionBitCount(), 16);
    assertEquals(p.getDistributionBitCount(), 16);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), 16);
    // Changing dist bits for explicit source should change nothing
    assertEquals(p.getPendingBucketCount(), 2);
    assertEquals(p.getFinishedBucketCount(), 1);
    assertEquals(p.getTotalBucketCount(), 3);
}

Example 14 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionIncreaseMultipleBits.

public void testRangeDistributionIncreaseMultipleBits() throws ParseException {
    int db = 16;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    // For this test, have 3 finished bucket, 2 pending and 1 active
    for (int i = 0; i < 3; ++i) {
        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    }
    VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];
    bpp[0] = iter.getNext();
    bpp[1] = iter.getNext();
    // Leave this hanging as active
    VisitorIterator.BucketProgress bpa = iter.getNext();
    iter.update(bpp[0].getSuperbucket(), new BucketId());
    iter.update(bpp[1].getSuperbucket(), new BucketId());
    iter.setDistributionBitCount(20);
    // ProgressToken doesn't change yet, since it had active buckets
    assertEquals(p.getDistributionBitCount(), 16);
    assertEquals(iter.getDistributionBitCount(), 20);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), 20);
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertTrue(iter.getBucketSource().shouldYield());
    assertEquals(p.getPendingBucketCount(), 2);
    assertEquals(p.getActiveBucketCount(), 1);
    // Finish active, triggering the consistency fixes
    iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertEquals(p.getDistributionBitCount(), 20);
    assertEquals(p.getPendingBucketCount(), 32);
    assertEquals(p.getActiveBucketCount(), 0);
    // Each bucket with db:16 becomes equal to 16 buckets with db:20, so
    // the bucket space position must be 16 * 6 = 96
    assertEquals(p.getBucketCursor(), 96);
    // Each finished bucket also covers less ground, so count is upped
    // accordingly
    assertEquals(p.getFinishedBucketCount(), 16 * 4);
    // when using 20 distribution bits
    for (int i = 0; i < 32; ++i) {
        long testKey = ProgressToken.makeNthBucketKey(i + 48, 20);
        VisitorIterator.BucketProgress bp = iter.getNext();
        assertEquals(bp.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(testKey)));
        iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    }
    assertEquals(p.getPendingBucketCount(), 0);
    assertEquals(p.getFinishedBucketCount(), 16 * 6);
    // Bucket source should now begin returning from bucket 20:0x6000
    assertEquals(iter.getNext().getSuperbucket(), new BucketId(20, 0x6000));
}

Example 15 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testProgressSerializationExplicit.

public void testProgressSerializationExplicit() throws ParseException {
    int distBits = 16;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, progress);
    assertEquals(progress.getDistributionBitCount(), distBits);
    assertTrue(iter.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);
    assertEquals(progress.getFinishedBucketCount(), 0);
    assertEquals(progress.getTotalBucketCount(), 3);
    assertEquals(progress.getPendingBucketCount(), 3);
    VisitorIterator.BucketProgress bp1 = iter.getNext();
    VisitorIterator.BucketProgress bp2 = iter.getNext();
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 2);
    // Buckets are ordered by their reverse bucket id key
    assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));
    assertEquals(bp1.getProgress(), new BucketId());
    // Put bucket 1234 back into pending
    iter.update(bp1.getSuperbucket(), new BucketId(36, 1234));
    assertEquals(progress.getPendingBucketCount(), 2);
    assertEquals(bp2.getSuperbucket(), new BucketId(32, 8009));
    assertEquals(bp2.getProgress(), new BucketId());
    {
        StringBuilder desired = new StringBuilder();
        desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n");
        desired.append(distBits);
        desired.append('\n');
        desired.append(0);
        desired.append('\n');
        desired.append(0);
        desired.append('\n');
        desired.append(3);
        desired.append('\n');
        // Pending/active buckets are written in an increasing (key, not
        // bucket-id!) order
        desired.append(Long.toHexString(new BucketId(32, 1234).getRawId()));
        desired.append(':');
        desired.append(Long.toHexString(new BucketId(36, 1234).getRawId()));
        desired.append('\n');
        desired.append(Long.toHexString(new BucketId(32, 8009).getRawId()));
        desired.append(":0\n");
        desired.append(Long.toHexString(new BucketId(32, 6789).getRawId()));
        desired.append(":0\n");
        assertEquals(desired.toString(), progress.toString());
        ProgressToken prog2 = new ProgressToken(progress.toString());
        assertEquals(prog2.getDistributionBitCount(), distBits);
        assertEquals(prog2.getTotalBucketCount(), 3);
        assertEquals(prog2.getFinishedBucketCount(), 0);
        VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection("id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, prog2);
        assertEquals(prog2.getPendingBucketCount(), 3);
        assertFalse(prog2.hasActive());
        assertTrue(iter2.hasNext());
        assertFalse(iter2.isDone());
        assertTrue(iter2.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);
        assertFalse(iter2.getBucketSource().hasNext());
        VisitorIterator.BucketProgress bp = iter2.getNext();
        assertEquals(bp.getSuperbucket(), new BucketId(32, 1234));
        assertEquals(bp.getProgress(), new BucketId(36, 1234));
        assertEquals(prog2.getPendingBucketCount(), 2);
        assertTrue(iter2.hasNext());
        assertFalse(iter2.isDone());
        bp = iter2.getNext();
        assertEquals(bp.getSuperbucket(), new BucketId(32, 8009));
        assertEquals(bp.getProgress(), new BucketId());
        assertEquals(prog2.getPendingBucketCount(), 1);
        assertTrue(iter2.hasNext());
        assertFalse(iter2.isDone());
        bp = iter2.getNext();
        assertEquals(prog2.getPendingBucketCount(), 0);
        assertEquals(bp.getSuperbucket(), new BucketId(32, 6789));
        assertEquals(bp.getProgress(), new BucketId());
        assertFalse(iter2.hasNext());
        // Active buckets
        assertFalse(iter2.isDone());
        assertEquals(prog2.getActiveBucketCount(), 3);
    }
    // Finish off all active buckets
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    bp1 = iter.getNext();
    assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));
    assertEquals(bp1.getProgress(), new BucketId(36, 1234));
    iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    bp1 = iter.getNext();
    assertEquals(bp1.getSuperbucket(), new BucketId(32, 6789));
    assertEquals(bp1.getProgress(), new BucketId());
    // Just to make sure Java serializes the long properly
    assertEquals(progress.toString(), "VDS bucket progress file (" + progress.percentFinished() + "% completed)\n" + "16\n" + "0\n" + "1\n" + "3\n" + "8000000000001f49:0\n" + "8000000000001a85:0\n");
    iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    // At this point, we've got one active but no pending, so hasNext == false,
    // but isDone is also == false
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
    iter.update(bp2.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertFalse(iter.hasNext());
    assertTrue(iter.isDone());
    assertTrue(progress.isFinished());
    assertEquals(progress.getActiveBucketCount(), 0);
    {
        StringBuilder finished = new StringBuilder();
        finished.append("VDS bucket progress file (100.0% completed)\n");
        finished.append(distBits);
        finished.append('\n');
        // Cursor (not used by explicit)
        finished.append(0);
        finished.append('\n');
        // Finished
        finished.append(3);
        finished.append('\n');
        // Total
        finished.append(3);
        finished.append('\n');
        assertEquals(finished.toString(), progress.toString());
    }
}