Examples with VisitorIterator com.yahoo.documentapi.VisitorIterator used on opensource projects

Example 16 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testImportInconsistentProgressDecrease.

public void testImportInconsistentProgressDecrease() throws ParseException {
    // Bucket progress "file" that upon time of changing from 4 to 7
    // distribution bits and writing the progress had an active bucket
    String input = "VDS bucket progress file\n" + "7\n" + "32\n" + "24\n" + "128\n" + "100000000000000c:0\n";
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken(input);
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 1, p);
    assertEquals(iter.getDistributionBitCount(), 7);
    // Now we're at 6 distribution bits
    iter.setDistributionBitCount(6);
    assertEquals(iter.getDistributionBitCount(), 6);
    assertEquals(p.getDistributionBitCount(), 6);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), 6);
    assertEquals(p.getTotalBucketCount(), 1 << 6);
    assertEquals(p.getFinishedBucketCount(), 24 >> 1);
    // Split 4 -> 7 bits, merge 7 -> 6 bits
    assertEquals(p.getPendingBucketCount(), 1 << 2);
    assertEquals(p.getActiveBucketCount(), 0);
    assertEquals(p.getBucketCursor(), 24 / 2 + (1 << 2));
    assertTrue(iter.hasNext());
}

Example 17 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testExplicitDistributionBitDecrease.

public void testExplicitDistributionBitDecrease() throws ParseException {
    int distBits = 20;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p);
    assertEquals(iter.getDistributionBitCount(), distBits);
    assertEquals(p.getDistributionBitCount(), distBits);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits);
    iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    iter.setDistributionBitCount(16);
    assertEquals(iter.getDistributionBitCount(), 16);
    assertEquals(p.getDistributionBitCount(), 16);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), 16);
    // Changing dist bits for explicit source should change nothing
    assertEquals(p.getPendingBucketCount(), 2);
    assertEquals(p.getFinishedBucketCount(), 1);
    assertEquals(p.getTotalBucketCount(), 3);
}

Example 18 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testProgressSerializationRange.

public void testProgressSerializationRange() throws ParseException {
    int distBits = 4;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    // docsel will be unknown --> entire bucket range will be covered
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, distBits, progress);
    assertEquals(progress.getDistributionBitCount(), distBits);
    assertTrue(iter.getBucketSource() instanceof VisitorIterator.DistributionRangeBucketSource);
    assertEquals(progress.getFinishedBucketCount(), 0);
    assertEquals(progress.getTotalBucketCount(), 1 << distBits);
    // First, get+update half of the buckets, marking them as done
    long bucketCount = 0;
    long bucketStop = 1 << (distBits - 1);
    while (iter.hasNext() && bucketCount != bucketStop) {
        VisitorIterator.BucketProgress ids = iter.getNext();
        iter.update(ids.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
        ++bucketCount;
    }
    assertEquals(bucketCount, bucketStop);
    // Should be no buckets in limbo at this point
    assertFalse(progress.hasActive());
    assertFalse(progress.hasPending());
    assertFalse(iter.isDone());
    assertTrue(iter.hasNext());
    assertEquals(progress.getFinishedBucketCount(), bucketCount);
    assertFalse(progress.isFinished());
    StringBuilder desired = new StringBuilder();
    desired.append("VDS bucket progress file (50.0% completed)\n");
    desired.append(distBits);
    desired.append('\n');
    // Finished == cursor for this
    desired.append(bucketCount);
    desired.append('\n');
    desired.append(bucketCount);
    desired.append('\n');
    desired.append(1 << distBits);
    desired.append('\n');
    assertEquals(desired.toString(), progress.toString());
    // Test import, in which case distribution bits are 1
    BucketIdFactory idFactory2 = new BucketIdFactory();
    // De-serialization with no pending buckets
    {
        ProgressToken progDs = new ProgressToken(progress.toString());
        assertEquals(progDs.getDistributionBitCount(), distBits);
        assertEquals(progDs.getTotalBucketCount(), 1 << distBits);
        assertEquals(progDs.getFinishedBucketCount(), bucketCount);
        VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory2, 1, progDs);
        assertFalse(progDs.hasPending());
        assertFalse(progDs.hasActive());
        assertTrue(iterDs.hasNext());
        assertFalse(iterDs.isDone());
        assertEquals(distBits, iterDs.getDistributionBitCount());
        assertEquals(distBits, progDs.getDistributionBitCount());
        // Iterator must start up on next bucket in range
        VisitorIterator.BucketProgress idDs = iterDs.getNext();
        long resumeKey = ProgressToken.makeNthBucketKey(bucketCount, distBits);
        assertEquals(idDs.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(resumeKey)));
        assertEquals(idDs.getProgress(), new BucketId());
    }
    // Now fetch a subset of the remaining buckets without finishing them,
    // keeping some in the active set and some in pending
    int pendingTotal = 1 << (distBits - 3);
    int activeTotal = 1 << (distBits - 3);
    Vector<VisitorIterator.BucketProgress> buckets = new Vector<VisitorIterator.BucketProgress>();
    // Pre-fetch, since otherwise we'd reuse pending buckets
    for (int i = 0; i < pendingTotal + activeTotal; ++i) {
        buckets.add(iter.getNext());
    }
    for (int i = 0; i < pendingTotal + activeTotal; ++i) {
        VisitorIterator.BucketProgress idTemp = buckets.get(i);
        if (i < activeTotal) {
            // Make them 50% done
            iter.update(idTemp.getSuperbucket(), new BucketId(distBits + 2, idTemp.getSuperbucket().getId() | (2 << distBits)));
        }
    // else: leave hanging as active
    }
    assertEquals(progress.getActiveBucketCount(), activeTotal);
    assertEquals(progress.getPendingBucketCount(), pendingTotal);
    // we can't reuse the existing string builder, since the bucket cursor
    // has changed
    desired = new StringBuilder();
    desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n");
    desired.append(distBits);
    desired.append('\n');
    desired.append(bucketCount + pendingTotal + activeTotal);
    desired.append('\n');
    desired.append(bucketCount);
    desired.append('\n');
    desired.append(1 << distBits);
    desired.append('\n');
    assertEquals(progress.getBuckets().entrySet().size(), pendingTotal + activeTotal);
    for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry : progress.getBuckets().entrySet()) {
        desired.append(Long.toHexString(ProgressToken.keyToBucketId(entry.getKey().getKey())));
        desired.append(':');
        desired.append(Long.toHexString(entry.getValue().getProgress().getRawId()));
        desired.append('\n');
    }
    assertEquals(progress.toString(), desired.toString());
    {
        // Deserialization with pending buckets
        ProgressToken progDs = new ProgressToken(progress.toString());
        assertEquals(progDs.getDistributionBitCount(), distBits);
        assertEquals(progDs.getTotalBucketCount(), 1 << distBits);
        assertEquals(progDs.getFinishedBucketCount(), bucketCount);
        VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory2, 1, progDs);
        // All started but nonfinished buckets get placed in pending upon
        // deserialization
        assertEquals(progDs.getPendingBucketCount(), pendingTotal + activeTotal);
        assertEquals(distBits, progDs.getDistributionBitCount());
        assertEquals(distBits, iterDs.getDistributionBitCount());
        assertFalse(progDs.hasActive());
        assertTrue(iterDs.hasNext());
        assertFalse(iterDs.isDone());
        assertEquals(progDs.getBucketCursor(), bucketCount + pendingTotal + activeTotal);
    }
    // Finish all the active buckets
    for (int i = activeTotal; i < activeTotal + pendingTotal; ++i) {
        iter.update(buckets.get(i).getSuperbucket(), ProgressToken.FINISHED_BUCKET);
        ++bucketCount;
    }
    assertEquals(progress.getActiveBucketCount(), 0);
    boolean consistentNext = true;
    // Get all pending/remaining sourced and finish them all
    while (!iter.isDone()) {
        if (!iter.hasNext()) {
            consistentNext = false;
            break;
        }
        VisitorIterator.BucketProgress bp = iter.getNext();
        iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
        ++bucketCount;
    }
    assertTrue(consistentNext);
    assertFalse(iter.hasNext());
    assertTrue(progress.isFinished());
    // Cumulative number of finished buckets must match 2^distbits
    assertEquals(bucketCount, 1 << distBits);
    StringBuilder finished = new StringBuilder();
    finished.append("VDS bucket progress file (100.0% completed)\n");
    finished.append(distBits);
    finished.append('\n');
    // Cursor
    finished.append(1 << distBits);
    finished.append('\n');
    // Finished
    finished.append(1 << distBits);
    finished.append('\n');
    // Total
    finished.append(1 << distBits);
    finished.append('\n');
    assertEquals(progress.toString(), finished.toString());
}

Example 19 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testIsBucketFinished.

public void testIsBucketFinished() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 4, p);
    assertFalse(p.isBucketFinished(new BucketId(32, 0)));
    // Finish superbucket 0x0000
    iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertTrue(p.isBucketFinished(new BucketId(32, 0)));
    // Cursor is 1, but bucket 0x1000 not yet returned
    assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));
    VisitorIterator.BucketProgress bp = iter.getNext();
    // Cursor 2, 0x1000 returned but is contained in state, so not finished
    assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));
    iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertTrue(p.isBucketFinished(new BucketId(32, 1 << 3)));
    // Only superbucket part is used
    // ...0000
    assertTrue(p.isBucketFinished(new BucketId(32, 0x12345670)));
    // ...1000
    assertTrue(p.isBucketFinished(new BucketId(32, 0x12345678)));
    // ...0001
    assertFalse(p.isBucketFinished(new BucketId(32, 0x12345671)));
    // ...1001
    assertFalse(p.isBucketFinished(new BucketId(32, 0x12345679)));
}

Example 20 with VisitorIterator

use of com.yahoo.documentapi.VisitorIterator in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionBitDecrease1.

public void testRangeDistributionBitDecrease1() throws ParseException {
    int db = 16;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.DistributionRangeBucketSource src = (VisitorIterator.DistributionRangeBucketSource) iter.getBucketSource();
    assertTrue(src.isLosslessResetPossible());
    // active buckets have been updated, 3 merges should be triggered
    for (int i = 0; i < 3; ++i) {
        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    }
    assertFalse(src.isLosslessResetPossible());
    VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[6];
    for (int i = 0; i < 6; ++i) {
        bpp[i] = iter.getNext();
    }
    // Leave this hanging as active
    VisitorIterator.BucketProgress bpa = iter.getNext();
    for (int i = 0; i < 6; ++i) {
        iter.update(bpp[i].getSuperbucket(), new BucketId());
    }
    assertEquals(p.getBucketCursor(), 10);
    iter.setDistributionBitCount(db - 1);
    assertEquals(iter.getDistributionBitCount(), db - 1);
    assertEquals(p.getDistributionBitCount(), db);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), db - 1);
    // The iterator is waiting patiently for all active buckets to be updated,
    // at which point it will performed the merging and actually updating the
    // progress token's distribution bit count
    assertTrue(iter.getBucketSource().shouldYield());
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(p.getActiveBucketCount(), 1);
    iter.update(bpa.getSuperbucket(), new BucketId());
    assertEquals(p.getDistributionBitCount(), db - 1);
    assertEquals(p.getActiveBucketCount(), 0);
    // 3 merges, P PP PP PP -> P P P P
    assertEquals(p.getPendingBucketCount(), 4);
    assertEquals(p.getFinishedBucketCount(), 1);
    assertEquals(p.getBucketCursor(), 5);
}