Examples with ProgressToken com.yahoo.documentapi.ProgressToken used on opensource projects

Example 26 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testIsBucketFinished.

public void testIsBucketFinished() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 4, p);
    assertFalse(p.isBucketFinished(new BucketId(32, 0)));
    // Finish superbucket 0x0000
    iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertTrue(p.isBucketFinished(new BucketId(32, 0)));
    // Cursor is 1, but bucket 0x1000 not yet returned
    assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));
    VisitorIterator.BucketProgress bp = iter.getNext();
    // Cursor 2, 0x1000 returned but is contained in state, so not finished
    assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));
    iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertTrue(p.isBucketFinished(new BucketId(32, 1 << 3)));
    // Only superbucket part is used
    // ...0000
    assertTrue(p.isBucketFinished(new BucketId(32, 0x12345670)));
    // ...1000
    assertTrue(p.isBucketFinished(new BucketId(32, 0x12345678)));
    // ...0001
    assertFalse(p.isBucketFinished(new BucketId(32, 0x12345671)));
    // ...1001
    assertFalse(p.isBucketFinished(new BucketId(32, 0x12345679)));
}

Example 27 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testExceptionOnWrongDocumentSelection.

public void testExceptionOnWrongDocumentSelection() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    // Since we don't store the actual original document selection in the
    // progress files, we can't really tell whether or not a "wrong" document
    // selection has been given, so we just do a best effort by checking
    // that the number of total buckets match up and that the bucket cursor
    // isn't set for explicit sources
    // Try to pass a known document selection to an unknown docsel iterator
    boolean caughtIt = false;
    try {
        ProgressToken p = new ProgressToken("VDS bucket progress file\n16\n3\n1\n3\n" + "8000000000001f49:0\n8000000000001a85:0\n");
        VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 16, p);
    } catch (IllegalArgumentException e) {
        caughtIt = true;
    }
    assertTrue(caughtIt);
    // Now try it the other way around
    caughtIt = false;
    try {
        ProgressToken p = new ProgressToken("VDS bucket progress file\n" + "10\n" + "503\n" + "500\n" + "1024\n" + "28000000000000be:0\n" + "28000000000002be:0\n" + "28000000000001be:0\n");
        VisitorIterator.createFromDocumentSelection("id.group=\"yahoo.com\" or id.user=555", idFactory, 16, p);
    } catch (IllegalArgumentException e) {
        caughtIt = true;
    }
    assertTrue(caughtIt);
}

Example 28 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionBitDecrease1.

public void testRangeDistributionBitDecrease1() throws ParseException {
    int db = 16;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.DistributionRangeBucketSource src = (VisitorIterator.DistributionRangeBucketSource) iter.getBucketSource();
    assertTrue(src.isLosslessResetPossible());
    // active buckets have been updated, 3 merges should be triggered
    for (int i = 0; i < 3; ++i) {
        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    }
    assertFalse(src.isLosslessResetPossible());
    VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[6];
    for (int i = 0; i < 6; ++i) {
        bpp[i] = iter.getNext();
    }
    // Leave this hanging as active
    VisitorIterator.BucketProgress bpa = iter.getNext();
    for (int i = 0; i < 6; ++i) {
        iter.update(bpp[i].getSuperbucket(), new BucketId());
    }
    assertEquals(p.getBucketCursor(), 10);
    iter.setDistributionBitCount(db - 1);
    assertEquals(iter.getDistributionBitCount(), db - 1);
    assertEquals(p.getDistributionBitCount(), db);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), db - 1);
    // The iterator is waiting patiently for all active buckets to be updated,
    // at which point it will performed the merging and actually updating the
    // progress token's distribution bit count
    assertTrue(iter.getBucketSource().shouldYield());
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(p.getActiveBucketCount(), 1);
    iter.update(bpa.getSuperbucket(), new BucketId());
    assertEquals(p.getDistributionBitCount(), db - 1);
    assertEquals(p.getActiveBucketCount(), 0);
    // 3 merges, P PP PP PP -> P P P P
    assertEquals(p.getPendingBucketCount(), 4);
    assertEquals(p.getFinishedBucketCount(), 1);
    assertEquals(p.getBucketCursor(), 5);
}

Example 29 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testDeserializedFinishedProgress.

public void testDeserializedFinishedProgress() {
    StringBuilder finished = new StringBuilder();
    // legacy; no completion percentage
    finished.append("VDS bucket progress file\n");
    finished.append(17);
    finished.append('\n');
    // Cursor
    finished.append(1L << 17);
    finished.append('\n');
    // Finished
    finished.append(1L << 17);
    finished.append('\n');
    // Total
    finished.append(1L << 17);
    finished.append('\n');
    ProgressToken token = new ProgressToken(finished.toString());
    assertEquals(token.getDistributionBitCount(), 17);
    assertEquals(token.getTotalBucketCount(), 1L << 17);
    assertEquals(token.getFinishedBucketCount(), 1L << 17);
    assertEquals(token.getBucketCursor(), 1L << 17);
    assertTrue(token.isFinished());
    ProgressToken token2 = new ProgressToken(token.serialize());
    assertEquals(17, token2.getDistributionBitCount());
    assertEquals(1L << 17, token2.getTotalBucketCount());
    assertEquals(1L << 17, token2.getFinishedBucketCount());
    assertEquals(1L << 17, token2.getBucketCursor());
    assertTrue(token2.isFinished());
}

Example 30 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionBitChangeWithoutDone.

// Test that intermittent changes in distribution are handled properly, e.g.
// changing from 11 -> 9 with X active and then before all those are flushed,
// the distribution goes up to 12
public void testRangeDistributionBitChangeWithoutDone() throws ParseException {
    int db = 11;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.DistributionRangeBucketSource src = (VisitorIterator.DistributionRangeBucketSource) iter.getBucketSource();
    VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];
    for (int i = 0; i < 4; ++i) {
        bpp[i] = iter.getNext();
    }
    for (int i = 0; i < 2; ++i) {
        iter.update(bpp[i].getSuperbucket(), new BucketId());
    }
    assertFalse(src.isLosslessResetPossible());
    // Now 2 pending, 2 active
    iter.setDistributionBitCount(9);
    assertEquals(p.getDistributionBitCount(), 11);
    assertEquals(p.getActiveBucketCount(), 2);
    assertEquals(p.getPendingBucketCount(), 2);
    assertTrue(iter.getBucketSource().shouldYield());
    // Update as pending, still with old count since there's 1 more active
    // with bpp[2]. Have progress so that lossless reset isn't possible
    iter.update(bpp[3].getSuperbucket(), new BucketId(15, bpp[3].getSuperbucket().getId()));
    iter.setDistributionBitCount(12);
    assertEquals(p.getActiveBucketCount(), 1);
    assertEquals(p.getPendingBucketCount(), 3);
    assertTrue(iter.getBucketSource().shouldYield());
    // Serialize before token is updated to 12 bits
    String serialized = p.toString();
    iter.update(bpp[2].getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertEquals(p.getActiveBucketCount(), 0);
    // All active buckets are at db=11, so they should be split once each
    assertEquals(p.getPendingBucketCount(), 3 * 2);
    assertFalse(iter.getBucketSource().shouldYield());
    assertEquals(p.getFinishedBucketCount(), 2);
    // Ensure we get a consistent progress token imported
    ProgressToken p2 = new ProgressToken(serialized);
    // Not yet updated
    assertEquals(p2.getDistributionBitCount(), 11);
    BucketIdFactory idFactory2 = new BucketIdFactory();
    VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory2, 1, p2);
    // Not yet updated, since we don't trust the initial BucketIdFactory
    assertEquals(iter2.getDistributionBitCount(), 11);
    assertEquals(p2.getDistributionBitCount(), 11);
    iter2.setDistributionBitCount(12);
    // Now it has been updated
    assertEquals(p2.getDistributionBitCount(), 12);
    assertEquals(p2.getPendingBucketCount(), 8);
    assertEquals(p2.getBucketCursor(), 8);
    assertEquals(p2.getFinishedBucketCount(), 0);
}