Examples with ProgressToken com.yahoo.documentapi.ProgressToken used on opensource projects

Example 1 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class ContinuationHitTest method decodingAcceptsLegacyNonUrlSafeTokens.

/**
 * Legacy Base64 encoder emitted MIME Base64. Ensure we handle tokens from that era.
 */
@Test
public void decodingAcceptsLegacyNonUrlSafeTokens() throws Exception {
    final String legacyBase64 = convertedToMimeBase64Chars(SINGLE_BUCKET_URL_SAFE_BASE64);
    final ProgressToken legacyToken = ContinuationHit.getToken(legacyBase64);
    assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, new ContinuationHit(legacyToken).getValue());
}

Example 2 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class ContinuationHitTest method decodingAcceptsLegacyMimeLineBrokenTokens.

/**
 * Legacy Base64 encoder would happily output line breaks after each MIME line
 * boundary. Ensure we handle these gracefully.
 */
@Test
public void decodingAcceptsLegacyMimeLineBrokenTokens() throws Exception {
    final String multiBucketLegacyToken = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz++gAAAAAAAAAA6jRWeJq8/vsA\r\n" + "AAAAAAAAAOo0VniavP7/AAAAAAAAAAA=";
    final ProgressToken legacyToken = ContinuationHit.getToken(multiBucketLegacyToken);
    assertEquals(MULTI_BUCKET_URL_SAFE_BASE64, new ContinuationHit(legacyToken).getValue());
}

Example 3 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testIterationSingleBucketUpdate.

public void testIterationSingleBucketUpdate() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.user = 1234", idFactory, 1, progress);
    assertFalse(progress.hasActive());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getFinishedBucketCount(), 0);
    assertEquals(progress.getTotalBucketCount(), 1);
    assertFalse(iter.isDone());
    assertTrue(iter.hasNext());
    assertEquals(iter.getRemainingBucketCount(), 1);
    VisitorIterator.BucketProgress b1 = iter.getNext();
    // Upon first getNext of a superbucket, progress == 0
    assertEquals(b1.getSuperbucket(), new BucketId(32, 1234));
    assertEquals(b1.getProgress(), new BucketId());
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(iter.getRemainingBucketCount(), 1);
    // Should only be one active bucket; the one we just got
    assertEquals(progress.getActiveBucketCount(), 1);
    // No pending yet
    assertFalse(progress.hasPending());
    // Update the bucket with a sub-bucket, moving it from active to pending
    BucketId sub = new BucketId(b1.getSuperbucket().getUsedBits() + 1, b1.getSuperbucket().getId());
    iter.update(b1.getSuperbucket(), sub);
    assertFalse(progress.hasActive());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(iter.getRemainingBucketCount(), 1);
    // Get the pending bucket
    VisitorIterator.BucketProgress b2 = iter.getNext();
    assertEquals(b2.getSuperbucket(), new BucketId(32, 1234));
    assertEquals(b2.getProgress(), new BucketId(33, 1234));
    assertFalse(iter.hasNext());
    assertEquals(progress.getActiveBucketCount(), 1);
    assertFalse(progress.hasPending());
    // Now update with progress==super, signalling that the bucket is done
    iter.update(b1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertFalse(progress.hasActive());
    assertFalse(progress.hasPending());
    assertFalse(iter.hasNext());
    assertTrue(iter.isDone());
    assertTrue(progress.isFinished());
    assertEquals(progress.getFinishedBucketCount(), 1);
    assertEquals(iter.getRemainingBucketCount(), 0);
}

Example 4 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testImportProgressWithOutdatedDistribution.

public void testImportProgressWithOutdatedDistribution() throws ParseException {
    String input = "VDS bucket progress file\n" + "10\n" + "503\n" + "500\n" + "1024\n" + "28000000000000be:0\n" + "28000000000002be:0\n" + "28000000000001be:0\n";
    int db = 12;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken(input);
    assertEquals(10, p.getDistributionBitCount());
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 1, p);
    iter.setDistributionBitCount(12);
    assertEquals(iter.getDistributionBitCount(), 12);
    assertEquals(p.getDistributionBitCount(), 12);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), 12);
    assertEquals(p.getTotalBucketCount(), 1 << 12);
    assertEquals(p.getFinishedBucketCount(), 500 << 2);
    assertEquals(p.getPendingBucketCount(), 3 << 2);
    assertEquals(p.getActiveBucketCount(), 0);
    assertEquals(p.getBucketCursor(), 503 << 2);
    assertTrue(iter.hasNext());
    ProgressToken p2 = new ProgressToken(p.serialize());
    assertEquals(p2.getDistributionBitCount(), 12);
    assertEquals(p2.getTotalBucketCount(), 1 << 12);
    assertEquals(p2.getFinishedBucketCount(), 500 << 2);
    assertEquals(p2.getPendingBucketCount(), 3 << 2);
    assertEquals(p2.getActiveBucketCount(), 0);
    assertEquals(p2.getBucketCursor(), 503 << 2);
}

Example 5 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testActiveUpdate.

/**
 * Test that doing update() on a bucket several times in a row (without re-fetching
 * from getNext first) works
 * @throws ParseException
 */
public void testActiveUpdate() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group = \"yahoo.com\"", idFactory, 16, progress);
    VisitorIterator.BucketProgress bp = iter.getNext();
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
    BucketId superbucket = bp.getSuperbucket();
    int usedBits = superbucket.getUsedBits();
    iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (2L << usedBits)));
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    bp = iter.getNext();
    assertEquals(bp.getSuperbucket(), superbucket);
    assertEquals(bp.getProgress(), new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
}