Examples with BucketIdFactory com.yahoo.document.BucketIdFactory used on opensource projects

Example 1 with BucketIdFactory

use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.

the class MessageBusVisitorSession method createVisitingProgress.

private VisitingProgress createVisitingProgress(VisitorParameters params) throws ParseException {
    ProgressToken progressToken;
    if (params.getResumeToken() != null) {
        progressToken = params.getResumeToken();
    } else {
        progressToken = new ProgressToken();
    }
    VisitorIterator visitorIterator;
    if (params.getBucketsToVisit() == null || params.getBucketsToVisit().isEmpty()) {
        // Use 1 distribution bit as a starting point. This will almost certainly
        // trigger a ERROR_WRONG_DISTRIBUTION reply immediately, meaning that we'll
        // get a fresh system state from the start. Since no buckets should ever
        // return with a OK result in such a case, we recognize this as a special
        // case in the iterator and simply reset its entire internal state using
        // the new db count rather than doing any splitting.
        BucketIdFactory bucketIdFactory = new BucketIdFactory();
        visitorIterator = VisitorIterator.createFromDocumentSelection(params.getDocumentSelection(), bucketIdFactory, 1, progressToken);
    } else {
        if (log.isLoggable(LogLevel.DEBUG)) {
            log.log(LogLevel.DEBUG, "parameters specify explicit bucket set " + "to visit; using it rather than document selection (" + params.getBucketsToVisit().size() + " buckets given)");
        }
        // Allow override of target buckets iff an explicit set of buckets
        // to visit is given by the visitor parameters. This was primarily
        // used for the defunct synchronization functionality, but since it's
        // so easy to support, don't deprecate it just yet.
        visitorIterator = VisitorIterator.createFromExplicitBucketSet(params.getBucketsToVisit(), 1, progressToken);
    }
    return new VisitingProgress(visitorIterator, progressToken);
}

Example 2 with BucketIdFactory

use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testIterationSingleBucketUpdate.

public void testIterationSingleBucketUpdate() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.user = 1234", idFactory, 1, progress);
    assertFalse(progress.hasActive());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getFinishedBucketCount(), 0);
    assertEquals(progress.getTotalBucketCount(), 1);
    assertFalse(iter.isDone());
    assertTrue(iter.hasNext());
    assertEquals(iter.getRemainingBucketCount(), 1);
    VisitorIterator.BucketProgress b1 = iter.getNext();
    // Upon first getNext of a superbucket, progress == 0
    assertEquals(b1.getSuperbucket(), new BucketId(32, 1234));
    assertEquals(b1.getProgress(), new BucketId());
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(iter.getRemainingBucketCount(), 1);
    // Should only be one active bucket; the one we just got
    assertEquals(progress.getActiveBucketCount(), 1);
    // No pending yet
    assertFalse(progress.hasPending());
    // Update the bucket with a sub-bucket, moving it from active to pending
    BucketId sub = new BucketId(b1.getSuperbucket().getUsedBits() + 1, b1.getSuperbucket().getId());
    iter.update(b1.getSuperbucket(), sub);
    assertFalse(progress.hasActive());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(iter.getRemainingBucketCount(), 1);
    // Get the pending bucket
    VisitorIterator.BucketProgress b2 = iter.getNext();
    assertEquals(b2.getSuperbucket(), new BucketId(32, 1234));
    assertEquals(b2.getProgress(), new BucketId(33, 1234));
    assertFalse(iter.hasNext());
    assertEquals(progress.getActiveBucketCount(), 1);
    assertFalse(progress.hasPending());
    // Now update with progress==super, signalling that the bucket is done
    iter.update(b1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertFalse(progress.hasActive());
    assertFalse(progress.hasPending());
    assertFalse(iter.hasNext());
    assertTrue(iter.isDone());
    assertTrue(progress.isFinished());
    assertEquals(progress.getFinishedBucketCount(), 1);
    assertEquals(iter.getRemainingBucketCount(), 0);
}

Example 3 with BucketIdFactory

use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testImportProgressWithOutdatedDistribution.

public void testImportProgressWithOutdatedDistribution() throws ParseException {
    String input = "VDS bucket progress file\n" + "10\n" + "503\n" + "500\n" + "1024\n" + "28000000000000be:0\n" + "28000000000002be:0\n" + "28000000000001be:0\n";
    int db = 12;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken(input);
    assertEquals(10, p.getDistributionBitCount());
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 1, p);
    iter.setDistributionBitCount(12);
    assertEquals(iter.getDistributionBitCount(), 12);
    assertEquals(p.getDistributionBitCount(), 12);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), 12);
    assertEquals(p.getTotalBucketCount(), 1 << 12);
    assertEquals(p.getFinishedBucketCount(), 500 << 2);
    assertEquals(p.getPendingBucketCount(), 3 << 2);
    assertEquals(p.getActiveBucketCount(), 0);
    assertEquals(p.getBucketCursor(), 503 << 2);
    assertTrue(iter.hasNext());
    ProgressToken p2 = new ProgressToken(p.serialize());
    assertEquals(p2.getDistributionBitCount(), 12);
    assertEquals(p2.getTotalBucketCount(), 1 << 12);
    assertEquals(p2.getFinishedBucketCount(), 500 << 2);
    assertEquals(p2.getPendingBucketCount(), 3 << 2);
    assertEquals(p2.getActiveBucketCount(), 0);
    assertEquals(p2.getBucketCursor(), 503 << 2);
}

Example 4 with BucketIdFactory

use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testActiveUpdate.

/**
 * Test that doing update() on a bucket several times in a row (without re-fetching
 * from getNext first) works
 * @throws ParseException
 */
public void testActiveUpdate() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group = \"yahoo.com\"", idFactory, 16, progress);
    VisitorIterator.BucketProgress bp = iter.getNext();
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
    BucketId superbucket = bp.getSuperbucket();
    int usedBits = superbucket.getUsedBits();
    iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (2L << usedBits)));
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    bp = iter.getNext();
    assertEquals(bp.getSuperbucket(), superbucket);
    assertEquals(bp.getProgress(), new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
}

Example 5 with BucketIdFactory

use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionBitIncreaseDecrease.

// Test that splitting and merging from and to the same db count gives
// back the initial state
public void testRangeDistributionBitIncreaseDecrease() throws ParseException {
    int db = 16;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.DistributionRangeBucketSource src = (VisitorIterator.DistributionRangeBucketSource) iter.getBucketSource();
    assertTrue(src.isLosslessResetPossible());
    // "Sabotage" resetting by having at least 1 finished
    iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];
    for (int i = 0; i < 4; ++i) {
        bpp[i] = iter.getNext();
    }
    for (int i = 0; i < 4; ++i) {
        iter.update(bpp[i].getSuperbucket(), new BucketId());
    }
    assertFalse(src.isLosslessResetPossible());
    iter.setDistributionBitCount(20);
    assertEquals(p.getDistributionBitCount(), 20);
    assertEquals(p.getPendingBucketCount(), 4 << 4);
    assertFalse(iter.getBucketSource().shouldYield());
    assertEquals(p.getBucketCursor(), 5 << 4);
    iter.setDistributionBitCount(16);
    assertEquals(p.getDistributionBitCount(), 16);
    assertEquals(p.getPendingBucketCount(), 4);
    assertFalse(iter.getBucketSource().shouldYield());
    assertEquals(p.getBucketCursor(), 5);
}