Examples with BucketId com.yahoo.document.BucketId used on opensource projects

Example 1 with BucketId

use of com.yahoo.document.BucketId in project vespa by vespa-engine.

the class MessageBusVisitorSession method handleCreateVisitorReply.

private void handleCreateVisitorReply(CreateVisitorReply reply) {
    CreateVisitorMessage msg = (CreateVisitorMessage) reply.getMessage();
    BucketId superbucket = msg.getBuckets().get(0);
    BucketId subBucketProgress = reply.getLastBucket();
    log.log(LogLevel.DEBUG, sessionName + ": received CreateVisitorReply for bucket " + superbucket + " with progress " + subBucketProgress);
    progress.getIterator().update(superbucket, subBucketProgress);
    params.getControlHandler().onProgress(progress.getToken());
    statistics.add(reply.getVisitorStatistics());
    params.getControlHandler().onVisitorStatistics(statistics);
    trace.getRoot().addChild(reply.getTrace().getRoot());
    if (params.getDynamicallyIncreaseMaxBucketsPerVisitor() && (reply.getVisitorStatistics().getDocumentsReturned() < params.getMaxFirstPassHits() / 2.0)) {
        // Attempt to increase parallelism to reduce latency of visiting
        // Ensure new count is within [1, 128]
        int newMaxBuckets = Math.max(Math.min((int) (params.getMaxBucketsPerVisitor() * params.getDynamicMaxBucketsIncreaseFactor()), 128), 1);
        params.setMaxBucketsPerVisitor(newMaxBuckets);
        log.log(LogLevel.DEBUG, sessionName + ": increasing max buckets per visitor to " + params.getMaxBucketsPerVisitor());
    }
}

Example 2 with BucketId

use of com.yahoo.document.BucketId in project vespa by vespa-engine.

the class MessageBusVisitorSession method handleErrorReply.

private void handleErrorReply(Reply reply) {
    CreateVisitorMessage msg = (CreateVisitorMessage) reply.getMessage();
    // Must reset bucket progress back to what it was before sending.
    BucketId bucket = msg.getBuckets().get(0);
    BucketId subProgress = msg.getBuckets().get(1);
    progress.getIterator().update(bucket, subProgress);
    String message = getErrorMessage(reply.getError(0));
    log.log(LogLevel.DEBUG, sessionName + ": received error reply for bucket " + bucket + " with message '" + message + "'");
    if (isFatalError(reply)) {
        if (params.skipBucketsOnFatalErrors()) {
            markBucketProgressAsFailed(bucket, subProgress, message);
        } else {
            reportVisitorError(message);
            transitionTo(new StateDescription(State.FAILED, message));
            // no additional visitors will be scheduled post-failure
            return;
        }
    }
    if (isErrorOfType(reply, DocumentProtocol.ERROR_WRONG_DISTRIBUTION)) {
        handleWrongDistributionReply((WrongDistributionReply) reply);
    } else {
        if (shouldReportError(reply)) {
            reportVisitorError(message);
        }
        // Wait 100ms before new visitor task is executed. Will prevent
        // visitors from being scheduled from caller.
        scheduleSendCreateVisitorsIfApplicable(100, TimeUnit.MILLISECONDS);
    }
}

Example 3 with BucketId

use of com.yahoo.document.BucketId in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testIterationSingleBucketUpdate.

public void testIterationSingleBucketUpdate() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.user = 1234", idFactory, 1, progress);
    assertFalse(progress.hasActive());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getFinishedBucketCount(), 0);
    assertEquals(progress.getTotalBucketCount(), 1);
    assertFalse(iter.isDone());
    assertTrue(iter.hasNext());
    assertEquals(iter.getRemainingBucketCount(), 1);
    VisitorIterator.BucketProgress b1 = iter.getNext();
    // Upon first getNext of a superbucket, progress == 0
    assertEquals(b1.getSuperbucket(), new BucketId(32, 1234));
    assertEquals(b1.getProgress(), new BucketId());
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(iter.getRemainingBucketCount(), 1);
    // Should only be one active bucket; the one we just got
    assertEquals(progress.getActiveBucketCount(), 1);
    // No pending yet
    assertFalse(progress.hasPending());
    // Update the bucket with a sub-bucket, moving it from active to pending
    BucketId sub = new BucketId(b1.getSuperbucket().getUsedBits() + 1, b1.getSuperbucket().getId());
    iter.update(b1.getSuperbucket(), sub);
    assertFalse(progress.hasActive());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(iter.getRemainingBucketCount(), 1);
    // Get the pending bucket
    VisitorIterator.BucketProgress b2 = iter.getNext();
    assertEquals(b2.getSuperbucket(), new BucketId(32, 1234));
    assertEquals(b2.getProgress(), new BucketId(33, 1234));
    assertFalse(iter.hasNext());
    assertEquals(progress.getActiveBucketCount(), 1);
    assertFalse(progress.hasPending());
    // Now update with progress==super, signalling that the bucket is done
    iter.update(b1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertFalse(progress.hasActive());
    assertFalse(progress.hasPending());
    assertFalse(iter.hasNext());
    assertTrue(iter.isDone());
    assertTrue(progress.isFinished());
    assertEquals(progress.getFinishedBucketCount(), 1);
    assertEquals(iter.getRemainingBucketCount(), 0);
}

Example 4 with BucketId

use of com.yahoo.document.BucketId in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testActiveUpdate.

/**
 * Test that doing update() on a bucket several times in a row (without re-fetching
 * from getNext first) works
 * @throws ParseException
 */
public void testActiveUpdate() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group = \"yahoo.com\"", idFactory, 16, progress);
    VisitorIterator.BucketProgress bp = iter.getNext();
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
    BucketId superbucket = bp.getSuperbucket();
    int usedBits = superbucket.getUsedBits();
    iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (2L << usedBits)));
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    bp = iter.getNext();
    assertEquals(bp.getSuperbucket(), superbucket);
    assertEquals(bp.getProgress(), new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
}

Example 5 with BucketId

use of com.yahoo.document.BucketId in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionBitIncreaseDecrease.

// Test that splitting and merging from and to the same db count gives
// back the initial state
public void testRangeDistributionBitIncreaseDecrease() throws ParseException {
    int db = 16;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    VisitorIterator.DistributionRangeBucketSource src = (VisitorIterator.DistributionRangeBucketSource) iter.getBucketSource();
    assertTrue(src.isLosslessResetPossible());
    // "Sabotage" resetting by having at least 1 finished
    iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];
    for (int i = 0; i < 4; ++i) {
        bpp[i] = iter.getNext();
    }
    for (int i = 0; i < 4; ++i) {
        iter.update(bpp[i].getSuperbucket(), new BucketId());
    }
    assertFalse(src.isLosslessResetPossible());
    iter.setDistributionBitCount(20);
    assertEquals(p.getDistributionBitCount(), 20);
    assertEquals(p.getPendingBucketCount(), 4 << 4);
    assertFalse(iter.getBucketSource().shouldYield());
    assertEquals(p.getBucketCursor(), 5 << 4);
    iter.setDistributionBitCount(16);
    assertEquals(p.getDistributionBitCount(), 16);
    assertEquals(p.getPendingBucketCount(), 4);
    assertFalse(iter.getBucketSource().shouldYield());
    assertEquals(p.getBucketCursor(), 5);
}