Examples with BucketIdFactory com.yahoo.document.BucketIdFactory used on opensource projects

Example 11 with BucketIdFactory

use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testNullAndSuperUpdate.

/**
 * Test that ensures doing update(superbucket, 0) simply puts the bucket back in
 * pending
 * @throws ParseException
 */
public void testNullAndSuperUpdate() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group = \"yahoo.com\"", idFactory, 16, progress);
    assertEquals(progress.getPendingBucketCount(), 1);
    VisitorIterator.BucketProgress bp = iter.getNext();
    assertEquals(bp.getProgress(), new BucketId());
    BucketId superbucket = bp.getSuperbucket();
    BucketId sub = bp.getProgress();
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
    // 0-bucket
    iter.update(superbucket, ProgressToken.NULL_BUCKET);
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    VisitorIterator.BucketProgress bp2 = iter.getNext();
    assertEquals(bp2.getSuperbucket(), superbucket);
    assertEquals(bp2.getProgress(), ProgressToken.NULL_BUCKET);
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
    // progress == super
    iter.update(superbucket, superbucket);
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    bp2 = iter.getNext();
    assertEquals(bp2.getSuperbucket(), superbucket);
    assertEquals(bp2.getProgress(), superbucket);
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
}

Example 12 with BucketIdFactory

use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testProgressEstimation.

public void testProgressEstimation() throws ParseException {
    int distBits = 4;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    // Create a range of [0, 16) superbuckets
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, distBits, progress);
    assertEquals(progress.getDistributionBitCount(), 4);
    double epsilon = 0.00001;
    assertEquals(progress.percentFinished(), 0, epsilon);
    VisitorIterator.BucketProgress bp = iter.getNext();
    // Finish first superbucket (6.25% total)
    iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertEquals(progress.percentFinished(), 6.25, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 1);
    bp = iter.getNext();
    VisitorIterator.BucketProgress bp3 = iter.getNext();
    VisitorIterator.BucketProgress bp4 = iter.getNext();
    // Finish second (12.5% total)
    iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertEquals(progress.percentFinished(), 12.5, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 2);
    // Finish third bucket 75% through (17.1875% total)
    iter.update(bp3.getSuperbucket(), new BucketId(distBits + 2, bp3.getSuperbucket().getId() | (1 << distBits)));
    assertEquals(progress.percentFinished(), 17.1875, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 2);
    // Finish fourth bucket 25% through (18.75% total)
    iter.update(bp4.getSuperbucket(), new BucketId(distBits + 2, bp4.getSuperbucket().getId()));
    assertEquals(progress.percentFinished(), 18.75, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 2);
    // Finish all buckets
    iter.update(bp4.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    iter.update(bp3.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertEquals(progress.percentFinished(), 25, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 4);
    while (iter.hasNext()) {
        bp = iter.getNext();
        iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    }
    assertEquals(progress.getFinishedBucketCount(), 16);
    assertEquals(progress.percentFinished(), 100, epsilon);
}

Example 13 with BucketIdFactory

use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.

the class Simulator method getIdealTarget.

private Integer getIdealTarget(String idString, String clusterState) {
    DocumentId did = new DocumentId(idString);
    BucketIdFactory factory = new BucketIdFactory();
    BucketId bid = factory.getBucketId(did);
    try {
        return policyFactory.getLastParameters().createDistribution(null).getIdealDistributorNode(new ClusterState(clusterState), bid, "uim");
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 14 with BucketIdFactory

use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.

the class BucketIdFactoryTestCase method testBidContainsDocId.

public void testBidContainsDocId() {
    DocumentId docId = new DocumentId("userdoc:recovery:18:99999");
    BucketIdFactory factory = new BucketIdFactory(32, 26, 6);
    BucketId bid = new BucketId(16, 0x12L);
    assert (bid.contains(docId, factory));
    // split on '0'
    bid = new BucketId(17, 0x12L);
    assert (bid.contains(docId, factory));
    // split on '1'
    bid = new BucketId(17, (1L << 16) + 0x12L);
    assert (!bid.contains(docId, factory));
}

Example 15 with BucketIdFactory

use of com.yahoo.document.BucketIdFactory in project vespa by vespa-engine.

the class BucketIdFactoryTestCase method testBucketGeneration.

public void testBucketGeneration() {
    BucketIdFactory factory = new BucketIdFactory(32, 26, 6);
    DocumentId doc1 = new DocumentId(new DocIdString("ns", "spec"));
    DocumentId doc2 = new DocumentId(new DocIdString("ns2", "spec"));
    DocumentId doc3 = new DocumentId(new DocIdString("ns", "spec2"));
    DocumentId userDoc1 = new DocumentId(new UserDocIdString("ns", 0x12, "spec"));
    DocumentId userDoc2 = new DocumentId(new UserDocIdString("ns2", 0x12, "spec2"));
    DocumentId userDoc3 = new DocumentId(new UserDocIdString("ns", 0x13, "spec"));
    DocumentId groupDoc1 = new DocumentId(new GroupDocIdString("ns", "yahoo.com", "spec"));
    DocumentId groupDoc2 = new DocumentId(new GroupDocIdString("ns2", "yahoo.com", "spec2"));
    DocumentId groupDoc3 = new DocumentId(new GroupDocIdString("ns", "yahoo", "spec"));
    DocumentId orderDoc1 = new DocumentId(new OrderDocIdString("ns", "13", 31, 19, 1268182861, "foo"));
    DocumentId orderDoc2 = new DocumentId(new OrderDocIdString("ns", "13", 31, 19, 1205110861, "foo"));
    DocumentId orderDoc3 = new DocumentId(new OrderDocIdString("ns", "13", 31, 19, 1205715661, "foo"));
    DocumentId orderDoc4 = new DocumentId(new OrderDocIdString("ns", "13", 4, 0, 2, "foo"));
    DocumentId orderDoc5 = new DocumentId(new OrderDocIdString("ns", "13", 4, 0, 4, "foo"));
    DocumentId orderDoc6 = new DocumentId(new OrderDocIdString("ns", "13", 4, 0, 11, "foo"));
    BucketId docBucket1 = factory.getBucketId(doc1);
    BucketId docBucket2 = factory.getBucketId(doc2);
    BucketId docBucket3 = factory.getBucketId(doc3);
    BucketId userDocBucket1 = factory.getBucketId(userDoc1);
    BucketId userDocBucket2 = factory.getBucketId(userDoc2);
    BucketId userDocBucket3 = factory.getBucketId(userDoc3);
    BucketId groupDocBucket1 = factory.getBucketId(groupDoc1);
    BucketId groupDocBucket2 = factory.getBucketId(groupDoc2);
    BucketId groupDocBucket3 = factory.getBucketId(groupDoc3);
    BucketId orderDocBucket1 = factory.getBucketId(orderDoc1);
    BucketId orderDocBucket2 = factory.getBucketId(orderDoc2);
    BucketId orderDocBucket3 = factory.getBucketId(orderDoc3);
    BucketId orderDocBucket4 = factory.getBucketId(orderDoc4);
    BucketId orderDocBucket5 = factory.getBucketId(orderDoc5);
    BucketId orderDocBucket6 = factory.getBucketId(orderDoc6);
    assertEquals(new Hex(0xe99703f200000012l), new Hex(userDocBucket1.getRawId()));
    assertEquals(new Hex(0xebfa518a00000012l), new Hex(userDocBucket2.getRawId()));
    assertEquals(new Hex(0xeac1850800000013l), new Hex(userDocBucket3.getRawId()));
    assertEquals(new Hex(0xe90ce4b09a1acd50l), new Hex(groupDocBucket1.getRawId()));
    assertEquals(new Hex(0xe9cedaa49a1acd50l), new Hex(groupDocBucket2.getRawId()));
    assertEquals(new Hex(0xe8cdb18bafe81f24l), new Hex(groupDocBucket3.getRawId()));
    assertEquals(new Hex(0xe980c9abd5fd8d11l), new Hex(docBucket1.getRawId()));
    assertEquals(new Hex(0xeafe870c5f9c37b9l), new Hex(docBucket2.getRawId()));
    assertEquals(new Hex(0xeaebe9473ecbcd69l), new Hex(docBucket3.getRawId()));
    assertEquals(new Hex(0xeae764e90000000dl), new Hex(orderDocBucket1.getRawId()));
    assertEquals(new Hex(0xeacb85f10000000dl), new Hex(orderDocBucket2.getRawId()));
    assertEquals(new Hex(0xea68ddf10000000dl), new Hex(orderDocBucket3.getRawId()));
    assertEquals(new Hex(0xe87526540000000dl), new Hex(orderDocBucket4.getRawId()));
    assertEquals(new Hex(0xea59f8f20000000dl), new Hex(orderDocBucket5.getRawId()));
    assertEquals(new Hex(0xe9eb703d0000000dl), new Hex(orderDocBucket6.getRawId()));
}