Examples with ProgressToken com.yahoo.documentapi.ProgressToken used on opensource projects

Example 11 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testRangeDistributionBitIncrease1NoPending.

/**
 * Test increasing the distribution bits for a full bucket space range
 * source with no finished, active or pending buckets
 * @throws ParseException upon docsel parse failure (shouldn't happen)
 */
public void testRangeDistributionBitIncrease1NoPending() throws ParseException {
    int db = 2;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    // Test for empty progress token. no splitting involved
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    assertEquals(p.getTotalBucketCount(), 4);
    iter.setDistributionBitCount(db + 1);
    assertEquals(p.getTotalBucketCount(), 8);
    assertEquals(p.getDistributionBitCount(), db + 1);
    assertEquals(iter.getDistributionBitCount(), db + 1);
    assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);
    int[] desired = new int[] { 0, 4, 2, 6, 1, 5, 3, 7 };
    for (int i = 0; i < 8; ++i) {
        VisitorIterator.BucketProgress bp = iter.getNext();
        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, desired[i]));
    }
}

Example 12 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testInconsistentState.

// Test that altering distribution bit count sets ProgressToken as
// inconsistent when there are active buckets
public void testInconsistentState() throws ParseException {
    int db = 16;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken p = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, db, p);
    // For this test, have 3 finished bucket, 2 pending and 1 active
    for (int i = 0; i < 3; ++i) {
        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    }
    VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];
    bpp[0] = iter.getNext();
    bpp[1] = iter.getNext();
    // Leave this hanging as active
    VisitorIterator.BucketProgress bpa = iter.getNext();
    iter.update(bpp[0].getSuperbucket(), new BucketId());
    iter.update(bpp[1].getSuperbucket(), new BucketId());
    assertFalse(p.isInconsistentState());
    iter.setDistributionBitCount(20);
    assertTrue(p.isInconsistentState());
    // Finish active, triggering the consistency fixes
    iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertFalse(p.isInconsistentState());
}

Example 13 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testNullAndSuperUpdate.

/**
 * Test that ensures doing update(superbucket, 0) simply puts the bucket back in
 * pending
 * @throws ParseException
 */
public void testNullAndSuperUpdate() throws ParseException {
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group = \"yahoo.com\"", idFactory, 16, progress);
    assertEquals(progress.getPendingBucketCount(), 1);
    VisitorIterator.BucketProgress bp = iter.getNext();
    assertEquals(bp.getProgress(), new BucketId());
    BucketId superbucket = bp.getSuperbucket();
    BucketId sub = bp.getProgress();
    assertFalse(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
    // 0-bucket
    iter.update(superbucket, ProgressToken.NULL_BUCKET);
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    VisitorIterator.BucketProgress bp2 = iter.getNext();
    assertEquals(bp2.getSuperbucket(), superbucket);
    assertEquals(bp2.getProgress(), ProgressToken.NULL_BUCKET);
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
    // progress == super
    iter.update(superbucket, superbucket);
    assertTrue(iter.hasNext());
    assertFalse(iter.isDone());
    assertEquals(progress.getPendingBucketCount(), 1);
    assertEquals(progress.getActiveBucketCount(), 0);
    bp2 = iter.getNext();
    assertEquals(bp2.getSuperbucket(), superbucket);
    assertEquals(bp2.getProgress(), superbucket);
    assertEquals(progress.getPendingBucketCount(), 0);
    assertEquals(progress.getActiveBucketCount(), 1);
}

Example 14 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VisitorIteratorTestCase method testProgressEstimation.

public void testProgressEstimation() throws ParseException {
    int distBits = 4;
    BucketIdFactory idFactory = new BucketIdFactory();
    ProgressToken progress = new ProgressToken();
    // Create a range of [0, 16) superbuckets
    VisitorIterator iter = VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, distBits, progress);
    assertEquals(progress.getDistributionBitCount(), 4);
    double epsilon = 0.00001;
    assertEquals(progress.percentFinished(), 0, epsilon);
    VisitorIterator.BucketProgress bp = iter.getNext();
    // Finish first superbucket (6.25% total)
    iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertEquals(progress.percentFinished(), 6.25, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 1);
    bp = iter.getNext();
    VisitorIterator.BucketProgress bp3 = iter.getNext();
    VisitorIterator.BucketProgress bp4 = iter.getNext();
    // Finish second (12.5% total)
    iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertEquals(progress.percentFinished(), 12.5, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 2);
    // Finish third bucket 75% through (17.1875% total)
    iter.update(bp3.getSuperbucket(), new BucketId(distBits + 2, bp3.getSuperbucket().getId() | (1 << distBits)));
    assertEquals(progress.percentFinished(), 17.1875, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 2);
    // Finish fourth bucket 25% through (18.75% total)
    iter.update(bp4.getSuperbucket(), new BucketId(distBits + 2, bp4.getSuperbucket().getId()));
    assertEquals(progress.percentFinished(), 18.75, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 2);
    // Finish all buckets
    iter.update(bp4.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    iter.update(bp3.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    assertEquals(progress.percentFinished(), 25, epsilon);
    assertEquals(progress.getFinishedBucketCount(), 4);
    while (iter.hasNext()) {
        bp = iter.getNext();
        iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
    }
    assertEquals(progress.getFinishedBucketCount(), 16);
    assertEquals(progress.percentFinished(), 100, epsilon);
}

Example 15 with ProgressToken

use of com.yahoo.documentapi.ProgressToken in project vespa by vespa-engine.

the class VdsVisit method doRun.

protected int doRun() {
    VisitorParameters visitorParameters = params.getVisitorParameters();
    // If progress file already exists, create resume token from it
    if (visitorParameters.getResumeFileName() != null && !"".equals(visitorParameters.getResumeFileName())) {
        try {
            File file = new File(visitorParameters.getResumeFileName());
            FileInputStream fos = new FileInputStream(file);
            StringBuilder builder = new StringBuilder();
            byte[] b = new byte[100000];
            int length;
            while ((length = fos.read(b)) > 0) {
                builder.append(new String(b, 0, length));
            }
            fos.close();
            visitorParameters.setResumeToken(new ProgressToken(builder.toString()));
            if (params.isVerbose()) {
                System.err.format("Resuming visitor already %.1f %% finished.\n", visitorParameters.getResumeToken().percentFinished());
            }
        } catch (FileNotFoundException e) {
        // Ignore; file has not been created yet but will be shortly.
        } catch (IOException e) {
            System.err.println("Could not open progress file: " + visitorParameters.getResumeFileName());
            e.printStackTrace(System.err);
            return 1;
        }
    }
    initShutdownHook();
    sessionAccessor = sessionAccessorFactory.createVisitorSessionAccessor();
    VdsVisitHandler handler;
    handler = new StdOutVisitorHandler(params.isPrintIdsOnly(), params.isVerbose(), params.isVerbose(), params.isVerbose(), params.getStatisticsParts() != null, params.getAbortOnClusterDown(), params.getProcessTime(), params.jsonOutput);
    if (visitorParameters.getResumeFileName() != null) {
        handler.setProgressFileName(visitorParameters.getResumeFileName());
    }
    visitorParameters.setControlHandler(handler.getControlHandler());
    if (visitorParameters.getRemoteDataHandler() == null) {
        visitorParameters.setLocalDataHandler(handler.getDataHandler());
    }
    if (params.getStatisticsParts() != null) {
        String[] parts = params.getStatisticsParts().split(",");
        for (String s : parts) {
            visitorParameters.setLibraryParameter(s, "true");
        }
    }
    try {
        session = sessionAccessor.createVisitorSession(visitorParameters);
        while (true) {
            try {
                if (session.waitUntilDone(params.getFullTimeout()))
                    break;
            } catch (InterruptedException e) {
            }
        }
        if (visitorParameters.getTraceLevel() > 0) {
            System.out.println(session.getTrace().toString());
        }
    } catch (ParseException e) {
        onDocumentSelectionException(e);
    } catch (IllegalArgumentException e) {
        onIllegalArgumentException(e);
    } catch (Exception e) {
        System.err.println("Document selection string was: " + visitorParameters.getDocumentSelection());
        System.err.println("Caught unexpected exception: ");
        e.printStackTrace(System.err);
        return 1;
    }
    if (visitorParameters.getControlHandler().getResult().code == VisitorControlHandler.CompletionCode.SUCCESS) {
        return 0;
    } else {
        return 1;
    }
}