Examples with VisitorParameters com.yahoo.documentapi.VisitorParameters used on opensource projects

Example 6 with VisitorParameters

use of com.yahoo.documentapi.VisitorParameters in project vespa by vespa-engine.

the class OperationHandlerImplTest method provided_visit_fieldset_is_propagated_to_visitor_parameters.

@Test
public void provided_visit_fieldset_is_propagated_to_visitor_parameters() throws Exception {
    VisitorParameters params = generatedParametersFromVisitOptions(optionsBuilder().fieldSet("document-type:bjarne").build());
    assertThat(params.fieldSet(), equalTo("document-type:bjarne"));
}

Example 7 with VisitorParameters

use of com.yahoo.documentapi.VisitorParameters in project vespa by vespa-engine.

the class OperationHandlerImplTest method too_high_wanted_document_count_is_bounded_to_upper_bound.

@Test
public void too_high_wanted_document_count_is_bounded_to_upper_bound() throws Exception {
    VisitorParameters params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(OperationHandlerImpl.WANTED_DOCUMENT_COUNT_UPPER_BOUND + 1));
    assertThat(params.getMaxTotalHits(), is((long) OperationHandlerImpl.WANTED_DOCUMENT_COUNT_UPPER_BOUND));
    params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(Integer.MAX_VALUE));
    assertThat(params.getMaxTotalHits(), is((long) OperationHandlerImpl.WANTED_DOCUMENT_COUNT_UPPER_BOUND));
}

Example 8 with VisitorParameters

use of com.yahoo.documentapi.VisitorParameters in project vespa by vespa-engine.

the class OperationHandlerImplTest method document_type_is_mapped_to_correct_bucket_space.

@Test
public void document_type_is_mapped_to_correct_bucket_space() throws Exception {
    OperationHandlerImplFixture fixture = new OperationHandlerImplFixture();
    fixture.bucketSpaces.put("document-type", "langbein");
    OperationHandlerImpl handler = fixture.createHandler();
    handler.visit(dummyVisitUri(), "", emptyVisitOptions());
    VisitorParameters parameters = fixture.assignedParameters.get();
    assertEquals("langbein", parameters.getBucketSpace());
}

Example 9 with VisitorParameters

use of com.yahoo.documentapi.VisitorParameters in project vespa by vespa-engine.

the class VdsVisit method doRun.

protected int doRun() {
    VisitorParameters visitorParameters = params.getVisitorParameters();
    // If progress file already exists, create resume token from it
    if (visitorParameters.getResumeFileName() != null && !"".equals(visitorParameters.getResumeFileName())) {
        try {
            File file = new File(visitorParameters.getResumeFileName());
            FileInputStream fos = new FileInputStream(file);
            StringBuilder builder = new StringBuilder();
            byte[] b = new byte[100000];
            int length;
            while ((length = fos.read(b)) > 0) {
                builder.append(new String(b, 0, length));
            }
            fos.close();
            visitorParameters.setResumeToken(new ProgressToken(builder.toString()));
            if (params.isVerbose()) {
                System.err.format("Resuming visitor already %.1f %% finished.\n", visitorParameters.getResumeToken().percentFinished());
            }
        } catch (FileNotFoundException e) {
        // Ignore; file has not been created yet but will be shortly.
        } catch (IOException e) {
            System.err.println("Could not open progress file: " + visitorParameters.getResumeFileName());
            e.printStackTrace(System.err);
            return 1;
        }
    }
    initShutdownHook();
    sessionAccessor = sessionAccessorFactory.createVisitorSessionAccessor();
    VdsVisitHandler handler;
    handler = new StdOutVisitorHandler(params.isPrintIdsOnly(), params.isVerbose(), params.isVerbose(), params.isVerbose(), params.getStatisticsParts() != null, params.getAbortOnClusterDown(), params.getProcessTime(), params.jsonOutput);
    if (visitorParameters.getResumeFileName() != null) {
        handler.setProgressFileName(visitorParameters.getResumeFileName());
    }
    visitorParameters.setControlHandler(handler.getControlHandler());
    if (visitorParameters.getRemoteDataHandler() == null) {
        visitorParameters.setLocalDataHandler(handler.getDataHandler());
    }
    if (params.getStatisticsParts() != null) {
        String[] parts = params.getStatisticsParts().split(",");
        for (String s : parts) {
            visitorParameters.setLibraryParameter(s, "true");
        }
    }
    try {
        session = sessionAccessor.createVisitorSession(visitorParameters);
        while (true) {
            try {
                if (session.waitUntilDone(params.getFullTimeout()))
                    break;
            } catch (InterruptedException e) {
            }
        }
        if (visitorParameters.getTraceLevel() > 0) {
            System.out.println(session.getTrace().toString());
        }
    } catch (ParseException e) {
        onDocumentSelectionException(e);
    } catch (IllegalArgumentException e) {
        onIllegalArgumentException(e);
    } catch (Exception e) {
        System.err.println("Document selection string was: " + visitorParameters.getDocumentSelection());
        System.err.println("Caught unexpected exception: ");
        e.printStackTrace(System.err);
        return 1;
    }
    if (visitorParameters.getControlHandler().getResult().code == VisitorControlHandler.CompletionCode.SUCCESS) {
        return 0;
    } else {
        return 1;
    }
}

Example 10 with VisitorParameters

use of com.yahoo.documentapi.VisitorParameters in project vespa by vespa-engine.

the class OperationHandlerImpl method createVisitorParameters.

private VisitorParameters createVisitorParameters(RestUri restUri, String documentSelection, VisitOptions options) throws RestApiException {
    StringBuilder selection = new StringBuilder();
    if (!documentSelection.isEmpty()) {
        // TODO shouldn't selection be wrapped in () itself ?
        selection.append("(").append(documentSelection).append(" and ");
    }
    selection.append(restUri.getDocumentType()).append(" and (id.namespace=='").append(restUri.getNamespace()).append("')");
    if (!documentSelection.isEmpty()) {
        selection.append(")");
    }
    VisitorParameters params = new VisitorParameters(selection.toString());
    // Only return fieldset that is part of the document.
    params.fieldSet(options.fieldSet.orElse(restUri.getDocumentType() + ":[document]"));
    params.setMaxBucketsPerVisitor(1);
    params.setMaxPending(32);
    params.setMaxFirstPassHits(1);
    params.setMaxTotalHits(options.wantedDocumentCount.map(n -> Math.min(Math.max(n, 1), WANTED_DOCUMENT_COUNT_UPPER_BOUND)).orElse(1));
    params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount(options.concurrency.orElse(1)));
    params.setToTimestamp(0L);
    params.setFromTimestamp(0L);
    params.setSessionTimeoutMs(VISIT_TIMEOUT_MS);
    // TODO document this as part of consistency doc
    params.visitInconsistentBuckets(true);
    params.setVisitorOrdering(VisitorOrdering.ASCENDING);
    BucketSpaceRoute bucketSpaceRoute = resolveBucketSpaceRoute(options.cluster, restUri.getDocumentType());
    params.setRoute(bucketSpaceRoute.getClusterRoute());
    params.setBucketSpace(bucketSpaceRoute.getBucketSpace());
    params.setTraceLevel(0);
    params.setPriority(DocumentProtocol.Priority.NORMAL_4);
    params.setVisitRemoves(false);
    if (options.continuation.isPresent()) {
        try {
            params.setResumeToken(ContinuationHit.getToken(options.continuation.get()));
        } catch (Exception e) {
            throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED));
        }
    }
    return params;
}