Examples with RunnerStatus org.opendatakit.briefcase.reused.job.RunnerStatus used on opensource projects

Example 1 with RunnerStatus

use of org.opendatakit.briefcase.reused.job.RunnerStatus in project briefcase by opendatakit.

the class PullFromAggregateIntegrationTest method setUp.

@Before
public void setUp() throws IOException {
    Files.createDirectories(briefcaseDir);
    prefs = new BriefcasePreferences(InMemoryPreferences.empty());
    prefs.setStorageDir(tmpDir);
    server = httpServer(serverPort);
    tracker = new PullFromAggregateTracker(form, e -> {
    });
    formMetadataPort = new InMemoryFormMetadataAdapter();
    pullOp = new PullFromAggregate(CommonsHttp.of(1), aggregateServer, briefcaseDir, true, e -> {
    }, formMetadataPort);
    runnerStatus = new TestRunnerStatus(false);
}

Example 2 with RunnerStatus

use of org.opendatakit.briefcase.reused.job.RunnerStatus in project briefcase by opendatakit.

the class PullFromCentral method pull.

/**
 * Pulls a form completely, writing the form file, form attachments,
 * submission files and their attachments to the local filesystem
 * under the Briefcase Storage directory.
 */
public Job<Void> pull(FormStatus form) {
    FormKey key = FormKey.from(form);
    PullFromCentralTracker tracker = new PullFromCentralTracker(form, onEventCallback);
    return run(rs -> tracker.trackStart()).thenRun(allOf(supply(runnerStatus -> getSubmissionIds(form, token, runnerStatus, tracker)), run(runnerStatus -> {
        downloadForm(form, token, runnerStatus, tracker);
        List<CentralAttachment> attachments = getFormAttachments(form, token, runnerStatus, tracker);
        int totalAttachments = attachments.size();
        AtomicInteger attachmentNumber = new AtomicInteger(1);
        attachments.parallelStream().forEach(attachment -> downloadFormAttachment(form, attachment, token, runnerStatus, tracker, attachmentNumber.getAndIncrement(), totalAttachments));
    }))).thenAccept((runnerStatus, pair) -> withDb(form.getFormDir(briefcaseDir), db -> {
        List<String> submissions = pair.getLeft();
        int totalSubmissions = submissions.size();
        AtomicInteger submissionNumber = new AtomicInteger(1);
        Set<String> submissionVersions = new HashSet<>();
        if (submissions.isEmpty())
            tracker.trackNoSubmissions();
        submissions.stream().map(instanceId -> Pair.of(submissionNumber.getAndIncrement(), instanceId)).forEach(submissionNumberId -> {
            int currentSubmissionNumber = submissionNumberId.getLeft();
            String instanceId = submissionNumberId.getRight();
            boolean inDb = db.hasRecordedInstance(instanceId) != null;
            Path downloadedSubmissionPath = form.getSubmissionFile(briefcaseDir, instanceId);
            if (!inDb || !downloadedSubmissionPath.toFile().exists()) {
                downloadSubmission(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
                if (downloadedSubmissionPath.toFile().exists()) {
                    XmlElement root = XmlElement.from(new String(readAllBytes(downloadedSubmissionPath)));
                    SubmissionMetaData metaData = new SubmissionMetaData(root);
                    metaData.getVersion().ifPresent(submissionVersions::add);
                }
            } else {
                tracker.trackSubmissionAlreadyDownloaded(currentSubmissionNumber, totalSubmissions);
            }
            List<CentralAttachment> attachments = getSubmissionAttachmentList(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
            int totalAttachments = attachments.size();
            AtomicInteger attachmentNumber = new AtomicInteger(1);
            attachments.stream().filter(attachment -> !inDb || !form.getSubmissionMediaFile(briefcaseDir, instanceId, attachment.getName()).toFile().exists()).forEach(attachment -> downloadSubmissionAttachment(form, instanceId, attachment, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions, attachmentNumber.getAndIncrement(), totalAttachments));
            if (!runnerStatus.isCancelled() && !inDb) {
                db.putRecordedInstanceDirectory(instanceId, form.getSubmissionDir(briefcaseDir, instanceId).toFile());
            }
        });
        tracker.trackEnd();
        formMetadataPort.execute(updateAsPulled(key, briefcaseDir, form.getFormDir(briefcaseDir), submissionVersions));
        EventBus.publish(PullEvent.Success.of(form, server));
    }));
}

Example 3 with RunnerStatus

use of org.opendatakit.briefcase.reused.job.RunnerStatus in project briefcase by opendatakit.

the class PullFromAggregate method getFormAttachments.

List<AggregateAttachment> getFormAttachments(FormStatus form, RunnerStatus runnerStatus, PullFromAggregateTracker tracker) {
    if (runnerStatus.isCancelled()) {
        tracker.trackCancellation("Get form attachments");
        return emptyList();
    }
    if (!form.getManifestUrl().filter(RequestBuilder::isUri).isPresent())
        return emptyList();
    tracker.trackStartGettingFormManifest();
    URL manifestUrl = form.getManifestUrl().map(RequestBuilder::url).get();
    Request<List<AggregateAttachment>> request = get(manifestUrl).asXmlElement().withResponseMapper(PullFromAggregate::parseMediaFiles).build();
    Response<List<AggregateAttachment>> response = http.execute(request);
    if (!response.isSuccess()) {
        tracker.trackErrorGettingFormManifest(response);
        return Collections.emptyList();
    }
    List<AggregateAttachment> attachments = response.get();
    List<AggregateAttachment> attachmentsToDownload = attachments.stream().filter(mediaFile -> mediaFile.needsUpdate(form.getFormMediaDir(briefcaseDir))).collect(toList());
    tracker.trackEndGettingFormManifest();
    tracker.trackIgnoredFormAttachments(attachmentsToDownload.size(), attachments.size());
    return attachmentsToDownload;
}

Example 4 with RunnerStatus

use of org.opendatakit.briefcase.reused.job.RunnerStatus in project briefcase by opendatakit.

the class PushToCentral method push.

/**
 * Pushes a form completely, sending the form file, form attachments,
 * submission files and their attachments to the Aggregate server.
 * <p>
 * It won't push a form and form attachments if it's already
 * present in the server.
 */
@SuppressWarnings("checkstyle:Indentation")
public Job<Void> push(FormStatus form) {
    PushToCentralTracker tracker = new PushToCentralTracker(form, onEventCallback);
    Job<Void> startTrackingJob = Job.run(runnerStatus -> tracker.trackStart());
    // Verify that the form is not encrypted
    Path formFile = form.getFormFile(briefcaseDir);
    try {
        if (FormDefinition.from(formFile).isFileEncryptedForm())
            return startTrackingJob.thenRun(rs -> {
                tracker.trackEncryptedForm();
                tracker.trackEnd();
            });
    } catch (RuntimeException e) {
        return startTrackingJob.thenRun(rs -> {
            tracker.trackCannotDetermineEncryption(e);
            tracker.trackEnd();
        });
    }
    FormMetadataPort formMetadataPort = FileSystemFormMetadataAdapter.at(briefcaseDir);
    return startTrackingJob.thenSupply(rs -> getMissingFormVersions(form, rs, tracker, formMetadataPort)).thenAccept(((rs, missingFormVersions) -> {
        String localFormVersion = form.getVersion().orElse("");
        boolean sendLocal = missingFormVersions.remove(localFormVersion);
        // First send a request to create the form. It doesn't matter what version is sent because it will be replaced
        // below. Will fail if a published form already exists with this id but then missing versions can still be added.
        createForm(form.getFormFile(briefcaseDir), rs, tracker);
        // Send all of the obsolete versions first so that we can send corresponding submissions. Skip attachments
        // because those versions won't be used by clients.
        missingFormVersions.stream().sorted().forEach(version -> {
            pushFormDraft(form.getFormFile(briefcaseDir), form.getFormId(), version, rs, tracker);
            publishDraft(form.getFormId(), version, rs, tracker);
        });
        if (sendLocal) {
            boolean formSent = pushFormDraft(formFile, form.getFormId(), localFormVersion, rs, tracker);
            if (formSent) {
                List<Path> formAttachments = getFormAttachments(form);
                AtomicInteger attachmentSeq = new AtomicInteger(1);
                int totalAttachments = formAttachments.size();
                formAttachments.forEach(attachment -> pushFormAttachment(form.getFormId(), attachment, rs, tracker, attachmentSeq.getAndIncrement(), totalAttachments));
                publishDraft(form.getFormId(), localFormVersion, rs, tracker);
            }
        }
    })).thenSupply(__ -> getSubmissions(form)).thenAccept((rs, submissions) -> {
        AtomicInteger submissionNumber = new AtomicInteger(1);
        int totalSubmissions = submissions.size();
        if (submissions.isEmpty())
            tracker.trackNoSubmissions();
        // We might not have had access to submission versions because e.g. submissions were manually added to the
        // Briefcase folder or were imported in <=v1.17 when submission versions weren't saved.
        Set<String> seenVersions = new HashSet<>();
        submissions.parallelStream().map(submission -> {
            XmlElement root = XmlElement.from(new String(readAllBytes(submission)));
            SubmissionMetaData metaData = new SubmissionMetaData(root);
            metaData.getVersion().ifPresent(seenVersions::add);
            return Triple.of(submission, submissionNumber.getAndIncrement(), metaData.getInstanceId());
        }).peek(triple -> {
            if (!triple.get3().isPresent())
                tracker.trackNoInstanceId(triple.get2(), totalSubmissions);
        }).filter(triple -> triple.get3().isPresent()).forEach(triple -> {
            Path submission = triple.get1();
            int currentSubmissionNumber = triple.get2();
            String instanceId = triple.get3().get();
            boolean submissionSent = pushSubmission(form.getFormId(), submission, rs, tracker, currentSubmissionNumber, totalSubmissions);
            if (submissionSent) {
                List<Path> submissionAttachments = getSubmissionAttachments(submission);
                AtomicInteger attachmentSeq = new AtomicInteger(1);
                int totalAttachments = submissionAttachments.size();
                submissionAttachments.parallelStream().forEach(attachment -> pushSubmissionAttachment(form.getFormId(), instanceId, attachment, rs, tracker, currentSubmissionNumber, totalSubmissions, attachmentSeq.getAndIncrement(), totalAttachments));
            }
        });
        formMetadataPort.execute(updateSubmissionVersions(FormKey.from(form), briefcaseDir, form.getFormDir(briefcaseDir), seenVersions));
        tracker.trackEnd();
    }).thenRun(rs -> EventBus.publish(new PushEvent.Success(form)));
}

Example 5 with RunnerStatus

use of org.opendatakit.briefcase.reused.job.RunnerStatus in project briefcase by opendatakit.

the class PushToAggregate method push.

/**
 * Pushes a form completely, sending the form file, form attachments,
 * submission files and their attachments to the Aggregate server.
 * <p>
 * By default, it won't push a form and form attachments if it's already
 * present in the server, which can be overriden with the {@link #forceSendForm}
 * field.
 */
public Job<Void> push(FormStatus form) {
    PushToAggregateTracker tracker = new PushToAggregateTracker(form, onEventCallback);
    return Job.run(runnerStatus -> {
        tracker.trackStart();
        tracker.trackForceSendForm(forceSendForm);
    }).thenSupply(runnerStatus -> !forceSendForm && checkFormExists(form.getFormId(), runnerStatus, tracker)).thenAccept(((runnerStatus, formExists) -> {
        if (!formExists) {
            Path formFile = form.getFormFile(briefcaseDir);
            List<Path> allAttachments = getFormAttachments(form);
            if (allAttachments.isEmpty()) {
                pushFormAndAttachments(form, emptyList(), runnerStatus, tracker);
            } else {
                AtomicInteger partsSeq = new AtomicInteger(1);
                List<List<Path>> attachmentGroups = createGroupsOfMaxSize(formFile, allAttachments, 10);
                attachmentGroups.forEach(attachments -> pushFormAndAttachments(form, attachments, runnerStatus, tracker, partsSeq.getAndIncrement(), attachmentGroups.size()));
            }
        }
    })).thenSupply(__ -> getSubmissions(form)).thenAccept((runnerStatus, submissions) -> {
        AtomicInteger submissionsSeq = new AtomicInteger(1);
        int totalSubmissions = submissions.size();
        if (submissions.isEmpty())
            tracker.trackNoSubmissions();
        submissions.parallelStream().forEach(submission -> {
            List<Path> allAttachments = getSubmissionAttachments(submission);
            if (allAttachments.isEmpty()) {
                pushSubmissionAndAttachments(submission, emptyList(), runnerStatus, tracker, submissionsSeq.getAndIncrement(), totalSubmissions);
            } else {
                AtomicInteger partsSeq = new AtomicInteger(1);
                List<List<Path>> attachmentGroups = createGroupsOfMaxSize(submission, allAttachments, 10);
                attachmentGroups.forEach(attachment -> pushSubmissionAndAttachments(submission, allAttachments, runnerStatus, tracker, submissionsSeq.getAndIncrement(), totalSubmissions, partsSeq.getAndIncrement(), attachmentGroups.size()));
            }
        });
        tracker.trackEnd();
    }).thenRun(rs -> EventBus.publish(new PushEvent.Success(form)));
}