Examples with Job org.opendatakit.briefcase.reused.job.Job used on opensource projects

Example 1 with Job

use of org.opendatakit.briefcase.reused.job.Job in project briefcase by opendatakit.

the class ExportPanel method export.

private void export() {
    Stream<Job<?>> allJobs = forms.getSelectedForms().stream().map(form -> {
        FormKey key = FormKey.from(form);
        FormMetadata formMetadata = formMetadataPort.fetch(key).orElseThrow(BriefcaseException::new);
        form.setStatusString("Starting to export form");
        String formId = form.getFormDefinition().getFormId();
        ExportConfiguration configuration = forms.getConfiguration(formId);
        FormDefinition formDef = FormDefinition.from((BriefcaseFormDefinition) form.getFormDefinition());
        // TODO Abstract away the subtype of RemoteServer. This should say Optional<RemoteServer>
        Optional<AggregateServer> savedPullSource = RemoteServer.readFromPrefs(appPreferences, pullPanelPrefs, form);
        Job<Void> pullJob = configuration.resolvePullBefore() && savedPullSource.isPresent() ? new PullFromAggregate(http, savedPullSource.get(), appPreferences.getBriefcaseDir().orElseThrow(BriefcaseException::new), false, EventBus::publish, formMetadataPort).pull(form, appPreferences.resolveStartFromLast() ? Optional.of(formMetadata.getCursor()) : Optional.empty()) : Job.noOpSupplier();
        Job<Void> exportJob = Job.run(runnerStatus -> ExportToCsv.export(formMetadataPort, formMetadata, form, formDef, appPreferences.getBriefcaseDir().orElseThrow(BriefcaseException::new), configuration, analytics));
        Job<Void> exportGeoJsonJob = configuration.resolveIncludeGeoJsonExport() ? Job.run(runnerStatus -> ExportToGeoJson.export(formMetadata, formDef, configuration, analytics)) : Job.noOp;
        return Job.run(runnerStatus -> form.clearStatusHistory()).thenRun(pullJob).thenRun(exportJob).thenRun(exportGeoJsonJob);
    });
    JobsRunner.launchAsync(allJobs).onComplete(form::unsetExporting).waitForCompletion();
}

Example 2 with Job

use of org.opendatakit.briefcase.reused.job.Job in project briefcase by opendatakit.

the class PullFromCentral method pull.

/**
 * Pulls a form completely, writing the form file, form attachments,
 * submission files and their attachments to the local filesystem
 * under the Briefcase Storage directory.
 */
public Job<Void> pull(FormStatus form) {
    FormKey key = FormKey.from(form);
    PullFromCentralTracker tracker = new PullFromCentralTracker(form, onEventCallback);
    return run(rs -> tracker.trackStart()).thenRun(allOf(supply(runnerStatus -> getSubmissionIds(form, token, runnerStatus, tracker)), run(runnerStatus -> {
        downloadForm(form, token, runnerStatus, tracker);
        List<CentralAttachment> attachments = getFormAttachments(form, token, runnerStatus, tracker);
        int totalAttachments = attachments.size();
        AtomicInteger attachmentNumber = new AtomicInteger(1);
        attachments.parallelStream().forEach(attachment -> downloadFormAttachment(form, attachment, token, runnerStatus, tracker, attachmentNumber.getAndIncrement(), totalAttachments));
    }))).thenAccept((runnerStatus, pair) -> withDb(form.getFormDir(briefcaseDir), db -> {
        List<String> submissions = pair.getLeft();
        int totalSubmissions = submissions.size();
        AtomicInteger submissionNumber = new AtomicInteger(1);
        Set<String> submissionVersions = new HashSet<>();
        if (submissions.isEmpty())
            tracker.trackNoSubmissions();
        submissions.stream().map(instanceId -> Pair.of(submissionNumber.getAndIncrement(), instanceId)).forEach(submissionNumberId -> {
            int currentSubmissionNumber = submissionNumberId.getLeft();
            String instanceId = submissionNumberId.getRight();
            boolean inDb = db.hasRecordedInstance(instanceId) != null;
            Path downloadedSubmissionPath = form.getSubmissionFile(briefcaseDir, instanceId);
            if (!inDb || !downloadedSubmissionPath.toFile().exists()) {
                downloadSubmission(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
                if (downloadedSubmissionPath.toFile().exists()) {
                    XmlElement root = XmlElement.from(new String(readAllBytes(downloadedSubmissionPath)));
                    SubmissionMetaData metaData = new SubmissionMetaData(root);
                    metaData.getVersion().ifPresent(submissionVersions::add);
                }
            } else {
                tracker.trackSubmissionAlreadyDownloaded(currentSubmissionNumber, totalSubmissions);
            }
            List<CentralAttachment> attachments = getSubmissionAttachmentList(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
            int totalAttachments = attachments.size();
            AtomicInteger attachmentNumber = new AtomicInteger(1);
            attachments.stream().filter(attachment -> !inDb || !form.getSubmissionMediaFile(briefcaseDir, instanceId, attachment.getName()).toFile().exists()).forEach(attachment -> downloadSubmissionAttachment(form, instanceId, attachment, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions, attachmentNumber.getAndIncrement(), totalAttachments));
            if (!runnerStatus.isCancelled() && !inDb) {
                db.putRecordedInstanceDirectory(instanceId, form.getSubmissionDir(briefcaseDir, instanceId).toFile());
            }
        });
        tracker.trackEnd();
        formMetadataPort.execute(updateAsPulled(key, briefcaseDir, form.getFormDir(briefcaseDir), submissionVersions));
        EventBus.publish(PullEvent.Success.of(form, server));
    }));
}

Example 3 with Job

use of org.opendatakit.briefcase.reused.job.Job in project briefcase by opendatakit.

the class PushToCentral method push.

/**
 * Pushes a form completely, sending the form file, form attachments,
 * submission files and their attachments to the Aggregate server.
 * <p>
 * It won't push a form and form attachments if it's already
 * present in the server.
 */
@SuppressWarnings("checkstyle:Indentation")
public Job<Void> push(FormStatus form) {
    PushToCentralTracker tracker = new PushToCentralTracker(form, onEventCallback);
    Job<Void> startTrackingJob = Job.run(runnerStatus -> tracker.trackStart());
    // Verify that the form is not encrypted
    Path formFile = form.getFormFile(briefcaseDir);
    try {
        if (FormDefinition.from(formFile).isFileEncryptedForm())
            return startTrackingJob.thenRun(rs -> {
                tracker.trackEncryptedForm();
                tracker.trackEnd();
            });
    } catch (RuntimeException e) {
        return startTrackingJob.thenRun(rs -> {
            tracker.trackCannotDetermineEncryption(e);
            tracker.trackEnd();
        });
    }
    FormMetadataPort formMetadataPort = FileSystemFormMetadataAdapter.at(briefcaseDir);
    return startTrackingJob.thenSupply(rs -> getMissingFormVersions(form, rs, tracker, formMetadataPort)).thenAccept(((rs, missingFormVersions) -> {
        String localFormVersion = form.getVersion().orElse("");
        boolean sendLocal = missingFormVersions.remove(localFormVersion);
        // First send a request to create the form. It doesn't matter what version is sent because it will be replaced
        // below. Will fail if a published form already exists with this id but then missing versions can still be added.
        createForm(form.getFormFile(briefcaseDir), rs, tracker);
        // Send all of the obsolete versions first so that we can send corresponding submissions. Skip attachments
        // because those versions won't be used by clients.
        missingFormVersions.stream().sorted().forEach(version -> {
            pushFormDraft(form.getFormFile(briefcaseDir), form.getFormId(), version, rs, tracker);
            publishDraft(form.getFormId(), version, rs, tracker);
        });
        if (sendLocal) {
            boolean formSent = pushFormDraft(formFile, form.getFormId(), localFormVersion, rs, tracker);
            if (formSent) {
                List<Path> formAttachments = getFormAttachments(form);
                AtomicInteger attachmentSeq = new AtomicInteger(1);
                int totalAttachments = formAttachments.size();
                formAttachments.forEach(attachment -> pushFormAttachment(form.getFormId(), attachment, rs, tracker, attachmentSeq.getAndIncrement(), totalAttachments));
                publishDraft(form.getFormId(), localFormVersion, rs, tracker);
            }
        }
    })).thenSupply(__ -> getSubmissions(form)).thenAccept((rs, submissions) -> {
        AtomicInteger submissionNumber = new AtomicInteger(1);
        int totalSubmissions = submissions.size();
        if (submissions.isEmpty())
            tracker.trackNoSubmissions();
        // We might not have had access to submission versions because e.g. submissions were manually added to the
        // Briefcase folder or were imported in <=v1.17 when submission versions weren't saved.
        Set<String> seenVersions = new HashSet<>();
        submissions.parallelStream().map(submission -> {
            XmlElement root = XmlElement.from(new String(readAllBytes(submission)));
            SubmissionMetaData metaData = new SubmissionMetaData(root);
            metaData.getVersion().ifPresent(seenVersions::add);
            return Triple.of(submission, submissionNumber.getAndIncrement(), metaData.getInstanceId());
        }).peek(triple -> {
            if (!triple.get3().isPresent())
                tracker.trackNoInstanceId(triple.get2(), totalSubmissions);
        }).filter(triple -> triple.get3().isPresent()).forEach(triple -> {
            Path submission = triple.get1();
            int currentSubmissionNumber = triple.get2();
            String instanceId = triple.get3().get();
            boolean submissionSent = pushSubmission(form.getFormId(), submission, rs, tracker, currentSubmissionNumber, totalSubmissions);
            if (submissionSent) {
                List<Path> submissionAttachments = getSubmissionAttachments(submission);
                AtomicInteger attachmentSeq = new AtomicInteger(1);
                int totalAttachments = submissionAttachments.size();
                submissionAttachments.parallelStream().forEach(attachment -> pushSubmissionAttachment(form.getFormId(), instanceId, attachment, rs, tracker, currentSubmissionNumber, totalSubmissions, attachmentSeq.getAndIncrement(), totalAttachments));
            }
        });
        formMetadataPort.execute(updateSubmissionVersions(FormKey.from(form), briefcaseDir, form.getFormDir(briefcaseDir), seenVersions));
        tracker.trackEnd();
    }).thenRun(rs -> EventBus.publish(new PushEvent.Success(form)));
}

Example 4 with Job

use of org.opendatakit.briefcase.reused.job.Job in project briefcase by opendatakit.

the class PullFromAggregate method pull.

/**
 * Pulls a form completely, writing the form file, form attachments,
 * submission files and their attachments to the local filesystem
 * under the Briefcase Storage directory.
 * <p>
 * A {@link Cursor} can be provided to define the starting point to
 * download the form's submissions.
 * <p>
 * Returns a Job that will produce a pull operation result.
 */
public Job<Void> pull(FormStatus form, Optional<Cursor> lastCursor) {
    FormKey key = FormKey.from(form);
    PullFromAggregateTracker tracker = new PullFromAggregateTracker(form, onEventCallback);
    // Download the form and attachments, and get the submissions list
    return run(rs -> tracker.trackStart()).thenSupply(rs -> downloadForm(form, rs, tracker)).thenAccept((rs, formXml) -> {
        if (formXml == null)
            return;
        List<AggregateAttachment> attachments = getFormAttachments(form, rs, tracker);
        int totalAttachments = attachments.size();
        AtomicInteger attachmentNumber = new AtomicInteger(1);
        attachments.parallelStream().forEach(attachment -> downloadFormAttachment(form, attachment, rs, tracker, attachmentNumber.getAndIncrement(), totalAttachments));
        List<InstanceIdBatch> instanceIdBatches = getSubmissionIds(form, lastCursor.orElse(Cursor.empty()), rs, tracker);
        // Build the submission key generator with the form's XML contents
        SubmissionKeyGenerator subKeyGen = SubmissionKeyGenerator.from(formXml);
        // Extract all the instance IDs from all the batches and download each instance
        List<String> ids = instanceIdBatches.stream().flatMap(batch -> batch.getInstanceIds().stream()).collect(toList());
        int totalSubmissions = ids.size();
        AtomicInteger submissionNumber = new AtomicInteger(1);
        Set<String> submissionVersions = new HashSet<>();
        if (ids.isEmpty())
            tracker.trackNoSubmissions();
        withDb(form.getFormDir(briefcaseDir), db -> {
            // We need to collect to be able to create a parallel stream again
            ids.parallelStream().map(instanceId -> Triple.of(submissionNumber.getAndIncrement(), instanceId, db.hasRecordedInstance(instanceId) == null)).peek(triple -> {
                if (!triple.get3())
                    tracker.trackSubmissionAlreadyDownloaded(triple.get1(), totalSubmissions);
            }).filter(Triple::get3).map(triple -> Pair.of(triple.get1(), downloadSubmission(form, triple.get2(), subKeyGen, rs, tracker, triple.get1(), totalSubmissions))).filter(p -> p.getRight() != null).forEach(pair -> {
                int currentSubmissionNumber = pair.getLeft();
                DownloadedSubmission submission = pair.getRight();
                submission.getFormVersion().ifPresent(submissionVersions::add);
                List<AggregateAttachment> submissionAttachments = submission.getAttachments();
                AtomicInteger submissionAttachmentNumber = new AtomicInteger(1);
                int totalSubmissionAttachments = submissionAttachments.size();
                submissionAttachments.parallelStream().forEach(attachment -> downloadSubmissionAttachment(form, submission, attachment, rs, tracker, currentSubmissionNumber, totalSubmissions, submissionAttachmentNumber.getAndIncrement(), totalSubmissionAttachments));
                if (!rs.isCancelled()) {
                    db.putRecordedInstanceDirectory(submission.getInstanceId(), form.getSubmissionDir(briefcaseDir, submission.getInstanceId()).toFile());
                }
            });
        });
        tracker.trackEnd();
        Cursor newCursor = getLastCursor(instanceIdBatches).orElse(Cursor.empty());
        formMetadataPort.execute(updateAsPulled(key, newCursor, briefcaseDir, form.getFormDir(briefcaseDir), submissionVersions));
        EventBus.publish(PullEvent.Success.of(form, server));
    });
}

Example 5 with Job

use of org.opendatakit.briefcase.reused.job.Job in project briefcase by opendatakit.

the class Export method export.

public static void export(Path storageDir, String formid, Path exportDir, String baseFilename, boolean exportMedia, boolean overwriteFiles, boolean pullBefore, Optional<LocalDate> startDate, Optional<LocalDate> endDate, Optional<Path> maybePemFile, boolean splitSelectMultiples, boolean includeGeoJsonExport, boolean removeGroupNames, boolean smartAppend) {
    CliEventsCompanion.attach(log);
    Path briefcaseDir = Common.getOrCreateBriefcaseDir(storageDir);
    FormCache formCache = FormCache.from(briefcaseDir);
    formCache.update();
    BriefcasePreferences appPreferences = BriefcasePreferences.appScoped();
    BriefcasePreferences exportPrefs = BriefcasePreferences.forClass(ExportPanel.class);
    BriefcasePreferences pullPrefs = BriefcasePreferences.forClass(ExportPanel.class);
    FormMetadataPort formMetadataPort = FileSystemFormMetadataAdapter.at(briefcaseDir);
    int maxHttpConnections = appPreferences.getMaxHttpConnections().orElse(DEFAULT_HTTP_CONNECTIONS);
    Http http = appPreferences.getHttpProxy().map(host -> CommonsHttp.of(maxHttpConnections, host)).orElseGet(() -> CommonsHttp.of(maxHttpConnections));
    Optional<BriefcaseFormDefinition> maybeFormDefinition = formCache.getForms().stream().filter(form -> form.getFormId().equals(formid)).findFirst();
    createDirectories(exportDir);
    BriefcaseFormDefinition formDefinition = maybeFormDefinition.orElseThrow(() -> new BriefcaseException("Form " + formid + " not found"));
    FormDefinition formDef = FormDefinition.from(formDefinition);
    System.out.println("Exporting form " + formDefinition.getFormName() + " (" + formDefinition.getFormId() + ") to: " + exportDir);
    DateRange dateRange = new DateRange(startDate, endDate);
    ExportConfiguration configuration = ExportConfiguration.Builder.empty().setExportFilename(baseFilename).setExportDir(exportDir).setPemFile(maybePemFile).setDateRange(dateRange).setPullBefore(pullBefore).setOverwriteFiles(overwriteFiles).setExportMedia(exportMedia).setSplitSelectMultiples(splitSelectMultiples).setIncludeGeoJsonExport(includeGeoJsonExport).setRemoveGroupNames(removeGroupNames).setSmartAppend(smartAppend).build();
    FormStatus formStatus = new FormStatus(formDefinition);
    FormKey key = FormKey.from(formStatus);
    FormMetadata formMetadata = formMetadataPort.fetch(key).orElseThrow(BriefcaseException::new);
    Job<Void> pullJob = Job.noOpSupplier();
    if (configuration.resolvePullBefore()) {
        Optional<AggregateServer> server = AggregateServer.readFromPrefs(appPreferences, pullPrefs, formStatus);
        if (server.isPresent()) {
            Optional<Cursor> lastCursor = appPreferences.resolveStartFromLast() ? formMetadataPort.query(lastCursorOf(key)) : Optional.empty();
            pullJob = new PullFromAggregate(http, server.get(), briefcaseDir, false, Export::onEvent, formMetadataPort).pull(formStatus, lastCursor);
        }
    }
    Job<Void> exportJob = Job.run(runnerStatus -> ExportToCsv.export(formMetadataPort, formMetadata, formStatus, formDef, briefcaseDir, configuration));
    Job<Void> exportGeoJsonJob = configuration.resolveIncludeGeoJsonExport() ? Job.run(runnerStatus -> ExportToGeoJson.export(formMetadata, formDef, configuration)) : Job.noOp;
    Job<Void> job = pullJob.thenRun(exportJob).thenRun(exportGeoJsonJob).thenRun(__ -> exportPrefs.put(buildExportDateTimePrefix(formDefinition.getFormId()), LocalDateTime.now().format(ISO_DATE_TIME)));
    JobsRunner.launchAsync(job, Export::onError).waitForCompletion();
    System.out.println();
    System.out.println("All operations completed");
    System.out.println();
}