Search in sources :

Example 1 with FormKey

use of org.opendatakit.briefcase.model.form.FormKey in project briefcase by opendatakit.

the class PushToCentralTest method setUp.

@Before
public void setUp() throws IOException {
    http = new FakeHttp();
    briefcaseDir = createTempDirectory("briefcase-test-");
    pushOp = new PushToCentral(http, server, briefcaseDir, token, this::onEvent);
    events = new ArrayList<>();
    runnerStatus = new TestRunnerStatus(false);
    tracker = new PushToCentralTracker(formStatus, this::onEvent);
    form = installForm(formStatus, getResourcePath("/org/opendatakit/briefcase/push/aggregate/push-form-test.xml"), briefcaseDir);
    formAttachment = installFormAttachment(formStatus, getResourcePath("/org/opendatakit/briefcase/push/aggregate/sparrow.png"), briefcaseDir);
    submission = installSubmission(formStatus, getResourcePath("/org/opendatakit/briefcase/push/aggregate/submission.xml"), briefcaseDir);
    submissionAttachment = installSubmissionAttachment(formStatus, getResourcePath("/org/opendatakit/briefcase/push/aggregate/1556532531101.jpg"), briefcaseDir, instanceId);
    formMetadataPort = new InMemoryFormMetadataAdapter();
    FormKey key = FormKey.from(formStatus);
    FormMetadata metadata = FormMetadata.of(key, formStatus.getFormDir(briefcaseDir), briefcaseDir);
    formMetadataPort.persist(metadata);
}
Also used : TestRunnerStatus(org.opendatakit.briefcase.reused.job.TestRunnerStatus) InMemoryFormMetadataAdapter(org.opendatakit.briefcase.model.form.InMemoryFormMetadataAdapter) FakeHttp(org.opendatakit.briefcase.reused.http.FakeHttp) FormKey(org.opendatakit.briefcase.model.form.FormKey) FormMetadata(org.opendatakit.briefcase.model.form.FormMetadata) Before(org.junit.Before)

Example 2 with FormKey

use of org.opendatakit.briefcase.model.form.FormKey in project briefcase by opendatakit.

the class ExportPanel method export.

private void export() {
    Stream<Job<?>> allJobs = forms.getSelectedForms().stream().map(form -> {
        FormKey key = FormKey.from(form);
        FormMetadata formMetadata = formMetadataPort.fetch(key).orElseThrow(BriefcaseException::new);
        form.setStatusString("Starting to export form");
        String formId = form.getFormDefinition().getFormId();
        ExportConfiguration configuration = forms.getConfiguration(formId);
        FormDefinition formDef = FormDefinition.from((BriefcaseFormDefinition) form.getFormDefinition());
        // TODO Abstract away the subtype of RemoteServer. This should say Optional<RemoteServer>
        Optional<AggregateServer> savedPullSource = RemoteServer.readFromPrefs(appPreferences, pullPanelPrefs, form);
        Job<Void> pullJob = configuration.resolvePullBefore() && savedPullSource.isPresent() ? new PullFromAggregate(http, savedPullSource.get(), appPreferences.getBriefcaseDir().orElseThrow(BriefcaseException::new), false, EventBus::publish, formMetadataPort).pull(form, appPreferences.resolveStartFromLast() ? Optional.of(formMetadata.getCursor()) : Optional.empty()) : Job.noOpSupplier();
        Job<Void> exportJob = Job.run(runnerStatus -> ExportToCsv.export(formMetadataPort, formMetadata, form, formDef, appPreferences.getBriefcaseDir().orElseThrow(BriefcaseException::new), configuration, analytics));
        Job<Void> exportGeoJsonJob = configuration.resolveIncludeGeoJsonExport() ? Job.run(runnerStatus -> ExportToGeoJson.export(formMetadata, formDef, configuration, analytics)) : Job.noOp;
        return Job.run(runnerStatus -> form.clearStatusHistory()).thenRun(pullJob).thenRun(exportJob).thenRun(exportGeoJsonJob);
    });
    JobsRunner.launchAsync(allJobs).onComplete(form::unsetExporting).waitForCompletion();
}
Also used : Builder.load(org.opendatakit.briefcase.export.ExportConfiguration.Builder.load) FormDefinition(org.opendatakit.briefcase.export.FormDefinition) LocalDateTime(java.time.LocalDateTime) BriefcasePreferences(org.opendatakit.briefcase.model.BriefcasePreferences) RemoteServer(org.opendatakit.briefcase.reused.transfer.RemoteServer) BriefcaseFormDefinition(org.opendatakit.briefcase.model.BriefcaseFormDefinition) ArrayList(java.util.ArrayList) ExportToGeoJson(org.opendatakit.briefcase.export.ExportToGeoJson) Builder.empty(org.opendatakit.briefcase.export.ExportConfiguration.Builder.empty) AnnotationProcessor(org.bushe.swing.event.annotation.AnnotationProcessor) FormMetadataPort(org.opendatakit.briefcase.model.form.FormMetadataPort) EventBus(org.bushe.swing.event.EventBus) ExportConfiguration(org.opendatakit.briefcase.export.ExportConfiguration) ISO_DATE_TIME(java.time.format.DateTimeFormatter.ISO_DATE_TIME) FormMetadata(org.opendatakit.briefcase.model.form.FormMetadata) BriefcaseException(org.opendatakit.briefcase.reused.BriefcaseException) AggregateServer(org.opendatakit.briefcase.reused.transfer.AggregateServer) ExportForms(org.opendatakit.briefcase.export.ExportForms) FormCache(org.opendatakit.briefcase.util.FormCache) FormKey(org.opendatakit.briefcase.model.form.FormKey) ExportForms.buildCustomConfPrefix(org.opendatakit.briefcase.export.ExportForms.buildCustomConfPrefix) FormStatus(org.opendatakit.briefcase.model.FormStatus) JobsRunner(org.opendatakit.briefcase.reused.job.JobsRunner) UI.errorMessage(org.opendatakit.briefcase.ui.reused.UI.errorMessage) Http(org.opendatakit.briefcase.reused.http.Http) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List) Stream(java.util.stream.Stream) PullFromAggregate(org.opendatakit.briefcase.pull.aggregate.PullFromAggregate) CacheUpdateEvent(org.opendatakit.briefcase.reused.CacheUpdateEvent) Optional(java.util.Optional) EventSubscriber(org.bushe.swing.event.annotation.EventSubscriber) ExportEvent(org.opendatakit.briefcase.export.ExportEvent) Analytics(org.opendatakit.briefcase.ui.reused.Analytics) Job(org.opendatakit.briefcase.reused.job.Job) ExportToCsv(org.opendatakit.briefcase.export.ExportToCsv) ExportConfiguration(org.opendatakit.briefcase.export.ExportConfiguration) EventBus(org.bushe.swing.event.EventBus) FormKey(org.opendatakit.briefcase.model.form.FormKey) FormMetadata(org.opendatakit.briefcase.model.form.FormMetadata) BriefcaseException(org.opendatakit.briefcase.reused.BriefcaseException) PullFromAggregate(org.opendatakit.briefcase.pull.aggregate.PullFromAggregate) Job(org.opendatakit.briefcase.reused.job.Job) FormDefinition(org.opendatakit.briefcase.export.FormDefinition) BriefcaseFormDefinition(org.opendatakit.briefcase.model.BriefcaseFormDefinition) AggregateServer(org.opendatakit.briefcase.reused.transfer.AggregateServer)

Example 3 with FormKey

use of org.opendatakit.briefcase.model.form.FormKey in project briefcase by opendatakit.

the class PullFromCentral method pull.

/**
 * Pulls a form completely, writing the form file, form attachments,
 * submission files and their attachments to the local filesystem
 * under the Briefcase Storage directory.
 */
public Job<Void> pull(FormStatus form) {
    FormKey key = FormKey.from(form);
    PullFromCentralTracker tracker = new PullFromCentralTracker(form, onEventCallback);
    return run(rs -> tracker.trackStart()).thenRun(allOf(supply(runnerStatus -> getSubmissionIds(form, token, runnerStatus, tracker)), run(runnerStatus -> {
        downloadForm(form, token, runnerStatus, tracker);
        List<CentralAttachment> attachments = getFormAttachments(form, token, runnerStatus, tracker);
        int totalAttachments = attachments.size();
        AtomicInteger attachmentNumber = new AtomicInteger(1);
        attachments.parallelStream().forEach(attachment -> downloadFormAttachment(form, attachment, token, runnerStatus, tracker, attachmentNumber.getAndIncrement(), totalAttachments));
    }))).thenAccept((runnerStatus, pair) -> withDb(form.getFormDir(briefcaseDir), db -> {
        List<String> submissions = pair.getLeft();
        int totalSubmissions = submissions.size();
        AtomicInteger submissionNumber = new AtomicInteger(1);
        Set<String> submissionVersions = new HashSet<>();
        if (submissions.isEmpty())
            tracker.trackNoSubmissions();
        submissions.stream().map(instanceId -> Pair.of(submissionNumber.getAndIncrement(), instanceId)).forEach(submissionNumberId -> {
            int currentSubmissionNumber = submissionNumberId.getLeft();
            String instanceId = submissionNumberId.getRight();
            boolean inDb = db.hasRecordedInstance(instanceId) != null;
            Path downloadedSubmissionPath = form.getSubmissionFile(briefcaseDir, instanceId);
            if (!inDb || !downloadedSubmissionPath.toFile().exists()) {
                downloadSubmission(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
                if (downloadedSubmissionPath.toFile().exists()) {
                    XmlElement root = XmlElement.from(new String(readAllBytes(downloadedSubmissionPath)));
                    SubmissionMetaData metaData = new SubmissionMetaData(root);
                    metaData.getVersion().ifPresent(submissionVersions::add);
                }
            } else {
                tracker.trackSubmissionAlreadyDownloaded(currentSubmissionNumber, totalSubmissions);
            }
            List<CentralAttachment> attachments = getSubmissionAttachmentList(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
            int totalAttachments = attachments.size();
            AtomicInteger attachmentNumber = new AtomicInteger(1);
            attachments.stream().filter(attachment -> !inDb || !form.getSubmissionMediaFile(briefcaseDir, instanceId, attachment.getName()).toFile().exists()).forEach(attachment -> downloadSubmissionAttachment(form, instanceId, attachment, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions, attachmentNumber.getAndIncrement(), totalAttachments));
            if (!runnerStatus.isCancelled() && !inDb) {
                db.putRecordedInstanceDirectory(instanceId, form.getSubmissionDir(briefcaseDir, instanceId).toFile());
            }
        });
        tracker.trackEnd();
        formMetadataPort.execute(updateAsPulled(key, briefcaseDir, form.getFormDir(briefcaseDir), submissionVersions));
        EventBus.publish(PullEvent.Success.of(form, server));
    }));
}
Also used : Job.run(org.opendatakit.briefcase.reused.job.Job.run) CentralAttachment(org.opendatakit.briefcase.reused.transfer.CentralAttachment) Response(org.opendatakit.briefcase.reused.http.response.Response) Job.supply(org.opendatakit.briefcase.reused.job.Job.supply) SubmissionMetaData(org.opendatakit.briefcase.export.SubmissionMetaData) HashSet(java.util.HashSet) FormMetadataPort(org.opendatakit.briefcase.model.form.FormMetadataPort) PullEvent(org.opendatakit.briefcase.pull.PullEvent) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) UncheckedFiles.readAllBytes(org.opendatakit.briefcase.reused.UncheckedFiles.readAllBytes) EventBus(org.bushe.swing.event.EventBus) Pair(org.opendatakit.briefcase.reused.Pair) DatabaseUtils.withDb(org.opendatakit.briefcase.util.DatabaseUtils.withDb) Path(java.nio.file.Path) FormKey(org.opendatakit.briefcase.model.form.FormKey) FormStatus(org.opendatakit.briefcase.model.FormStatus) FormStatusEvent(org.opendatakit.briefcase.model.FormStatusEvent) CentralServer(org.opendatakit.briefcase.reused.transfer.CentralServer) Job.allOf(org.opendatakit.briefcase.reused.job.Job.allOf) Collections.emptyList(java.util.Collections.emptyList) Set(java.util.Set) RunnerStatus(org.opendatakit.briefcase.reused.job.RunnerStatus) FormMetadataCommands.updateAsPulled(org.opendatakit.briefcase.model.form.FormMetadataCommands.updateAsPulled) Http(org.opendatakit.briefcase.reused.http.Http) UncheckedFiles.createDirectories(org.opendatakit.briefcase.reused.UncheckedFiles.createDirectories) Consumer(java.util.function.Consumer) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List) Job(org.opendatakit.briefcase.reused.job.Job) XmlElement(org.opendatakit.briefcase.export.XmlElement) Path(java.nio.file.Path) SubmissionMetaData(org.opendatakit.briefcase.export.SubmissionMetaData) HashSet(java.util.HashSet) Set(java.util.Set) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) XmlElement(org.opendatakit.briefcase.export.XmlElement) Collections.emptyList(java.util.Collections.emptyList) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List) FormKey(org.opendatakit.briefcase.model.form.FormKey)

Example 4 with FormKey

use of org.opendatakit.briefcase.model.form.FormKey in project briefcase by opendatakit.

the class ExportToCsvScenario method runExport.

void runExport(boolean overwrite, boolean exportMedia, LocalDate startDate, LocalDate endDate, Path pemFile, boolean splitSelectMultiples) {
    ExportConfiguration configuration = ExportConfiguration.Builder.empty().setExportDir(outputDir.resolve("new")).setPemFile(pemFile).setStartDate(startDate).setEndDate(endDate).setOverwriteFiles(overwrite).setExportMedia(exportMedia).setSplitSelectMultiples(splitSelectMultiples).build();
    FormKey formKey = FormKey.of(formDef.getFormName(), formDef.getFormId());
    FormMetadata formMetadata = new FormMetadata(formKey, briefcaseDir, formDef.getFormDir().resolve(stripIllegalChars(formDef.getFormName()) + ".xml"), true, Cursor.empty(), Optional.empty(), Collections.emptySet());
    ExportToCsv.export(new InMemoryFormMetadataAdapter(), formMetadata, formStatus, formDef, briefcaseDir, configuration);
}
Also used : InMemoryFormMetadataAdapter(org.opendatakit.briefcase.model.form.InMemoryFormMetadataAdapter) FormKey(org.opendatakit.briefcase.model.form.FormKey) FormMetadata(org.opendatakit.briefcase.model.form.FormMetadata)

Example 5 with FormKey

use of org.opendatakit.briefcase.model.form.FormKey in project briefcase by opendatakit.

the class PullFromAggregate method pull.

/**
 * Pulls a form completely, writing the form file, form attachments,
 * submission files and their attachments to the local filesystem
 * under the Briefcase Storage directory.
 * <p>
 * A {@link Cursor} can be provided to define the starting point to
 * download the form's submissions.
 * <p>
 * Returns a Job that will produce a pull operation result.
 */
public Job<Void> pull(FormStatus form, Optional<Cursor> lastCursor) {
    FormKey key = FormKey.from(form);
    PullFromAggregateTracker tracker = new PullFromAggregateTracker(form, onEventCallback);
    // Download the form and attachments, and get the submissions list
    return run(rs -> tracker.trackStart()).thenSupply(rs -> downloadForm(form, rs, tracker)).thenAccept((rs, formXml) -> {
        if (formXml == null)
            return;
        List<AggregateAttachment> attachments = getFormAttachments(form, rs, tracker);
        int totalAttachments = attachments.size();
        AtomicInteger attachmentNumber = new AtomicInteger(1);
        attachments.parallelStream().forEach(attachment -> downloadFormAttachment(form, attachment, rs, tracker, attachmentNumber.getAndIncrement(), totalAttachments));
        List<InstanceIdBatch> instanceIdBatches = getSubmissionIds(form, lastCursor.orElse(Cursor.empty()), rs, tracker);
        // Build the submission key generator with the form's XML contents
        SubmissionKeyGenerator subKeyGen = SubmissionKeyGenerator.from(formXml);
        // Extract all the instance IDs from all the batches and download each instance
        List<String> ids = instanceIdBatches.stream().flatMap(batch -> batch.getInstanceIds().stream()).collect(toList());
        int totalSubmissions = ids.size();
        AtomicInteger submissionNumber = new AtomicInteger(1);
        Set<String> submissionVersions = new HashSet<>();
        if (ids.isEmpty())
            tracker.trackNoSubmissions();
        withDb(form.getFormDir(briefcaseDir), db -> {
            // We need to collect to be able to create a parallel stream again
            ids.parallelStream().map(instanceId -> Triple.of(submissionNumber.getAndIncrement(), instanceId, db.hasRecordedInstance(instanceId) == null)).peek(triple -> {
                if (!triple.get3())
                    tracker.trackSubmissionAlreadyDownloaded(triple.get1(), totalSubmissions);
            }).filter(Triple::get3).map(triple -> Pair.of(triple.get1(), downloadSubmission(form, triple.get2(), subKeyGen, rs, tracker, triple.get1(), totalSubmissions))).filter(p -> p.getRight() != null).forEach(pair -> {
                int currentSubmissionNumber = pair.getLeft();
                DownloadedSubmission submission = pair.getRight();
                submission.getFormVersion().ifPresent(submissionVersions::add);
                List<AggregateAttachment> submissionAttachments = submission.getAttachments();
                AtomicInteger submissionAttachmentNumber = new AtomicInteger(1);
                int totalSubmissionAttachments = submissionAttachments.size();
                submissionAttachments.parallelStream().forEach(attachment -> downloadSubmissionAttachment(form, submission, attachment, rs, tracker, currentSubmissionNumber, totalSubmissions, submissionAttachmentNumber.getAndIncrement(), totalSubmissionAttachments));
                if (!rs.isCancelled()) {
                    db.putRecordedInstanceDirectory(submission.getInstanceId(), form.getSubmissionDir(briefcaseDir, submission.getInstanceId()).toFile());
                }
            });
        });
        tracker.trackEnd();
        Cursor newCursor = getLastCursor(instanceIdBatches).orElse(Cursor.empty());
        formMetadataPort.execute(updateAsPulled(key, newCursor, briefcaseDir, form.getFormDir(briefcaseDir), submissionVersions));
        EventBus.publish(PullEvent.Success.of(form, server));
    });
}
Also used : Job.run(org.opendatakit.briefcase.reused.job.Job.run) TRUNCATE_EXISTING(java.nio.file.StandardOpenOption.TRUNCATE_EXISTING) URL(java.net.URL) Response(org.opendatakit.briefcase.reused.http.response.Response) RequestBuilder(org.opendatakit.briefcase.reused.http.RequestBuilder) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) FormMetadataPort(org.opendatakit.briefcase.model.form.FormMetadataPort) PullEvent(org.opendatakit.briefcase.pull.PullEvent) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) EventBus(org.bushe.swing.event.EventBus) AggregateServer(org.opendatakit.briefcase.reused.transfer.AggregateServer) Pair(org.opendatakit.briefcase.reused.Pair) DatabaseUtils.withDb(org.opendatakit.briefcase.util.DatabaseUtils.withDb) Path(java.nio.file.Path) FormKey(org.opendatakit.briefcase.model.form.FormKey) RemoteFormDefinition(org.opendatakit.briefcase.model.RemoteFormDefinition) BinaryOperator.maxBy(java.util.function.BinaryOperator.maxBy) OptionalProduct(org.opendatakit.briefcase.reused.OptionalProduct) FormStatus(org.opendatakit.briefcase.model.FormStatus) FormStatusEvent(org.opendatakit.briefcase.model.FormStatusEvent) Request(org.opendatakit.briefcase.reused.http.Request) Collections.emptyList(java.util.Collections.emptyList) Set(java.util.Set) RunnerStatus(org.opendatakit.briefcase.reused.job.RunnerStatus) FormMetadataCommands.updateAsPulled(org.opendatakit.briefcase.model.form.FormMetadataCommands.updateAsPulled) Http(org.opendatakit.briefcase.reused.http.Http) UncheckedFiles.createDirectories(org.opendatakit.briefcase.reused.UncheckedFiles.createDirectories) Consumer(java.util.function.Consumer) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List) UncheckedFiles.write(org.opendatakit.briefcase.reused.UncheckedFiles.write) CREATE(java.nio.file.StandardOpenOption.CREATE) Triple(org.opendatakit.briefcase.reused.Triple) Optional(java.util.Optional) Job(org.opendatakit.briefcase.reused.job.Job) Collections(java.util.Collections) RequestBuilder.get(org.opendatakit.briefcase.reused.http.RequestBuilder.get) XmlElement(org.opendatakit.briefcase.export.XmlElement) FormKey(org.opendatakit.briefcase.model.form.FormKey) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) HashSet(java.util.HashSet)

Aggregations

FormKey (org.opendatakit.briefcase.model.form.FormKey)7 EventBus (org.bushe.swing.event.EventBus)4 FormStatus (org.opendatakit.briefcase.model.FormStatus)4 FormMetadataPort (org.opendatakit.briefcase.model.form.FormMetadataPort)4 Http (org.opendatakit.briefcase.reused.http.Http)4 Job (org.opendatakit.briefcase.reused.job.Job)4 Path (java.nio.file.Path)3 List (java.util.List)3 Optional (java.util.Optional)3 Collectors.toList (java.util.stream.Collectors.toList)3 FormStatusEvent (org.opendatakit.briefcase.model.FormStatusEvent)3 FormMetadata (org.opendatakit.briefcase.model.form.FormMetadata)3 LocalDateTime (java.time.LocalDateTime)2 ISO_DATE_TIME (java.time.format.DateTimeFormatter.ISO_DATE_TIME)2 ArrayList (java.util.ArrayList)2 Collections.emptyList (java.util.Collections.emptyList)2 HashSet (java.util.HashSet)2 Set (java.util.Set)2 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)2 Consumer (java.util.function.Consumer)2