Examples with FormMetadata org.opendatakit.briefcase.model.form.FormMetadata used on opensource projects

Example 1 with FormMetadata

use of org.opendatakit.briefcase.model.form.FormMetadata in project briefcase by opendatakit.

the class PushToCentralTest method setUp.

@Before
public void setUp() throws IOException {
    http = new FakeHttp();
    briefcaseDir = createTempDirectory("briefcase-test-");
    pushOp = new PushToCentral(http, server, briefcaseDir, token, this::onEvent);
    events = new ArrayList<>();
    runnerStatus = new TestRunnerStatus(false);
    tracker = new PushToCentralTracker(formStatus, this::onEvent);
    form = installForm(formStatus, getResourcePath("/org/opendatakit/briefcase/push/aggregate/push-form-test.xml"), briefcaseDir);
    formAttachment = installFormAttachment(formStatus, getResourcePath("/org/opendatakit/briefcase/push/aggregate/sparrow.png"), briefcaseDir);
    submission = installSubmission(formStatus, getResourcePath("/org/opendatakit/briefcase/push/aggregate/submission.xml"), briefcaseDir);
    submissionAttachment = installSubmissionAttachment(formStatus, getResourcePath("/org/opendatakit/briefcase/push/aggregate/1556532531101.jpg"), briefcaseDir, instanceId);
    formMetadataPort = new InMemoryFormMetadataAdapter();
    FormKey key = FormKey.from(formStatus);
    FormMetadata metadata = FormMetadata.of(key, formStatus.getFormDir(briefcaseDir), briefcaseDir);
    formMetadataPort.persist(metadata);
}

Example 2 with FormMetadata

use of org.opendatakit.briefcase.model.form.FormMetadata in project briefcase by opendatakit.

the class ExportPanel method export.

private void export() {
    Stream<Job<?>> allJobs = forms.getSelectedForms().stream().map(form -> {
        FormKey key = FormKey.from(form);
        FormMetadata formMetadata = formMetadataPort.fetch(key).orElseThrow(BriefcaseException::new);
        form.setStatusString("Starting to export form");
        String formId = form.getFormDefinition().getFormId();
        ExportConfiguration configuration = forms.getConfiguration(formId);
        FormDefinition formDef = FormDefinition.from((BriefcaseFormDefinition) form.getFormDefinition());
        // TODO Abstract away the subtype of RemoteServer. This should say Optional<RemoteServer>
        Optional<AggregateServer> savedPullSource = RemoteServer.readFromPrefs(appPreferences, pullPanelPrefs, form);
        Job<Void> pullJob = configuration.resolvePullBefore() && savedPullSource.isPresent() ? new PullFromAggregate(http, savedPullSource.get(), appPreferences.getBriefcaseDir().orElseThrow(BriefcaseException::new), false, EventBus::publish, formMetadataPort).pull(form, appPreferences.resolveStartFromLast() ? Optional.of(formMetadata.getCursor()) : Optional.empty()) : Job.noOpSupplier();
        Job<Void> exportJob = Job.run(runnerStatus -> ExportToCsv.export(formMetadataPort, formMetadata, form, formDef, appPreferences.getBriefcaseDir().orElseThrow(BriefcaseException::new), configuration, analytics));
        Job<Void> exportGeoJsonJob = configuration.resolveIncludeGeoJsonExport() ? Job.run(runnerStatus -> ExportToGeoJson.export(formMetadata, formDef, configuration, analytics)) : Job.noOp;
        return Job.run(runnerStatus -> form.clearStatusHistory()).thenRun(pullJob).thenRun(exportJob).thenRun(exportGeoJsonJob);
    });
    JobsRunner.launchAsync(allJobs).onComplete(form::unsetExporting).waitForCompletion();
}

Example 3 with FormMetadata

use of org.opendatakit.briefcase.model.form.FormMetadata in project briefcase by opendatakit.

the class ExportToCsvScenario method runExport.

void runExport(boolean overwrite, boolean exportMedia, LocalDate startDate, LocalDate endDate, Path pemFile, boolean splitSelectMultiples) {
    ExportConfiguration configuration = ExportConfiguration.Builder.empty().setExportDir(outputDir.resolve("new")).setPemFile(pemFile).setStartDate(startDate).setEndDate(endDate).setOverwriteFiles(overwrite).setExportMedia(exportMedia).setSplitSelectMultiples(splitSelectMultiples).build();
    FormKey formKey = FormKey.of(formDef.getFormName(), formDef.getFormId());
    FormMetadata formMetadata = new FormMetadata(formKey, briefcaseDir, formDef.getFormDir().resolve(stripIllegalChars(formDef.getFormName()) + ".xml"), true, Cursor.empty(), Optional.empty(), Collections.emptySet());
    ExportToCsv.export(new InMemoryFormMetadataAdapter(), formMetadata, formStatus, formDef, briefcaseDir, configuration);
}

Example 4 with FormMetadata

use of org.opendatakit.briefcase.model.form.FormMetadata in project briefcase by opendatakit.

the class ExportToGeoJson method export.

/**
 * Export a form's submissions into a GeoJSON file.
 *
 * @param formDef       the {@link FormDefinition} form definition of the form to be exported
 * @param configuration the {@link ExportConfiguration} export configuration
 * @return an {@link ExportOutcome} with the export operation's outcome
 * @see ExportConfiguration
 */
private static ExportOutcome export(FormMetadata formMetadata, FormDefinition formDef, ExportConfiguration configuration, Optional<Analytics> analytics) {
    // Create an export tracker object with the total number of submissions we have to export
    ExportProcessTracker exportTracker = new ExportProcessTracker(formDef);
    exportTracker.start();
    SubmissionExportErrorCallback onParsingError = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName()));
    SubmissionExportErrorCallback onInvalidSubmission = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName())).andThen((path, message) -> analytics.ifPresent(ga -> ga.event("Export", "Export", "invalid submission", null)));
    List<Path> submissionFiles = getListOfSubmissionFiles(formMetadata, formDef, configuration.getDateRange(), configuration.resolveSmartAppend(), onParsingError);
    exportTracker.trackTotal(submissionFiles.size());
    createDirectories(configuration.getExportDir());
    // Generate csv lines grouped by the fqdn of the model they belong to
    Stream<Submission> validSubmissions = ExportTools.getValidSubmissions(formDef, configuration, submissionFiles, onParsingError, onInvalidSubmission);
    Stream<Feature> features = validSubmissions.peek(s -> exportTracker.incAndReport()).flatMap(submission -> GeoJson.toFeatures(formDef.getModel(), submission));
    Path output = configuration.getExportDir().resolve(configuration.getFilenameBase(formDef.getFormName()) + ".geojson");
    GeoJson.write(output, features);
    exportTracker.end();
    ExportOutcome exportOutcome = exportTracker.computeOutcome();
    if (exportOutcome == ALL_EXPORTED)
        EventBus.publish(ExportEvent.successForm(formDef, (int) exportTracker.total));
    if (exportOutcome == SOME_SKIPPED)
        EventBus.publish(ExportEvent.partialSuccessForm(formDef, (int) exportTracker.exported, (int) exportTracker.total));
    if (exportOutcome == ALL_SKIPPED)
        EventBus.publish(ExportEvent.failure(formDef, "All submissions have been skipped"));
    return exportOutcome;
}

Example 5 with FormMetadata

use of org.opendatakit.briefcase.model.form.FormMetadata in project briefcase by opendatakit.

the class ExportToCsv method export.

/**
 * Export a form's submissions into some CSV files.
 * <p>
 * If the form has repeat groups, each repeat group will be exported into a separate CSV file.
 *
 * @param formDef       the {@link FormDefinition} form definition of the form to be exported
 * @param configuration the {@link ExportConfiguration} export configuration
 * @return an {@link ExportOutcome} with the export operation's outcome
 * @see ExportConfiguration
 */
private static ExportOutcome export(FormMetadataPort formMetadataPort, FormMetadata formMetadata, FormStatus formStatus, FormDefinition formDef, Path briefcaseDir, ExportConfiguration configuration, Optional<Analytics> analytics) {
    // Create an export tracker object with the total number of submissions we have to export
    ExportProcessTracker exportTracker = new ExportProcessTracker(formDef);
    exportTracker.start();
    SubmissionExportErrorCallback onParsingError = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName()));
    SubmissionExportErrorCallback onInvalidSubmission = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName())).andThen((path, message) -> analytics.ifPresent(ga -> ga.event("Export", "Export", "invalid submission", null)));
    List<Path> submissionFiles = getListOfSubmissionFiles(formMetadata, formDef, configuration.getDateRange(), configuration.resolveSmartAppend(), onParsingError);
    exportTracker.trackTotal(submissionFiles.size());
    createDirectories(configuration.getExportDir());
    List<Csv> csvs = Csv.getCsvs(formDef, configuration);
    csvs.forEach(Csv::prepareOutputFiles);
    if (formDef.getModel().hasAuditField()) {
        Path audit = configuration.getAuditPath(formDef.getFormName());
        if (!exists(audit) || configuration.resolveOverwriteExistingFiles())
            write(audit, "instance ID, event, node, start, end\n", CREATE, WRITE, TRUNCATE_EXISTING);
    }
    // Generate csv lines grouped by the fqdn of the model they belong to
    Map<String, CsvLines> csvLinesPerModel = ExportTools.getValidSubmissions(formDef, configuration, submissionFiles, onParsingError, onInvalidSubmission).peek(s -> exportTracker.incAndReport()).flatMap(submission -> csvs.stream().map(Csv::getMapper).map(mapper -> mapper.apply(submission))).collect(groupingByConcurrent(CsvLines::getModelFqn, reducing(CsvLines.empty(), CsvLines::merge)));
    // TODO We should have an extra step to produce the side effect of writing media files to disk to avoid having side-effects while generating the CSV output of binary fields
    // Write lines to each output Csv
    csvs.forEach(csv -> csv.appendLines(Optional.ofNullable(csvLinesPerModel.get(csv.getModelFqn())).orElse(CsvLines.empty())));
    exportTracker.end();
    Optional.ofNullable(csvLinesPerModel.get(formDef.getModel().fqn())).orElse(CsvLines.empty()).getLastLine().ifPresent(line -> {
        formMetadataPort.execute(updateLastExportedSubmission(formMetadata.getKey(), line.getInstanceId(), line.getSubmissionDate(), OffsetDateTime.now(), briefcaseDir, formStatus.getFormDir(briefcaseDir)));
    });
    ExportOutcome exportOutcome = exportTracker.computeOutcome();
    if (exportOutcome == ALL_EXPORTED)
        EventBus.publish(ExportEvent.successForm(formDef, (int) exportTracker.total));
    if (exportOutcome == SOME_SKIPPED)
        EventBus.publish(ExportEvent.partialSuccessForm(formDef, (int) exportTracker.exported, (int) exportTracker.total));
    if (exportOutcome == ALL_SKIPPED)
        EventBus.publish(ExportEvent.failure(formDef, "All submissions have been skipped"));
    return exportOutcome;
}