Search in sources :

Example 1 with SOME_SKIPPED

use of org.opendatakit.briefcase.export.ExportOutcome.SOME_SKIPPED in project briefcase by opendatakit.

the class ExportToGeoJson method export.

/**
 * Export a form's submissions into a GeoJSON file.
 *
 * @param formDef       the {@link FormDefinition} form definition of the form to be exported
 * @param configuration the {@link ExportConfiguration} export configuration
 * @return an {@link ExportOutcome} with the export operation's outcome
 * @see ExportConfiguration
 */
private static ExportOutcome export(FormMetadata formMetadata, FormDefinition formDef, ExportConfiguration configuration, Optional<Analytics> analytics) {
    // Create an export tracker object with the total number of submissions we have to export
    ExportProcessTracker exportTracker = new ExportProcessTracker(formDef);
    exportTracker.start();
    SubmissionExportErrorCallback onParsingError = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName()));
    SubmissionExportErrorCallback onInvalidSubmission = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName())).andThen((path, message) -> analytics.ifPresent(ga -> ga.event("Export", "Export", "invalid submission", null)));
    List<Path> submissionFiles = getListOfSubmissionFiles(formMetadata, formDef, configuration.getDateRange(), configuration.resolveSmartAppend(), onParsingError);
    exportTracker.trackTotal(submissionFiles.size());
    createDirectories(configuration.getExportDir());
    // Generate csv lines grouped by the fqdn of the model they belong to
    Stream<Submission> validSubmissions = ExportTools.getValidSubmissions(formDef, configuration, submissionFiles, onParsingError, onInvalidSubmission);
    Stream<Feature> features = validSubmissions.peek(s -> exportTracker.incAndReport()).flatMap(submission -> GeoJson.toFeatures(formDef.getModel(), submission));
    Path output = configuration.getExportDir().resolve(configuration.getFilenameBase(formDef.getFormName()) + ".geojson");
    GeoJson.write(output, features);
    exportTracker.end();
    ExportOutcome exportOutcome = exportTracker.computeOutcome();
    if (exportOutcome == ALL_EXPORTED)
        EventBus.publish(ExportEvent.successForm(formDef, (int) exportTracker.total));
    if (exportOutcome == SOME_SKIPPED)
        EventBus.publish(ExportEvent.partialSuccessForm(formDef, (int) exportTracker.exported, (int) exportTracker.total));
    if (exportOutcome == ALL_SKIPPED)
        EventBus.publish(ExportEvent.failure(formDef, "All submissions have been skipped"));
    return exportOutcome;
}
Also used : SOME_SKIPPED(org.opendatakit.briefcase.export.ExportOutcome.SOME_SKIPPED) Logger(org.slf4j.Logger) UncheckedFiles.deleteRecursive(org.opendatakit.briefcase.reused.UncheckedFiles.deleteRecursive) LoggerFactory(org.slf4j.LoggerFactory) UncheckedFiles.copy(org.opendatakit.briefcase.reused.UncheckedFiles.copy) Feature(org.geojson.Feature) UncheckedFiles.createDirectories(org.opendatakit.briefcase.reused.UncheckedFiles.createDirectories) UncheckedFiles.exists(org.opendatakit.briefcase.reused.UncheckedFiles.exists) List(java.util.List) ALL_EXPORTED(org.opendatakit.briefcase.export.ExportOutcome.ALL_EXPORTED) Stream(java.util.stream.Stream) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) EventBus(org.bushe.swing.event.EventBus) FormMetadata(org.opendatakit.briefcase.model.form.FormMetadata) Optional(java.util.Optional) ALL_SKIPPED(org.opendatakit.briefcase.export.ExportOutcome.ALL_SKIPPED) Analytics(org.opendatakit.briefcase.ui.reused.Analytics) SubmissionParser.getListOfSubmissionFiles(org.opendatakit.briefcase.export.SubmissionParser.getListOfSubmissionFiles) Path(java.nio.file.Path) Path(java.nio.file.Path) Feature(org.geojson.Feature)

Example 2 with SOME_SKIPPED

use of org.opendatakit.briefcase.export.ExportOutcome.SOME_SKIPPED in project briefcase by opendatakit.

the class ExportToCsv method export.

/**
 * Export a form's submissions into some CSV files.
 * <p>
 * If the form has repeat groups, each repeat group will be exported into a separate CSV file.
 *
 * @param formDef       the {@link FormDefinition} form definition of the form to be exported
 * @param configuration the {@link ExportConfiguration} export configuration
 * @return an {@link ExportOutcome} with the export operation's outcome
 * @see ExportConfiguration
 */
private static ExportOutcome export(FormMetadataPort formMetadataPort, FormMetadata formMetadata, FormStatus formStatus, FormDefinition formDef, Path briefcaseDir, ExportConfiguration configuration, Optional<Analytics> analytics) {
    // Create an export tracker object with the total number of submissions we have to export
    ExportProcessTracker exportTracker = new ExportProcessTracker(formDef);
    exportTracker.start();
    SubmissionExportErrorCallback onParsingError = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName()));
    SubmissionExportErrorCallback onInvalidSubmission = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName())).andThen((path, message) -> analytics.ifPresent(ga -> ga.event("Export", "Export", "invalid submission", null)));
    List<Path> submissionFiles = getListOfSubmissionFiles(formMetadata, formDef, configuration.getDateRange(), configuration.resolveSmartAppend(), onParsingError);
    exportTracker.trackTotal(submissionFiles.size());
    createDirectories(configuration.getExportDir());
    List<Csv> csvs = Csv.getCsvs(formDef, configuration);
    csvs.forEach(Csv::prepareOutputFiles);
    if (formDef.getModel().hasAuditField()) {
        Path audit = configuration.getAuditPath(formDef.getFormName());
        if (!exists(audit) || configuration.resolveOverwriteExistingFiles())
            write(audit, "instance ID, event, node, start, end\n", CREATE, WRITE, TRUNCATE_EXISTING);
    }
    // Generate csv lines grouped by the fqdn of the model they belong to
    Map<String, CsvLines> csvLinesPerModel = ExportTools.getValidSubmissions(formDef, configuration, submissionFiles, onParsingError, onInvalidSubmission).peek(s -> exportTracker.incAndReport()).flatMap(submission -> csvs.stream().map(Csv::getMapper).map(mapper -> mapper.apply(submission))).collect(groupingByConcurrent(CsvLines::getModelFqn, reducing(CsvLines.empty(), CsvLines::merge)));
    // TODO We should have an extra step to produce the side effect of writing media files to disk to avoid having side-effects while generating the CSV output of binary fields
    // Write lines to each output Csv
    csvs.forEach(csv -> csv.appendLines(Optional.ofNullable(csvLinesPerModel.get(csv.getModelFqn())).orElse(CsvLines.empty())));
    exportTracker.end();
    Optional.ofNullable(csvLinesPerModel.get(formDef.getModel().fqn())).orElse(CsvLines.empty()).getLastLine().ifPresent(line -> {
        formMetadataPort.execute(updateLastExportedSubmission(formMetadata.getKey(), line.getInstanceId(), line.getSubmissionDate(), OffsetDateTime.now(), briefcaseDir, formStatus.getFormDir(briefcaseDir)));
    });
    ExportOutcome exportOutcome = exportTracker.computeOutcome();
    if (exportOutcome == ALL_EXPORTED)
        EventBus.publish(ExportEvent.successForm(formDef, (int) exportTracker.total));
    if (exportOutcome == SOME_SKIPPED)
        EventBus.publish(ExportEvent.partialSuccessForm(formDef, (int) exportTracker.exported, (int) exportTracker.total));
    if (exportOutcome == ALL_SKIPPED)
        EventBus.publish(ExportEvent.failure(formDef, "All submissions have been skipped"));
    return exportOutcome;
}
Also used : UncheckedFiles.deleteRecursive(org.opendatakit.briefcase.reused.UncheckedFiles.deleteRecursive) TRUNCATE_EXISTING(java.nio.file.StandardOpenOption.TRUNCATE_EXISTING) LoggerFactory(org.slf4j.LoggerFactory) UncheckedFiles.copy(org.opendatakit.briefcase.reused.UncheckedFiles.copy) FormMetadataCommands.updateLastExportedSubmission(org.opendatakit.briefcase.model.form.FormMetadataCommands.updateLastExportedSubmission) ALL_EXPORTED(org.opendatakit.briefcase.export.ExportOutcome.ALL_EXPORTED) FormMetadataPort(org.opendatakit.briefcase.model.form.FormMetadataPort) EventBus(org.bushe.swing.event.EventBus) Map(java.util.Map) FormMetadata(org.opendatakit.briefcase.model.form.FormMetadata) ALL_SKIPPED(org.opendatakit.briefcase.export.ExportOutcome.ALL_SKIPPED) Path(java.nio.file.Path) WRITE(java.nio.file.StandardOpenOption.WRITE) SOME_SKIPPED(org.opendatakit.briefcase.export.ExportOutcome.SOME_SKIPPED) Collectors.groupingByConcurrent(java.util.stream.Collectors.groupingByConcurrent) FormStatus(org.opendatakit.briefcase.model.FormStatus) Logger(org.slf4j.Logger) Collectors.reducing(java.util.stream.Collectors.reducing) UncheckedFiles.createDirectories(org.opendatakit.briefcase.reused.UncheckedFiles.createDirectories) UncheckedFiles.exists(org.opendatakit.briefcase.reused.UncheckedFiles.exists) List(java.util.List) UncheckedFiles.write(org.opendatakit.briefcase.reused.UncheckedFiles.write) OffsetDateTime(java.time.OffsetDateTime) CREATE(java.nio.file.StandardOpenOption.CREATE) Optional(java.util.Optional) Analytics(org.opendatakit.briefcase.ui.reused.Analytics) SubmissionParser.getListOfSubmissionFiles(org.opendatakit.briefcase.export.SubmissionParser.getListOfSubmissionFiles) Path(java.nio.file.Path)

Aggregations

Path (java.nio.file.Path)2 List (java.util.List)2 Optional (java.util.Optional)2 EventBus (org.bushe.swing.event.EventBus)2 ALL_EXPORTED (org.opendatakit.briefcase.export.ExportOutcome.ALL_EXPORTED)2 ALL_SKIPPED (org.opendatakit.briefcase.export.ExportOutcome.ALL_SKIPPED)2 SOME_SKIPPED (org.opendatakit.briefcase.export.ExportOutcome.SOME_SKIPPED)2 SubmissionParser.getListOfSubmissionFiles (org.opendatakit.briefcase.export.SubmissionParser.getListOfSubmissionFiles)2 FormMetadata (org.opendatakit.briefcase.model.form.FormMetadata)2 UncheckedFiles.copy (org.opendatakit.briefcase.reused.UncheckedFiles.copy)2 UncheckedFiles.createDirectories (org.opendatakit.briefcase.reused.UncheckedFiles.createDirectories)2 UncheckedFiles.deleteRecursive (org.opendatakit.briefcase.reused.UncheckedFiles.deleteRecursive)2 UncheckedFiles.exists (org.opendatakit.briefcase.reused.UncheckedFiles.exists)2 Analytics (org.opendatakit.briefcase.ui.reused.Analytics)2 Logger (org.slf4j.Logger)2 LoggerFactory (org.slf4j.LoggerFactory)2 CREATE (java.nio.file.StandardOpenOption.CREATE)1 TRUNCATE_EXISTING (java.nio.file.StandardOpenOption.TRUNCATE_EXISTING)1 WRITE (java.nio.file.StandardOpenOption.WRITE)1 OffsetDateTime (java.time.OffsetDateTime)1