use of org.opendatakit.briefcase.export.ExportOutcome.SOME_SKIPPED in project briefcase by opendatakit.
the class ExportToGeoJson method export.
/**
* Export a form's submissions into a GeoJSON file.
*
* @param formDef the {@link FormDefinition} form definition of the form to be exported
* @param configuration the {@link ExportConfiguration} export configuration
* @return an {@link ExportOutcome} with the export operation's outcome
* @see ExportConfiguration
*/
private static ExportOutcome export(FormMetadata formMetadata, FormDefinition formDef, ExportConfiguration configuration, Optional<Analytics> analytics) {
// Create an export tracker object with the total number of submissions we have to export
ExportProcessTracker exportTracker = new ExportProcessTracker(formDef);
exportTracker.start();
SubmissionExportErrorCallback onParsingError = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName()));
SubmissionExportErrorCallback onInvalidSubmission = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName())).andThen((path, message) -> analytics.ifPresent(ga -> ga.event("Export", "Export", "invalid submission", null)));
List<Path> submissionFiles = getListOfSubmissionFiles(formMetadata, formDef, configuration.getDateRange(), configuration.resolveSmartAppend(), onParsingError);
exportTracker.trackTotal(submissionFiles.size());
createDirectories(configuration.getExportDir());
// Generate csv lines grouped by the fqdn of the model they belong to
Stream<Submission> validSubmissions = ExportTools.getValidSubmissions(formDef, configuration, submissionFiles, onParsingError, onInvalidSubmission);
Stream<Feature> features = validSubmissions.peek(s -> exportTracker.incAndReport()).flatMap(submission -> GeoJson.toFeatures(formDef.getModel(), submission));
Path output = configuration.getExportDir().resolve(configuration.getFilenameBase(formDef.getFormName()) + ".geojson");
GeoJson.write(output, features);
exportTracker.end();
ExportOutcome exportOutcome = exportTracker.computeOutcome();
if (exportOutcome == ALL_EXPORTED)
EventBus.publish(ExportEvent.successForm(formDef, (int) exportTracker.total));
if (exportOutcome == SOME_SKIPPED)
EventBus.publish(ExportEvent.partialSuccessForm(formDef, (int) exportTracker.exported, (int) exportTracker.total));
if (exportOutcome == ALL_SKIPPED)
EventBus.publish(ExportEvent.failure(formDef, "All submissions have been skipped"));
return exportOutcome;
}
use of org.opendatakit.briefcase.export.ExportOutcome.SOME_SKIPPED in project briefcase by opendatakit.
the class ExportToCsv method export.
/**
* Export a form's submissions into some CSV files.
* <p>
* If the form has repeat groups, each repeat group will be exported into a separate CSV file.
*
* @param formDef the {@link FormDefinition} form definition of the form to be exported
* @param configuration the {@link ExportConfiguration} export configuration
* @return an {@link ExportOutcome} with the export operation's outcome
* @see ExportConfiguration
*/
private static ExportOutcome export(FormMetadataPort formMetadataPort, FormMetadata formMetadata, FormStatus formStatus, FormDefinition formDef, Path briefcaseDir, ExportConfiguration configuration, Optional<Analytics> analytics) {
// Create an export tracker object with the total number of submissions we have to export
ExportProcessTracker exportTracker = new ExportProcessTracker(formDef);
exportTracker.start();
SubmissionExportErrorCallback onParsingError = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName()));
SubmissionExportErrorCallback onInvalidSubmission = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName())).andThen((path, message) -> analytics.ifPresent(ga -> ga.event("Export", "Export", "invalid submission", null)));
List<Path> submissionFiles = getListOfSubmissionFiles(formMetadata, formDef, configuration.getDateRange(), configuration.resolveSmartAppend(), onParsingError);
exportTracker.trackTotal(submissionFiles.size());
createDirectories(configuration.getExportDir());
List<Csv> csvs = Csv.getCsvs(formDef, configuration);
csvs.forEach(Csv::prepareOutputFiles);
if (formDef.getModel().hasAuditField()) {
Path audit = configuration.getAuditPath(formDef.getFormName());
if (!exists(audit) || configuration.resolveOverwriteExistingFiles())
write(audit, "instance ID, event, node, start, end\n", CREATE, WRITE, TRUNCATE_EXISTING);
}
// Generate csv lines grouped by the fqdn of the model they belong to
Map<String, CsvLines> csvLinesPerModel = ExportTools.getValidSubmissions(formDef, configuration, submissionFiles, onParsingError, onInvalidSubmission).peek(s -> exportTracker.incAndReport()).flatMap(submission -> csvs.stream().map(Csv::getMapper).map(mapper -> mapper.apply(submission))).collect(groupingByConcurrent(CsvLines::getModelFqn, reducing(CsvLines.empty(), CsvLines::merge)));
// TODO We should have an extra step to produce the side effect of writing media files to disk to avoid having side-effects while generating the CSV output of binary fields
// Write lines to each output Csv
csvs.forEach(csv -> csv.appendLines(Optional.ofNullable(csvLinesPerModel.get(csv.getModelFqn())).orElse(CsvLines.empty())));
exportTracker.end();
Optional.ofNullable(csvLinesPerModel.get(formDef.getModel().fqn())).orElse(CsvLines.empty()).getLastLine().ifPresent(line -> {
formMetadataPort.execute(updateLastExportedSubmission(formMetadata.getKey(), line.getInstanceId(), line.getSubmissionDate(), OffsetDateTime.now(), briefcaseDir, formStatus.getFormDir(briefcaseDir)));
});
ExportOutcome exportOutcome = exportTracker.computeOutcome();
if (exportOutcome == ALL_EXPORTED)
EventBus.publish(ExportEvent.successForm(formDef, (int) exportTracker.total));
if (exportOutcome == SOME_SKIPPED)
EventBus.publish(ExportEvent.partialSuccessForm(formDef, (int) exportTracker.exported, (int) exportTracker.total));
if (exportOutcome == ALL_SKIPPED)
EventBus.publish(ExportEvent.failure(formDef, "All submissions have been skipped"));
return exportOutcome;
}
Aggregations