use of org.opendatakit.briefcase.model.form.FormMetadataPort in project briefcase by opendatakit.
the class ExportPanel method export.
private void export() {
Stream<Job<?>> allJobs = forms.getSelectedForms().stream().map(form -> {
FormKey key = FormKey.from(form);
FormMetadata formMetadata = formMetadataPort.fetch(key).orElseThrow(BriefcaseException::new);
form.setStatusString("Starting to export form");
String formId = form.getFormDefinition().getFormId();
ExportConfiguration configuration = forms.getConfiguration(formId);
FormDefinition formDef = FormDefinition.from((BriefcaseFormDefinition) form.getFormDefinition());
// TODO Abstract away the subtype of RemoteServer. This should say Optional<RemoteServer>
Optional<AggregateServer> savedPullSource = RemoteServer.readFromPrefs(appPreferences, pullPanelPrefs, form);
Job<Void> pullJob = configuration.resolvePullBefore() && savedPullSource.isPresent() ? new PullFromAggregate(http, savedPullSource.get(), appPreferences.getBriefcaseDir().orElseThrow(BriefcaseException::new), false, EventBus::publish, formMetadataPort).pull(form, appPreferences.resolveStartFromLast() ? Optional.of(formMetadata.getCursor()) : Optional.empty()) : Job.noOpSupplier();
Job<Void> exportJob = Job.run(runnerStatus -> ExportToCsv.export(formMetadataPort, formMetadata, form, formDef, appPreferences.getBriefcaseDir().orElseThrow(BriefcaseException::new), configuration, analytics));
Job<Void> exportGeoJsonJob = configuration.resolveIncludeGeoJsonExport() ? Job.run(runnerStatus -> ExportToGeoJson.export(formMetadata, formDef, configuration, analytics)) : Job.noOp;
return Job.run(runnerStatus -> form.clearStatusHistory()).thenRun(pullJob).thenRun(exportJob).thenRun(exportGeoJsonJob);
});
JobsRunner.launchAsync(allJobs).onComplete(form::unsetExporting).waitForCompletion();
}
use of org.opendatakit.briefcase.model.form.FormMetadataPort in project briefcase by opendatakit.
the class PullFormFromAggregate method pullFormFromAggregate.
public static void pullFormFromAggregate(Path storageDir, Optional<String> formId, String username, String password, URL server, boolean resumeLastPull, Optional<LocalDate> startFromDate, boolean includeIncomplete, Optional<Integer> maybeMaxHttpConnections) {
CliEventsCompanion.attach(log);
Path briefcaseDir = Common.getOrCreateBriefcaseDir(storageDir);
FormCache formCache = FormCache.from(briefcaseDir);
formCache.update();
BriefcasePreferences appPreferences = BriefcasePreferences.appScoped();
FormMetadataPort formMetadataPort = FileSystemFormMetadataAdapter.at(briefcaseDir);
int maxHttpConnections = Optionals.race(maybeMaxHttpConnections, appPreferences.getMaxHttpConnections()).orElse(DEFAULT_HTTP_CONNECTIONS);
Http http = appPreferences.getHttpProxy().map(host -> CommonsHttp.of(maxHttpConnections, host)).orElseGet(() -> CommonsHttp.of(maxHttpConnections));
AggregateServer aggregateServer = AggregateServer.authenticated(server, new Credentials(username, password));
Response<List<RemoteFormDefinition>> response = http.execute(aggregateServer.getFormListRequest());
if (!response.isSuccess()) {
System.err.println(response.isRedirection() ? "Error connecting to Aggregate: Redirection detected" : response.isUnauthorized() ? "Error connecting to Aggregate: Wrong credentials" : response.isNotFound() ? "Error connecting to Aggregate: Aggregate not found" : "Error connecting to Aggregate");
return;
}
List<FormStatus> filteredForms = response.orElseThrow(BriefcaseException::new).stream().filter(f -> formId.map(id -> f.getFormId().equals(id)).orElse(true)).map(FormStatus::new).collect(toList());
if (formId.isPresent() && filteredForms.isEmpty())
throw new BriefcaseException("Form " + formId.get() + " not found");
TransferForms forms = TransferForms.empty();
forms.load(filteredForms);
forms.selectAll();
PullFromAggregate pullOp = new PullFromAggregate(http, aggregateServer, briefcaseDir, includeIncomplete, PullFormFromAggregate::onEvent, formMetadataPort);
JobsRunner.launchAsync(forms.map(form -> pullOp.pull(form, resolveCursor(resumeLastPull, startFromDate, form, formMetadataPort))), PullFormFromAggregate::onError).waitForCompletion();
System.out.println();
System.out.println("All operations completed");
System.out.println();
}
use of org.opendatakit.briefcase.model.form.FormMetadataPort in project briefcase by opendatakit.
the class ImportFromODK method importODK.
public static void importODK(Path storageDir, Path odkDir, Optional<String> formId) {
CliEventsCompanion.attach(log);
Path briefcaseDir = Common.getOrCreateBriefcaseDir(storageDir);
FormCache formCache = FormCache.from(briefcaseDir);
formCache.update();
FormMetadataPort formMetadataPort = FileSystemFormMetadataAdapter.at(briefcaseDir);
TransferForms forms = TransferForms.from(FileSystemUtils.getODKFormList(odkDir.toFile()).stream().map(FormStatus::new).filter(form -> formId.map(id -> form.getFormDefinition().getFormId().equals(id)).orElse(true)).collect(toList()));
forms.selectAll();
if (formId.isPresent() && forms.isEmpty())
throw new BriefcaseException("Form " + formId.get() + " not found");
PullFromCollect.pullForms(formMetadataPort, forms, briefcaseDir, odkDir, ImportFromODK::onEvent).waitForCompletion();
}
use of org.opendatakit.briefcase.model.form.FormMetadataPort in project briefcase by opendatakit.
the class PullFormFromCentral method pullFromCentral.
private static void pullFromCentral(Args args) {
CliEventsCompanion.attach(log);
Path briefcaseDir = Common.getOrCreateBriefcaseDir(args.get(STORAGE_DIR));
FormCache formCache = FormCache.from(briefcaseDir);
formCache.update();
BriefcasePreferences appPreferences = BriefcasePreferences.appScoped();
FormMetadataPort formMetadataPort = FileSystemFormMetadataAdapter.at(briefcaseDir);
int maxHttpConnections = Optionals.race(args.getOptional(MAX_HTTP_CONNECTIONS), appPreferences.getMaxHttpConnections()).orElse(DEFAULT_HTTP_CONNECTIONS);
Http http = appPreferences.getHttpProxy().map(host -> CommonsHttp.of(maxHttpConnections, host)).orElseGet(() -> CommonsHttp.of(maxHttpConnections));
CentralServer server = CentralServer.of(args.get(SERVER_URL), args.get(PROJECT_ID), new Credentials(args.get(CREDENTIALS_EMAIL), args.get(CREDENTIALS_PASSWORD)));
String token = http.execute(server.getSessionTokenRequest()).orElseThrow(() -> new BriefcaseException("Can't authenticate with ODK Central"));
Response<List<RemoteFormDefinition>> response = http.execute(server.getFormsListRequest(token));
if (!response.isSuccess()) {
System.err.println(response.isRedirection() ? "Error connecting to Central: Redirection detected" : response.isUnauthorized() ? "Error connecting to Central: Wrong credentials" : response.isNotFound() ? "Error connecting to Central: Central not found" : "Error connecting to Central");
return;
}
Optional<String> formId = args.getOptional(FORM_ID);
List<FormStatus> filteredForms = response.orElseThrow(BriefcaseException::new).stream().filter(f -> formId.map(id -> f.getFormId().equals(id)).orElse(true)).map(FormStatus::new).collect(toList());
if (formId.isPresent() && filteredForms.isEmpty())
throw new BriefcaseException("Form " + formId.get() + " not found");
TransferForms forms = TransferForms.empty();
forms.load(filteredForms);
forms.selectAll();
PullFromCentral pullOp = new PullFromCentral(http, server, briefcaseDir, token, PullFormFromCentral::onEvent, formMetadataPort);
JobsRunner.launchAsync(forms.map(pullOp::pull), PullFormFromCentral::onError).waitForCompletion();
System.out.println();
System.out.println("All operations completed");
System.out.println();
}
use of org.opendatakit.briefcase.model.form.FormMetadataPort in project briefcase by opendatakit.
the class ExportToCsv method export.
/**
* Export a form's submissions into some CSV files.
* <p>
* If the form has repeat groups, each repeat group will be exported into a separate CSV file.
*
* @param formDef the {@link FormDefinition} form definition of the form to be exported
* @param configuration the {@link ExportConfiguration} export configuration
* @return an {@link ExportOutcome} with the export operation's outcome
* @see ExportConfiguration
*/
private static ExportOutcome export(FormMetadataPort formMetadataPort, FormMetadata formMetadata, FormStatus formStatus, FormDefinition formDef, Path briefcaseDir, ExportConfiguration configuration, Optional<Analytics> analytics) {
// Create an export tracker object with the total number of submissions we have to export
ExportProcessTracker exportTracker = new ExportProcessTracker(formDef);
exportTracker.start();
SubmissionExportErrorCallback onParsingError = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName()));
SubmissionExportErrorCallback onInvalidSubmission = buildParsingErrorCallback(configuration.getErrorsDir(formDef.getFormName())).andThen((path, message) -> analytics.ifPresent(ga -> ga.event("Export", "Export", "invalid submission", null)));
List<Path> submissionFiles = getListOfSubmissionFiles(formMetadata, formDef, configuration.getDateRange(), configuration.resolveSmartAppend(), onParsingError);
exportTracker.trackTotal(submissionFiles.size());
createDirectories(configuration.getExportDir());
List<Csv> csvs = Csv.getCsvs(formDef, configuration);
csvs.forEach(Csv::prepareOutputFiles);
if (formDef.getModel().hasAuditField()) {
Path audit = configuration.getAuditPath(formDef.getFormName());
if (!exists(audit) || configuration.resolveOverwriteExistingFiles())
write(audit, "instance ID, event, node, start, end\n", CREATE, WRITE, TRUNCATE_EXISTING);
}
// Generate csv lines grouped by the fqdn of the model they belong to
Map<String, CsvLines> csvLinesPerModel = ExportTools.getValidSubmissions(formDef, configuration, submissionFiles, onParsingError, onInvalidSubmission).peek(s -> exportTracker.incAndReport()).flatMap(submission -> csvs.stream().map(Csv::getMapper).map(mapper -> mapper.apply(submission))).collect(groupingByConcurrent(CsvLines::getModelFqn, reducing(CsvLines.empty(), CsvLines::merge)));
// TODO We should have an extra step to produce the side effect of writing media files to disk to avoid having side-effects while generating the CSV output of binary fields
// Write lines to each output Csv
csvs.forEach(csv -> csv.appendLines(Optional.ofNullable(csvLinesPerModel.get(csv.getModelFqn())).orElse(CsvLines.empty())));
exportTracker.end();
Optional.ofNullable(csvLinesPerModel.get(formDef.getModel().fqn())).orElse(CsvLines.empty()).getLastLine().ifPresent(line -> {
formMetadataPort.execute(updateLastExportedSubmission(formMetadata.getKey(), line.getInstanceId(), line.getSubmissionDate(), OffsetDateTime.now(), briefcaseDir, formStatus.getFormDir(briefcaseDir)));
});
ExportOutcome exportOutcome = exportTracker.computeOutcome();
if (exportOutcome == ALL_EXPORTED)
EventBus.publish(ExportEvent.successForm(formDef, (int) exportTracker.total));
if (exportOutcome == SOME_SKIPPED)
EventBus.publish(ExportEvent.partialSuccessForm(formDef, (int) exportTracker.exported, (int) exportTracker.total));
if (exportOutcome == ALL_SKIPPED)
EventBus.publish(ExportEvent.failure(formDef, "All submissions have been skipped"));
return exportOutcome;
}
Aggregations