use of org.opendatakit.briefcase.model.form.FormKey in project briefcase by opendatakit.
the class PushToCentralTest method setUp.
@Before
public void setUp() throws IOException {
http = new FakeHttp();
briefcaseDir = createTempDirectory("briefcase-test-");
pushOp = new PushToCentral(http, server, briefcaseDir, token, this::onEvent);
events = new ArrayList<>();
runnerStatus = new TestRunnerStatus(false);
tracker = new PushToCentralTracker(formStatus, this::onEvent);
form = installForm(formStatus, getResourcePath("/org/opendatakit/briefcase/push/aggregate/push-form-test.xml"), briefcaseDir);
formAttachment = installFormAttachment(formStatus, getResourcePath("/org/opendatakit/briefcase/push/aggregate/sparrow.png"), briefcaseDir);
submission = installSubmission(formStatus, getResourcePath("/org/opendatakit/briefcase/push/aggregate/submission.xml"), briefcaseDir);
submissionAttachment = installSubmissionAttachment(formStatus, getResourcePath("/org/opendatakit/briefcase/push/aggregate/1556532531101.jpg"), briefcaseDir, instanceId);
formMetadataPort = new InMemoryFormMetadataAdapter();
FormKey key = FormKey.from(formStatus);
FormMetadata metadata = FormMetadata.of(key, formStatus.getFormDir(briefcaseDir), briefcaseDir);
formMetadataPort.persist(metadata);
}
use of org.opendatakit.briefcase.model.form.FormKey in project briefcase by opendatakit.
the class ExportPanel method export.
private void export() {
Stream<Job<?>> allJobs = forms.getSelectedForms().stream().map(form -> {
FormKey key = FormKey.from(form);
FormMetadata formMetadata = formMetadataPort.fetch(key).orElseThrow(BriefcaseException::new);
form.setStatusString("Starting to export form");
String formId = form.getFormDefinition().getFormId();
ExportConfiguration configuration = forms.getConfiguration(formId);
FormDefinition formDef = FormDefinition.from((BriefcaseFormDefinition) form.getFormDefinition());
// TODO Abstract away the subtype of RemoteServer. This should say Optional<RemoteServer>
Optional<AggregateServer> savedPullSource = RemoteServer.readFromPrefs(appPreferences, pullPanelPrefs, form);
Job<Void> pullJob = configuration.resolvePullBefore() && savedPullSource.isPresent() ? new PullFromAggregate(http, savedPullSource.get(), appPreferences.getBriefcaseDir().orElseThrow(BriefcaseException::new), false, EventBus::publish, formMetadataPort).pull(form, appPreferences.resolveStartFromLast() ? Optional.of(formMetadata.getCursor()) : Optional.empty()) : Job.noOpSupplier();
Job<Void> exportJob = Job.run(runnerStatus -> ExportToCsv.export(formMetadataPort, formMetadata, form, formDef, appPreferences.getBriefcaseDir().orElseThrow(BriefcaseException::new), configuration, analytics));
Job<Void> exportGeoJsonJob = configuration.resolveIncludeGeoJsonExport() ? Job.run(runnerStatus -> ExportToGeoJson.export(formMetadata, formDef, configuration, analytics)) : Job.noOp;
return Job.run(runnerStatus -> form.clearStatusHistory()).thenRun(pullJob).thenRun(exportJob).thenRun(exportGeoJsonJob);
});
JobsRunner.launchAsync(allJobs).onComplete(form::unsetExporting).waitForCompletion();
}
use of org.opendatakit.briefcase.model.form.FormKey in project briefcase by opendatakit.
the class PullFromCentral method pull.
/**
* Pulls a form completely, writing the form file, form attachments,
* submission files and their attachments to the local filesystem
* under the Briefcase Storage directory.
*/
public Job<Void> pull(FormStatus form) {
FormKey key = FormKey.from(form);
PullFromCentralTracker tracker = new PullFromCentralTracker(form, onEventCallback);
return run(rs -> tracker.trackStart()).thenRun(allOf(supply(runnerStatus -> getSubmissionIds(form, token, runnerStatus, tracker)), run(runnerStatus -> {
downloadForm(form, token, runnerStatus, tracker);
List<CentralAttachment> attachments = getFormAttachments(form, token, runnerStatus, tracker);
int totalAttachments = attachments.size();
AtomicInteger attachmentNumber = new AtomicInteger(1);
attachments.parallelStream().forEach(attachment -> downloadFormAttachment(form, attachment, token, runnerStatus, tracker, attachmentNumber.getAndIncrement(), totalAttachments));
}))).thenAccept((runnerStatus, pair) -> withDb(form.getFormDir(briefcaseDir), db -> {
List<String> submissions = pair.getLeft();
int totalSubmissions = submissions.size();
AtomicInteger submissionNumber = new AtomicInteger(1);
Set<String> submissionVersions = new HashSet<>();
if (submissions.isEmpty())
tracker.trackNoSubmissions();
submissions.stream().map(instanceId -> Pair.of(submissionNumber.getAndIncrement(), instanceId)).forEach(submissionNumberId -> {
int currentSubmissionNumber = submissionNumberId.getLeft();
String instanceId = submissionNumberId.getRight();
boolean inDb = db.hasRecordedInstance(instanceId) != null;
Path downloadedSubmissionPath = form.getSubmissionFile(briefcaseDir, instanceId);
if (!inDb || !downloadedSubmissionPath.toFile().exists()) {
downloadSubmission(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
if (downloadedSubmissionPath.toFile().exists()) {
XmlElement root = XmlElement.from(new String(readAllBytes(downloadedSubmissionPath)));
SubmissionMetaData metaData = new SubmissionMetaData(root);
metaData.getVersion().ifPresent(submissionVersions::add);
}
} else {
tracker.trackSubmissionAlreadyDownloaded(currentSubmissionNumber, totalSubmissions);
}
List<CentralAttachment> attachments = getSubmissionAttachmentList(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
int totalAttachments = attachments.size();
AtomicInteger attachmentNumber = new AtomicInteger(1);
attachments.stream().filter(attachment -> !inDb || !form.getSubmissionMediaFile(briefcaseDir, instanceId, attachment.getName()).toFile().exists()).forEach(attachment -> downloadSubmissionAttachment(form, instanceId, attachment, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions, attachmentNumber.getAndIncrement(), totalAttachments));
if (!runnerStatus.isCancelled() && !inDb) {
db.putRecordedInstanceDirectory(instanceId, form.getSubmissionDir(briefcaseDir, instanceId).toFile());
}
});
tracker.trackEnd();
formMetadataPort.execute(updateAsPulled(key, briefcaseDir, form.getFormDir(briefcaseDir), submissionVersions));
EventBus.publish(PullEvent.Success.of(form, server));
}));
}
use of org.opendatakit.briefcase.model.form.FormKey in project briefcase by opendatakit.
the class ExportToCsvScenario method runExport.
void runExport(boolean overwrite, boolean exportMedia, LocalDate startDate, LocalDate endDate, Path pemFile, boolean splitSelectMultiples) {
ExportConfiguration configuration = ExportConfiguration.Builder.empty().setExportDir(outputDir.resolve("new")).setPemFile(pemFile).setStartDate(startDate).setEndDate(endDate).setOverwriteFiles(overwrite).setExportMedia(exportMedia).setSplitSelectMultiples(splitSelectMultiples).build();
FormKey formKey = FormKey.of(formDef.getFormName(), formDef.getFormId());
FormMetadata formMetadata = new FormMetadata(formKey, briefcaseDir, formDef.getFormDir().resolve(stripIllegalChars(formDef.getFormName()) + ".xml"), true, Cursor.empty(), Optional.empty(), Collections.emptySet());
ExportToCsv.export(new InMemoryFormMetadataAdapter(), formMetadata, formStatus, formDef, briefcaseDir, configuration);
}
use of org.opendatakit.briefcase.model.form.FormKey in project briefcase by opendatakit.
the class PullFromAggregate method pull.
/**
* Pulls a form completely, writing the form file, form attachments,
* submission files and their attachments to the local filesystem
* under the Briefcase Storage directory.
* <p>
* A {@link Cursor} can be provided to define the starting point to
* download the form's submissions.
* <p>
* Returns a Job that will produce a pull operation result.
*/
public Job<Void> pull(FormStatus form, Optional<Cursor> lastCursor) {
FormKey key = FormKey.from(form);
PullFromAggregateTracker tracker = new PullFromAggregateTracker(form, onEventCallback);
// Download the form and attachments, and get the submissions list
return run(rs -> tracker.trackStart()).thenSupply(rs -> downloadForm(form, rs, tracker)).thenAccept((rs, formXml) -> {
if (formXml == null)
return;
List<AggregateAttachment> attachments = getFormAttachments(form, rs, tracker);
int totalAttachments = attachments.size();
AtomicInteger attachmentNumber = new AtomicInteger(1);
attachments.parallelStream().forEach(attachment -> downloadFormAttachment(form, attachment, rs, tracker, attachmentNumber.getAndIncrement(), totalAttachments));
List<InstanceIdBatch> instanceIdBatches = getSubmissionIds(form, lastCursor.orElse(Cursor.empty()), rs, tracker);
// Build the submission key generator with the form's XML contents
SubmissionKeyGenerator subKeyGen = SubmissionKeyGenerator.from(formXml);
// Extract all the instance IDs from all the batches and download each instance
List<String> ids = instanceIdBatches.stream().flatMap(batch -> batch.getInstanceIds().stream()).collect(toList());
int totalSubmissions = ids.size();
AtomicInteger submissionNumber = new AtomicInteger(1);
Set<String> submissionVersions = new HashSet<>();
if (ids.isEmpty())
tracker.trackNoSubmissions();
withDb(form.getFormDir(briefcaseDir), db -> {
// We need to collect to be able to create a parallel stream again
ids.parallelStream().map(instanceId -> Triple.of(submissionNumber.getAndIncrement(), instanceId, db.hasRecordedInstance(instanceId) == null)).peek(triple -> {
if (!triple.get3())
tracker.trackSubmissionAlreadyDownloaded(triple.get1(), totalSubmissions);
}).filter(Triple::get3).map(triple -> Pair.of(triple.get1(), downloadSubmission(form, triple.get2(), subKeyGen, rs, tracker, triple.get1(), totalSubmissions))).filter(p -> p.getRight() != null).forEach(pair -> {
int currentSubmissionNumber = pair.getLeft();
DownloadedSubmission submission = pair.getRight();
submission.getFormVersion().ifPresent(submissionVersions::add);
List<AggregateAttachment> submissionAttachments = submission.getAttachments();
AtomicInteger submissionAttachmentNumber = new AtomicInteger(1);
int totalSubmissionAttachments = submissionAttachments.size();
submissionAttachments.parallelStream().forEach(attachment -> downloadSubmissionAttachment(form, submission, attachment, rs, tracker, currentSubmissionNumber, totalSubmissions, submissionAttachmentNumber.getAndIncrement(), totalSubmissionAttachments));
if (!rs.isCancelled()) {
db.putRecordedInstanceDirectory(submission.getInstanceId(), form.getSubmissionDir(briefcaseDir, submission.getInstanceId()).toFile());
}
});
});
tracker.trackEnd();
Cursor newCursor = getLastCursor(instanceIdBatches).orElse(Cursor.empty());
formMetadataPort.execute(updateAsPulled(key, newCursor, briefcaseDir, form.getFormDir(briefcaseDir), submissionVersions));
EventBus.publish(PullEvent.Success.of(form, server));
});
}
Aggregations