use of org.opendatakit.briefcase.export.SubmissionMetaData in project briefcase by opendatakit.
the class PullFromCentral method pull.
/**
* Pulls a form completely, writing the form file, form attachments,
* submission files and their attachments to the local filesystem
* under the Briefcase Storage directory.
*/
public Job<Void> pull(FormStatus form) {
FormKey key = FormKey.from(form);
PullFromCentralTracker tracker = new PullFromCentralTracker(form, onEventCallback);
return run(rs -> tracker.trackStart()).thenRun(allOf(supply(runnerStatus -> getSubmissionIds(form, token, runnerStatus, tracker)), run(runnerStatus -> {
downloadForm(form, token, runnerStatus, tracker);
List<CentralAttachment> attachments = getFormAttachments(form, token, runnerStatus, tracker);
int totalAttachments = attachments.size();
AtomicInteger attachmentNumber = new AtomicInteger(1);
attachments.parallelStream().forEach(attachment -> downloadFormAttachment(form, attachment, token, runnerStatus, tracker, attachmentNumber.getAndIncrement(), totalAttachments));
}))).thenAccept((runnerStatus, pair) -> withDb(form.getFormDir(briefcaseDir), db -> {
List<String> submissions = pair.getLeft();
int totalSubmissions = submissions.size();
AtomicInteger submissionNumber = new AtomicInteger(1);
Set<String> submissionVersions = new HashSet<>();
if (submissions.isEmpty())
tracker.trackNoSubmissions();
submissions.stream().map(instanceId -> Pair.of(submissionNumber.getAndIncrement(), instanceId)).forEach(submissionNumberId -> {
int currentSubmissionNumber = submissionNumberId.getLeft();
String instanceId = submissionNumberId.getRight();
boolean inDb = db.hasRecordedInstance(instanceId) != null;
Path downloadedSubmissionPath = form.getSubmissionFile(briefcaseDir, instanceId);
if (!inDb || !downloadedSubmissionPath.toFile().exists()) {
downloadSubmission(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
if (downloadedSubmissionPath.toFile().exists()) {
XmlElement root = XmlElement.from(new String(readAllBytes(downloadedSubmissionPath)));
SubmissionMetaData metaData = new SubmissionMetaData(root);
metaData.getVersion().ifPresent(submissionVersions::add);
}
} else {
tracker.trackSubmissionAlreadyDownloaded(currentSubmissionNumber, totalSubmissions);
}
List<CentralAttachment> attachments = getSubmissionAttachmentList(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
int totalAttachments = attachments.size();
AtomicInteger attachmentNumber = new AtomicInteger(1);
attachments.stream().filter(attachment -> !inDb || !form.getSubmissionMediaFile(briefcaseDir, instanceId, attachment.getName()).toFile().exists()).forEach(attachment -> downloadSubmissionAttachment(form, instanceId, attachment, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions, attachmentNumber.getAndIncrement(), totalAttachments));
if (!runnerStatus.isCancelled() && !inDb) {
db.putRecordedInstanceDirectory(instanceId, form.getSubmissionDir(briefcaseDir, instanceId).toFile());
}
});
tracker.trackEnd();
formMetadataPort.execute(updateAsPulled(key, briefcaseDir, form.getFormDir(briefcaseDir), submissionVersions));
EventBus.publish(PullEvent.Success.of(form, server));
}));
}
use of org.opendatakit.briefcase.export.SubmissionMetaData in project briefcase by opendatakit.
the class TransferTestHelpers method installSubmission.
public static Path installSubmission(FormStatus form, Path source, Path briefcaseDir) throws IOException {
String instanceId = new SubmissionMetaData(XmlElement.from(new String(readAllBytes(source)))).getInstanceId().orElseThrow(RuntimeException::new);
Path submissionDir = form.getSubmissionDir(briefcaseDir, instanceId);
createDirectories(submissionDir);
return copy(source, form.getSubmissionFile(briefcaseDir, instanceId));
}
use of org.opendatakit.briefcase.export.SubmissionMetaData in project briefcase by opendatakit.
the class PushToCentral method push.
/**
* Pushes a form completely, sending the form file, form attachments,
* submission files and their attachments to the Aggregate server.
* <p>
* It won't push a form and form attachments if it's already
* present in the server.
*/
@SuppressWarnings("checkstyle:Indentation")
public Job<Void> push(FormStatus form) {
PushToCentralTracker tracker = new PushToCentralTracker(form, onEventCallback);
Job<Void> startTrackingJob = Job.run(runnerStatus -> tracker.trackStart());
// Verify that the form is not encrypted
Path formFile = form.getFormFile(briefcaseDir);
try {
if (FormDefinition.from(formFile).isFileEncryptedForm())
return startTrackingJob.thenRun(rs -> {
tracker.trackEncryptedForm();
tracker.trackEnd();
});
} catch (RuntimeException e) {
return startTrackingJob.thenRun(rs -> {
tracker.trackCannotDetermineEncryption(e);
tracker.trackEnd();
});
}
FormMetadataPort formMetadataPort = FileSystemFormMetadataAdapter.at(briefcaseDir);
return startTrackingJob.thenSupply(rs -> getMissingFormVersions(form, rs, tracker, formMetadataPort)).thenAccept(((rs, missingFormVersions) -> {
String localFormVersion = form.getVersion().orElse("");
boolean sendLocal = missingFormVersions.remove(localFormVersion);
// First send a request to create the form. It doesn't matter what version is sent because it will be replaced
// below. Will fail if a published form already exists with this id but then missing versions can still be added.
createForm(form.getFormFile(briefcaseDir), rs, tracker);
// Send all of the obsolete versions first so that we can send corresponding submissions. Skip attachments
// because those versions won't be used by clients.
missingFormVersions.stream().sorted().forEach(version -> {
pushFormDraft(form.getFormFile(briefcaseDir), form.getFormId(), version, rs, tracker);
publishDraft(form.getFormId(), version, rs, tracker);
});
if (sendLocal) {
boolean formSent = pushFormDraft(formFile, form.getFormId(), localFormVersion, rs, tracker);
if (formSent) {
List<Path> formAttachments = getFormAttachments(form);
AtomicInteger attachmentSeq = new AtomicInteger(1);
int totalAttachments = formAttachments.size();
formAttachments.forEach(attachment -> pushFormAttachment(form.getFormId(), attachment, rs, tracker, attachmentSeq.getAndIncrement(), totalAttachments));
publishDraft(form.getFormId(), localFormVersion, rs, tracker);
}
}
})).thenSupply(__ -> getSubmissions(form)).thenAccept((rs, submissions) -> {
AtomicInteger submissionNumber = new AtomicInteger(1);
int totalSubmissions = submissions.size();
if (submissions.isEmpty())
tracker.trackNoSubmissions();
// We might not have had access to submission versions because e.g. submissions were manually added to the
// Briefcase folder or were imported in <=v1.17 when submission versions weren't saved.
Set<String> seenVersions = new HashSet<>();
submissions.parallelStream().map(submission -> {
XmlElement root = XmlElement.from(new String(readAllBytes(submission)));
SubmissionMetaData metaData = new SubmissionMetaData(root);
metaData.getVersion().ifPresent(seenVersions::add);
return Triple.of(submission, submissionNumber.getAndIncrement(), metaData.getInstanceId());
}).peek(triple -> {
if (!triple.get3().isPresent())
tracker.trackNoInstanceId(triple.get2(), totalSubmissions);
}).filter(triple -> triple.get3().isPresent()).forEach(triple -> {
Path submission = triple.get1();
int currentSubmissionNumber = triple.get2();
String instanceId = triple.get3().get();
boolean submissionSent = pushSubmission(form.getFormId(), submission, rs, tracker, currentSubmissionNumber, totalSubmissions);
if (submissionSent) {
List<Path> submissionAttachments = getSubmissionAttachments(submission);
AtomicInteger attachmentSeq = new AtomicInteger(1);
int totalAttachments = submissionAttachments.size();
submissionAttachments.parallelStream().forEach(attachment -> pushSubmissionAttachment(form.getFormId(), instanceId, attachment, rs, tracker, currentSubmissionNumber, totalSubmissions, attachmentSeq.getAndIncrement(), totalAttachments));
}
});
formMetadataPort.execute(updateSubmissionVersions(FormKey.from(form), briefcaseDir, form.getFormDir(briefcaseDir), seenVersions));
tracker.trackEnd();
}).thenRun(rs -> EventBus.publish(new PushEvent.Success(form)));
}
Aggregations