Examples with Pair org.opendatakit.briefcase.reused.Pair used on opensource projects

Example 1 with Pair

use of org.opendatakit.briefcase.reused.Pair in project briefcase by opendatakit.

the class PullFromCentral method pull.

/**
 * Pulls a form completely, writing the form file, form attachments,
 * submission files and their attachments to the local filesystem
 * under the Briefcase Storage directory.
 */
public Job<Void> pull(FormStatus form) {
    FormKey key = FormKey.from(form);
    PullFromCentralTracker tracker = new PullFromCentralTracker(form, onEventCallback);
    return run(rs -> tracker.trackStart()).thenRun(allOf(supply(runnerStatus -> getSubmissionIds(form, token, runnerStatus, tracker)), run(runnerStatus -> {
        downloadForm(form, token, runnerStatus, tracker);
        List<CentralAttachment> attachments = getFormAttachments(form, token, runnerStatus, tracker);
        int totalAttachments = attachments.size();
        AtomicInteger attachmentNumber = new AtomicInteger(1);
        attachments.parallelStream().forEach(attachment -> downloadFormAttachment(form, attachment, token, runnerStatus, tracker, attachmentNumber.getAndIncrement(), totalAttachments));
    }))).thenAccept((runnerStatus, pair) -> withDb(form.getFormDir(briefcaseDir), db -> {
        List<String> submissions = pair.getLeft();
        int totalSubmissions = submissions.size();
        AtomicInteger submissionNumber = new AtomicInteger(1);
        Set<String> submissionVersions = new HashSet<>();
        if (submissions.isEmpty())
            tracker.trackNoSubmissions();
        submissions.stream().map(instanceId -> Pair.of(submissionNumber.getAndIncrement(), instanceId)).forEach(submissionNumberId -> {
            int currentSubmissionNumber = submissionNumberId.getLeft();
            String instanceId = submissionNumberId.getRight();
            boolean inDb = db.hasRecordedInstance(instanceId) != null;
            Path downloadedSubmissionPath = form.getSubmissionFile(briefcaseDir, instanceId);
            if (!inDb || !downloadedSubmissionPath.toFile().exists()) {
                downloadSubmission(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
                if (downloadedSubmissionPath.toFile().exists()) {
                    XmlElement root = XmlElement.from(new String(readAllBytes(downloadedSubmissionPath)));
                    SubmissionMetaData metaData = new SubmissionMetaData(root);
                    metaData.getVersion().ifPresent(submissionVersions::add);
                }
            } else {
                tracker.trackSubmissionAlreadyDownloaded(currentSubmissionNumber, totalSubmissions);
            }
            List<CentralAttachment> attachments = getSubmissionAttachmentList(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
            int totalAttachments = attachments.size();
            AtomicInteger attachmentNumber = new AtomicInteger(1);
            attachments.stream().filter(attachment -> !inDb || !form.getSubmissionMediaFile(briefcaseDir, instanceId, attachment.getName()).toFile().exists()).forEach(attachment -> downloadSubmissionAttachment(form, instanceId, attachment, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions, attachmentNumber.getAndIncrement(), totalAttachments));
            if (!runnerStatus.isCancelled() && !inDb) {
                db.putRecordedInstanceDirectory(instanceId, form.getSubmissionDir(briefcaseDir, instanceId).toFile());
            }
        });
        tracker.trackEnd();
        formMetadataPort.execute(updateAsPulled(key, briefcaseDir, form.getFormDir(briefcaseDir), submissionVersions));
        EventBus.publish(PullEvent.Success.of(form, server));
    }));
}

Example 2 with Pair

use of org.opendatakit.briefcase.reused.Pair in project briefcase by opendatakit.

the class SubmissionParser method getListOfSubmissionFiles.

/**
 * Returns an sorted {@link List} of {@link Path} instances pointing to all the
 * submissions of a form that belong to the given {@link DateRange}.
 * <p>
 * Each file gets briefly parsed to obtain their submission date and use it as
 * the sorting criteria and for filtering.
 */
public static List<Path> getListOfSubmissionFiles(FormMetadata formMetadata, FormDefinition formDef, DateRange dateRange, boolean smartAppend, SubmissionExportErrorCallback onParsingError) {
    Path instancesDir = formDef.getFormDir().resolve("instances");
    if (!Files.exists(instancesDir) || !Files.isReadable(instancesDir))
        return Collections.emptyList();
    // TODO Migrate this code to Try<Pair<Path, Option<OffsetDate>>> to be able to filter failed parsing attempts
    List<Pair<Path, OffsetDateTime>> paths = new ArrayList<>();
    list(instancesDir).filter(UncheckedFiles::isInstanceDir).forEach(instanceDir -> {
        Path submissionFile = instanceDir.resolve("submission.xml");
        try {
            Optional<OffsetDateTime> submissionDate = readSubmissionDate(submissionFile);
            paths.add(Pair.of(submissionFile, submissionDate.orElse(OffsetDateTime.of(1970, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC))));
        } catch (Throwable t) {
            log.error("Parse error attempting to read instance date", t);
            EventBus.publish(ExportEvent.failureSubmission(formDef, instanceDir.getFileName().toString(), t));
            onParsingError.accept(submissionFile, "Parse error attempting to read instance date");
        }
    });
    return paths.parallelStream().filter(pair -> {
        boolean inRange = dateRange.contains(pair.getRight());
        boolean afterLastExportedSubmission = !smartAppend || formMetadata.getLastExportedSubmission().map(s -> s.isBefore(pair.getRight())).orElse(true);
        return inRange && afterLastExportedSubmission;
    }).map(Pair::getLeft).collect(toList());
}

Example 3 with Pair

use of org.opendatakit.briefcase.reused.Pair in project briefcase by opendatakit.

the class CsvFieldMappers method aggregatedAuditFile.

private static Stream<Pair<String, String>> aggregatedAuditFile(String formName, String localId, Path workingDir, ExportConfiguration configuration, XmlElement e) {
    if (!e.hasValue())
        return empty(e.fqn() + "-aggregated");
    Path sourceFile = workingDir.resolve(e.getValue());
    // When the source file doesn't exist, we return an empty string
    if (!exists(sourceFile))
        return Stream.of(Pair.of(e.fqn() + "-aggregated", ""));
    // Process the audit file contents and append the instance ID column to all lines
    List<String> sourceLines = lines(sourceFile).collect(toList());
    // We prepend the submission's instance ID to all body lines
    List<String> bodyLines = sourceLines.subList(1, sourceLines.size()).stream().map(line -> localId + "," + line).collect(toList());
    Path destinationFile = configuration.getAuditPath(formName);
    write(destinationFile, bodyLines, APPEND);
    return Stream.of(Pair.of(e.fqn() + "-aggregated", destinationFile.getFileName().toString()));
}

Example 4 with Pair

use of org.opendatakit.briefcase.reused.Pair in project briefcase by opendatakit.

the class CsvFieldMappers method binary.

private static Stream<Pair<String, String>> binary(XmlElement element, Path workingDir, ExportConfiguration configuration) {
    if (!element.hasValue())
        return empty(element.fqn());
    String sourceFilename = element.getValue();
    if (!configuration.resolveExportMedia())
        return Stream.of(Pair.of(element.fqn(), sourceFilename));
    if (!Files.exists(configuration.getExportMediaPath()))
        createDirectories(configuration.getExportMediaPath());
    Path sourceFile = workingDir.resolve(sourceFilename);
    // When the source file doesn't exist, we return the input value
    if (!exists(sourceFile))
        return Stream.of(Pair.of(element.fqn(), Paths.get("media").resolve(sourceFilename).toString()));
    // When the destination file doesn't exist, we copy the source file
    // there and return its path relative to the instance folder
    Path destinationFile = configuration.getExportMediaPath().resolve(sourceFilename);
    if (!exists(destinationFile)) {
        copy(sourceFile, destinationFile);
        return Stream.of(Pair.of(element.fqn(), Paths.get("media").resolve(destinationFile.getFileName()).toString()));
    }
    // When the destination file has the same hash as the source file,
    // we don't do any side-effect and return its path relative to the
    // instance folder
    Boolean sameHash = OptionalProduct.all(getMd5Hash(sourceFile), getMd5Hash(destinationFile)).map(Objects::equals).orElse(false);
    if (sameHash)
        return Stream.of(Pair.of(element.fqn(), Paths.get("media").resolve(destinationFile.getFileName()).toString()));
    // When the hashes are different, we compute the next sequential suffix for
    // the a new destination file to avoid overwriting the one we found already
    // there. We try every number in the sequence until we find one that won't
    // produce a destination file that exists in the output directory.
    String namePart = stripFileExtension(sourceFilename);
    String extPart = getFileExtension(sourceFilename).map(extension -> "." + extension).orElse("");
    int sequenceSuffix = 2;
    Path sequentialDestinationFile;
    do {
        sequentialDestinationFile = configuration.getExportMediaPath().resolve(String.format("%s-%d%s", namePart, sequenceSuffix++, extPart));
    } while (exists(sequentialDestinationFile));
    // Now that we have a valid destination file, we copy the source file
    // there and return its path relative to the instance folder
    copy(sourceFile, sequentialDestinationFile);
    return Stream.of(Pair.of(element.fqn(), Paths.get("media").resolve(sequentialDestinationFile.getFileName()).toString()));
}

Example 5 with Pair

use of org.opendatakit.briefcase.reused.Pair in project briefcase by opendatakit.

the class GeoJsonTest method getFeatures.

@SafeVarargs
private static List<Feature> getFeatures(Pair<DataType, String>... contents) throws XmlPullParserException, IOException {
    AtomicInteger seq = new AtomicInteger(1);
    List<Pair<ModelBuilder, String>> fieldsAndContents = Stream.of(contents).map(pair1 -> {
        String fieldName1 = "some-field-" + seq.getAndIncrement();
        return pair1.map(type -> ModelBuilder.field(fieldName1, type), c -> c);
    }).collect(toList());
    Model model = ModelBuilder.instance(fieldsAndContents.stream().map(Pair::getLeft).collect(toList())).build();
    StringBuilder xml = new StringBuilder("<data instanceID=\"uuid:39f3dd36-161e-45cb-a1a4-395831d253a7\">");
    fieldsAndContents.forEach(pair -> xml.append("<").append(pair.getLeft().getName()).append(">").append(pair.getRight()).append("</").append(pair.getLeft().getName()).append(">"));
    xml.append("</data>");
    XmlElement root = ModelBuilder.parseXmlElement(xml.toString());
    Submission submission = Submission.notValidated(Paths.get("/some/path"), Paths.get("/some/path"), root, new SubmissionMetaData(root), Optional.empty(), Optional.empty());
    return GeoJson.toFeatures(model, submission).collect(Collectors.toList());
}