Examples with BriefcaseFormDefinition org.opendatakit.briefcase.model.BriefcaseFormDefinition used on opensource projects

Example 6 with BriefcaseFormDefinition

use of org.opendatakit.briefcase.model.BriefcaseFormDefinition in project briefcase by opendatakit.

the class TransferFromODK method doAction.

@Override
public boolean doAction() {
    boolean allSuccessful = true;
    for (FormStatus fs : formsToTransfer) {
        boolean isSuccessful = true;
        try {
            if (terminationFuture.isCancelled()) {
                fs.setStatusString("Aborted. Skipping fetch of form and submissions...", true);
                EventBus.publish(new FormStatusEvent(fs));
                return false;
            }
            BriefcaseFormDefinition briefcaseLfd = doResolveOdkCollectFormDefinition(fs);
            if (briefcaseLfd == null) {
                allSuccessful = isSuccessful = false;
                continue;
            }
            OdkCollectFormDefinition odkFormDef = (OdkCollectFormDefinition) fs.getFormDefinition();
            File odkFormDefFile = odkFormDef.getFormDefinitionFile();
            final String odkFormName = odkFormDefFile.getName().substring(0, odkFormDefFile.getName().lastIndexOf("."));
            DatabaseUtils formDatabase = null;
            try {
                formDatabase = DatabaseUtils.newInstance(briefcaseLfd.getFormDirectory());
                File destinationFormInstancesDir;
                try {
                    destinationFormInstancesDir = FileSystemUtils.getFormInstancesDirectory(briefcaseLfd.getFormDirectory());
                } catch (FileSystemException e) {
                    allSuccessful = isSuccessful = false;
                    String msg = "unable to create instances folder";
                    log.error(msg, e);
                    fs.setStatusString(msg + ": " + e.getMessage(), false);
                    EventBus.publish(new FormStatusEvent(fs));
                    continue;
                }
                // we have the needed directory structure created...
                fs.setStatusString("preparing to retrieve instance data", true);
                EventBus.publish(new FormStatusEvent(fs));
                // construct up the list of folders that might have ODK form data.
                File odkFormInstancesDir = new File(odkFormDefFile.getParentFile().getParentFile(), "instances");
                // rely on ODK naming conventions to identify form data files...
                File[] odkFormInstanceDirs = odkFormInstancesDir.listFiles(pathname -> {
                    boolean beginsWithFormName = pathname.getName().startsWith(odkFormName);
                    if (!beginsWithFormName)
                        return false;
                    // skip the separator character, as it varies between 1.1.5, 1.1.6 and 1.1.7
                    String afterName = pathname.getName().substring(odkFormName.length() + 1);
                    // aftername should be a reasonable date though we allow extra stuff at the end...
                    // protects against someone having "formname" and "formname_2"
                    // and us mistaking "formname_2_2009-01-02_15_10_03" as containing
                    // instance data for "formname" instead of "formname_2"
                    boolean outcome = afterName.matches("^[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}.*");
                    return outcome;
                });
                if (odkFormInstanceDirs != null) {
                    int instanceCount = 1;
                    for (File dir : odkFormInstanceDirs) {
                        if (terminationFuture.isCancelled()) {
                            allSuccessful = isSuccessful = false;
                            fs.setStatusString("aborting retrieving submissions...", true);
                            EventBus.publish(new FormStatusEvent(fs));
                            return false;
                        }
                        // 1.1.8 -- submission is saved as submission.xml.
                        // full instance data is stored as directoryName.xml (as is the convention in 1.1.5, 1.1.7)
                        String instanceId = null;
                        File fullXml = new File(dir, dir.getName() + ".xml");
                        File xml = new File(dir, "submission.xml");
                        if (!xml.exists() && fullXml.exists()) {
                            // e.g., 1.1.5, 1.1.7
                            xml = fullXml;
                        }
                        // rename it to match the directory name.
                        if (!xml.exists()) {
                            File[] xmlFiles = dir.listFiles(fileEndsWithXml);
                            if (xmlFiles.length == 1) {
                                try {
                                    FileUtils.moveFile(xmlFiles[0], xml);
                                } catch (IOException e) {
                                    allSuccessful = isSuccessful = false;
                                    String msg = "unable to rename form instance xml";
                                    log.error(msg, e);
                                    fs.setStatusString(msg + ": " + e.getMessage(), false);
                                    EventBus.publish(new FormStatusEvent(fs));
                                    continue;
                                }
                            }
                        }
                        if (xml.exists()) {
                            // Check if the instance has an instanceID
                            try {
                                XmlManipulationUtils.FormInstanceMetadata formInstanceMetadata = XmlManipulationUtils.getFormInstanceMetadata(XmlManipulationUtils.parseXml(xml).getRootElement());
                                instanceId = formInstanceMetadata.instanceId;
                            } catch (ParsingException e) {
                                log.error("failed to get instance id from submission", e);
                            }
                            // OK, we can copy the directory off...
                            // Briefcase instances directory name is arbitrary.
                            // Rename the xml within that to always be "submission.xml"
                            // to remove the correspondence to the directory name.
                            File scratchInstance = FileSystemUtils.getFormSubmissionDirectory(destinationFormInstancesDir, dir.getName());
                            String safeName = scratchInstance.getName();
                            int i = 2;
                            boolean same = false;
                            while (scratchInstance.exists()) {
                                File[] contents = scratchInstance.listFiles(fileEndsWithXml);
                                if (contents == null || contents.length == 0)
                                    break;
                                if (contents.length == 1) {
                                    String itsInstanceId = null;
                                    try {
                                        XmlManipulationUtils.FormInstanceMetadata formInstanceMetadata = XmlManipulationUtils.getFormInstanceMetadata(XmlManipulationUtils.parseXml(contents[0]).getRootElement());
                                        itsInstanceId = formInstanceMetadata.instanceId;
                                        // if yes don't copy it, skip to next file
                                        if (itsInstanceId != null && itsInstanceId.equals(instanceId) && FileSystemUtils.getMd5Hash(xml).equals(FileSystemUtils.getMd5Hash(contents[0]))) {
                                            same = true;
                                            break;
                                        }
                                    } catch (ParsingException e) {
                                        log.error("failed to parse submission", e);
                                    }
                                }
                                scratchInstance = new File(destinationFormInstancesDir, safeName + "-" + Integer.toString(i));
                                i++;
                            }
                            if (same) {
                                fs.setStatusString("already present - skipping: " + xml.getName(), true);
                                EventBus.publish(new FormStatusEvent(fs));
                                continue;
                            }
                            try {
                                FileUtils.copyDirectory(dir, scratchInstance);
                            } catch (IOException e) {
                                allSuccessful = isSuccessful = false;
                                String msg = "unable to copy saved instance";
                                log.error(msg, e);
                                fs.setStatusString(msg + ": " + e.getMessage(), false);
                                EventBus.publish(new FormStatusEvent(fs));
                                continue;
                            }
                            if (xml.equals(fullXml)) {
                                // need to rename
                                File odkSubmissionFile = new File(scratchInstance, fullXml.getName());
                                File scratchSubmissionFile = new File(scratchInstance, "submission.xml");
                                try {
                                    FileUtils.moveFile(odkSubmissionFile, scratchSubmissionFile);
                                } catch (IOException e) {
                                    allSuccessful = isSuccessful = false;
                                    String msg = "unable to rename submission file to submission.xml";
                                    log.error(msg, e);
                                    fs.setStatusString(msg + ": " + e.getMessage(), false);
                                    EventBus.publish(new FormStatusEvent(fs));
                                    continue;
                                }
                            } else {
                                // delete the full xml file (keep only the submission.xml)
                                File odkSubmissionFile = new File(scratchInstance, fullXml.getName());
                                odkSubmissionFile.delete();
                            }
                            fs.setStatusString(String.format("retrieving (%1$d)", instanceCount), true);
                            EventBus.publish(new FormStatusEvent(fs));
                            ++instanceCount;
                        }
                    }
                }
            } catch (SQLException | FileSystemException e) {
                allSuccessful = isSuccessful = false;
                String msg = "unable to open form database";
                log.error(msg, e);
                fs.setStatusString(msg + ": " + e.getMessage(), false);
                EventBus.publish(new FormStatusEvent(fs));
                continue;
            } finally {
                if (formDatabase != null) {
                    try {
                        formDatabase.close();
                    } catch (SQLException e) {
                        allSuccessful = isSuccessful = false;
                        String msg = "unable to close form database";
                        log.error(msg, e);
                        fs.setStatusString(msg + ": " + e.getMessage(), false);
                        EventBus.publish(new FormStatusEvent(fs));
                        continue;
                    }
                }
            }
        } finally {
            if (isSuccessful) {
                fs.setStatusString(ServerFetcher.SUCCESS_STATUS, true);
                EventBus.publish(new FormStatusEvent(fs));
            } else {
                fs.setStatusString(ServerFetcher.FAILED_STATUS, true);
                EventBus.publish(new FormStatusEvent(fs));
            }
        }
    }
    return allSuccessful;
}

Example 7 with BriefcaseFormDefinition

use of org.opendatakit.briefcase.model.BriefcaseFormDefinition in project briefcase by opendatakit.

the class ServerUploader method uploadFormAndSubmissionFiles.

public boolean uploadFormAndSubmissionFiles(List<FormStatus> formsToTransfer) {
    boolean allSuccessful = true;
    for (FormStatus formToTransfer : formsToTransfer) {
        BriefcaseFormDefinition briefcaseLfd = (BriefcaseFormDefinition) formToTransfer.getFormDefinition();
        boolean thisFormSuccessful = true;
        if (isCancelled()) {
            formToTransfer.setStatusString("Aborted upload.", true);
            EventBus.publish(new FormStatusEvent(formToTransfer));
            return false;
        }
        if (!formToTransfer.isSuccessful()) {
            formToTransfer.setStatusString("Skipping upload -- download failed", false);
            EventBus.publish(new FormStatusEvent(formToTransfer));
            continue;
        }
        File briefcaseFormDefFile = FileSystemUtils.getFormDefinitionFileIfExists(briefcaseLfd.getFormDirectory());
        if (briefcaseFormDefFile == null) {
            formToTransfer.setStatusString("Form does not exist", true);
            EventBus.publish(new FormStatusEvent(formToTransfer));
            continue;
        }
        File briefcaseFormMediaDir = FileSystemUtils.getMediaDirectoryIfExists(briefcaseLfd.getFormDirectory());
        boolean outcome;
        outcome = uploadForm(formToTransfer, briefcaseFormDefFile, briefcaseFormMediaDir);
        thisFormSuccessful = thisFormSuccessful & outcome;
        allSuccessful = allSuccessful & outcome;
        URI u = getUploadSubmissionUri(formToTransfer);
        if (u == null) {
            // error already logged...
            continue;
        }
        Set<File> briefcaseInstances = FileSystemUtils.getFormSubmissionDirectories(briefcaseLfd.getFormDirectory());
        DatabaseUtils formDatabase = null;
        try {
            formDatabase = DatabaseUtils.newInstance(briefcaseLfd.getFormDirectory());
            // make sure all the local instances are in the database...
            formDatabase.updateInstanceLists(briefcaseInstances);
            // exclude submissions the server reported as already submitted
            subtractServerInstances(formToTransfer, formDatabase, briefcaseInstances);
            int i = 1;
            for (File briefcaseInstance : briefcaseInstances) {
                outcome = uploadSubmission(formDatabase, formToTransfer, u, i++, briefcaseInstances.size(), briefcaseInstance);
                thisFormSuccessful = thisFormSuccessful & outcome;
                allSuccessful = allSuccessful & outcome;
                // and stop this loop quickly if we're cancelled...
                if (isCancelled()) {
                    break;
                }
            }
        } catch (SQLException | FileSystemException e) {
            thisFormSuccessful = false;
            allSuccessful = false;
            String msg = "unable to open form database";
            log.error(msg, e);
            formToTransfer.setStatusString(msg + ": " + e.getMessage(), false);
            EventBus.publish(new FormStatusEvent(formToTransfer));
        } finally {
            if (formDatabase != null) {
                try {
                    formDatabase.close();
                } catch (SQLException e) {
                    thisFormSuccessful = false;
                    allSuccessful = false;
                    String msg = "unable to close form database";
                    log.warn(msg, e);
                    formToTransfer.setStatusString(msg + ": " + e.getMessage(), false);
                    EventBus.publish(new FormStatusEvent(formToTransfer));
                }
            }
        }
        if (isCancelled()) {
            formToTransfer.setStatusString("Aborted upload.", true);
            EventBus.publish(new FormStatusEvent(formToTransfer));
        } else if (thisFormSuccessful) {
            formToTransfer.setStatusString("Successful upload!", true);
            EventBus.publish(new FormStatusEvent(formToTransfer));
        } else {
            formToTransfer.setStatusString("Partially successful upload...", true);
            EventBus.publish(new FormStatusEvent(formToTransfer));
        }
    }
    return allSuccessful;
}

Example 8 with BriefcaseFormDefinition

use of org.opendatakit.briefcase.model.BriefcaseFormDefinition in project briefcase by opendatakit.

the class ServerFetcher method downloadFormAndSubmissionFiles.

public boolean downloadFormAndSubmissionFiles(List<FormStatus> formsToTransfer) {
    boolean allSuccessful = true;
    // boolean error = false;
    int total = formsToTransfer.size();
    for (int i = 0; i < total; i++) {
        FormStatus fs = formsToTransfer.get(i);
        if (isCancelled()) {
            fs.setStatusString("Aborted. Skipping fetch of form and submissions...", true);
            EventBus.publish(new FormStatusEvent(fs));
            return false;
        }
        RemoteFormDefinition fd = getRemoteFormDefinition(fs);
        fs.setStatusString("Fetching form definition", true);
        EventBus.publish(new FormStatusEvent(fs));
        try {
            File tmpdl = FileSystemUtils.getTempFormDefinitionFile();
            AggregateUtils.commonDownloadFile(serverInfo, tmpdl, fd.getDownloadUrl());
            fs.setStatusString("resolving against briefcase form definitions", true);
            EventBus.publish(new FormStatusEvent(fs));
            boolean successful = false;
            BriefcaseFormDefinition briefcaseLfd;
            DatabaseUtils formDatabase = null;
            try {
                try {
                    briefcaseLfd = BriefcaseFormDefinition.resolveAgainstBriefcaseDefn(tmpdl);
                    if (briefcaseLfd.needsMediaUpdate()) {
                        if (fd.getManifestUrl() != null) {
                            File mediaDir = FileSystemUtils.getMediaDirectory(briefcaseLfd.getFormDirectory());
                            String error = downloadManifestAndMediaFiles(mediaDir, fs);
                            if (error != null) {
                                allSuccessful = false;
                                fs.setStatusString("Error fetching form definition: " + error, false);
                                EventBus.publish(new FormStatusEvent(fs));
                                continue;
                            }
                        }
                    }
                    formDatabase = DatabaseUtils.newInstance(briefcaseLfd.getFormDirectory());
                } catch (BadFormDefinition e) {
                    allSuccessful = false;
                    String msg = "Error parsing form definition";
                    log.error(msg, e);
                    fs.setStatusString(msg + ": " + e.getMessage(), false);
                    EventBus.publish(new FormStatusEvent(fs));
                    continue;
                }
                fs.setStatusString("preparing to retrieve instance data", true);
                EventBus.publish(new FormStatusEvent(fs));
                File formInstancesDir = FileSystemUtils.getFormInstancesDirectory(briefcaseLfd.getFormDirectory());
                // this will publish events
                successful = downloadAllSubmissionsForForm(formInstancesDir, formDatabase, briefcaseLfd, fs);
            } catch (SQLException | FileSystemException e) {
                allSuccessful = false;
                String msg = "unable to open form database";
                log.error(msg, e);
                fs.setStatusString(msg + ": " + e.getMessage(), false);
                EventBus.publish(new FormStatusEvent(fs));
                continue;
            } finally {
                if (formDatabase != null) {
                    try {
                        formDatabase.close();
                    } catch (SQLException e) {
                        allSuccessful = false;
                        String msg = "unable to close form database";
                        log.error(msg, e);
                        fs.setStatusString(msg + ": " + e.getMessage(), false);
                        EventBus.publish(new FormStatusEvent(fs));
                        continue;
                    }
                }
            }
            allSuccessful = allSuccessful && successful;
            // on success, we haven't actually set a success event (because we don't know we're done)
            if (successful) {
                fs.setStatusString(SUCCESS_STATUS, true);
                EventBus.publish(new FormStatusEvent(fs));
            } else {
                fs.setStatusString(FAILED_STATUS, true);
                EventBus.publish(new FormStatusEvent(fs));
            }
        } catch (SocketTimeoutException se) {
            allSuccessful = false;
            log.error("error accessing URL", se);
            fs.setStatusString("Communications to the server timed out. Detailed message: " + se.getLocalizedMessage() + " while accessing: " + fd.getDownloadUrl() + " A network login screen may be interfering with the transmission to the server.", false);
            EventBus.publish(new FormStatusEvent(fs));
        } catch (IOException e) {
            allSuccessful = false;
            log.error("error accessing form download URL", e);
            fs.setStatusString("Unexpected error: " + e.getLocalizedMessage() + " while accessing: " + fd.getDownloadUrl() + " A network login screen may be interfering with the transmission to the server.", false);
            EventBus.publish(new FormStatusEvent(fs));
        } catch (FileSystemException | TransmissionException | URISyntaxException e) {
            allSuccessful = false;
            log.error("error accessing form download URL", e);
            fs.setStatusString("Unexpected error: " + e.getLocalizedMessage() + " while accessing: " + fd.getDownloadUrl(), false);
            EventBus.publish(new FormStatusEvent(fs));
        }
    }
    return allSuccessful;
}

Example 9 with BriefcaseFormDefinition

use of org.opendatakit.briefcase.model.BriefcaseFormDefinition in project briefcase by opendatakit.

the class PushTransferPanel method updateFormStatuses.

public void updateFormStatuses() {
    List<FormStatus> statuses = new ArrayList<>();
    List<BriefcaseFormDefinition> forms = FileSystemUtils.getBriefcaseFormList();
    for (BriefcaseFormDefinition f : forms) {
        statuses.add(new FormStatus(FormStatus.TransferType.UPLOAD, f));
    }
    formTransferTable.setFormStatusList(statuses);
}

Example 10 with BriefcaseFormDefinition

use of org.opendatakit.briefcase.model.BriefcaseFormDefinition in project briefcase by opendatakit.

the class Export method export.

public static void export(String storageDir, String formid, Path exportPath, String baseFilename, boolean includeMediaFiles, boolean overwriteFiles, Optional<LocalDate> startDate, Optional<LocalDate> endDate, Optional<Path> maybePemFile) {
    CliEventsCompanion.attach(log);
    bootCache(storageDir);
    Optional<BriefcaseFormDefinition> maybeFormDefinition = FileSystemUtils.getBriefcaseFormList().stream().filter(form -> form.getFormId().equals(formid)).findFirst();
    BriefcaseFormDefinition formDefinition = maybeFormDefinition.orElseThrow(() -> new FormNotFoundException(formid));
    if (formDefinition.isFileEncryptedForm() || formDefinition.isFieldEncryptedForm()) {
        Path pemFile = maybePemFile.filter(Files::exists).orElseThrow(() -> new BriefcaseException("Missing pem file configuration"));
        try (PEMReader rdr = new PEMReader(new BufferedReader(new InputStreamReader(Files.newInputStream(pemFile), "UTF-8")))) {
            Object o = Optional.ofNullable(rdr.readObject()).orElseThrow(() -> new BriefcaseException("Can't parse Pem file"));
            Optional<PrivateKey> privKey;
            if (o instanceof KeyPair)
                privKey = Optional.of(((KeyPair) o).getPrivate());
            else if (o instanceof PrivateKey)
                privKey = Optional.of((PrivateKey) o);
            else
                privKey = Optional.empty();
            formDefinition.setPrivateKey(privKey.orElseThrow(() -> new BriefcaseException("No private key found on Pem file")));
            EventBus.publish(new ExportProgressEvent("Successfully parsed Pem file", formDefinition));
        } catch (IOException e) {
            throw new BriefcaseException("Can't parse Pem file");
        }
    }
    System.out.println("Exporting form " + formDefinition.getFormName() + " (" + formDefinition.getFormId() + ") to: " + exportPath);
    ExportToCsv.export(exportPath, formDefinition, baseFilename, includeMediaFiles, overwriteFiles, startDate, endDate);
    BriefcasePreferences.forClass(ExportPanel.class).put(buildExportDateTimePrefix(formDefinition.getFormId()), LocalDateTime.now().format(ISO_DATE_TIME));
}