Examples with FormInstanceMetadata org.opendatakit.briefcase.util.XmlManipulationUtils.FormInstanceMetadata used on opensource projects

Example 1 with FormInstanceMetadata

use of org.opendatakit.briefcase.util.XmlManipulationUtils.FormInstanceMetadata in project briefcase by opendatakit.

the class ExportToCsv method processInstance.

private boolean processInstance(File instanceDir) {
    File submission = new File(instanceDir, "submission.xml");
    if (!submission.exists() || !submission.isFile()) {
        EventBus.publish(new ExportProgressEvent("Submission not found for instance directory: " + instanceDir.getPath(), briefcaseLfd));
        return false;
    }
    processedInstances++;
    EventBus.publish(new ExportProgressEvent("Processing instance: " + instanceDir.getName(), briefcaseLfd));
    EventBus.publish(new ExportProgressPercentageEvent((processedInstances * 100.0) / totalInstances, briefcaseLfd));
    // If we are encrypted, be sure the temporary directory
    // that will hold the unencrypted files is created.
    // If we aren't encrypted, the temporary directory
    // is the same as the instance directory.
    File unEncryptedDir;
    if (briefcaseLfd.isFileEncryptedForm()) {
        // create the temp directory that will hold the unencrypted
        // files. Do this in the outputDir so that the briefcase storage location
        // can be a read-only network mount. issue 676.
        Path path;
        try {
            path = Files.createTempDirectory(Paths.get(outputDir.toURI()), ".temp");
        } catch (IOException e) {
            String msg = "Unable to create temp directory.";
            log.error(msg, e);
            EventBus.publish(new ExportProgressEvent(msg + " Cause : " + e.toString(), briefcaseLfd));
            return false;
        }
        unEncryptedDir = path.toFile();
    } else {
        unEncryptedDir = instanceDir;
    }
    // parse the xml document (this is the manifest if encrypted)...
    Document doc;
    boolean isValidated = false;
    try {
        doc = XmlManipulationUtils.parseXml(submission);
    } catch (ParsingException | FileSystemException e) {
        log.error("Error parsing submission", e);
        EventBus.publish(new ExportProgressEvent(("Error parsing submission " + instanceDir.getName()) + " Cause: " + e.toString(), briefcaseLfd));
        return false;
    }
    String submissionDate = null;
    // extract the submissionDate, if present, from the attributes
    // of the root element of the submission or submission manifest (if encrypted).
    submissionDate = doc.getRootElement().getAttributeValue(null, "submissionDate");
    if (submissionDate == null || submissionDate.length() == 0) {
        submissionDate = null;
    } else {
        Date theDate = WebUtils.parseDate(submissionDate);
        DateFormat formatter = DateFormat.getDateTimeInstance();
        submissionDate = formatter.format(theDate);
        // just return true to skip records out of range
        if (startDate != null && theDate.before(startDate)) {
            log.info("Submission date is before specified, skipping: " + instanceDir.getName());
            return true;
        }
        if (endDate != null && theDate.after(endDate)) {
            log.info("Submission date is after specified, skipping: " + instanceDir.getName());
            return true;
        }
        // don't export records without dates if either date is set
        if ((startDate != null || endDate != null) && submissionDate == null) {
            log.info("No submission date found, skipping: " + instanceDir.getName());
            return true;
        }
    }
    // failure.
    try {
        if (briefcaseLfd.isFileEncryptedForm()) {
            // NOTE: this changes the value of 'doc'
            try {
                FileSystemUtils.DecryptOutcome outcome = FileSystemUtils.decryptAndValidateSubmission(doc, briefcaseLfd.getPrivateKey(), instanceDir, unEncryptedDir);
                doc = outcome.submission;
                isValidated = outcome.isValidated;
            } catch (ParsingException | CryptoException | FileSystemException e) {
                // Was unable to parse file or decrypt file or a file system error occurred
                // Hence skip this instance
                EventBus.publish(new ExportProgressEvent("Error decrypting submission " + instanceDir.getName() + " Cause: " + e.toString() + " skipping....", briefcaseLfd));
                log.info("Error decrypting submission " + instanceDir.getName() + " Cause: " + e.toString());
                // update total number of files skipped
                totalFilesSkipped++;
                return true;
            }
        }
        String instanceId = null;
        String base64EncryptedFieldKey = null;
        // find an instanceId to use...
        try {
            FormInstanceMetadata sim = XmlManipulationUtils.getFormInstanceMetadata(doc.getRootElement());
            instanceId = sim.instanceId;
            base64EncryptedFieldKey = sim.base64EncryptedFieldKey;
        } catch (ParsingException e) {
            log.error("Could not extract metadata from submission", e);
            EventBus.publish(new ExportProgressEvent("Could not extract metadata from submission " + submission.getAbsolutePath(), briefcaseLfd));
            return false;
        }
        if (instanceId == null || instanceId.length() == 0) {
            // if we have no instanceID, and there isn't any in the file,
            // use the checksum as the id.
            // NOTE: encrypted submissions always have instanceIDs.
            // This is for legacy non-OpenRosa forms.
            long checksum;
            try {
                checksum = FileUtils.checksumCRC32(submission);
            } catch (IOException e1) {
                String msg = "Failed during computing of crc";
                log.error(msg, e1);
                EventBus.publish(new ExportProgressEvent(msg + ": " + e1.getMessage(), briefcaseLfd));
                return false;
            }
            instanceId = "crc32:" + Long.toString(checksum);
        }
        if (terminationFuture.isCancelled()) {
            EventBus.publish(new ExportProgressEvent("Aborted", briefcaseLfd));
            return false;
        }
        EncryptionInformation ei = null;
        if (base64EncryptedFieldKey != null) {
            try {
                ei = new EncryptionInformation(base64EncryptedFieldKey, instanceId, briefcaseLfd.getPrivateKey());
            } catch (CryptoException e) {
                log.error("Error establishing field decryption", e);
                EventBus.publish(new ExportProgressEvent("Error establishing field decryption for submission " + instanceDir.getName() + " Cause: " + e.toString(), briefcaseLfd));
                return false;
            }
        }
        // emit the csv record...
        try {
            OutputStreamWriter osw = fileMap.get(briefcaseLfd.getSubmissionElement());
            emitString(osw, true, submissionDate);
            emitSubmissionCsv(osw, ei, doc.getRootElement(), briefcaseLfd.getSubmissionElement(), briefcaseLfd.getSubmissionElement(), false, instanceId, unEncryptedDir);
            emitString(osw, false, instanceId);
            if (briefcaseLfd.isFileEncryptedForm()) {
                emitString(osw, false, Boolean.toString(isValidated));
                if (!isValidated) {
                    EventBus.publish(new ExportProgressEvent("Decrypted submission " + instanceDir.getName() + " may be missing attachments and could not be validated.", briefcaseLfd));
                }
            }
            osw.append("\n");
            return true;
        } catch (IOException e) {
            String msg = "Failed writing csv";
            log.error(msg, e);
            EventBus.publish(new ExportProgressEvent(msg + ": " + e.getMessage(), briefcaseLfd));
            return false;
        }
    } finally {
        if (briefcaseLfd.isFileEncryptedForm()) {
            // destroy the temp directory and its contents...
            try {
                FileUtils.deleteDirectory(unEncryptedDir);
            } catch (IOException e) {
                String msg = "Unable to remove decrypted files";
                log.error(msg, e);
                EventBus.publish(new ExportProgressEvent(msg + ": " + e.getMessage(), briefcaseLfd));
                return false;
            }
        }
    }
}

Example 2 with FormInstanceMetadata

use of org.opendatakit.briefcase.util.XmlManipulationUtils.FormInstanceMetadata in project briefcase by opendatakit.

the class FileSystemUtils method decryptAndValidateSubmission.

public static DecryptOutcome decryptAndValidateSubmission(Document doc, PrivateKey rsaPrivateKey, File instanceDir, File unEncryptedDir) throws ParsingException, FileSystemException, CryptoException {
    Element rootElement = doc.getRootElement();
    String base64EncryptedSymmetricKey;
    String instanceIdMetadata = null;
    List<String> mediaNames = new ArrayList<>();
    String encryptedSubmissionFile;
    String base64EncryptedElementSignature;
    {
        Element base64Key = null;
        Element base64Signature = null;
        Element encryptedXml = null;
        for (int i = 0; i < rootElement.getChildCount(); ++i) {
            if (rootElement.getType(i) == Node.ELEMENT) {
                Element child = rootElement.getElement(i);
                String name = child.getName();
                if (name.equals("base64EncryptedKey")) {
                    base64Key = child;
                } else if (name.equals("base64EncryptedElementSignature")) {
                    base64Signature = child;
                } else if (name.equals("encryptedXmlFile")) {
                    encryptedXml = child;
                } else if (name.equals("media")) {
                    Element media = child;
                    for (int j = 0; j < media.getChildCount(); ++j) {
                        if (media.getType(j) == Node.ELEMENT) {
                            Element mediaChild = media.getElement(j);
                            String mediaFileElementName = mediaChild.getName();
                            if (mediaFileElementName.equals("file")) {
                                String mediaName = XFormParser.getXMLText(mediaChild, true);
                                if (mediaName == null || mediaName.length() == 0) {
                                    mediaNames.add(null);
                                } else {
                                    mediaNames.add(mediaName);
                                }
                            }
                        }
                    }
                }
            }
        }
        // verify base64Key
        if (base64Key == null) {
            throw new ParsingException("Missing base64EncryptedKey element in encrypted form.");
        }
        base64EncryptedSymmetricKey = XFormParser.getXMLText(base64Key, true);
        // get instanceID out of OpenRosa meta block
        instanceIdMetadata = XmlManipulationUtils.getOpenRosaInstanceId(rootElement);
        if (instanceIdMetadata == null) {
            throw new ParsingException("Missing instanceID within meta block of encrypted form.");
        }
        // get submission filename
        if (encryptedXml == null) {
            throw new ParsingException("Missing encryptedXmlFile element in encrypted form.");
        }
        encryptedSubmissionFile = XFormParser.getXMLText(encryptedXml, true);
        if (base64Signature == null) {
            throw new ParsingException("Missing base64EncryptedElementSignature element in encrypted form.");
        }
        base64EncryptedElementSignature = XFormParser.getXMLText(base64Signature, true);
    }
    if (instanceIdMetadata == null || base64EncryptedSymmetricKey == null || base64EncryptedElementSignature == null || encryptedSubmissionFile == null) {
        throw new ParsingException("Missing one or more required elements of encrypted form.");
    }
    FormInstanceMetadata fim;
    try {
        fim = XmlManipulationUtils.getFormInstanceMetadata(rootElement);
    } catch (ParsingException e) {
        String msg = "Unable to extract form instance metadata from submission manifest";
        log.error(msg, e);
        throw new ParsingException(msg + ". Cause: " + e.toString());
    }
    if (!instanceIdMetadata.equals(fim.instanceId)) {
        throw new ParsingException("InstanceID within metadata does not match that on top level element.");
    }
    boolean isValidated = FileSystemUtils.decryptSubmissionFiles(base64EncryptedSymmetricKey, fim, mediaNames, encryptedSubmissionFile, base64EncryptedElementSignature, rsaPrivateKey, instanceDir, unEncryptedDir);
    // and change doc to be the decrypted submission document
    File decryptedSubmission = new File(unEncryptedDir, "submission.xml");
    doc = XmlManipulationUtils.parseXml(decryptedSubmission);
    if (doc == null) {
        return null;
    }
    // verify that the metadata matches between the manifest and the submission
    rootElement = doc.getRootElement();
    FormInstanceMetadata sim = XmlManipulationUtils.getFormInstanceMetadata(rootElement);
    if (!fim.xparam.equals(sim.xparam)) {
        throw new ParsingException("FormId or version in decrypted submission does not match that in manifest!");
    }
    if (!fim.instanceId.equals(sim.instanceId)) {
        throw new ParsingException("InstanceId in decrypted submission does not match that in manifest!");
    }
    return new DecryptOutcome(doc, isValidated);
}

Example 3 with FormInstanceMetadata

use of org.opendatakit.briefcase.util.XmlManipulationUtils.FormInstanceMetadata in project briefcase by opendatakit.

the class FileSystemUtils method decryptSubmissionFiles.

private static boolean decryptSubmissionFiles(String base64EncryptedSymmetricKey, FormInstanceMetadata fim, List<String> mediaNames, String encryptedSubmissionFile, String base64EncryptedElementSignature, PrivateKey rsaPrivateKey, File instanceDir, File unencryptedDir) throws FileSystemException, CryptoException, ParsingException {
    EncryptionInformation ei = new EncryptionInformation(base64EncryptedSymmetricKey, fim.instanceId, rsaPrivateKey);
    byte[] elementDigest;
    try {
        // construct the base64-encoded RSA-encrypted symmetric key
        Cipher pkCipher;
        pkCipher = Cipher.getInstance(ASYMMETRIC_ALGORITHM);
        // extract digest
        pkCipher.init(Cipher.DECRYPT_MODE, rsaPrivateKey);
        byte[] encryptedElementSignature = Base64.decodeBase64(base64EncryptedElementSignature);
        elementDigest = pkCipher.doFinal(encryptedElementSignature);
    } catch (NoSuchAlgorithmException | BadPaddingException | IllegalBlockSizeException | InvalidKeyException | NoSuchPaddingException e) {
        String msg = "Error decrypting base64EncryptedElementSignature";
        log.error(msg, e);
        throw new CryptoException(msg + " Cause: " + e.toString());
    }
    // NOTE: will decrypt only the files in the media list, plus the encryptedSubmissionFile
    File[] allFiles = instanceDir.listFiles();
    List<File> filesToProcess = new ArrayList<>();
    for (File f : allFiles) {
        if (mediaNames.contains(f.getName())) {
            filesToProcess.add(f);
        } else if (encryptedSubmissionFile.equals(f.getName())) {
            filesToProcess.add(f);
        }
    }
    // should have all media files plus one submission.xml.enc file
    if (filesToProcess.size() != mediaNames.size() + 1) {
        // figure out what we're missing...
        int lostFileCount = 0;
        List<String> missing = new ArrayList<>();
        for (String name : mediaNames) {
            if (name == null) {
                // this was lost due to an pre-ODK Aggregate 1.4.5 mark-as-complete action
                ++lostFileCount;
                continue;
            }
            File f = new File(instanceDir, name);
            if (!filesToProcess.contains(f)) {
                missing.add(name);
            }
        }
        StringBuilder b = new StringBuilder();
        for (String name : missing) {
            b.append(" ").append(name);
        }
        if (!filesToProcess.contains(new File(instanceDir, encryptedSubmissionFile))) {
            b.append(" ").append(encryptedSubmissionFile);
            throw new FileSystemException("Error decrypting: " + instanceDir.getName() + " Missing files:" + b.toString());
        } else {
            // ignore the fact that we don't have the lost files
            if (filesToProcess.size() + lostFileCount != mediaNames.size() + 1) {
                throw new FileSystemException("Error decrypting: " + instanceDir.getName() + " Missing files:" + b.toString());
            }
        }
    }
    // decrypt the media files IN ORDER.
    for (String mediaName : mediaNames) {
        String displayedName = (mediaName == null) ? "<missing .enc file>" : mediaName;
        File f = (mediaName == null) ? null : new File(instanceDir, mediaName);
        try {
            decryptFile(ei, f, unencryptedDir);
        } catch (InvalidKeyException | NoSuchPaddingException | InvalidAlgorithmParameterException | NoSuchAlgorithmException e) {
            String msg = "Error decrypting:" + displayedName;
            log.error(msg, e);
            throw new CryptoException(msg + " Cause: " + e.toString());
        } catch (IOException e) {
            String msg = "Error decrypting:" + displayedName;
            log.error(msg, e);
            throw new FileSystemException(msg + " Cause: " + e.toString());
        }
    }
    // decrypt the submission file
    File f = new File(instanceDir, encryptedSubmissionFile);
    try {
        decryptFile(ei, f, unencryptedDir);
    } catch (InvalidKeyException | NoSuchPaddingException | InvalidAlgorithmParameterException | NoSuchAlgorithmException e) {
        log.error("Error decrypting file", e);
        throw new CryptoException("Error decrypting:" + f.getName() + " Cause: " + e.toString());
    } catch (IOException e) {
        log.error("Error decrypting file", e);
        throw new FileSystemException("Error decrypting:" + f.getName() + " Cause: " + e.toString());
    }
    // get the FIM for the decrypted submission file
    File submissionFile = new File(unencryptedDir, encryptedSubmissionFile.substring(0, encryptedSubmissionFile.lastIndexOf(".enc")));
    FormInstanceMetadata submissionFim;
    try {
        Document subDoc = XmlManipulationUtils.parseXml(submissionFile);
        submissionFim = XmlManipulationUtils.getFormInstanceMetadata(subDoc.getRootElement());
    } catch (ParsingException | FileSystemException e) {
        log.error("Error decrypting submission", e);
        throw new FileSystemException("Error decrypting: " + submissionFile.getName() + " Cause: " + e);
    }
    boolean same = submissionFim.xparam.formId.equals(fim.xparam.formId);
    if (!same) {
        throw new FileSystemException("Error decrypting:" + unencryptedDir.getName() + " Cause: form instance metadata differs from that in manifest");
    }
    // Construct the element signature string
    StringBuilder b = new StringBuilder();
    appendElementSignatureSource(b, fim.xparam.formId);
    if (fim.xparam.modelVersion != null) {
        appendElementSignatureSource(b, fim.xparam.modelVersion);
    }
    appendElementSignatureSource(b, base64EncryptedSymmetricKey);
    appendElementSignatureSource(b, fim.instanceId);
    boolean missingFile = false;
    for (String encFilename : mediaNames) {
        if (encFilename == null) {
            missingFile = true;
            continue;
        }
        File decryptedFile = new File(unencryptedDir, encFilename.substring(0, encFilename.lastIndexOf(".enc")));
        if (decryptedFile.getName().endsWith(".missing")) {
            // this is a missing file -- we will not be able to
            // confirm the signature of the submission.
            missingFile = true;
            continue;
        }
        String md5 = FileSystemUtils.getMd5Hash(decryptedFile);
        appendElementSignatureSource(b, decryptedFile.getName() + "::" + md5);
    }
    String md5 = FileSystemUtils.getMd5Hash(submissionFile);
    appendElementSignatureSource(b, submissionFile.getName() + "::" + md5);
    // compute the digest of the element signature string
    byte[] messageDigest;
    try {
        MessageDigest md = MessageDigest.getInstance("MD5");
        md.update(b.toString().getBytes("UTF-8"));
        messageDigest = md.digest();
    } catch (NoSuchAlgorithmException | UnsupportedEncodingException e) {
        String msg = "Error computing xml signature";
        log.error(msg, e);
        throw new CryptoException(msg + " Cause: " + e);
    }
    same = true;
    for (int i = 0; i < messageDigest.length; ++i) {
        if (messageDigest[i] != elementDigest[i]) {
            same = false;
            break;
        }
    }
    return same;
}