Examples with ExportProgressEvent org.opendatakit.briefcase.model.ExportProgressEvent used on opensource projects

Example 1 with ExportProgressEvent

use of org.opendatakit.briefcase.model.ExportProgressEvent in project briefcase by opendatakit.

the class FormsTableUnitTest method appends_to_a_forms_status_history_when_export_events_are_sent.

@Test
@Ignore
public void appends_to_a_forms_status_history_when_export_events_are_sent() {
    FormStatus theForm = buildFormStatus(1);
    ExportForms forms = new ExportForms(Collections.singletonList(theForm), ExportConfiguration.empty(), new HashMap<>(), new HashMap<>(), new HashMap<>());
    TestFormsTableViewModel viewModel = new TestFormsTableViewModel(forms);
    new FormsTable(forms, new TestFormsTableView(viewModel), viewModel);
    // TODO Event publishing happens asynchronously. We have to work this test a little more to stop ignoring it
    EventBus.publish(new ExportProgressEvent("some progress", (BriefcaseFormDefinition) theForm.getFormDefinition()));
    EventBus.publish(new ExportFailedEvent((BriefcaseFormDefinition) theForm.getFormDefinition()));
    EventBus.publish(new ExportSucceededEvent((BriefcaseFormDefinition) theForm.getFormDefinition()));
    EventBus.publish(new ExportSucceededWithErrorsEvent((BriefcaseFormDefinition) theForm.getFormDefinition()));
    assertThat(theForm.getStatusHistory(), Matchers.containsString("some progress"));
    assertThat(theForm.getStatusHistory(), Matchers.containsString("Failed."));
    assertThat(theForm.getStatusHistory(), Matchers.containsString("Succeeded."));
    assertThat(theForm.getStatusHistory(), Matchers.containsString("Succeeded, but with errors."));
}

Example 2 with ExportProgressEvent

use of org.opendatakit.briefcase.model.ExportProgressEvent in project briefcase by opendatakit.

the class ExportToCsv method processInstance.

private boolean processInstance(File instanceDir) {
    File submission = new File(instanceDir, "submission.xml");
    if (!submission.exists() || !submission.isFile()) {
        EventBus.publish(new ExportProgressEvent("Submission not found for instance directory: " + instanceDir.getPath(), briefcaseLfd));
        return false;
    }
    processedInstances++;
    EventBus.publish(new ExportProgressEvent("Processing instance: " + instanceDir.getName(), briefcaseLfd));
    EventBus.publish(new ExportProgressPercentageEvent((processedInstances * 100.0) / totalInstances, briefcaseLfd));
    // If we are encrypted, be sure the temporary directory
    // that will hold the unencrypted files is created.
    // If we aren't encrypted, the temporary directory
    // is the same as the instance directory.
    File unEncryptedDir;
    if (briefcaseLfd.isFileEncryptedForm()) {
        // create the temp directory that will hold the unencrypted
        // files. Do this in the outputDir so that the briefcase storage location
        // can be a read-only network mount. issue 676.
        Path path;
        try {
            path = Files.createTempDirectory(Paths.get(outputDir.toURI()), ".temp");
        } catch (IOException e) {
            String msg = "Unable to create temp directory.";
            log.error(msg, e);
            EventBus.publish(new ExportProgressEvent(msg + " Cause : " + e.toString(), briefcaseLfd));
            return false;
        }
        unEncryptedDir = path.toFile();
    } else {
        unEncryptedDir = instanceDir;
    }
    // parse the xml document (this is the manifest if encrypted)...
    Document doc;
    boolean isValidated = false;
    try {
        doc = XmlManipulationUtils.parseXml(submission);
    } catch (ParsingException | FileSystemException e) {
        log.error("Error parsing submission", e);
        EventBus.publish(new ExportProgressEvent(("Error parsing submission " + instanceDir.getName()) + " Cause: " + e.toString(), briefcaseLfd));
        return false;
    }
    String submissionDate = null;
    // extract the submissionDate, if present, from the attributes
    // of the root element of the submission or submission manifest (if encrypted).
    submissionDate = doc.getRootElement().getAttributeValue(null, "submissionDate");
    if (submissionDate == null || submissionDate.length() == 0) {
        submissionDate = null;
    } else {
        Date theDate = WebUtils.parseDate(submissionDate);
        DateFormat formatter = DateFormat.getDateTimeInstance();
        submissionDate = formatter.format(theDate);
        // just return true to skip records out of range
        if (startDate != null && theDate.before(startDate)) {
            log.info("Submission date is before specified, skipping: " + instanceDir.getName());
            return true;
        }
        if (endDate != null && theDate.after(endDate)) {
            log.info("Submission date is after specified, skipping: " + instanceDir.getName());
            return true;
        }
        // don't export records without dates if either date is set
        if ((startDate != null || endDate != null) && submissionDate == null) {
            log.info("No submission date found, skipping: " + instanceDir.getName());
            return true;
        }
    }
    // failure.
    try {
        if (briefcaseLfd.isFileEncryptedForm()) {
            // NOTE: this changes the value of 'doc'
            try {
                FileSystemUtils.DecryptOutcome outcome = FileSystemUtils.decryptAndValidateSubmission(doc, briefcaseLfd.getPrivateKey(), instanceDir, unEncryptedDir);
                doc = outcome.submission;
                isValidated = outcome.isValidated;
            } catch (ParsingException | CryptoException | FileSystemException e) {
                // Was unable to parse file or decrypt file or a file system error occurred
                // Hence skip this instance
                EventBus.publish(new ExportProgressEvent("Error decrypting submission " + instanceDir.getName() + " Cause: " + e.toString() + " skipping....", briefcaseLfd));
                log.info("Error decrypting submission " + instanceDir.getName() + " Cause: " + e.toString());
                // update total number of files skipped
                totalFilesSkipped++;
                return true;
            }
        }
        String instanceId = null;
        String base64EncryptedFieldKey = null;
        // find an instanceId to use...
        try {
            FormInstanceMetadata sim = XmlManipulationUtils.getFormInstanceMetadata(doc.getRootElement());
            instanceId = sim.instanceId;
            base64EncryptedFieldKey = sim.base64EncryptedFieldKey;
        } catch (ParsingException e) {
            log.error("Could not extract metadata from submission", e);
            EventBus.publish(new ExportProgressEvent("Could not extract metadata from submission " + submission.getAbsolutePath(), briefcaseLfd));
            return false;
        }
        if (instanceId == null || instanceId.length() == 0) {
            // if we have no instanceID, and there isn't any in the file,
            // use the checksum as the id.
            // NOTE: encrypted submissions always have instanceIDs.
            // This is for legacy non-OpenRosa forms.
            long checksum;
            try {
                checksum = FileUtils.checksumCRC32(submission);
            } catch (IOException e1) {
                String msg = "Failed during computing of crc";
                log.error(msg, e1);
                EventBus.publish(new ExportProgressEvent(msg + ": " + e1.getMessage(), briefcaseLfd));
                return false;
            }
            instanceId = "crc32:" + Long.toString(checksum);
        }
        if (terminationFuture.isCancelled()) {
            EventBus.publish(new ExportProgressEvent("Aborted", briefcaseLfd));
            return false;
        }
        EncryptionInformation ei = null;
        if (base64EncryptedFieldKey != null) {
            try {
                ei = new EncryptionInformation(base64EncryptedFieldKey, instanceId, briefcaseLfd.getPrivateKey());
            } catch (CryptoException e) {
                log.error("Error establishing field decryption", e);
                EventBus.publish(new ExportProgressEvent("Error establishing field decryption for submission " + instanceDir.getName() + " Cause: " + e.toString(), briefcaseLfd));
                return false;
            }
        }
        // emit the csv record...
        try {
            OutputStreamWriter osw = fileMap.get(briefcaseLfd.getSubmissionElement());
            emitString(osw, true, submissionDate);
            emitSubmissionCsv(osw, ei, doc.getRootElement(), briefcaseLfd.getSubmissionElement(), briefcaseLfd.getSubmissionElement(), false, instanceId, unEncryptedDir);
            emitString(osw, false, instanceId);
            if (briefcaseLfd.isFileEncryptedForm()) {
                emitString(osw, false, Boolean.toString(isValidated));
                if (!isValidated) {
                    EventBus.publish(new ExportProgressEvent("Decrypted submission " + instanceDir.getName() + " may be missing attachments and could not be validated.", briefcaseLfd));
                }
            }
            osw.append("\n");
            return true;
        } catch (IOException e) {
            String msg = "Failed writing csv";
            log.error(msg, e);
            EventBus.publish(new ExportProgressEvent(msg + ": " + e.getMessage(), briefcaseLfd));
            return false;
        }
    } finally {
        if (briefcaseLfd.isFileEncryptedForm()) {
            // destroy the temp directory and its contents...
            try {
                FileUtils.deleteDirectory(unEncryptedDir);
            } catch (IOException e) {
                String msg = "Unable to remove decrypted files";
                log.error(msg, e);
                EventBus.publish(new ExportProgressEvent(msg + ": " + e.getMessage(), briefcaseLfd));
                return false;
            }
        }
    }
}

Example 3 with ExportProgressEvent

use of org.opendatakit.briefcase.model.ExportProgressEvent in project briefcase by opendatakit.

the class ExportToCsv method emitSubmissionCsv.

private boolean emitSubmissionCsv(OutputStreamWriter osw, EncryptionInformation ei, Element submissionElement, TreeElement primarySet, TreeElement treeElement, boolean first, String uniquePath, File instanceDir) throws IOException {
    // OK -- group with at least one element -- assume no value...
    // TreeElement list has the begin and end tags for the nested groups.
    // Swallow the end tag by looking to see if the prior and current
    // field names are the same.
    TreeElement prior = null;
    for (int i = 0; i < treeElement.getNumChildren(); ++i) {
        TreeElement current = (TreeElement) treeElement.getChildAt(i);
        log.debug(" element name: " + current.getName());
        if ((prior != null) && (prior.getName().equals(current.getName()))) {
            // it is the end-group tag... seems to happen with two adjacent repeat
            // groups
            log.info("repeating tag at " + i + " skipping " + current.getName());
            prior = current;
        } else {
            Element ec = findElement(submissionElement, current.getName());
            switch(current.getDataType()) {
                case org.javarosa.core.model.Constants.DATATYPE_TEXT:
                /**
                 * Text question
                 * type.
                 */
                case org.javarosa.core.model.Constants.DATATYPE_INTEGER:
                /**
                 * Numeric
                 * question type. These are numbers without decimal points
                 */
                case org.javarosa.core.model.Constants.DATATYPE_DECIMAL:
                /**
                 * Decimal
                 * question type. These are numbers with decimals
                 */
                case org.javarosa.core.model.Constants.DATATYPE_CHOICE:
                /**
                 * This is a
                 * question with alist of options where not more than one option can
                 * be selected at a time.
                 */
                case org.javarosa.core.model.Constants.DATATYPE_CHOICE_LIST:
                /**
                 * This is a
                 * question with alist of options where more than one option can be
                 * selected at a time.
                 */
                case org.javarosa.core.model.Constants.DATATYPE_BOOLEAN:
                /**
                 * Question with
                 * true and false answers.
                 */
                case org.javarosa.core.model.Constants.DATATYPE_BARCODE:
                /**
                 * Question with
                 * barcode string answer.
                 */
                default:
                case org.javarosa.core.model.Constants.DATATYPE_UNSUPPORTED:
                    if (ec == null) {
                        emitString(osw, first, null);
                    } else {
                        emitString(osw, first, getSubmissionValue(ei, current, ec));
                    }
                    first = false;
                    break;
                case org.javarosa.core.model.Constants.DATATYPE_DATE:
                    /**
                     * Date question type. This has only date component without time.
                     */
                    if (ec == null) {
                        emitString(osw, first, null);
                    } else {
                        String value = getSubmissionValue(ei, current, ec);
                        if (value == null || value.length() == 0) {
                            emitString(osw, first, null);
                        } else {
                            Date date = WebUtils.parseDate(value);
                            DateFormat formatter = DateFormat.getDateInstance();
                            emitString(osw, first, formatter.format(date));
                        }
                    }
                    first = false;
                    break;
                case org.javarosa.core.model.Constants.DATATYPE_TIME:
                    /**
                     * Time question type. This has only time element without date
                     */
                    if (ec == null) {
                        emitString(osw, first, null);
                    } else {
                        String value = getSubmissionValue(ei, current, ec);
                        if (value == null || value.length() == 0) {
                            emitString(osw, first, null);
                        } else {
                            Date date = WebUtils.parseDate(value);
                            DateFormat formatter = DateFormat.getTimeInstance();
                            emitString(osw, first, formatter.format(date));
                        }
                    }
                    first = false;
                    break;
                case org.javarosa.core.model.Constants.DATATYPE_DATE_TIME:
                    /**
                     * Date and Time question type. This has both the date and time
                     * components
                     */
                    if (ec == null) {
                        emitString(osw, first, null);
                    } else {
                        String value = getSubmissionValue(ei, current, ec);
                        if (value == null || value.length() == 0) {
                            emitString(osw, first, null);
                        } else {
                            Date date = WebUtils.parseDate(value);
                            DateFormat formatter = DateFormat.getDateTimeInstance();
                            emitString(osw, first, formatter.format(date));
                        }
                    }
                    first = false;
                    break;
                case org.javarosa.core.model.Constants.DATATYPE_GEOPOINT:
                    /**
                     * Question with location answer.
                     */
                    String compositeValue = (ec == null) ? null : getSubmissionValue(ei, current, ec);
                    compositeValue = (compositeValue == null) ? null : compositeValue.trim();
                    // emit separate lat, long, alt, acc columns...
                    if (compositeValue == null || compositeValue.length() == 0) {
                        for (int count = 0; count < 4; ++count) {
                            emitString(osw, first, null);
                            first = false;
                        }
                    } else {
                        String[] values = compositeValue.split(" ");
                        for (String value : values) {
                            emitString(osw, first, value);
                            first = false;
                        }
                        for (int count = values.length; count < 4; ++count) {
                            emitString(osw, first, null);
                            first = false;
                        }
                    }
                    break;
                case org.javarosa.core.model.Constants.DATATYPE_BINARY:
                    /**
                     * Question with external binary answer.
                     */
                    String binaryFilename = getSubmissionValue(ei, current, ec);
                    if (binaryFilename == null || binaryFilename.length() == 0) {
                        emitString(osw, first, null);
                        first = false;
                    } else {
                        if (exportMedia) {
                            if (!outputMediaDir.exists()) {
                                if (!outputMediaDir.mkdir()) {
                                    EventBus.publish(new ExportProgressEvent("Unable to create destination media directory", briefcaseLfd));
                                    return false;
                                }
                            }
                            int dotIndex = binaryFilename.lastIndexOf(".");
                            String namePart = (dotIndex == -1) ? binaryFilename : binaryFilename.substring(0, dotIndex);
                            String extPart = (dotIndex == -1) ? "" : binaryFilename.substring(dotIndex);
                            File binaryFile = new File(instanceDir, binaryFilename);
                            String destBinaryFilename = binaryFilename;
                            int version = 1;
                            File destFile = new File(outputMediaDir, destBinaryFilename);
                            boolean exists = false;
                            String binaryFileHash = null;
                            String destFileHash = null;
                            if (destFile.exists() && binaryFile.exists()) {
                                binaryFileHash = FileSystemUtils.getMd5Hash(binaryFile);
                                while (destFile.exists()) {
                                    if (fileHashMap.containsKey(destFile.getName())) {
                                        destFileHash = fileHashMap.get(destFile.getName());
                                    } else {
                                        destFileHash = FileSystemUtils.getMd5Hash(destFile);
                                        if (destFileHash != null) {
                                            fileHashMap.put(destFile.getName(), destFileHash);
                                        }
                                    }
                                    if (binaryFileHash != null && destFileHash != null && destFileHash.equals(binaryFileHash)) {
                                        exists = true;
                                        break;
                                    }
                                    destBinaryFilename = namePart + "-" + (++version) + extPart;
                                    destFile = new File(outputMediaDir, destBinaryFilename);
                                }
                            }
                            if (binaryFile.exists() && exists == false) {
                                FileUtils.copyFile(binaryFile, destFile);
                            }
                            emitString(osw, first, MEDIA_DIR + File.separator + destFile.getName());
                        } else {
                            emitString(osw, first, binaryFilename);
                        }
                        first = false;
                    }
                    break;
                case org.javarosa.core.model.Constants.DATATYPE_NULL:
                    /*
           * for nodes that
           * have no data,
           * or data type
           * otherwise
           * unknown
           */
                    if (current.isRepeatable()) {
                        if (prior == null || !current.getName().equals(prior.getName())) {
                            // repeatable group...
                            if (ec == null) {
                                emitString(osw, first, null);
                                first = false;
                            } else {
                                String uniqueGroupPath = uniquePath + "/" + getFullName(current, primarySet);
                                emitString(osw, first, uniqueGroupPath);
                                first = false;
                                // first time processing this repeat group (ignore templates)
                                List<Element> ecl = findElementList(submissionElement, current.getName());
                                emitRepeatingGroupCsv(ei, ecl, current, uniquePath, uniqueGroupPath, instanceDir);
                            }
                        }
                    } else if (current.getNumChildren() == 0 && current != briefcaseLfd.getSubmissionElement()) {
                        // assume fields that don't have children are string fields.
                        if (ec == null) {
                            emitString(osw, first, null);
                            first = false;
                        } else {
                            emitString(osw, first, getSubmissionValue(ei, current, ec));
                            first = false;
                        }
                    } else {
                        /* one or more children -- this is a non-repeating group */
                        first = emitSubmissionCsv(osw, ei, ec, primarySet, current, first, uniquePath, instanceDir);
                    }
                    break;
            }
            prior = current;
        }
    }
    return first;
}

Example 4 with ExportProgressEvent

use of org.opendatakit.briefcase.model.ExportProgressEvent in project briefcase by opendatakit.

the class ExportToCsv method processFormDefinition.

private boolean processFormDefinition() {
    TreeElement submission = briefcaseLfd.getSubmissionElement();
    String formName = baseFilename;
    File topLevelCsv = new File(outputDir, safeFilename(formName) + ".csv");
    log.info("Trying to create top level CSV file at {}", topLevelCsv);
    boolean exists = topLevelCsv.exists();
    FileOutputStream os;
    try {
        os = new FileOutputStream(topLevelCsv, !overwrite);
        OutputStreamWriter osw = new OutputStreamWriter(os, "UTF-8");
        fileMap.put(submission, osw);
        // only write headers if overwrite is set, or creating file for the first time
        if (overwrite || !exists) {
            emitString(osw, true, "SubmissionDate");
            emitCsvHeaders(osw, submission, submission, false);
            emitString(osw, false, "KEY");
            if (briefcaseLfd.isFileEncryptedForm()) {
                emitString(osw, false, "isValidated");
            }
            osw.append("\n");
        } else {
            populateRepeatGroupsIntoFileMap(submission, submission);
        }
    } catch (IOException e) {
        String msg = "Unable to create csv file";
        log.error(msg, e);
        EventBus.publish(new ExportProgressEvent(msg, briefcaseLfd));
        for (OutputStreamWriter w : fileMap.values()) {
            try {
                w.close();
            } catch (IOException e1) {
                log.warn("failed to close writer", e1);
            }
        }
        fileMap.clear();
        return false;
    }
    return true;
}

Example 5 with ExportProgressEvent

use of org.opendatakit.briefcase.model.ExportProgressEvent in project briefcase by opendatakit.

the class ExportToCsv method doAction.

@Override
public boolean doAction() {
    boolean allSuccessful = true;
    File instancesDir;
    try {
        instancesDir = FileSystemUtils.getFormInstancesDirectory(briefcaseLfd.getFormDirectory());
    } catch (FileSystemException e) {
        String msg = "Unable to access instances directory of form";
        log.error(msg, e);
        EventBus.publish(new ExportProgressEvent(msg, briefcaseLfd));
        return false;
    }
    if (!outputDir.exists()) {
        if (!outputDir.mkdir()) {
            EventBus.publish(new ExportProgressEvent("Unable to create destination directory", briefcaseLfd));
            return false;
        }
    }
    if (!processFormDefinition()) {
        // weren't able to initialize the csv file...
        return false;
    }
    File[] instances = instancesDir.listFiles(file -> file.isDirectory() && new File(file, "submission.xml").exists());
    totalInstances = instances.length;
    // assume it to be latest.
    if (instances != null) {
        Arrays.sort(instances, (f1, f2) -> {
            try {
                if (f1.isDirectory() && f2.isDirectory()) {
                    File submission1 = new File(f1, "submission.xml");
                    String submissionDate1String = XmlManipulationUtils.parseXml(submission1).getRootElement().getAttributeValue(null, "submissionDate");
                    File submission2 = new File(f2, "submission.xml");
                    String submissionDate2String = XmlManipulationUtils.parseXml(submission2).getRootElement().getAttributeValue(null, "submissionDate");
                    Date submissionDate1 = StringUtils.isNotEmptyNotNull(submissionDate1String) ? WebUtils.parseDate(submissionDate1String) : new Date();
                    Date submissionDate2 = StringUtils.isNotEmptyNotNull(submissionDate2String) ? WebUtils.parseDate(submissionDate2String) : new Date();
                    return submissionDate1.compareTo(submissionDate2);
                }
            } catch (ParsingException | FileSystemException e) {
                log.error("failed to sort submissions", e);
            }
            return 0;
        });
    }
    for (File instanceDir : instances) {
        if (terminationFuture.isCancelled()) {
            EventBus.publish(new ExportProgressEvent("Aborted", briefcaseLfd));
            allSuccessful = false;
            break;
        }
        if (instanceDir.getName().startsWith("."))
            // Mac OSX
            continue;
        allSuccessful = allSuccessful && processInstance(instanceDir);
    }
    for (OutputStreamWriter w : fileMap.values()) {
        try {
            w.flush();
            w.close();
        } catch (IOException e) {
            String msg = "Error flushing csv file";
            EventBus.publish(new ExportProgressEvent(msg, briefcaseLfd));
            log.error(msg, e);
            allSuccessful = false;
        }
    }
    return allSuccessful;
}