Examples with RemoteFormDefinition org.opendatakit.briefcase.model.RemoteFormDefinition used on opensource projects

Example 1 with RemoteFormDefinition

use of org.opendatakit.briefcase.model.RemoteFormDefinition in project briefcase by opendatakit.

the class RetrieveAvailableFormsFromServer method doAction.

public void doAction() throws XmlDocumentFetchException, ParsingException {
    List<RemoteFormDefinition> formDefs = Collections.emptyList();
    formDefs = ServerFetcher.retrieveAvailableFormsFromServer(originServerInfo, terminationFuture);
    for (IFormDefinition fd : formDefs) {
        formStatuses.add(new FormStatus(FormStatus.TransferType.GATHER, fd));
    }
}

Example 2 with RemoteFormDefinition

use of org.opendatakit.briefcase.model.RemoteFormDefinition in project briefcase by opendatakit.

the class ServerFetcher method downloadManifestAndMediaFiles.

private String downloadManifestAndMediaFiles(File mediaDir, FormStatus fs) {
    RemoteFormDefinition fd = getRemoteFormDefinition(fs);
    if (fd.getManifestUrl() == null)
        return null;
    fs.setStatusString("Fetching form manifest", true);
    EventBus.publish(new FormStatusEvent(fs));
    List<MediaFile> files = new ArrayList<>();
    AggregateUtils.DocumentFetchResult result;
    try {
        DocumentDescription formManifestDescription = new DocumentDescription("Fetch of manifest failed. Detailed reason: ", "Fetch of manifest failed ", "form manifest", terminationFuture);
        result = AggregateUtils.getXmlDocument(fd.getManifestUrl(), serverInfo, false, formManifestDescription, null);
    } catch (XmlDocumentFetchException e) {
        return e.getMessage();
    }
    try {
        files = XmlManipulationUtils.parseFormManifestResponse(result.isOpenRosaResponse, result.doc);
    } catch (ParsingException e) {
        return e.getMessage();
    }
    // OK we now have the full set of files to download...
    log.info("Downloading " + files.size() + " media files.");
    int mCount = 0;
    if (files.size() > 0) {
        for (MediaFile m : files) {
            ++mCount;
            fs.setStatusString(String.format(" (getting %1$d of %2$d media files)", mCount, files.size()), true);
            EventBus.publish(new FormStatusEvent(fs));
            try {
                downloadMediaFileIfChanged(mediaDir, m, fs);
            } catch (Exception e) {
                return e.getLocalizedMessage();
            }
        }
    }
    return null;
}

Example 3 with RemoteFormDefinition

use of org.opendatakit.briefcase.model.RemoteFormDefinition in project briefcase by opendatakit.

the class XmlManipulationUtils method parseFormListResponse.

public static final List<RemoteFormDefinition> parseFormListResponse(boolean isOpenRosaResponse, Document formListDoc) throws ParsingException {
    // This gets a list of available forms from the specified server.
    List<RemoteFormDefinition> formList = new ArrayList<>();
    if (isOpenRosaResponse) {
        // Attempt OpenRosa 1.0 parsing
        Element xformsElement = formListDoc.getRootElement();
        if (!xformsElement.getName().equals("xforms")) {
            log.error("Parsing OpenRosa reply -- root element is not <xforms> :" + xformsElement.getName());
            throw new ParsingException(BAD_OPENROSA_FORMLIST);
        }
        String namespace = xformsElement.getNamespace();
        if (!isXformsListNamespacedElement(xformsElement)) {
            log.error("Parsing OpenRosa reply -- root element namespace is incorrect:" + namespace);
            throw new ParsingException(BAD_OPENROSA_FORMLIST);
        }
        int nElements = xformsElement.getChildCount();
        for (int i = 0; i < nElements; ++i) {
            if (xformsElement.getType(i) != Element.ELEMENT) {
                // e.g., whitespace (text)
                continue;
            }
            Element xformElement = (Element) xformsElement.getElement(i);
            if (!isXformsListNamespacedElement(xformElement)) {
                // someone else's extension?
                continue;
            }
            String name = xformElement.getName();
            if (!name.equalsIgnoreCase("xform")) {
                // someone else's extension?
                continue;
            }
            // this is something we know how to interpret
            String formId = null;
            String formName = null;
            String version = null;
            String majorMinorVersion = null;
            String description = null;
            String downloadUrl = null;
            String manifestUrl = null;
            // don't process descriptionUrl
            int fieldCount = xformElement.getChildCount();
            for (int j = 0; j < fieldCount; ++j) {
                if (xformElement.getType(j) != Element.ELEMENT) {
                    // whitespace
                    continue;
                }
                Element child = xformElement.getElement(j);
                if (!isXformsListNamespacedElement(child)) {
                    // someone else's extension?
                    continue;
                }
                String tag = child.getName();
                if (tag.equals("formID")) {
                    formId = XFormParser.getXMLText(child, true);
                    if (formId != null && formId.length() == 0) {
                        formId = null;
                    }
                } else if (tag.equals("name")) {
                    formName = XFormParser.getXMLText(child, true);
                    if (formName != null && formName.length() == 0) {
                        formName = null;
                    }
                } else if (tag.equals("version")) {
                    version = XFormParser.getXMLText(child, true);
                    if (version != null && version.length() == 0) {
                        version = null;
                    }
                } else if (tag.equals("majorMinorVersion")) {
                    majorMinorVersion = XFormParser.getXMLText(child, true);
                    if (majorMinorVersion != null && majorMinorVersion.length() == 0) {
                        majorMinorVersion = null;
                    }
                } else if (tag.equals("descriptionText")) {
                    description = XFormParser.getXMLText(child, true);
                    if (description != null && description.length() == 0) {
                        description = null;
                    }
                } else if (tag.equals("downloadUrl")) {
                    downloadUrl = XFormParser.getXMLText(child, true);
                    if (downloadUrl != null && downloadUrl.length() == 0) {
                        downloadUrl = null;
                    }
                } else if (tag.equals("manifestUrl")) {
                    manifestUrl = XFormParser.getXMLText(child, true);
                    if (manifestUrl != null && manifestUrl.length() == 0) {
                        manifestUrl = null;
                    }
                }
            }
            if (formId == null || downloadUrl == null || formName == null) {
                log.error("Parsing OpenRosa reply -- Forms list entry " + Integer.toString(i) + " is missing one or more tags: formId, name, or downloadUrl");
                formList.clear();
                throw new ParsingException(BAD_OPENROSA_FORMLIST);
            }
            String versionString = null;
            if (version != null && version.length() != 0) {
                versionString = version;
            } else if (majorMinorVersion != null && majorMinorVersion.length() != 0) {
                int idx = majorMinorVersion.indexOf(".");
                if (idx == -1) {
                    versionString = majorMinorVersion;
                } else {
                    versionString = majorMinorVersion.substring(0, idx);
                }
            }
            try {
                if (versionString != null) {
                    // verify that  the version string is a long integer value...
                    Long.parseLong(versionString);
                }
            } catch (Exception e) {
                log.error("Parsing OpenRosa reply -- Forms list entry " + Integer.toString(i) + " has an invalid version string: " + versionString, e);
                formList.clear();
                throw new ParsingException(BAD_OPENROSA_FORMLIST);
            }
            formList.add(new RemoteFormDefinition(formName, formId, versionString, downloadUrl, manifestUrl));
        }
    } else {
        // Aggregate 0.9.x mode...
        // populate HashMap with form names and urls
        Element formsElement = formListDoc.getRootElement();
        int formsCount = formsElement.getChildCount();
        for (int i = 0; i < formsCount; ++i) {
            if (formsElement.getType(i) != Element.ELEMENT) {
                // whitespace
                continue;
            }
            Element child = formsElement.getElement(i);
            String tag = child.getName();
            if (tag.equalsIgnoreCase("form")) {
                String formName = XFormParser.getXMLText(child, true);
                if (formName != null && formName.length() == 0) {
                    formName = null;
                }
                String downloadUrl = child.getAttributeValue(null, "url");
                downloadUrl = downloadUrl.trim();
                if (downloadUrl != null && downloadUrl.length() == 0) {
                    downloadUrl = null;
                }
                if (downloadUrl == null || formName == null) {
                    log.error("Parsing OpenRosa reply -- Forms list entry " + Integer.toString(i) + " is missing form name or url attribute");
                    formList.clear();
                    throw new ParsingException(BAD_LEGACY_FORMLIST);
                }
                // Since this is ODK Aggregate 0.9.8 or higher, we know that the
                // formId is
                // given as a parameter of the URL...
                String formId = null;
                try {
                    URL url = new URL(downloadUrl);
                    String qs = url.getQuery();
                    if (qs.startsWith(ODK_ID_PARAMETER_EQUALS)) {
                        formId = qs.substring(ODK_ID_PARAMETER_EQUALS.length());
                    }
                } catch (MalformedURLException e) {
                    log.warn("bad download url", e);
                }
                if (formId == null) {
                    throw new ParsingException("Unable to extract formId from download URL of legacy 0.9.8 server");
                }
                formList.add(new RemoteFormDefinition(formName, formId, null, downloadUrl, null));
            }
        }
    }
    return formList;
}

Example 4 with RemoteFormDefinition

use of org.opendatakit.briefcase.model.RemoteFormDefinition in project briefcase by opendatakit.

the class ServerFetcher method downloadAllSubmissionsForForm.

private boolean downloadAllSubmissionsForForm(File formInstancesDir, DatabaseUtils formDatabase, BriefcaseFormDefinition lfd, FormStatus fs) {
    int submissionCount = 1;
    int chunkCount = 1;
    boolean allSuccessful = true;
    RemoteFormDefinition fd = getRemoteFormDefinition(fs);
    ExecutorService execSvc = getFetchExecutorService();
    CompletionService<SubmissionChunk> chunkCompleter = new ExecutorCompletionService<>(execSvc);
    CompletionService<String> submissionCompleter = new ExecutorCompletionService<>(execSvc);
    String oldWebsafeCursorString;
    String websafeCursorString = "";
    chunkCompleter.submit(new SubmissionChunkDownload(fs, fd.getFormId(), websafeCursorString));
    boolean cursorFinished;
    try {
        do {
            if (isCancelled()) {
                fs.setStatusString("aborting fetching submission chunks...", true);
                EventBus.publish(new FormStatusEvent(fs));
                return false;
            }
            fs.setStatusString("processing chunk " + chunkCount + "...", true);
            EventBus.publish(new FormStatusEvent(fs));
            // remember what we had...
            oldWebsafeCursorString = websafeCursorString;
            SubmissionChunk chunk;
            try {
                chunk = chunkCompleter.take().get();
                chunkCount += 1;
                websafeCursorString = chunk.websafeCursorString;
                cursorFinished = oldWebsafeCursorString.equals(websafeCursorString);
            } catch (InterruptedException | ExecutionException e) {
                return false;
            }
            if (!cursorFinished) {
                // submit another chunk request so it's ready by the time we finish processing this chunk
                chunkCompleter.submit(new SubmissionChunkDownload(fs, fd.getFormId(), websafeCursorString));
            }
            // submit a download job for each uri in the chunk
            for (String uri : chunk.uriList) {
                if (isCancelled()) {
                    fs.setStatusString("aborting requesting submissions...", true);
                    EventBus.publish(new FormStatusEvent(fs));
                    return false;
                }
                submissionCompleter.submit(new SubmissionDownload(formInstancesDir, formDatabase, lfd, fs, uri));
            }
            // call take() and get() exactly once for each download submitted above (we don't need the uri)
            for (int i = 0; i < chunk.uriList.size(); i++) {
                if (isCancelled()) {
                    fs.setStatusString("aborting processing submissions...", true);
                    EventBus.publish(new FormStatusEvent(fs));
                    return false;
                }
                try {
                    submissionCompleter.take().get();
                    fs.setStatusString(String.format("fetched instance %s...", submissionCount++), true);
                    EventBus.publish(new FormStatusEvent(fs));
                } catch (InterruptedException | ExecutionException e) {
                    log.error("failure during submission download", e);
                    allSuccessful = false;
                    fs.setStatusString("Submission not retrieved: " + e.getMessage(), false);
                    EventBus.publish(new FormStatusEvent(fs));
                // but try to get the next one...
                }
            }
        } while (!cursorFinished);
    } finally {
        execSvc.shutdown();
        try {
            execSvc.awaitTermination(1, TimeUnit.MINUTES);
        } catch (InterruptedException e) {
            log.warn("interrupted while waiting for pull to complete");
        }
    }
    return allSuccessful;
}

Example 5 with RemoteFormDefinition

use of org.opendatakit.briefcase.model.RemoteFormDefinition in project briefcase by opendatakit.

the class ServerFetcher method downloadFormAndSubmissionFiles.

public boolean downloadFormAndSubmissionFiles(List<FormStatus> formsToTransfer) {
    boolean allSuccessful = true;
    // boolean error = false;
    int total = formsToTransfer.size();
    for (int i = 0; i < total; i++) {
        FormStatus fs = formsToTransfer.get(i);
        if (isCancelled()) {
            fs.setStatusString("Aborted. Skipping fetch of form and submissions...", true);
            EventBus.publish(new FormStatusEvent(fs));
            return false;
        }
        RemoteFormDefinition fd = getRemoteFormDefinition(fs);
        fs.setStatusString("Fetching form definition", true);
        EventBus.publish(new FormStatusEvent(fs));
        try {
            File tmpdl = FileSystemUtils.getTempFormDefinitionFile();
            AggregateUtils.commonDownloadFile(serverInfo, tmpdl, fd.getDownloadUrl());
            fs.setStatusString("resolving against briefcase form definitions", true);
            EventBus.publish(new FormStatusEvent(fs));
            boolean successful = false;
            BriefcaseFormDefinition briefcaseLfd;
            DatabaseUtils formDatabase = null;
            try {
                try {
                    briefcaseLfd = BriefcaseFormDefinition.resolveAgainstBriefcaseDefn(tmpdl);
                    if (briefcaseLfd.needsMediaUpdate()) {
                        if (fd.getManifestUrl() != null) {
                            File mediaDir = FileSystemUtils.getMediaDirectory(briefcaseLfd.getFormDirectory());
                            String error = downloadManifestAndMediaFiles(mediaDir, fs);
                            if (error != null) {
                                allSuccessful = false;
                                fs.setStatusString("Error fetching form definition: " + error, false);
                                EventBus.publish(new FormStatusEvent(fs));
                                continue;
                            }
                        }
                    }
                    formDatabase = DatabaseUtils.newInstance(briefcaseLfd.getFormDirectory());
                } catch (BadFormDefinition e) {
                    allSuccessful = false;
                    String msg = "Error parsing form definition";
                    log.error(msg, e);
                    fs.setStatusString(msg + ": " + e.getMessage(), false);
                    EventBus.publish(new FormStatusEvent(fs));
                    continue;
                }
                fs.setStatusString("preparing to retrieve instance data", true);
                EventBus.publish(new FormStatusEvent(fs));
                File formInstancesDir = FileSystemUtils.getFormInstancesDirectory(briefcaseLfd.getFormDirectory());
                // this will publish events
                successful = downloadAllSubmissionsForForm(formInstancesDir, formDatabase, briefcaseLfd, fs);
            } catch (SQLException | FileSystemException e) {
                allSuccessful = false;
                String msg = "unable to open form database";
                log.error(msg, e);
                fs.setStatusString(msg + ": " + e.getMessage(), false);
                EventBus.publish(new FormStatusEvent(fs));
                continue;
            } finally {
                if (formDatabase != null) {
                    try {
                        formDatabase.close();
                    } catch (SQLException e) {
                        allSuccessful = false;
                        String msg = "unable to close form database";
                        log.error(msg, e);
                        fs.setStatusString(msg + ": " + e.getMessage(), false);
                        EventBus.publish(new FormStatusEvent(fs));
                        continue;
                    }
                }
            }
            allSuccessful = allSuccessful && successful;
            // on success, we haven't actually set a success event (because we don't know we're done)
            if (successful) {
                fs.setStatusString(SUCCESS_STATUS, true);
                EventBus.publish(new FormStatusEvent(fs));
            } else {
                fs.setStatusString(FAILED_STATUS, true);
                EventBus.publish(new FormStatusEvent(fs));
            }
        } catch (SocketTimeoutException se) {
            allSuccessful = false;
            log.error("error accessing URL", se);
            fs.setStatusString("Communications to the server timed out. Detailed message: " + se.getLocalizedMessage() + " while accessing: " + fd.getDownloadUrl() + " A network login screen may be interfering with the transmission to the server.", false);
            EventBus.publish(new FormStatusEvent(fs));
        } catch (IOException e) {
            allSuccessful = false;
            log.error("error accessing form download URL", e);
            fs.setStatusString("Unexpected error: " + e.getLocalizedMessage() + " while accessing: " + fd.getDownloadUrl() + " A network login screen may be interfering with the transmission to the server.", false);
            EventBus.publish(new FormStatusEvent(fs));
        } catch (FileSystemException | TransmissionException | URISyntaxException e) {
            allSuccessful = false;
            log.error("error accessing form download URL", e);
            fs.setStatusString("Unexpected error: " + e.getLocalizedMessage() + " while accessing: " + fd.getDownloadUrl(), false);
            EventBus.publish(new FormStatusEvent(fs));
        }
    }
    return allSuccessful;
}