Examples with XmlDocumentFetchException org.opendatakit.briefcase.model.XmlDocumentFetchException used on opensource projects

Example 1 with XmlDocumentFetchException

use of org.opendatakit.briefcase.model.XmlDocumentFetchException in project briefcase by opendatakit.

the class AggregateUtils method getXmlDocument.

/**
 * Common method for returning a parsed xml document given a url and the http
 * context and client objects involved in the web connection. The document is
 * the body of the response entity and should be xml.
 *
 * @param urlString
 * @return
 */
public static final DocumentFetchResult getXmlDocument(String urlString, ServerConnectionInfo serverInfo, boolean alwaysResetCredentials, DocumentDescription description, ResponseAction action) throws XmlDocumentFetchException {
    log.info("Parsing URL {}", urlString);
    URI u = null;
    try {
        URL url = new URL(urlString);
        u = url.toURI();
    } catch (MalformedURLException e) {
        String msg = description.getFetchDocFailed() + "Invalid url. Failed with error: " + e.getMessage();
        if (!urlString.toLowerCase().startsWith("http://") && !urlString.toLowerCase().startsWith("https://")) {
            msg += ". Did you forget to prefix the address with 'http://' or 'https://' ?";
        }
        log.warn(msg, e);
        throw new XmlDocumentFetchException(msg);
    } catch (URISyntaxException e) {
        String msg = description.getFetchDocFailed() + "Invalid uri. Failed with error: " + e.getMessage();
        log.warn(msg, e);
        throw new XmlDocumentFetchException(msg);
    }
    // set up request...
    HttpGet req = WebUtils.createOpenRosaHttpGet(u);
    int[] validStatusList = { 200 };
    return httpRetrieveXmlDocument(req, validStatusList, serverInfo, alwaysResetCredentials, description, action);
}

Example 2 with XmlDocumentFetchException

use of org.opendatakit.briefcase.model.XmlDocumentFetchException in project briefcase by opendatakit.

the class AggregateUtils method uploadFilesToServer.

public static final boolean uploadFilesToServer(ServerConnectionInfo serverInfo, URI u, String distinguishedFileTagName, File file, List<File> files, DocumentDescription description, SubmissionResponseAction action, TerminationFuture terminationFuture, FormStatus formToTransfer) {
    boolean allSuccessful = true;
    formToTransfer.setStatusString("Preparing for upload of " + description.getDocumentDescriptionType() + " with " + files.size() + " media attachments", true);
    EventBus.publish(new FormStatusEvent(formToTransfer));
    // handles case where there are no media files
    boolean first = true;
    int lastJ = 0;
    int j = 0;
    while (j < files.size() || first) {
        lastJ = j;
        first = false;
        if (terminationFuture.isCancelled()) {
            formToTransfer.setStatusString("Aborting upload of " + description.getDocumentDescriptionType() + " with " + files.size() + " media attachments", true);
            EventBus.publish(new FormStatusEvent(formToTransfer));
            return false;
        }
        HttpPost httppost = WebUtils.createOpenRosaHttpPost(u);
        long byteCount = 0L;
        // mime post
        MultipartEntityBuilder builder = MultipartEntityBuilder.create();
        // add the submission file first...
        FileBody fb = new FileBody(file, ContentType.TEXT_XML);
        builder.addPart(distinguishedFileTagName, fb);
        log.info("added " + distinguishedFileTagName + ": " + file.getName());
        byteCount += file.length();
        for (; j < files.size(); j++) {
            File f = files.get(j);
            log.info("Trying file {}", f);
            String fileName = f.getName();
            int idx = fileName.lastIndexOf(".");
            String extension = "";
            if (idx != -1) {
                extension = fileName.substring(idx + 1);
            }
            // we only need to deal with the content type determination...
            if (extension.equals("xml")) {
                fb = new FileBody(f, ContentType.TEXT_XML);
                builder.addPart(f.getName(), fb);
                byteCount += f.length();
                log.info("added xml file " + f.getName());
            } else if (extension.equals("jpg")) {
                fb = new FileBody(f, ContentType.create("image/jpeg"));
                builder.addPart(f.getName(), fb);
                byteCount += f.length();
                log.info("added image file " + f.getName());
            } else if (extension.equals("3gpp")) {
                fb = new FileBody(f, ContentType.create("audio/3gpp"));
                builder.addPart(f.getName(), fb);
                byteCount += f.length();
                log.info("added audio file " + f.getName());
            } else if (extension.equals("3gp")) {
                fb = new FileBody(f, ContentType.create("video/3gpp"));
                builder.addPart(f.getName(), fb);
                byteCount += f.length();
                log.info("added video file " + f.getName());
            } else if (extension.equals("mp4")) {
                fb = new FileBody(f, ContentType.create("video/mp4"));
                builder.addPart(f.getName(), fb);
                byteCount += f.length();
                log.info("added video file " + f.getName());
            } else if (extension.equals("csv")) {
                fb = new FileBody(f, ContentType.create("text/csv"));
                builder.addPart(f.getName(), fb);
                byteCount += f.length();
                log.info("added csv file " + f.getName());
            } else if (extension.equals("xls")) {
                fb = new FileBody(f, ContentType.create("application/vnd.ms-excel"));
                builder.addPart(f.getName(), fb);
                byteCount += f.length();
                log.info("added xls file " + f.getName());
            } else {
                fb = new FileBody(f, ContentType.create("application/octet-stream"));
                builder.addPart(f.getName(), fb);
                byteCount += f.length();
                log.warn("added unrecognized file (application/octet-stream)");
            }
            // we've added at least one attachment to the request...
            if (j + 1 < files.size()) {
                if ((j - lastJ + 1) > 100 || byteCount + files.get(j + 1).length() > 10000000L) {
                    // more than 100 attachments or the next file would exceed the 10MB threshold...
                    log.info("Extremely long post is being split into multiple posts");
                    try {
                        StringBody sb = new StringBody("yes", ContentType.DEFAULT_TEXT.withCharset(Charset.forName("UTF-8")));
                        builder.addPart("*isIncomplete*", sb);
                    } catch (Exception e) {
                        log.error("impossible condition", e);
                        throw new IllegalStateException("never happens");
                    }
                    // advance over the last attachment added...
                    ++j;
                    break;
                }
            }
        }
        httppost.setEntity(builder.build());
        int[] validStatusList = { 201 };
        try {
            if (j != files.size()) {
                formToTransfer.setStatusString("Uploading " + description.getDocumentDescriptionType() + " and media files " + (lastJ + 1) + " through " + (j + 1) + " of " + files.size() + " media attachments", true);
            } else if (j == 0) {
                formToTransfer.setStatusString("Uploading " + description.getDocumentDescriptionType() + " with no media attachments", true);
            } else {
                formToTransfer.setStatusString("Uploading " + description.getDocumentDescriptionType() + " and " + (j - lastJ) + ((lastJ != 0) ? " remaining" : "") + " media attachments", true);
            }
            EventBus.publish(new FormStatusEvent(formToTransfer));
            httpRetrieveXmlDocument(httppost, validStatusList, serverInfo, false, description, action);
        } catch (XmlDocumentFetchException e) {
            allSuccessful = false;
            log.error("upload failed", e);
            formToTransfer.setStatusString("UPLOAD FAILED: " + e.getMessage(), false);
            EventBus.publish(new FormStatusEvent(formToTransfer));
            if (description.isCancelled())
                return false;
        }
    }
    return allSuccessful;
}

Example 3 with XmlDocumentFetchException

use of org.opendatakit.briefcase.model.XmlDocumentFetchException in project briefcase by opendatakit.

the class ServerFetcher method downloadManifestAndMediaFiles.

private String downloadManifestAndMediaFiles(File mediaDir, FormStatus fs) {
    RemoteFormDefinition fd = getRemoteFormDefinition(fs);
    if (fd.getManifestUrl() == null)
        return null;
    fs.setStatusString("Fetching form manifest", true);
    EventBus.publish(new FormStatusEvent(fs));
    List<MediaFile> files = new ArrayList<>();
    AggregateUtils.DocumentFetchResult result;
    try {
        DocumentDescription formManifestDescription = new DocumentDescription("Fetch of manifest failed. Detailed reason: ", "Fetch of manifest failed ", "form manifest", terminationFuture);
        result = AggregateUtils.getXmlDocument(fd.getManifestUrl(), serverInfo, false, formManifestDescription, null);
    } catch (XmlDocumentFetchException e) {
        return e.getMessage();
    }
    try {
        files = XmlManipulationUtils.parseFormManifestResponse(result.isOpenRosaResponse, result.doc);
    } catch (ParsingException e) {
        return e.getMessage();
    }
    // OK we now have the full set of files to download...
    log.info("Downloading " + files.size() + " media files.");
    int mCount = 0;
    if (files.size() > 0) {
        for (MediaFile m : files) {
            ++mCount;
            fs.setStatusString(String.format(" (getting %1$d of %2$d media files)", mCount, files.size()), true);
            EventBus.publish(new FormStatusEvent(fs));
            try {
                downloadMediaFileIfChanged(mediaDir, m, fs);
            } catch (Exception e) {
                return e.getLocalizedMessage();
            }
        }
    }
    return null;
}

Example 4 with XmlDocumentFetchException

use of org.opendatakit.briefcase.model.XmlDocumentFetchException in project briefcase by opendatakit.

the class ServerUploader method subtractServerInstances.

// remove any instances already completed on server
private void subtractServerInstances(FormStatus fs, DatabaseUtils formDatabase, Set<File> instancesToUpload) {
    /*
     * The /view/submissionList interface returns the list of COMPLETED submissions
     * on the server. Fetch this list and filter out the locally-held submissions
     * with the same instanceIds.  We know the server is already content with what
     * it has, so we don't need to send any of these to the server, as that POST
     * request will be treated as a no-op.
     */
    String baseUrl = serverInfo.getUrl() + "/view/submissionList";
    String oldWebsafeCursorString = "not-empty";
    String websafeCursorString = "";
    for (; !oldWebsafeCursorString.equals(websafeCursorString); ) {
        if (isCancelled()) {
            fs.setStatusString("aborting retrieval of instanceIds of submissions on server...", true);
            EventBus.publish(new FormStatusEvent(fs));
            return;
        }
        fs.setStatusString("retrieving next chunk of instanceIds from server...", true);
        EventBus.publish(new FormStatusEvent(fs));
        Map<String, String> params = new HashMap<>();
        params.put("numEntries", Integer.toString(MAX_ENTRIES));
        params.put("formId", fs.getFormDefinition().getFormId());
        params.put("cursor", websafeCursorString);
        String fullUrl = WebUtils.createLinkWithProperties(baseUrl, params);
        // remember what we had...
        oldWebsafeCursorString = websafeCursorString;
        AggregateUtils.DocumentFetchResult result;
        try {
            DocumentDescription submissionChunkDescription = new DocumentDescription("Fetch of instanceIds (submission download chunk) failed.  Detailed error: ", "Fetch of instanceIds (submission download chunk) failed.", "submission download chunk", terminationFuture);
            result = AggregateUtils.getXmlDocument(fullUrl, serverInfo, false, submissionChunkDescription, null);
        } catch (XmlDocumentFetchException e) {
            fs.setStatusString("Not all submissions retrieved: Error fetching list of instanceIds: " + e.getMessage(), false);
            EventBus.publish(new FormStatusEvent(fs));
            return;
        }
        SubmissionChunk chunk;
        try {
            chunk = XmlManipulationUtils.parseSubmissionDownloadListResponse(result.doc);
        } catch (ParsingException e) {
            fs.setStatusString("Not all instanceIds retrieved: Error parsing the submission download chunk: " + e.getMessage(), false);
            EventBus.publish(new FormStatusEvent(fs));
            return;
        }
        websafeCursorString = chunk.websafeCursorString;
        for (String uri : chunk.uriList) {
            File f = formDatabase.hasRecordedInstance(uri);
            if (f != null) {
                instancesToUpload.remove(f);
            }
        }
    }
}

Example 5 with XmlDocumentFetchException

use of org.opendatakit.briefcase.model.XmlDocumentFetchException in project briefcase by opendatakit.

the class ServerFetcher method downloadSubmission.

private void downloadSubmission(File formInstancesDir, DatabaseUtils formDatabase, BriefcaseFormDefinition lfd, FormStatus fs, String uri) throws Exception {
    File instanceFolder = formDatabase.hasRecordedInstance(uri);
    if (instanceFolder != null) {
        // check if the submission file is present in the folder before skipping
        File instance = new File(instanceFolder, "submission.xml");
        File instanceEncrypted = new File(instanceFolder, "submission.xml.enc");
        if (instance.exists() || instanceEncrypted.exists()) {
            log.info("already present - skipping fetch: " + uri);
            return;
        }
    }
    String formId = lfd.getSubmissionKey(uri);
    if (isCancelled()) {
        fs.setStatusString("aborting fetch of submission...", true);
        EventBus.publish(new FormStatusEvent(fs));
        throw new SubmissionDownloadException("Transfer cancelled by user.");
    }
    String baseUrl = serverInfo.getUrl() + "/view/downloadSubmission";
    Map<String, String> params = new HashMap<>();
    params.put("formId", formId);
    String fullUrl = WebUtils.createLinkWithProperties(baseUrl, params);
    AggregateUtils.DocumentFetchResult result;
    try {
        DocumentDescription submissionDescription = new DocumentDescription("Fetch of a submission failed.  Detailed error: ", "Fetch of a submission failed.", "submission", terminationFuture);
        result = AggregateUtils.getXmlDocument(fullUrl, serverInfo, false, submissionDescription, null);
    } catch (XmlDocumentFetchException e) {
        throw new SubmissionDownloadException(e.getMessage());
    }
    // and parse the document...
    SubmissionManifest submissionManifest;
    try {
        submissionManifest = XmlManipulationUtils.parseDownloadSubmissionResponse(result.doc);
    } catch (ParsingException e) {
        throw new SubmissionDownloadException(e.getMessage());
    }
    String msg = "Fetched instanceID=" + submissionManifest.instanceID;
    log.info(msg);
    if (FileSystemUtils.hasFormSubmissionDirectory(formInstancesDir, submissionManifest.instanceID)) {
        // create instance directory...
        File instanceDir = FileSystemUtils.assertFormSubmissionDirectory(formInstancesDir, submissionManifest.instanceID);
        // fetch attachments
        for (MediaFile m : submissionManifest.attachmentList) {
            downloadMediaFileIfChanged(instanceDir, m, fs);
        }
        // write submission file -- we rely on instanceId being unique...
        File submissionFile = new File(instanceDir, "submission.xml");
        OutputStreamWriter fo = new OutputStreamWriter(new FileOutputStream(submissionFile), "UTF-8");
        fo.write(submissionManifest.submissionXml);
        fo.close();
        // if we get here, we know that this is a completed submission
        // (because it was in /view/submissionList) and that we safely
        // copied it into the storage area (because we didn't get any
        // exceptions).
        formDatabase.assertRecordedInstanceDirectory(uri, instanceDir);
    } else {
        // create instance directory...
        File instanceDir = FileSystemUtils.assertFormSubmissionDirectory(formInstancesDir, submissionManifest.instanceID);
        // fetch attachments
        for (MediaFile m : submissionManifest.attachmentList) {
            downloadMediaFileIfChanged(instanceDir, m, fs);
        }
        // write submission file
        File submissionFile = new File(instanceDir, "submission.xml");
        OutputStreamWriter fo = new OutputStreamWriter(new FileOutputStream(submissionFile), "UTF-8");
        fo.write(submissionManifest.submissionXml);
        fo.close();
        // if we get here, we know that this is a completed submission
        // (because it was in /view/submissionList) and that we safely
        // copied it into the storage area (because we didn't get any
        // exceptions).
        formDatabase.assertRecordedInstanceDirectory(uri, instanceDir);
    }
}