Examples with MultipartEntityBuilder org.opendatakit.httpclientandroidlib.entity.mime.MultipartEntityBuilder used on opensource projects

Example 1 with MultipartEntityBuilder

use of org.opendatakit.httpclientandroidlib.entity.mime.MultipartEntityBuilder in project collect by opendatakit.

the class InstanceServerUploader method uploadOneSubmission.

/**
 * Uploads to urlString the submission identified by id with filepath of instance
 *
 * @param urlString    destination URL
 * @param toUpdate     - Instance URL for recording status update.
 * @param localContext - context (e.g., credentials, cookies) for client connection
 * @param uriRemap     - mapping of Uris to avoid redirects on subsequent invocations
 * @return false if credentials are required and we should terminate immediately.
 */
private boolean uploadOneSubmission(String urlString, String id, String instanceFilePath, Uri toUpdate, HttpContext localContext, Map<Uri, Uri> uriRemap, Outcome outcome) {
    Collect.getInstance().getActivityLogger().logAction(this, urlString, instanceFilePath);
    File instanceFile = new File(instanceFilePath);
    ContentValues cv = new ContentValues();
    Uri submissionUri = Uri.parse(urlString);
    HttpClient httpclient = WebUtils.createHttpClient(CONNECTION_TIMEOUT);
    ResponseMessageParser messageParser = null;
    boolean openRosaServer = false;
    if (uriRemap.containsKey(submissionUri)) {
        // we already issued a head request and got a response,
        // so we know the proper URL to send the submission to
        // and the proper scheme. We also know that it was an
        // OpenRosa compliant server.
        openRosaServer = true;
        submissionUri = uriRemap.get(submissionUri);
        // if https then enable preemptive basic auth...
        if (submissionUri.getScheme().equals("https")) {
            WebUtils.enablePreemptiveBasicAuth(localContext, submissionUri.getHost());
        }
        Timber.i("Using Uri remap for submission %s. Now: %s", id, submissionUri.toString());
    } else {
        if (submissionUri.getHost() == null) {
            Timber.i("Host name may not be null");
            outcome.messagesByInstanceId.put(id, fail + "Host name may not be null");
            cv.put(InstanceColumns.STATUS, InstanceProviderAPI.STATUS_SUBMISSION_FAILED);
            Collect.getInstance().getContentResolver().update(toUpdate, cv, null, null);
            return true;
        }
        // if https then enable preemptive basic auth...
        if (submissionUri.getScheme() != null && submissionUri.getScheme().equals("https")) {
            WebUtils.enablePreemptiveBasicAuth(localContext, submissionUri.getHost());
        }
        URI uri;
        try {
            uri = URI.create(submissionUri.toString());
        } catch (IllegalArgumentException e) {
            Timber.i(e);
            outcome.messagesByInstanceId.put(id, Collect.getInstance().getString(R.string.url_error));
            return false;
        }
        // Issue a head request to confirm the server is an OpenRosa server and see if auth
        // is required
        // http://docs.opendatakit.org/openrosa-form-submission/#extended-transmission-considerations
        HttpHead httpHead = WebUtils.createOpenRosaHttpHead(uri);
        // prepare response
        final HttpResponse response;
        try {
            Timber.i("Issuing HEAD request for %s to: %s", id, submissionUri.toString());
            response = httpclient.execute(httpHead, localContext);
            int statusCode = response.getStatusLine().getStatusCode();
            if (statusCode == HttpStatus.SC_UNAUTHORIZED) {
                // clear the cookies -- should not be necessary?
                Collect.getInstance().getCookieStore().clear();
                WebUtils.discardEntityBytes(response);
                // we need authentication, so stop and return what we've
                // done so far.
                outcome.authRequestingServer = submissionUri;
                return false;
            } else if (statusCode == 204) {
                Header[] locations = response.getHeaders("Location");
                WebUtils.discardEntityBytes(response);
                if (locations != null && locations.length == 1) {
                    try {
                        Uri newURI = Uri.parse(URLDecoder.decode(locations[0].getValue(), "utf-8"));
                        if (submissionUri.getHost().equalsIgnoreCase(newURI.getHost())) {
                            openRosaServer = true;
                            // Re-add params if server didn't respond with params
                            if (newURI.getQuery() == null) {
                                newURI = newURI.buildUpon().encodedQuery(submissionUri.getEncodedQuery()).build();
                            }
                            uriRemap.put(submissionUri, newURI);
                            submissionUri = newURI;
                        } else {
                            // Don't follow a redirection attempt to a different host.
                            // We can't tell if this is a spoof or not.
                            outcome.messagesByInstanceId.put(id, fail + "Unexpected redirection attempt to a different " + "host: " + newURI.toString());
                            cv.put(InstanceColumns.STATUS, InstanceProviderAPI.STATUS_SUBMISSION_FAILED);
                            Collect.getInstance().getContentResolver().update(toUpdate, cv, null, null);
                            return true;
                        }
                    } catch (Exception e) {
                        Timber.e(e, "Exception thrown parsing URI for url %s", urlString);
                        outcome.messagesByInstanceId.put(id, fail + urlString + " " + e.toString());
                        cv.put(InstanceColumns.STATUS, InstanceProviderAPI.STATUS_SUBMISSION_FAILED);
                        Collect.getInstance().getContentResolver().update(toUpdate, cv, null, null);
                        return true;
                    }
                }
            } else {
                // may be a server that does not handle
                WebUtils.discardEntityBytes(response);
                Timber.w("Status code on Head request: %d", statusCode);
                if (statusCode >= HttpStatus.SC_OK && statusCode < HttpStatus.SC_MULTIPLE_CHOICES) {
                    outcome.messagesByInstanceId.put(id, fail + "Invalid status code on Head request.  If you have a " + "web proxy, you may need to login to your network. ");
                    cv.put(InstanceColumns.STATUS, InstanceProviderAPI.STATUS_SUBMISSION_FAILED);
                    Collect.getInstance().getContentResolver().update(toUpdate, cv, null, null);
                    return true;
                }
            }
        } catch (ClientProtocolException | ConnectTimeoutException | UnknownHostException | SocketTimeoutException | NoHttpResponseException | SocketException e) {
            if (e instanceof ClientProtocolException) {
                outcome.messagesByInstanceId.put(id, fail + "Client Protocol Exception");
                Timber.i(e, "Client Protocol Exception");
            } else if (e instanceof ConnectTimeoutException) {
                outcome.messagesByInstanceId.put(id, fail + "Connection Timeout");
                Timber.i(e, "Connection Timeout");
            } else if (e instanceof UnknownHostException) {
                outcome.messagesByInstanceId.put(id, fail + e.toString() + " :: Network Connection Failed");
                Timber.i(e, "Network Connection Failed");
            } else if (e instanceof SocketTimeoutException) {
                outcome.messagesByInstanceId.put(id, fail + "Connection Timeout");
                Timber.i(e, "Connection timeout");
            } else {
                outcome.messagesByInstanceId.put(id, fail + "Network Connection Refused");
                Timber.i(e, "Network Connection Refused");
            }
            cv.put(InstanceColumns.STATUS, InstanceProviderAPI.STATUS_SUBMISSION_FAILED);
            Collect.getInstance().getContentResolver().update(toUpdate, cv, null, null);
            return true;
        } catch (Exception e) {
            String msg = e.getMessage();
            if (msg == null) {
                msg = e.toString();
            }
            outcome.messagesByInstanceId.put(id, fail + "Generic Exception: " + msg);
            Timber.e(e);
            cv.put(InstanceColumns.STATUS, InstanceProviderAPI.STATUS_SUBMISSION_FAILED);
            Collect.getInstance().getContentResolver().update(toUpdate, cv, null, null);
            return true;
        }
    }
    // At this point, we may have updated the uri to use https.
    // This occurs only if the Location header keeps the host name
    // the same. If it specifies a different host name, we error
    // out.
    // 
    // And we may have set authentication cookies in our
    // cookiestore (referenced by localContext) that will enable
    // authenticated publication to the server.
    // 
    // get instance file
    // Under normal operations, we upload the instanceFile to
    // the server.  However, during the save, there is a failure
    // window that may mark the submission as complete but leave
    // the file-to-be-uploaded with the name "submission.xml" and
    // the plaintext submission files on disk.  In this case,
    // upload the submission.xml and all the files in the directory.
    // This means the plaintext files and the encrypted files
    // will be sent to the server and the server will have to
    // figure out what to do with them.
    File submissionFile = new File(instanceFile.getParentFile(), "submission.xml");
    if (submissionFile.exists()) {
        Timber.w("submission.xml will be uploaded instead of %s", instanceFile.getAbsolutePath());
    } else {
        submissionFile = instanceFile;
    }
    if (!instanceFile.exists() && !submissionFile.exists()) {
        outcome.messagesByInstanceId.put(id, fail + "instance XML file does not exist!");
        cv.put(InstanceColumns.STATUS, InstanceProviderAPI.STATUS_SUBMISSION_FAILED);
        Collect.getInstance().getContentResolver().update(toUpdate, cv, null, null);
        return true;
    }
    // find all files in parent directory
    File[] allFiles = instanceFile.getParentFile().listFiles();
    // add media files
    List<File> files = new ArrayList<File>();
    if (allFiles != null) {
        for (File f : allFiles) {
            String fileName = f.getName();
            if (fileName.startsWith(".")) {
                // ignore invisible files
                continue;
            } else if (fileName.equals(instanceFile.getName())) {
                // the xml file has already been added
                continue;
            } else if (fileName.equals(submissionFile.getName())) {
                // the xml file has already been added
                continue;
            }
            String extension = getFileExtension(fileName);
            if (openRosaServer) {
                files.add(f);
            } else if (extension.equals("jpg")) {
                // legacy 0.9x
                files.add(f);
            } else if (extension.equals("3gpp")) {
                // legacy 0.9x
                files.add(f);
            } else if (extension.equals("3gp")) {
                // legacy 0.9x
                files.add(f);
            } else if (extension.equals("mp4")) {
                // legacy 0.9x
                files.add(f);
            } else if (extension.equals("osm")) {
                // legacy 0.9x
                files.add(f);
            } else {
                Timber.w("unrecognized file type %s", f.getName());
            }
        }
    } else {
        return false;
    }
    boolean first = true;
    int j = 0;
    int lastJ;
    while (j < files.size() || first) {
        lastJ = j;
        first = false;
        MimeTypeMap m = MimeTypeMap.getSingleton();
        long byteCount = 0L;
        // mime post
        MultipartEntityBuilder builder = MultipartEntityBuilder.create();
        // add the submission file first...
        FileBody fb = new FileBody(submissionFile, ContentType.TEXT_XML);
        builder.addPart("xml_submission_file", fb);
        Timber.i("added xml_submission_file: %s", submissionFile.getName());
        byteCount += submissionFile.length();
        for (; j < files.size(); j++) {
            File f = files.get(j);
            // we will be processing every one of these, so
            // we only need to deal with the content type determination...
            ContentType contentType = ContentTypeMapping.of(f.getName());
            if (contentType == null) {
                String mime = m.getMimeTypeFromExtension(getFileExtension(f.getName()));
                if (mime != null) {
                    contentType = ContentType.create(mime);
                } else {
                    Timber.w("No specific MIME type found for file: %s", f.getName());
                    contentType = ContentType.APPLICATION_OCTET_STREAM;
                }
            }
            fb = new FileBody(f, contentType);
            builder.addPart(f.getName(), fb);
            byteCount += f.length();
            Timber.i("added file of type '%s' %s", contentType, f.getName());
            // we've added at least one attachment to the request...
            if (j + 1 < files.size()) {
                if ((j - lastJ + 1 > 100) || (byteCount + files.get(j + 1).length() > 10000000L)) {
                    // the next file would exceed the 10MB threshold...
                    Timber.i("Extremely long post is being split into multiple posts");
                    try {
                        StringBody sb = new StringBody("yes", ContentType.TEXT_PLAIN.withCharset(Charset.forName("UTF-8")));
                        builder.addPart("*isIncomplete*", sb);
                    } catch (Exception e) {
                        Timber.e(e);
                    }
                    // advance over the last attachment added...
                    ++j;
                    break;
                }
            }
        }
        HttpPost httppost = WebUtils.createOpenRosaHttpPost(submissionUri);
        httppost.setEntity(builder.build());
        // prepare response and return uploaded
        HttpResponse response;
        try {
            Timber.i("Issuing POST request for %s to: %s", id, submissionUri.toString());
            response = httpclient.execute(httppost, localContext);
            int responseCode = response.getStatusLine().getStatusCode();
            HttpEntity httpEntity = response.getEntity();
            messageParser = new ResponseMessageParser(httpEntity);
            WebUtils.discardEntityBytes(response);
            Timber.i("Response code:%d", responseCode);
            // If it wasn't, the submission has failed.
            if (responseCode != HttpStatus.SC_CREATED && responseCode != HttpStatus.SC_ACCEPTED) {
                if (responseCode == HttpStatus.SC_OK) {
                    outcome.messagesByInstanceId.put(id, fail + "Network login failure? Again?");
                } else if (responseCode == HttpStatus.SC_UNAUTHORIZED) {
                    // clear the cookies -- should not be necessary?
                    Collect.getInstance().getCookieStore().clear();
                    outcome.messagesByInstanceId.put(id, fail + response.getStatusLine().getReasonPhrase() + " (" + responseCode + ") at " + urlString);
                } else {
                    // If response from server is valid use that else use default messaging
                    if (messageParser.isValid()) {
                        outcome.messagesByInstanceId.put(id, fail + messageParser.getMessageResponse());
                    } else {
                        outcome.messagesByInstanceId.put(id, fail + response.getStatusLine().getReasonPhrase() + " (" + responseCode + ") at " + urlString);
                    }
                }
                cv.put(InstanceColumns.STATUS, InstanceProviderAPI.STATUS_SUBMISSION_FAILED);
                Collect.getInstance().getContentResolver().update(toUpdate, cv, null, null);
                return true;
            }
        } catch (IOException e) {
            if (e instanceof UnknownHostException || e instanceof HttpHostConnectException || e instanceof SocketException || e instanceof NoHttpResponseException || e instanceof SocketTimeoutException || e instanceof ConnectTimeoutException) {
                Timber.i(e);
            } else {
                Timber.e(e);
            }
            String msg = e.getMessage();
            if (msg == null) {
                msg = e.toString();
            }
            outcome.messagesByInstanceId.put(id, fail + "Generic Exception: " + msg);
            cv.put(InstanceColumns.STATUS, InstanceProviderAPI.STATUS_SUBMISSION_FAILED);
            Collect.getInstance().getContentResolver().update(toUpdate, cv, null, null);
            return true;
        }
    }
    // If response from server is valid use that else use default messaging
    if (messageParser.isValid()) {
        outcome.messagesByInstanceId.put(id, messageParser.getMessageResponse());
    } else {
        // Default messaging
        outcome.messagesByInstanceId.put(id, Collect.getInstance().getString(R.string.success));
    }
    cv.put(InstanceColumns.STATUS, InstanceProviderAPI.STATUS_SUBMITTED);
    Collect.getInstance().getContentResolver().update(toUpdate, cv, null, null);
    return true;
}