Examples with Response org.jsoup.Connection.Response used on opensource projects

Example 16 with Response

use of org.jsoup.Connection.Response in project ripme by RipMeApp.

the class FuraffinityRipper method getDescription.

public String getDescription(String page) {
    try {
        // Fetch the image page
        Response resp = Http.url(page).referrer(this.url).response();
        cookies.putAll(resp.cookies());
        // Try to find the description
        Elements els = resp.parse().select("td[class=alt1][width=\"70%\"]");
        if (els.size() == 0) {
            logger.debug("No description at " + page);
            throw new IOException("No description found");
        }
        logger.debug("Description found!");
        Document documentz = resp.parse();
        // This is where the description is.
        Element ele = documentz.select("td[class=alt1][width=\"70%\"]").get(0);
        // Would break completely if FurAffinity changed site layout.
        documentz.outputSettings(new Document.OutputSettings().prettyPrint(false));
        ele.select("br").append("\\n");
        ele.select("p").prepend("\\n\\n");
        logger.debug("Returning description at " + page);
        String tempPage = Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false));
        // Overridden saveText takes first line and makes it the file name.
        return documentz.select("meta[property=og:title]").attr("content") + "\n" + tempPage;
    } catch (IOException ioe) {
        logger.info("Failed to get description " + page + " : '" + ioe.getMessage() + "'");
        return null;
    }
}

Example 17 with Response

use of org.jsoup.Connection.Response in project ripme by RipMeApp.

the class AerisdiesRipper method getFirstPage.

@Override
public Document getFirstPage() throws IOException {
    if (albumDoc == null) {
        Response resp = Http.url(url).response();
        cookies.putAll(resp.cookies());
        albumDoc = resp.parse();
    }
    return albumDoc;
}

Example 18 with Response

use of org.jsoup.Connection.Response in project ripme by RipMeApp.

the class DeviantartRipper method smallToFull.

/**
 * If largest resolution for image at 'thumb' is found, starts downloading
 * and returns null.
 * If it finds a larger resolution on another page, returns the image URL.
 * @param thumb Thumbnail URL
 * @param page Page the thumbnail is retrieved from
 * @return Highest-resolution version of the image based on thumbnail URL and the page.
 */
private String smallToFull(String thumb, String page) {
    try {
        // Fetch the image page
        Response resp = Http.url(page).referrer(this.url).cookies(cookies).response();
        cookies.putAll(resp.cookies());
        Document doc = resp.parse();
        Elements els = doc.select("img.dev-content-full");
        String fsimage = null;
        // Get the largest resolution image on the page
        if (els.size() > 0) {
            // Large image
            fsimage = els.get(0).attr("src");
            logger.info("Found large-scale: " + fsimage);
            if (fsimage.contains("//orig")) {
                return fsimage;
            }
        }
        // Try to find the download button
        els = doc.select("a.dev-page-download");
        if (els.size() > 0) {
            // Full-size image
            String downloadLink = els.get(0).attr("href");
            logger.info("Found download button link: " + downloadLink);
            HttpURLConnection con = (HttpURLConnection) new URL(downloadLink).openConnection();
            con.setRequestProperty("Referer", this.url.toString());
            String cookieString = "";
            for (Map.Entry<String, String> entry : cookies.entrySet()) {
                cookieString = cookieString + entry.getKey() + "=" + entry.getValue() + "; ";
            }
            cookieString = cookieString.substring(0, cookieString.length() - 1);
            con.setRequestProperty("Cookie", cookieString);
            con.setRequestProperty("User-Agent", USER_AGENT);
            con.setInstanceFollowRedirects(true);
            con.connect();
            int code = con.getResponseCode();
            String location = con.getURL().toString();
            con.disconnect();
            if (location.contains("//orig")) {
                fsimage = location;
                logger.info("Found image download: " + location);
            }
        }
        if (fsimage != null) {
            return fsimage;
        }
        throw new IOException("No download page found");
    } catch (IOException ioe) {
        try {
            logger.info("Failed to get full size download image at " + page + " : '" + ioe.getMessage() + "'");
            String lessThanFull = thumbToFull(thumb, false);
            logger.info("Falling back to less-than-full-size image " + lessThanFull);
            return lessThanFull;
        } catch (Exception e) {
            return null;
        }
    }
}

Example 19 with Response

use of org.jsoup.Connection.Response in project ripme by RipMeApp.

the class DeviantartRipper method getDescription.

/**
 * Attempts to download description for image.
 * Comes in handy when people put entire stories in their description.
 * If no description was found, returns null.
 * @param url The URL the description will be retrieved from
 * @param page The gallery page the URL was found on
 * @return A String[] with first object being the description, and the second object being image file name if found.
 */
@Override
public String[] getDescription(String url, Document page) {
    if (isThisATest()) {
        return null;
    }
    try {
        // Fetch the image page
        Response resp = Http.url(url).referrer(this.url).cookies(cookies).response();
        cookies.putAll(resp.cookies());
        // Try to find the description
        Document documentz = resp.parse();
        Element ele = documentz.select("div.dev-description").first();
        if (ele == null) {
            throw new IOException("No description found");
        }
        documentz.outputSettings(new Document.OutputSettings().prettyPrint(false));
        ele.select("br").append("\\n");
        ele.select("p").prepend("\\n\\n");
        String fullSize = null;
        Element thumb = page.select("div.zones-container span.thumb[href=\"" + url + "\"]").get(0);
        if (!thumb.attr("data-super-full-img").isEmpty()) {
            fullSize = thumb.attr("data-super-full-img");
            String[] split = fullSize.split("/");
            fullSize = split[split.length - 1];
        } else {
            String spanUrl = thumb.attr("href");
            fullSize = jsonToImage(page, spanUrl.substring(spanUrl.lastIndexOf('-') + 1));
            if (fullSize != null) {
                String[] split = fullSize.split("/");
                fullSize = split[split.length - 1];
            }
        }
        if (fullSize == null) {
            return new String[] { Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false)) };
        }
        fullSize = fullSize.substring(0, fullSize.lastIndexOf("."));
        return new String[] { Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false)), fullSize };
    // TODO Make this not make a newline if someone just types \n into the description.
    } catch (IOException ioe) {
        logger.info("Failed to get description at " + url + ": '" + ioe.getMessage() + "'");
        return null;
    }
}

Example 20 with Response

use of org.jsoup.Connection.Response in project ripme by RipMeApp.

the class DeviantartRipper method loginToDeviantart.

/**
 * Logs into deviant art. Required to rip full-size NSFW content.
 * @return Map of cookies containing session data.
 */
private Map<String, String> loginToDeviantart() throws IOException {
    // Populate postData fields
    Map<String, String> postData = new HashMap<>();
    String username = Utils.getConfigString("deviantart.username", new String(Base64.decode("Z3JhYnB5")));
    String password = Utils.getConfigString("deviantart.password", new String(Base64.decode("ZmFrZXJz")));
    if (username == null || password == null) {
        throw new IOException("could not find username or password in config");
    }
    Response resp = Http.url("http://www.deviantart.com/").response();
    for (Element input : resp.parse().select("form#form-login input[type=hidden]")) {
        postData.put(input.attr("name"), input.attr("value"));
    }
    postData.put("username", username);
    postData.put("password", password);
    postData.put("remember_me", "1");
    // Send login request
    resp = Http.url("https://www.deviantart.com/users/login").userAgent(USER_AGENT).data(postData).cookies(resp.cookies()).method(Method.POST).response();
    // Assert we are logged in
    if (resp.hasHeader("Location") && resp.header("Location").contains("password")) {
        // Wrong password
        throw new IOException("Wrong password");
    }
    if (resp.url().toExternalForm().contains("bad_form")) {
        throw new IOException("Login form was incorrectly submitted");
    }
    if (resp.cookie("auth_secure") == null || resp.cookie("auth") == null) {
        throw new IOException("No auth_secure or auth cookies received");
    }
    // We are logged in, save the cookies
    return resp.cookies();
}