Examples with DumpDownloader org.wikivoyage.listings.input.DumpDownloader used on opensource projects

Example 1 with DumpDownloader

use of org.wikivoyage.listings.input.DumpDownloader in project wikivoyage-listings by baturin.

the class DumpDownloaderTest method isNotPartialDump.

/**
 * Test if the partial dump is detected correctly
 */
@Test
public void isNotPartialDump() throws Exception {
    String dumpStatus = "", partialDumpStatus = "";
    dumpStatus = IOUtils.toString(this.getClass().getResourceAsStream("/dumpstatus.json"), "UTF-8");
    partialDumpStatus = IOUtils.toString(this.getClass().getResourceAsStream("/partial_dumpstatus.json"), "UTF-8");
    DumpDownloader downloader = new DumpDownloader();
    Assert.assertFalse(downloader.isPartialDump(dumpStatus));
    Assert.assertTrue(downloader.isPartialDump(partialDumpStatus));
}

Example 2 with DumpDownloader

use of org.wikivoyage.listings.input.DumpDownloader in project wikivoyage-listings by baturin.

the class Main method main.

public static void main(String[] args) {
    HashMap<String, OutputFormat> formats = new HashMap<>();
    formats.put("csv", new CSV());
    formats.put("osmand-xml", new OsmXml(false));
    formats.put("osmand-xml-user-defined", new OsmXml(true));
    formats.put("xml", new Xml());
    formats.put("obf", new OBF(false, "tmp", "tmp/pois.xml"));
    formats.put("obf-user-defined", new OBF(true, "tmp", "tmp/pois.xml"));
    formats.put("sql", new SQL());
    formats.put("gpx", new GPX());
    formats.put("osmand.gpx", new OsmAndGPX());
    formats.put("kml", new KML());
    formats.put("validation-report", new ValidationReport());
    CommandLine cl = new CommandLine();
    String[] formatNames = formats.keySet().toArray(new String[formats.keySet().size()]);
    cl.parse(args, formatNames);
    fileNames = new FileNames(cl.listingsDir, cl.dumpsCacheDir, cl.workingDir);
    try {
        if (cl.help) {
            cl.printHelp((String[]) formats.keySet().toArray());
        } else if (cl.dailyUpdate) {
            dailyUpdate(cl, formats);
        } else {
            String inputFilename;
            // To embed in files to provide freshness information to users (example: "Generated from Wikivoyage 2016/07/20 data").
            String dumpDate = "";
            createWorkingDir();
            if (cl.inputFile != null) {
                inputFilename = cl.inputFile;
                log.info("Take POIs from '" + inputFilename + "'");
            } else {
                DumpDownloader downloader = new DumpDownloader();
                if (cl.inputUrl != null) {
                    inputFilename = fileNames.workingDirPath("dump.xml.bz2");
                    downloader.downloadDumpFromUrl(cl.inputUrl, inputFilename);
                } else {
                    createDumpsCacheDir();
                    List<String> dumpDates = downloader.listDumps(cl.inputLatest);
                    dumpDate = dumpDates.get(0);
                    inputFilename = fileNames.dumpCacheFilename(cl.inputLatest, dumpDate);
                    if (!FileUtils.fileExists(inputFilename)) {
                        String dumpUrl = downloader.dumpUrl(cl.inputLatest, dumpDate);
                        downloader.downloadDumpFromUrl(dumpUrl, inputFilename);
                    } else {
                        log.info("Use cached dump");
                    }
                }
            }
            if (cl.outputFormat != null) {
                OutputFormat format = formats.get(cl.outputFormat);
                generateFileForFormat(inputFilename, cl.outputFilename, format, dumpDate);
            }
            UnrecognizeTemplateCounter.getInstance().logUnrecognizeTemplatesSummary();
            log.info("Finished");
        }
    } catch (Exception e) {
        System.err.println("Failure");
        e.printStackTrace();
    }
}

Example 3 with DumpDownloader

use of org.wikivoyage.listings.input.DumpDownloader in project wikivoyage-listings by baturin.

the class Main method dailyUpdate.

private static void dailyUpdate(CommandLine cl, HashMap<String, OutputFormat> formats) throws IOException, FileUtilsException {
    createWorkingDir();
    createListingsDir();
    createDumpsCacheDir();
    DumpDownloader downloader = new DumpDownloader();
    for (String language : Languages.getLanguageCodes()) {
        log.info("Processing language " + language);
        List<String> dumpDates = downloader.listDumps(language);
        if (dumpDates.size() == 0) {
            continue;
        }
        String latestDumpDate = dumpDates.get(0);
        if (cl.latestCount != null) {
            log.info("Processing the latest " + cl.latestCount + " dumps");
            dumpDates = dumpDates.subList(0, cl.latestCount);
        }
        for (String dumpDate : dumpDates) {
            log.info("Processing dump " + dumpDate);
            try {
                processDump(downloader, language, latestDumpDate, dumpDate, formats, !cl.doNotUseIntermediateFile);
            } catch (Exception e) {
                log.info("Failed to create dump " + dumpDate);
                log.debug("Exception: ", e);
                StringWriter sw = new StringWriter();
                e.printStackTrace(new PrintWriter(sw));
                log.debug("Stack trace: " + sw.toString());
            }
        }
    }
    System.exit(0);
}