Examples with OutputFormat org.wikivoyage.listings.output.OutputFormat used on opensource projects

Example 1 with OutputFormat

use of org.wikivoyage.listings.output.OutputFormat in project wikivoyage-listings by baturin.

the class Main method processDump.

private static void processDump(DumpDownloader downloader, String language, String latestDumpDate, String dumpDate, HashMap<String, OutputFormat> formats, boolean useIntermediateFile) throws IOException, FileUtilsException, InterruptedException, WriteOutputException {
    boolean allFileExists = true;
    for (OutputFormat format : formats.values()) {
        String fileName = fileNames.getListingPath(language, dumpDate, format.getDefaultExtension(), true);
        if (!FileUtils.fileExists(fileName)) {
            allFileExists = false;
            break;
        }
    }
    if (allFileExists) {
        log.info("All files already exist for '" + language + "-" + dumpDate + "'");
        return;
    }
    log.info("Create POIs for '" + dumpDate + "'");
    String dumpUrl = downloader.dumpUrl(language, dumpDate);
    String dumpPath = fileNames.dumpCacheFilename(language, dumpDate);
    if (!FileUtils.fileExists(dumpPath)) {
        downloader.downloadDumpFromUrl(dumpUrl, dumpPath);
    }
    // Prepare to iterate over listings.
    Iterable<Listing> listings = new ListingsIterable(dumpPath);
    // Write listings to intermediate file.
    if (useIntermediateFile) {
        log.info("Write intermediate file with parsed listings");
        String javaSerialFile = fileNames.workingDirPath("serialized-pois.bin");
        FileUtils.removeFile(javaSerialFile);
        new JavaSerializedObject().write(listings, javaSerialFile, dumpDate);
        listings = new JavaSerializedIterable(javaSerialFile);
    }
    listings = validate(listings);
    // Write listings to all the output formats.
    for (OutputFormat format : formats.values()) {
        writeFormat(listings, language, dumpDate, latestDumpDate, format);
    }
}

Example 2 with OutputFormat

use of org.wikivoyage.listings.output.OutputFormat in project wikivoyage-listings by baturin.

the class Main method main.

public static void main(String[] args) {
    HashMap<String, OutputFormat> formats = new HashMap<>();
    formats.put("csv", new CSV());
    formats.put("osmand-xml", new OsmXml(false));
    formats.put("osmand-xml-user-defined", new OsmXml(true));
    formats.put("xml", new Xml());
    formats.put("obf", new OBF(false, "tmp", "tmp/pois.xml"));
    formats.put("obf-user-defined", new OBF(true, "tmp", "tmp/pois.xml"));
    formats.put("sql", new SQL());
    formats.put("gpx", new GPX());
    formats.put("osmand.gpx", new OsmAndGPX());
    formats.put("kml", new KML());
    formats.put("validation-report", new ValidationReport());
    CommandLine cl = new CommandLine();
    String[] formatNames = formats.keySet().toArray(new String[formats.keySet().size()]);
    cl.parse(args, formatNames);
    fileNames = new FileNames(cl.listingsDir, cl.dumpsCacheDir, cl.workingDir);
    try {
        if (cl.help) {
            cl.printHelp((String[]) formats.keySet().toArray());
        } else if (cl.dailyUpdate) {
            dailyUpdate(cl, formats);
        } else {
            String inputFilename;
            // To embed in files to provide freshness information to users (example: "Generated from Wikivoyage 2016/07/20 data").
            String dumpDate = "";
            createWorkingDir();
            if (cl.inputFile != null) {
                inputFilename = cl.inputFile;
                log.info("Take POIs from '" + inputFilename + "'");
            } else {
                DumpDownloader downloader = new DumpDownloader();
                if (cl.inputUrl != null) {
                    inputFilename = fileNames.workingDirPath("dump.xml.bz2");
                    downloader.downloadDumpFromUrl(cl.inputUrl, inputFilename);
                } else {
                    createDumpsCacheDir();
                    List<String> dumpDates = downloader.listDumps(cl.inputLatest);
                    dumpDate = dumpDates.get(0);
                    inputFilename = fileNames.dumpCacheFilename(cl.inputLatest, dumpDate);
                    if (!FileUtils.fileExists(inputFilename)) {
                        String dumpUrl = downloader.dumpUrl(cl.inputLatest, dumpDate);
                        downloader.downloadDumpFromUrl(dumpUrl, inputFilename);
                    } else {
                        log.info("Use cached dump");
                    }
                }
            }
            if (cl.outputFormat != null) {
                OutputFormat format = formats.get(cl.outputFormat);
                generateFileForFormat(inputFilename, cl.outputFilename, format, dumpDate);
            }
            UnrecognizeTemplateCounter.getInstance().logUnrecognizeTemplatesSummary();
            log.info("Finished");
        }
    } catch (Exception e) {
        System.err.println("Failure");
        e.printStackTrace();
    }
}