Examples with JavaSerializedIterable org.wikivoyage.listings.input.JavaSerializedIterable used on opensource projects

Example 1 with JavaSerializedIterable

use of org.wikivoyage.listings.input.JavaSerializedIterable in project wikivoyage-listings by baturin.

the class Main method processDump.

private static void processDump(DumpDownloader downloader, String language, String latestDumpDate, String dumpDate, HashMap<String, OutputFormat> formats, boolean useIntermediateFile) throws IOException, FileUtilsException, InterruptedException, WriteOutputException {
    boolean allFileExists = true;
    for (OutputFormat format : formats.values()) {
        String fileName = fileNames.getListingPath(language, dumpDate, format.getDefaultExtension(), true);
        if (!FileUtils.fileExists(fileName)) {
            allFileExists = false;
            break;
        }
    }
    if (allFileExists) {
        log.info("All files already exist for '" + language + "-" + dumpDate + "'");
        return;
    }
    log.info("Create POIs for '" + dumpDate + "'");
    String dumpUrl = downloader.dumpUrl(language, dumpDate);
    String dumpPath = fileNames.dumpCacheFilename(language, dumpDate);
    if (!FileUtils.fileExists(dumpPath)) {
        downloader.downloadDumpFromUrl(dumpUrl, dumpPath);
    }
    // Prepare to iterate over listings.
    Iterable<Listing> listings = new ListingsIterable(dumpPath);
    // Write listings to intermediate file.
    if (useIntermediateFile) {
        log.info("Write intermediate file with parsed listings");
        String javaSerialFile = fileNames.workingDirPath("serialized-pois.bin");
        FileUtils.removeFile(javaSerialFile);
        new JavaSerializedObject().write(listings, javaSerialFile, dumpDate);
        listings = new JavaSerializedIterable(javaSerialFile);
    }
    listings = validate(listings);
    // Write listings to all the output formats.
    for (OutputFormat format : formats.values()) {
        writeFormat(listings, language, dumpDate, latestDumpDate, format);
    }
}