Examples with CBZip2OutputStream org.apache.tools.bzip2.CBZip2OutputStream used on opensource projects

Example 1 with CBZip2OutputStream

use of org.apache.tools.bzip2.CBZip2OutputStream in project OsmAnd-tools by osmandapp.

the class WikipediaByCountryDivider method generateCountrySqlite.

protected static void generateCountrySqlite(String folder, boolean skip) throws SQLException, IOException, InterruptedException, XmlPullParserException {
    Connection conn = (Connection) DBDialect.SQLITE.getDatabaseConnection(folder + "wiki.sqlite", log);
    OsmandRegions regs = new OsmandRegions();
    regs.prepareFile(new File("resources/countries-info/regions.ocbf").getAbsolutePath());
    Map<String, LinkedList<BinaryMapDataObject>> mapObjects = regs.cacheAllCountries();
    File rgns = new File(folder, "regions");
    rgns.mkdirs();
    Map<String, String> preferredRegionLanguages = new LinkedHashMap<>();
    for (String key : mapObjects.keySet()) {
        if (key == null) {
            continue;
        }
        WorldRegion wr = regs.getRegionDataByDownloadName(key);
        if (wr == null) {
            System.out.println("Missing language for world region '" + key + "'!");
        } else {
            String regionLang = wr.getParams().getRegionLang();
            preferredRegionLanguages.put(key.toLowerCase(), regionLang);
        }
    }
    ResultSet rs = conn.createStatement().executeQuery("SELECT DISTINCT regionName  FROM wiki_region");
    while (rs.next()) {
        String lcRegionName = rs.getString(1);
        if (lcRegionName == null) {
            continue;
        }
        String regionName = Algorithms.capitalizeFirstLetterAndLowercase(lcRegionName);
        String preferredLang = preferredRegionLanguages.get(lcRegionName);
        if (preferredLang == null) {
            preferredLang = "";
        }
        LinkedList<BinaryMapDataObject> list = mapObjects.get(lcRegionName.toLowerCase());
        boolean hasWiki = false;
        if (list != null) {
            for (BinaryMapDataObject o : list) {
                Integer rl = o.getMapIndex().getRule("region_wiki", "yes");
                if (o.containsAdditionalType(rl)) {
                    hasWiki = true;
                    break;
                }
            }
        }
        if (!hasWiki) {
            System.out.println("Skip " + lcRegionName.toLowerCase() + " doesn't generate wiki");
            continue;
        }
        File fl = new File(rgns, regionName + ".sqlite");
        File osmBz2 = new File(rgns, regionName + "_" + IndexConstants.BINARY_MAP_VERSION + ".wiki.osm.bz2");
        File obfFile = new File(rgns, regionName + "_" + IndexConstants.BINARY_MAP_VERSION + ".wiki.obf");
        if (obfFile.exists() && skip) {
            continue;
        }
        fl.delete();
        osmBz2.delete();
        obfFile.delete();
        System.out.println("Generate " + fl.getName());
        Connection loc = (Connection) DBDialect.SQLITE.getDatabaseConnection(fl.getAbsolutePath(), log);
        loc.createStatement().execute("CREATE TABLE wiki_content(id long, lat double, lon double, lang text, wikiId long, title text, zipContent blob)");
        PreparedStatement insertWikiContent = loc.prepareStatement("INSERT INTO wiki_content VALUES(?, ?, ?, ?, ?, ?, ?)");
        ResultSet rps = conn.createStatement().executeQuery("SELECT WC.id, WC.lat, WC.lon, WC.lang, WC.wikiId, WC.title, WC.zipContent " + " FROM wiki_content WC INNER JOIN wiki_region WR " + " ON WC.id = WR.id AND WR.regionName = '" + rs.getString(1) + "' ORDER BY WC.id");
        FileOutputStream out = new FileOutputStream(osmBz2);
        out.write('B');
        out.write('Z');
        CBZip2OutputStream bzipStream = new CBZip2OutputStream(out);
        XmlSerializer serializer = new org.kxml2.io.KXmlSerializer();
        serializer.setOutput(bzipStream, "UTF-8");
        serializer.startDocument("UTF-8", true);
        serializer.startTag(null, "osm");
        serializer.attribute(null, "version", "0.6");
        serializer.attribute(null, "generator", "OsmAnd");
        serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true);
        // indentation as 3 spaces
        // serializer.setProperty(
        // "http://xmlpull.org/v1/doc/properties.html#serializer-indentation", "   ");
        // // also set the line separator
        // serializer.setProperty(
        // "http://xmlpull.org/v1/doc/properties.html#serializer-line-separator", "\n");
        int cnt = 1;
        long prevOsmId = -1;
        StringBuilder content = new StringBuilder();
        String nameUnique = null;
        boolean preferredAdded = false;
        boolean nameAdded = false;
        while (rps.next()) {
            long osmId = -rps.getLong(1);
            double lat = rps.getDouble(2);
            double lon = rps.getDouble(3);
            long wikiId = rps.getLong(5);
            String wikiLang = rps.getString(4);
            String title = rps.getString(6);
            byte[] bytes = rps.getBytes(7);
            GZIPInputStream gzin = new GZIPInputStream(new ByteArrayInputStream(bytes));
            BufferedReader br = new BufferedReader(new InputStreamReader(gzin));
            content.setLength(0);
            String s;
            while ((s = br.readLine()) != null) {
                content.append(s);
            }
            String contentStr = content.toString();
            contentStr = contentStr.replace((char) 9, ' ');
            contentStr = contentStr.replace((char) 0, ' ');
            contentStr = contentStr.replace((char) 22, ' ');
            contentStr = contentStr.replace((char) 27, ' ');
            insertWikiContent.setLong(1, osmId);
            insertWikiContent.setDouble(2, lat);
            insertWikiContent.setDouble(3, lon);
            insertWikiContent.setString(4, wikiLang);
            insertWikiContent.setLong(5, wikiId);
            insertWikiContent.setString(6, title);
            insertWikiContent.setBytes(7, bytes);
            insertWikiContent.addBatch();
            if (cnt++ % BATCH_SIZE == 0) {
                insertWikiContent.executeBatch();
            }
            if (osmId != prevOsmId) {
                if (prevOsmId != -1) {
                    closeOsmWikiNode(serializer, nameUnique, nameAdded);
                }
                prevOsmId = osmId;
                nameAdded = false;
                nameUnique = null;
                preferredAdded = false;
                serializer.startTag(null, "node");
                serializer.attribute(null, "visible", "true");
                serializer.attribute(null, "id", (osmId) + "");
                serializer.attribute(null, "lat", lat + "");
                serializer.attribute(null, "lon", lon + "");
            }
            if (wikiLang.equals("en")) {
                nameAdded = true;
                addTag(serializer, "name", title);
                addTag(serializer, "wiki_id", wikiId + "");
                addTag(serializer, "content", contentStr);
                addTag(serializer, "wiki_lang:en", "yes");
            } else {
                addTag(serializer, "name:" + wikiLang, title);
                addTag(serializer, "wiki_id:" + wikiLang, wikiId + "");
                addTag(serializer, "wiki_lang:" + wikiLang, "yes");
                if (!preferredAdded) {
                    nameUnique = title;
                    preferredAdded = preferredLang.contains(wikiLang);
                }
                addTag(serializer, "content:" + wikiLang, contentStr);
            }
        }
        if (prevOsmId != -1) {
            closeOsmWikiNode(serializer, nameUnique, nameAdded);
        }
        insertWikiContent.executeBatch();
        loc.close();
        serializer.endDocument();
        serializer.flush();
        bzipStream.close();
        System.out.println("Processed " + cnt + " pois");
        generateObf(osmBz2, obfFile);
    }
    conn.close();
}

Example 2 with CBZip2OutputStream

use of org.apache.tools.bzip2.CBZip2OutputStream in project OsmAnd-tools by osmandapp.

the class GenerateRegionTags method process.

private static void process(File inputFile, File targetFile, OsmandRegions or) throws IOException, XmlPullParserException, XMLStreamException {
    InputStream fis = new FileInputStream(inputFile);
    if (inputFile.getName().endsWith(".gz")) {
        fis = new GZIPInputStream(fis);
    } else if (inputFile.getName().endsWith(".bz2")) {
        if (fis.read() != 'B' || fis.read() != 'Z') {
            throw new RuntimeException(// $NON-NLS-1$
            "The source stream must start with the characters BZ if it is to be read as a BZip2 stream.");
        }
        fis = new CBZip2InputStream(fis);
    }
    OsmBaseStorage bs = new OsmBaseStorage();
    bs.parseOSM(fis, new ConsoleProgressImplementation());
    LOG.info("File was read");
    iterateOverEntities(bs.getRegisteredEntities(), or);
    OsmStorageWriter w = new OsmStorageWriter();
    OutputStream output = new FileOutputStream(targetFile);
    if (targetFile.getName().endsWith(".gz")) {
        output = new GZIPOutputStream(output);
    } else if (targetFile.getName().endsWith(".bz2")) {
        output.write("BZ".getBytes());
        output = new CBZip2OutputStream(output);
    }
    LOG.info("Entities processed. About to save the file.");
    w.saveStorage(output, bs, null, true);
    output.close();
    fis.close();
}

Example 3 with CBZip2OutputStream

use of org.apache.tools.bzip2.CBZip2OutputStream in project gradle by gradle.

the class Bzip2Archiver method getCompressor.

public static ArchiveOutputStreamFactory getCompressor() {
    // get rid of ArchiveOutputStreamFactory in favor of the writable Resource
    return new ArchiveOutputStreamFactory() {

        public OutputStream createArchiveOutputStream(File destination) throws FileNotFoundException {
            OutputStream outStr = new BufferedOutputStream(new FileOutputStream(destination));
            try {
                outStr.write('B');
                outStr.write('Z');
                return new CBZip2OutputStream(outStr);
            } catch (Exception e) {
                IOUtils.closeQuietly(outStr);
                String message = String.format("Unable to create bzip2 output stream for file %s", destination);
                throw new RuntimeException(message, e);
            }
        }
    };
}

Example 4 with CBZip2OutputStream

use of org.apache.tools.bzip2.CBZip2OutputStream in project OsmAnd-tools by osmandapp.

the class OsmExtractionUI method saveCountry.

public void saveCountry(final File f, final OsmBaseStorage storage) {
    final OsmStorageWriter writer = new OsmStorageWriter();
    try {
        // $NON-NLS-1$
        final ProgressDialog dlg = new ProgressDialog(frame, Messages.getString("OsmExtractionUI.SAVING_OSM_FILE"));
        dlg.setRunnable(new Runnable() {

            @Override
            public void run() {
                try {
                    OutputStream output = new FileOutputStream(f);
                    try {
                        if (f.getName().endsWith(".bz2")) {
                            // $NON-NLS-1$
                            output.write('B');
                            output.write('Z');
                            output = new CBZip2OutputStream(output);
                        }
                        writer.saveStorage(output, storage, null, false);
                    } finally {
                        output.close();
                    }
                } catch (IOException e) {
                    throw new IllegalArgumentException(e);
                } catch (XMLStreamException e) {
                    throw new IllegalArgumentException(e);
                }
            }
        });
        dlg.run();
    } catch (InterruptedException e1) {
        // $NON-NLS-1$
        log.error("Interrupted", e1);
    } catch (InvocationTargetException e1) {
        // $NON-NLS-1$
        ExceptionHandler.handle("Log file is not found", e1.getCause());
    }
}

Example 5 with CBZip2OutputStream

use of org.apache.tools.bzip2.CBZip2OutputStream in project Osmand by osmandapp.

the class FavouritesDbHelper method backup.

private void backup(File backupFile, File externalFile) {
    try {
        File f = new File(backupFile.getParentFile(), backupFile.getName());
        FileOutputStream fout = new FileOutputStream(f);
        fout.write('B');
        fout.write('Z');
        CBZip2OutputStream out = new CBZip2OutputStream(fout);
        FileInputStream fis = new FileInputStream(externalFile);
        Algorithms.streamCopy(fis, out);
        fis.close();
        out.close();
        fout.close();
    } catch (Exception e) {
        log.warn("Backup failed", e);
    }
}