Search in sources :

Example 1 with WorldRegion

use of net.osmand.map.WorldRegion in project OsmAnd-tools by osmandapp.

the class WikipediaByCountryDivider method generateCountrySqlite.

protected static void generateCountrySqlite(String folder, boolean skip) throws SQLException, IOException, InterruptedException, XmlPullParserException {
    Connection conn = (Connection) DBDialect.SQLITE.getDatabaseConnection(folder + "wiki.sqlite", log);
    OsmandRegions regs = new OsmandRegions();
    regs.prepareFile(new File("resources/countries-info/regions.ocbf").getAbsolutePath());
    Map<String, LinkedList<BinaryMapDataObject>> mapObjects = regs.cacheAllCountries();
    File rgns = new File(folder, "regions");
    rgns.mkdirs();
    Map<String, String> preferredRegionLanguages = new LinkedHashMap<>();
    for (String key : mapObjects.keySet()) {
        if (key == null) {
            continue;
        }
        WorldRegion wr = regs.getRegionDataByDownloadName(key);
        if (wr == null) {
            System.out.println("Missing language for world region '" + key + "'!");
        } else {
            String regionLang = wr.getParams().getRegionLang();
            preferredRegionLanguages.put(key.toLowerCase(), regionLang);
        }
    }
    ResultSet rs = conn.createStatement().executeQuery("SELECT DISTINCT regionName  FROM wiki_region");
    while (rs.next()) {
        String lcRegionName = rs.getString(1);
        if (lcRegionName == null) {
            continue;
        }
        String regionName = Algorithms.capitalizeFirstLetterAndLowercase(lcRegionName);
        String preferredLang = preferredRegionLanguages.get(lcRegionName);
        if (preferredLang == null) {
            preferredLang = "";
        }
        LinkedList<BinaryMapDataObject> list = mapObjects.get(lcRegionName.toLowerCase());
        boolean hasWiki = false;
        if (list != null) {
            for (BinaryMapDataObject o : list) {
                Integer rl = o.getMapIndex().getRule("region_wiki", "yes");
                if (o.containsAdditionalType(rl)) {
                    hasWiki = true;
                    break;
                }
            }
        }
        if (!hasWiki) {
            System.out.println("Skip " + lcRegionName.toLowerCase() + " doesn't generate wiki");
            continue;
        }
        File fl = new File(rgns, regionName + ".sqlite");
        File osmBz2 = new File(rgns, regionName + "_" + IndexConstants.BINARY_MAP_VERSION + ".wiki.osm.bz2");
        File obfFile = new File(rgns, regionName + "_" + IndexConstants.BINARY_MAP_VERSION + ".wiki.obf");
        if (obfFile.exists() && skip) {
            continue;
        }
        fl.delete();
        osmBz2.delete();
        obfFile.delete();
        System.out.println("Generate " + fl.getName());
        Connection loc = (Connection) DBDialect.SQLITE.getDatabaseConnection(fl.getAbsolutePath(), log);
        loc.createStatement().execute("CREATE TABLE wiki_content(id long, lat double, lon double, lang text, wikiId long, title text, zipContent blob)");
        PreparedStatement insertWikiContent = loc.prepareStatement("INSERT INTO wiki_content VALUES(?, ?, ?, ?, ?, ?, ?)");
        ResultSet rps = conn.createStatement().executeQuery("SELECT WC.id, WC.lat, WC.lon, WC.lang, WC.wikiId, WC.title, WC.zipContent " + " FROM wiki_content WC INNER JOIN wiki_region WR " + " ON WC.id = WR.id AND WR.regionName = '" + rs.getString(1) + "' ORDER BY WC.id");
        FileOutputStream out = new FileOutputStream(osmBz2);
        out.write('B');
        out.write('Z');
        CBZip2OutputStream bzipStream = new CBZip2OutputStream(out);
        XmlSerializer serializer = new org.kxml2.io.KXmlSerializer();
        serializer.setOutput(bzipStream, "UTF-8");
        serializer.startDocument("UTF-8", true);
        serializer.startTag(null, "osm");
        serializer.attribute(null, "version", "0.6");
        serializer.attribute(null, "generator", "OsmAnd");
        serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true);
        // indentation as 3 spaces
        // serializer.setProperty(
        // "http://xmlpull.org/v1/doc/properties.html#serializer-indentation", "   ");
        // // also set the line separator
        // serializer.setProperty(
        // "http://xmlpull.org/v1/doc/properties.html#serializer-line-separator", "\n");
        int cnt = 1;
        long prevOsmId = -1;
        StringBuilder content = new StringBuilder();
        String nameUnique = null;
        boolean preferredAdded = false;
        boolean nameAdded = false;
        while (rps.next()) {
            long osmId = -rps.getLong(1);
            double lat = rps.getDouble(2);
            double lon = rps.getDouble(3);
            long wikiId = rps.getLong(5);
            String wikiLang = rps.getString(4);
            String title = rps.getString(6);
            byte[] bytes = rps.getBytes(7);
            GZIPInputStream gzin = new GZIPInputStream(new ByteArrayInputStream(bytes));
            BufferedReader br = new BufferedReader(new InputStreamReader(gzin));
            content.setLength(0);
            String s;
            while ((s = br.readLine()) != null) {
                content.append(s);
            }
            String contentStr = content.toString();
            contentStr = contentStr.replace((char) 9, ' ');
            contentStr = contentStr.replace((char) 0, ' ');
            contentStr = contentStr.replace((char) 22, ' ');
            contentStr = contentStr.replace((char) 27, ' ');
            insertWikiContent.setLong(1, osmId);
            insertWikiContent.setDouble(2, lat);
            insertWikiContent.setDouble(3, lon);
            insertWikiContent.setString(4, wikiLang);
            insertWikiContent.setLong(5, wikiId);
            insertWikiContent.setString(6, title);
            insertWikiContent.setBytes(7, bytes);
            insertWikiContent.addBatch();
            if (cnt++ % BATCH_SIZE == 0) {
                insertWikiContent.executeBatch();
            }
            if (osmId != prevOsmId) {
                if (prevOsmId != -1) {
                    closeOsmWikiNode(serializer, nameUnique, nameAdded);
                }
                prevOsmId = osmId;
                nameAdded = false;
                nameUnique = null;
                preferredAdded = false;
                serializer.startTag(null, "node");
                serializer.attribute(null, "visible", "true");
                serializer.attribute(null, "id", (osmId) + "");
                serializer.attribute(null, "lat", lat + "");
                serializer.attribute(null, "lon", lon + "");
            }
            if (wikiLang.equals("en")) {
                nameAdded = true;
                addTag(serializer, "name", title);
                addTag(serializer, "wiki_id", wikiId + "");
                addTag(serializer, "content", contentStr);
                addTag(serializer, "wiki_lang:en", "yes");
            } else {
                addTag(serializer, "name:" + wikiLang, title);
                addTag(serializer, "wiki_id:" + wikiLang, wikiId + "");
                addTag(serializer, "wiki_lang:" + wikiLang, "yes");
                if (!preferredAdded) {
                    nameUnique = title;
                    preferredAdded = preferredLang.contains(wikiLang);
                }
                addTag(serializer, "content:" + wikiLang, contentStr);
            }
        }
        if (prevOsmId != -1) {
            closeOsmWikiNode(serializer, nameUnique, nameAdded);
        }
        insertWikiContent.executeBatch();
        loc.close();
        serializer.endDocument();
        serializer.flush();
        bzipStream.close();
        System.out.println("Processed " + cnt + " pois");
        generateObf(osmBz2, obfFile);
    }
    conn.close();
}
Also used : LinkedHashMap(java.util.LinkedHashMap) GZIPInputStream(java.util.zip.GZIPInputStream) OsmandRegions(net.osmand.map.OsmandRegions) BinaryMapDataObject(net.osmand.binary.BinaryMapDataObject) CBZip2OutputStream(org.apache.tools.bzip2.CBZip2OutputStream) ResultSet(java.sql.ResultSet) WorldRegion(net.osmand.map.WorldRegion) InputStreamReader(java.io.InputStreamReader) Connection(java.sql.Connection) PreparedStatement(java.sql.PreparedStatement) LinkedList(java.util.LinkedList) ByteArrayInputStream(java.io.ByteArrayInputStream) FileOutputStream(java.io.FileOutputStream) BufferedReader(java.io.BufferedReader) File(java.io.File) XmlSerializer(org.xmlpull.v1.XmlSerializer)

Example 2 with WorldRegion

use of net.osmand.map.WorldRegion in project OsmAnd-tools by osmandapp.

the class CalculateCountryForChangesets method initCountriesTable.

private static OsmandRegions initCountriesTable(Connection conn, boolean empty, Map<WorldRegion, Integer> map) throws IOException, SQLException {
    OsmandRegions or = new OsmandRegions();
    File regions = new File("OsmAndMapCreator/regions.ocbf");
    if (!regions.exists()) {
        regions = new File("regions.ocbf");
    }
    or.prepareFile(regions.getAbsolutePath());
    or.cacheAllCountries();
    WorldRegion worldRegion = or.getWorldRegion();
    if (empty) {
        int id = 0;
        PreparedStatement ps = conn.prepareStatement("INSERT INTO countries(id, parentid, name, fullname, downloadname, clat, clon, map)" + " VALUES(?, ?, ?, ?, ?, ?, ?, ?)");
        LinkedList<WorldRegion> queue = new LinkedList<WorldRegion>();
        queue.add(worldRegion);
        while (!queue.isEmpty()) {
            WorldRegion wr = queue.pollFirst();
            id++;
            map.put(wr, id);
            ps.setInt(1, id);
            WorldRegion parent = wr.getSuperregion();
            if (parent != null) {
                ps.setInt(2, map.get(parent));
            } else {
                ps.setInt(2, 0);
            }
            ps.setString(3, wr.getLocaleName());
            ps.setString(4, wr.getRegionId());
            ps.setString(5, wr.getRegionDownloadName());
            if (wr.getRegionCenter() != null) {
                ps.setDouble(6, wr.getRegionCenter().getLatitude());
                ps.setDouble(7, wr.getRegionCenter().getLongitude());
            } else {
                ps.setDouble(6, 0);
                ps.setDouble(7, 0);
            }
            ps.setInt(8, wr.isRegionMapDownload() ? 1 : 0);
            ps.addBatch();
            List<WorldRegion> lst = wr.getSubregions();
            if (lst != null) {
                queue.addAll(lst);
            }
        }
        ps.executeBatch();
        ps.close();
    }
    map.clear();
    ResultSet rs = conn.createStatement().executeQuery("select id, fullname from countries");
    while (rs.next()) {
        int id = rs.getInt(1);
        WorldRegion rd;
        if (rs.getString(2).equals("world")) {
            rd = worldRegion;
        } else {
            rd = or.getRegionData(rs.getString(2));
        }
        if (rd == null) {
            throw new UnsupportedOperationException(rs.getString(2) + " not found");
        }
        map.put(rd, id);
    }
    return or;
}
Also used : OsmandRegions(net.osmand.map.OsmandRegions) WorldRegion(net.osmand.map.WorldRegion) ResultSet(java.sql.ResultSet) PreparedStatement(java.sql.PreparedStatement) File(java.io.File) LinkedList(java.util.LinkedList)

Example 3 with WorldRegion

use of net.osmand.map.WorldRegion in project OsmAnd-tools by osmandapp.

the class CalculateCountryForChangesets method calculateCountries.

private static void calculateCountries() throws Exception {
    // jdbc:postgresql://user:secret@localhost
    Connection conn = DriverManager.getConnection("jdbc:postgresql://localhost:5433/changeset", System.getenv("DB_USER"), System.getenv("DB_PWD"));
    try {
        Statement stat = conn.createStatement();
        ResultSet rs = stat.executeQuery("SELECT COUNT(*) FROM countries");
        boolean empty = !rs.next() || rs.getInt(1) == 0;
        rs.close();
        Map<WorldRegion, Integer> map = new LinkedHashMap<WorldRegion, Integer>();
        OsmandRegions or = initCountriesTable(conn, empty, map);
        PreparedStatement ps = conn.prepareStatement("INSERT INTO changeset_country(changesetid, countryid, small)" + " VALUES(?, ?, ?)");
        rs = stat.executeQuery("select id, minlat, minlon, maxlat, maxlon from changesets C " + " where (maxlat <> 0 or minlat <> 0 or maxlon <> 0 or minlon <> 0) and " + "not exists (select 1 from changeset_country CC where CC.changesetid=C.id) limit " + FETCH_LIMIT);
        int batch = 0;
        int batchInd = 1;
        while (rs.next()) {
            double minlat = rs.getDouble(2);
            double minlon = rs.getDouble(3);
            double maxlat = rs.getDouble(4);
            double maxlon = rs.getDouble(5);
            String changesetId = rs.getString(1);
            int lx = MapUtils.get31TileNumberX(minlon);
            int rx = MapUtils.get31TileNumberX(maxlon);
            int ty = MapUtils.get31TileNumberY(maxlat);
            int by = MapUtils.get31TileNumberY(minlat);
            List<BinaryMapDataObject> objs = or.queryBbox(lx, rx, ty, by);
            int cid = 0;
            for (BinaryMapDataObject o : objs) {
                if (!or.intersect(o, lx, ty, rx, by)) {
                    continue;
                }
                String full = or.getFullName(o);
                WorldRegion reg = or.getRegionData(full);
                if (reg.isRegionMapDownload() && !full.toLowerCase().startsWith("world_")) {
                    cid++;
                    if (cid > MAX_COUNTRY_SIZE) {
                        continue;
                    }
                    // System.out.println(changesetId  + " " + full + " " + reg.getLocaleName() + " " + map.get(reg));
                    if (map.get(reg) == null) {
                        throw new UnsupportedOperationException("Not found " + changesetId + " " + full);
                    }
                    boolean small = true;
                    List<WorldRegion> subs = reg.getSubregions();
                    if (subs != null) {
                        for (WorldRegion sub : subs) {
                            if (sub.isRegionMapDownload()) {
                                small = false;
                                break;
                            }
                        }
                    }
                    ps.setString(1, changesetId);
                    ps.setInt(2, map.get(reg));
                    ps.setInt(3, small ? 1 : 0);
                    ps.addBatch();
                }
            }
            if (batch++ > BATCH_SIZE) {
                System.out.println("Execute batch " + (batchInd++) + " by " + BATCH_SIZE);
                ps.executeBatch();
                batch = 0;
            }
        }
        ps.executeBatch();
    } finally {
        conn.close();
    }
}
Also used : WorldRegion(net.osmand.map.WorldRegion) PreparedStatement(java.sql.PreparedStatement) Statement(java.sql.Statement) Connection(java.sql.Connection) PreparedStatement(java.sql.PreparedStatement) LinkedHashMap(java.util.LinkedHashMap) OsmandRegions(net.osmand.map.OsmandRegions) BinaryMapDataObject(net.osmand.binary.BinaryMapDataObject) ResultSet(java.sql.ResultSet)

Example 4 with WorldRegion

use of net.osmand.map.WorldRegion in project OsmAnd-tools by osmandapp.

the class ObfRegionSplitter method splitRegionRouteData.

private Map<String, TLongObjectHashMap<RouteDataObject>> splitRegionRouteData(ObfFileInMemory fl, OsmandRegions osmandRegions) throws IOException {
    Map<String, TLongObjectHashMap<RouteDataObject>> result = new HashMap<>();
    TLongObjectHashMap<RouteDataObject> routingData = fl.getRoutingData();
    for (RouteDataObject obj : routingData.valueCollection()) {
        // if(obj.getPointsLength() == 0) {
        // continue;
        // }
        int x = obj.getPoint31XTile(0);
        int y = obj.getPoint31YTile(0);
        List<BinaryMapDataObject> l = osmandRegions.query(x, y);
        for (BinaryMapDataObject b : l) {
            if (osmandRegions.contain(b, x, y)) {
                String dw = osmandRegions.getDownloadName(b);
                WorldRegion wr = osmandRegions.getRegionDataByDownloadName(dw);
                if (dw == null || wr == null) {
                    continue;
                }
                if (!Algorithms.isEmpty(dw) && wr.isRegionMapDownload()) {
                    TLongObjectHashMap<RouteDataObject> mp = result.get(dw);
                    if (mp == null) {
                        mp = new TLongObjectHashMap<>();
                        result.put(dw, mp);
                    }
                    mp.put(obj.getId(), obj);
                }
            }
        }
    }
    return result;
}
Also used : WorldRegion(net.osmand.map.WorldRegion) BinaryMapDataObject(net.osmand.binary.BinaryMapDataObject) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) TLongObjectHashMap(gnu.trove.map.hash.TLongObjectHashMap) TLongObjectHashMap(gnu.trove.map.hash.TLongObjectHashMap) RouteDataObject(net.osmand.binary.RouteDataObject)

Example 5 with WorldRegion

use of net.osmand.map.WorldRegion in project OsmAnd-tools by osmandapp.

the class ObfRegionSplitter method splitRegionPoiData.

private Map<String, TLongObjectHashMap<Map<String, Amenity>>> splitRegionPoiData(ObfFileInMemory fl, OsmandRegions osmandRegions) throws IOException {
    Map<String, TLongObjectHashMap<Map<String, Amenity>>> result = new HashMap<>();
    TLongObjectHashMap<Map<String, Amenity>> poiData = fl.getPoiObjects();
    for (Map<String, Amenity> objMap : poiData.valueCollection()) {
        Amenity obj = objMap.values().iterator().next();
        int x = MapUtils.get31TileNumberX(obj.getLocation().getLongitude());
        int y = MapUtils.get31TileNumberY(obj.getLocation().getLatitude());
        List<BinaryMapDataObject> l = osmandRegions.query(x, y);
        for (BinaryMapDataObject b : l) {
            if (osmandRegions.contain(b, x, y)) {
                String dw = osmandRegions.getDownloadName(b);
                WorldRegion wr = osmandRegions.getRegionDataByDownloadName(dw);
                if (dw == null || wr == null) {
                    continue;
                }
                if (!Algorithms.isEmpty(dw) && wr.isRegionMapDownload()) {
                    TLongObjectHashMap<Map<String, Amenity>> mp = result.get(dw);
                    if (mp == null) {
                        mp = new TLongObjectHashMap<>();
                        result.put(dw, mp);
                    }
                    mp.put(obj.getId(), objMap);
                }
            }
        }
    }
    return result;
}
Also used : Amenity(net.osmand.data.Amenity) WorldRegion(net.osmand.map.WorldRegion) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) TLongObjectHashMap(gnu.trove.map.hash.TLongObjectHashMap) TLongObjectHashMap(gnu.trove.map.hash.TLongObjectHashMap) BinaryMapDataObject(net.osmand.binary.BinaryMapDataObject) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) TLongObjectHashMap(gnu.trove.map.hash.TLongObjectHashMap) Map(java.util.Map)

Aggregations

WorldRegion (net.osmand.map.WorldRegion)26 BinaryMapDataObject (net.osmand.binary.BinaryMapDataObject)16 LinkedHashMap (java.util.LinkedHashMap)12 OsmandRegions (net.osmand.map.OsmandRegions)11 LinkedList (java.util.LinkedList)8 TLongObjectHashMap (gnu.trove.map.hash.TLongObjectHashMap)7 HashMap (java.util.HashMap)7 PreparedStatement (java.sql.PreparedStatement)6 ResultSet (java.sql.ResultSet)6 Connection (java.sql.Connection)4 ArrayList (java.util.ArrayList)4 IndexItem (net.osmand.plus.download.IndexItem)4 File (java.io.File)3 IOException (java.io.IOException)3 Map (java.util.Map)3 Paint (android.graphics.Paint)2 SpannableString (android.text.SpannableString)2 TextPaint (android.text.TextPaint)2 Statement (java.sql.Statement)2 List (java.util.List)2