Examples with BinaryMapDataObject net.osmand.binary.BinaryMapDataObject used on opensource projects

Example 1 with BinaryMapDataObject

use of net.osmand.binary.BinaryMapDataObject in project OsmAnd-tools by osmandapp.

the class IndexUploader method copyMapIndex.

private static void copyMapIndex(File roadOnlyFile, MapIndex part, BinaryMapIndexReader index, CodedOutputStream ous, RandomAccessFile routf) throws IOException, RTreeException {
    final List<MapRoot> rts = part.getRoots();
    BinaryMapIndexWriter writer = new BinaryMapIndexWriter(routf, ous);
    writer.startWriteMapIndex(part.getName());
    boolean first = true;
    for (MapRoot r : rts) {
        final TLongObjectHashMap<BinaryMapDataObject> objects = new TLongObjectHashMap<BinaryMapDataObject>();
        File nonpackRtree = new File(roadOnlyFile.getParentFile(), "nonpack" + r.getMinZoom() + "." + roadOnlyFile.getName() + ".rtree");
        File packRtree = new File(roadOnlyFile.getParentFile(), "pack" + r.getMinZoom() + "." + roadOnlyFile.getName() + ".rtree");
        RTree rtree = null;
        try {
            rtree = new RTree(nonpackRtree.getAbsolutePath());
            final SearchRequest<BinaryMapDataObject> req = buildSearchRequest(r, objects, rtree);
            index.searchMapIndex(req, part);
            rtree = AbstractIndexPartCreator.packRtreeFile(rtree, nonpackRtree.getAbsolutePath(), packRtree.getAbsolutePath());
            TLongObjectHashMap<BinaryFileReference> treeHeader = new TLongObjectHashMap<BinaryFileReference>();
            long rootIndex = rtree.getFileHdr().getRootIndex();
            rtree.Node root = rtree.getReadNode(rootIndex);
            Rect rootBounds = calcBounds(root);
            if (rootBounds != null) {
                if (first) {
                    writer.writeMapEncodingRules(part.decodingRules);
                    first = false;
                }
                writer.startWriteMapLevelIndex(r.getMinZoom(), r.getMaxZoom(), rootBounds.getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY());
                IndexVectorMapCreator.writeBinaryMapTree(root, rootBounds, rtree, writer, treeHeader);
                writeBinaryMapBlock(root, rootBounds, rtree, writer, treeHeader, objects, r.getMapZoom(), false);
                writer.endWriteMapLevelIndex();
            }
        } finally {
            if (rtree != null) {
                RandomAccessFile file = rtree.getFileHdr().getFile();
                file.close();
            }
            nonpackRtree.delete();
            packRtree.delete();
            RTree.clearCache();
        }
    }
    writer.endWriteMapIndex();
}

Example 2 with BinaryMapDataObject

use of net.osmand.binary.BinaryMapDataObject in project OsmAnd-tools by osmandapp.

the class IndexUploader method writeBinaryMapBlock.

public static void writeBinaryMapBlock(Node parent, Rect parentBounds, RTree r, BinaryMapIndexWriter writer, TLongObjectHashMap<BinaryFileReference> bounds, TLongObjectHashMap<BinaryMapDataObject> objects, MapZooms.MapZoomPair pair, boolean doNotSimplify) throws IOException, RTreeException {
    rtree.Element[] e = parent.getAllElements();
    MapDataBlock.Builder dataBlock = null;
    BinaryFileReference ref = bounds.get(parent.getNodeIndex());
    long baseId = 0;
    Map<String, Integer> tempStringTable = new LinkedHashMap<String, Integer>();
    for (int i = 0; i < parent.getTotalElements(); i++) {
        if (e[i].getElementType() == rtree.Node.LEAF_NODE) {
            long id = e[i].getPtr();
            if (objects.containsKey(id)) {
                long cid = id;
                BinaryMapDataObject mdo = objects.get(id);
                if (dataBlock == null) {
                    baseId = cid;
                    dataBlock = writer.createWriteMapDataBlock(baseId);
                    tempStringTable.clear();
                }
                int[] typeUse = mdo.getTypes();
                int[] addtypeUse = mdo.getAdditionalTypes();
                byte[] coordinates = new byte[8 * mdo.getPointsLength()];
                for (int t = 0; t < mdo.getPointsLength(); t++) {
                    Algorithms.putIntToBytes(coordinates, 8 * t, mdo.getPoint31XTile(t));
                    Algorithms.putIntToBytes(coordinates, 8 * t + 4, mdo.getPoint31YTile(t));
                }
                byte[] innerPolygonTypes = new byte[0];
                int[][] pip = mdo.getPolygonInnerCoordinates();
                if (pip != null && pip.length > 0) {
                    ByteArrayOutputStream bous = new ByteArrayOutputStream();
                    for (int s = 0; s < pip.length; s++) {
                        int[] st = pip[s];
                        for (int t = 0; t < st.length; t++) {
                            Algorithms.writeInt(bous, st[t]);
                        }
                        Algorithms.writeInt(bous, 0);
                        Algorithms.writeInt(bous, 0);
                    }
                    innerPolygonTypes = bous.toByteArray();
                }
                MapData mapData = writer.writeMapData(cid - baseId, parentBounds.getMinX(), parentBounds.getMinY(), mdo.isArea(), coordinates, innerPolygonTypes, typeUse, addtypeUse, null, mdo.getOrderedObjectNames(), tempStringTable, dataBlock, !doNotSimplify && pair.getMaxZoom() > 15);
                if (mapData != null) {
                    dataBlock.addDataObjects(mapData);
                }
            } else {
                // $NON-NLS-1$
                log.error("Something goes wrong with id = " + id);
            }
        }
    }
    if (dataBlock != null) {
        writer.writeMapDataBlock(dataBlock, tempStringTable, ref);
    }
    for (int i = 0; i < parent.getTotalElements(); i++) {
        if (e[i].getElementType() != rtree.Node.LEAF_NODE) {
            long ptr = e[i].getPtr();
            rtree.Node ns = r.getReadNode(ptr);
            writeBinaryMapBlock(ns, e[i].getRect(), r, writer, bounds, objects, pair, doNotSimplify);
        }
    }
}

Example 3 with BinaryMapDataObject

use of net.osmand.binary.BinaryMapDataObject in project OsmAnd-tools by osmandapp.

the class WikipediaByCountryDivider method generateCountrySqlite.

protected static void generateCountrySqlite(String folder, boolean skip) throws SQLException, IOException, InterruptedException, XmlPullParserException {
    Connection conn = (Connection) DBDialect.SQLITE.getDatabaseConnection(folder + "wiki.sqlite", log);
    OsmandRegions regs = new OsmandRegions();
    regs.prepareFile(new File("resources/countries-info/regions.ocbf").getAbsolutePath());
    Map<String, LinkedList<BinaryMapDataObject>> mapObjects = regs.cacheAllCountries();
    File rgns = new File(folder, "regions");
    rgns.mkdirs();
    Map<String, String> preferredRegionLanguages = new LinkedHashMap<>();
    for (String key : mapObjects.keySet()) {
        if (key == null) {
            continue;
        }
        WorldRegion wr = regs.getRegionDataByDownloadName(key);
        if (wr == null) {
            System.out.println("Missing language for world region '" + key + "'!");
        } else {
            String regionLang = wr.getParams().getRegionLang();
            preferredRegionLanguages.put(key.toLowerCase(), regionLang);
        }
    }
    ResultSet rs = conn.createStatement().executeQuery("SELECT DISTINCT regionName  FROM wiki_region");
    while (rs.next()) {
        String lcRegionName = rs.getString(1);
        if (lcRegionName == null) {
            continue;
        }
        String regionName = Algorithms.capitalizeFirstLetterAndLowercase(lcRegionName);
        String preferredLang = preferredRegionLanguages.get(lcRegionName);
        if (preferredLang == null) {
            preferredLang = "";
        }
        LinkedList<BinaryMapDataObject> list = mapObjects.get(lcRegionName.toLowerCase());
        boolean hasWiki = false;
        if (list != null) {
            for (BinaryMapDataObject o : list) {
                Integer rl = o.getMapIndex().getRule("region_wiki", "yes");
                if (o.containsAdditionalType(rl)) {
                    hasWiki = true;
                    break;
                }
            }
        }
        if (!hasWiki) {
            System.out.println("Skip " + lcRegionName.toLowerCase() + " doesn't generate wiki");
            continue;
        }
        File fl = new File(rgns, regionName + ".sqlite");
        File osmBz2 = new File(rgns, regionName + "_" + IndexConstants.BINARY_MAP_VERSION + ".wiki.osm.bz2");
        File obfFile = new File(rgns, regionName + "_" + IndexConstants.BINARY_MAP_VERSION + ".wiki.obf");
        if (obfFile.exists() && skip) {
            continue;
        }
        fl.delete();
        osmBz2.delete();
        obfFile.delete();
        System.out.println("Generate " + fl.getName());
        Connection loc = (Connection) DBDialect.SQLITE.getDatabaseConnection(fl.getAbsolutePath(), log);
        loc.createStatement().execute("CREATE TABLE wiki_content(id long, lat double, lon double, lang text, wikiId long, title text, zipContent blob)");
        PreparedStatement insertWikiContent = loc.prepareStatement("INSERT INTO wiki_content VALUES(?, ?, ?, ?, ?, ?, ?)");
        ResultSet rps = conn.createStatement().executeQuery("SELECT WC.id, WC.lat, WC.lon, WC.lang, WC.wikiId, WC.title, WC.zipContent " + " FROM wiki_content WC INNER JOIN wiki_region WR " + " ON WC.id = WR.id AND WR.regionName = '" + rs.getString(1) + "' ORDER BY WC.id");
        FileOutputStream out = new FileOutputStream(osmBz2);
        out.write('B');
        out.write('Z');
        CBZip2OutputStream bzipStream = new CBZip2OutputStream(out);
        XmlSerializer serializer = new org.kxml2.io.KXmlSerializer();
        serializer.setOutput(bzipStream, "UTF-8");
        serializer.startDocument("UTF-8", true);
        serializer.startTag(null, "osm");
        serializer.attribute(null, "version", "0.6");
        serializer.attribute(null, "generator", "OsmAnd");
        serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true);
        // indentation as 3 spaces
        // serializer.setProperty(
        // "http://xmlpull.org/v1/doc/properties.html#serializer-indentation", "   ");
        // // also set the line separator
        // serializer.setProperty(
        // "http://xmlpull.org/v1/doc/properties.html#serializer-line-separator", "\n");
        int cnt = 1;
        long prevOsmId = -1;
        StringBuilder content = new StringBuilder();
        String nameUnique = null;
        boolean preferredAdded = false;
        boolean nameAdded = false;
        while (rps.next()) {
            long osmId = -rps.getLong(1);
            double lat = rps.getDouble(2);
            double lon = rps.getDouble(3);
            long wikiId = rps.getLong(5);
            String wikiLang = rps.getString(4);
            String title = rps.getString(6);
            byte[] bytes = rps.getBytes(7);
            GZIPInputStream gzin = new GZIPInputStream(new ByteArrayInputStream(bytes));
            BufferedReader br = new BufferedReader(new InputStreamReader(gzin));
            content.setLength(0);
            String s;
            while ((s = br.readLine()) != null) {
                content.append(s);
            }
            String contentStr = content.toString();
            contentStr = contentStr.replace((char) 9, ' ');
            contentStr = contentStr.replace((char) 0, ' ');
            contentStr = contentStr.replace((char) 22, ' ');
            contentStr = contentStr.replace((char) 27, ' ');
            insertWikiContent.setLong(1, osmId);
            insertWikiContent.setDouble(2, lat);
            insertWikiContent.setDouble(3, lon);
            insertWikiContent.setString(4, wikiLang);
            insertWikiContent.setLong(5, wikiId);
            insertWikiContent.setString(6, title);
            insertWikiContent.setBytes(7, bytes);
            insertWikiContent.addBatch();
            if (cnt++ % BATCH_SIZE == 0) {
                insertWikiContent.executeBatch();
            }
            if (osmId != prevOsmId) {
                if (prevOsmId != -1) {
                    closeOsmWikiNode(serializer, nameUnique, nameAdded);
                }
                prevOsmId = osmId;
                nameAdded = false;
                nameUnique = null;
                preferredAdded = false;
                serializer.startTag(null, "node");
                serializer.attribute(null, "visible", "true");
                serializer.attribute(null, "id", (osmId) + "");
                serializer.attribute(null, "lat", lat + "");
                serializer.attribute(null, "lon", lon + "");
            }
            if (wikiLang.equals("en")) {
                nameAdded = true;
                addTag(serializer, "name", title);
                addTag(serializer, "wiki_id", wikiId + "");
                addTag(serializer, "content", contentStr);
                addTag(serializer, "wiki_lang:en", "yes");
            } else {
                addTag(serializer, "name:" + wikiLang, title);
                addTag(serializer, "wiki_id:" + wikiLang, wikiId + "");
                addTag(serializer, "wiki_lang:" + wikiLang, "yes");
                if (!preferredAdded) {
                    nameUnique = title;
                    preferredAdded = preferredLang.contains(wikiLang);
                }
                addTag(serializer, "content:" + wikiLang, contentStr);
            }
        }
        if (prevOsmId != -1) {
            closeOsmWikiNode(serializer, nameUnique, nameAdded);
        }
        insertWikiContent.executeBatch();
        loc.close();
        serializer.endDocument();
        serializer.flush();
        bzipStream.close();
        System.out.println("Processed " + cnt + " pois");
        generateObf(osmBz2, obfFile);
    }
    conn.close();
}

Example 4 with BinaryMapDataObject

use of net.osmand.binary.BinaryMapDataObject in project OsmAnd-tools by osmandapp.

the class CalculateCountryForChangesets method calculateCountries.

private static void calculateCountries() throws Exception {
    // jdbc:postgresql://user:secret@localhost
    Connection conn = DriverManager.getConnection("jdbc:postgresql://localhost:5433/changeset", System.getenv("DB_USER"), System.getenv("DB_PWD"));
    try {
        Statement stat = conn.createStatement();
        ResultSet rs = stat.executeQuery("SELECT COUNT(*) FROM countries");
        boolean empty = !rs.next() || rs.getInt(1) == 0;
        rs.close();
        Map<WorldRegion, Integer> map = new LinkedHashMap<WorldRegion, Integer>();
        OsmandRegions or = initCountriesTable(conn, empty, map);
        PreparedStatement ps = conn.prepareStatement("INSERT INTO changeset_country(changesetid, countryid, small)" + " VALUES(?, ?, ?)");
        rs = stat.executeQuery("select id, minlat, minlon, maxlat, maxlon from changesets C " + " where (maxlat <> 0 or minlat <> 0 or maxlon <> 0 or minlon <> 0) and " + "not exists (select 1 from changeset_country CC where CC.changesetid=C.id) limit " + FETCH_LIMIT);
        int batch = 0;
        int batchInd = 1;
        while (rs.next()) {
            double minlat = rs.getDouble(2);
            double minlon = rs.getDouble(3);
            double maxlat = rs.getDouble(4);
            double maxlon = rs.getDouble(5);
            String changesetId = rs.getString(1);
            int lx = MapUtils.get31TileNumberX(minlon);
            int rx = MapUtils.get31TileNumberX(maxlon);
            int ty = MapUtils.get31TileNumberY(maxlat);
            int by = MapUtils.get31TileNumberY(minlat);
            List<BinaryMapDataObject> objs = or.queryBbox(lx, rx, ty, by);
            int cid = 0;
            for (BinaryMapDataObject o : objs) {
                if (!or.intersect(o, lx, ty, rx, by)) {
                    continue;
                }
                String full = or.getFullName(o);
                WorldRegion reg = or.getRegionData(full);
                if (reg.isRegionMapDownload() && !full.toLowerCase().startsWith("world_")) {
                    cid++;
                    if (cid > MAX_COUNTRY_SIZE) {
                        continue;
                    }
                    // System.out.println(changesetId  + " " + full + " " + reg.getLocaleName() + " " + map.get(reg));
                    if (map.get(reg) == null) {
                        throw new UnsupportedOperationException("Not found " + changesetId + " " + full);
                    }
                    boolean small = true;
                    List<WorldRegion> subs = reg.getSubregions();
                    if (subs != null) {
                        for (WorldRegion sub : subs) {
                            if (sub.isRegionMapDownload()) {
                                small = false;
                                break;
                            }
                        }
                    }
                    ps.setString(1, changesetId);
                    ps.setInt(2, map.get(reg));
                    ps.setInt(3, small ? 1 : 0);
                    ps.addBatch();
                }
            }
            if (batch++ > BATCH_SIZE) {
                System.out.println("Execute batch " + (batchInd++) + " by " + BATCH_SIZE);
                ps.executeBatch();
                batch = 0;
            }
        }
        ps.executeBatch();
    } finally {
        conn.close();
    }
}

Example 5 with BinaryMapDataObject

use of net.osmand.binary.BinaryMapDataObject in project OsmAnd-tools by osmandapp.

the class ObfDiffGenerator method compareMapData.

private void compareMapData(ObfFileInMemory fStart, ObfFileInMemory fEnd, boolean print, Set<EntityId> modifiedObjIds) {
    fStart.filterAllZoomsBelow(13);
    fEnd.filterAllZoomsBelow(13);
    MapIndex mi = fEnd.getMapIndex();
    int deleteId;
    Integer rl = mi.getRule(OSMAND_CHANGE_TAG, OSMAND_CHANGE_VALUE);
    if (rl != null) {
        deleteId = rl;
    } else {
        deleteId = mi.decodingRules.size() + 1;
        mi.initMapEncodingRule(0, deleteId, OSMAND_CHANGE_TAG, OSMAND_CHANGE_VALUE);
    }
    for (MapZoomPair mz : fStart.getZooms()) {
        TLongObjectHashMap<BinaryMapDataObject> startData = fStart.get(mz);
        TLongObjectHashMap<BinaryMapDataObject> endData = fEnd.get(mz);
        if (print) {
            System.out.println("Compare map " + mz);
        }
        if (endData == null) {
            continue;
        }
        for (Long idx : startData.keys()) {
            BinaryMapDataObject objE = endData.get(idx);
            BinaryMapDataObject objS = startData.get(idx);
            EntityId thisEntityId = getMapEntityId(objS.getId());
            if (print) {
                if (objE == null) {
                    System.out.println("Map " + idx + " is missing in (2): " + toString(objS));
                } else {
                    if (// !objS.getMapIndex().decodeType(objS.getTypes()[0]).tag.equals(OSMAND_CHANGE_TAG) &&
                    !objE.compareBinary(objS, COORDINATES_PRECISION_COMPARE)) {
                        System.out.println("Map " + idx + " is not equal: " + toString(objS) + " != " + toString(objE));
                    }
                    endData.remove(idx);
                }
            } else {
                if (objE == null) {
                    if (modifiedObjIds == null || modifiedObjIds.contains(thisEntityId) || thisEntityId == null) {
                        BinaryMapDataObject obj = new BinaryMapDataObject(idx, objS.getCoordinates(), null, objS.getObjectType(), objS.isArea(), new int[] { deleteId }, null);
                        endData.put(idx, obj);
                    }
                } else if (objE.compareBinary(objS, COORDINATES_PRECISION_COMPARE)) {
                    endData.remove(idx);
                }
            }
        }
        if (print) {
            for (BinaryMapDataObject e : endData.valueCollection()) {
                System.out.println("Map " + e.getId() + " is missing in (1): " + toString(e));
            }
        }
    }
}