Examples with BinaryFileReference net.osmand.data.preparation.BinaryFileReference used on opensource projects

Example 1 with BinaryFileReference

use of net.osmand.data.preparation.BinaryFileReference in project OsmAnd-tools by osmandapp.

the class IndexUploader method copyMapIndex.

private static void copyMapIndex(File roadOnlyFile, MapIndex part, BinaryMapIndexReader index, CodedOutputStream ous, RandomAccessFile routf) throws IOException, RTreeException {
    final List<MapRoot> rts = part.getRoots();
    BinaryMapIndexWriter writer = new BinaryMapIndexWriter(routf, ous);
    writer.startWriteMapIndex(part.getName());
    boolean first = true;
    for (MapRoot r : rts) {
        final TLongObjectHashMap<BinaryMapDataObject> objects = new TLongObjectHashMap<BinaryMapDataObject>();
        File nonpackRtree = new File(roadOnlyFile.getParentFile(), "nonpack" + r.getMinZoom() + "." + roadOnlyFile.getName() + ".rtree");
        File packRtree = new File(roadOnlyFile.getParentFile(), "pack" + r.getMinZoom() + "." + roadOnlyFile.getName() + ".rtree");
        RTree rtree = null;
        try {
            rtree = new RTree(nonpackRtree.getAbsolutePath());
            final SearchRequest<BinaryMapDataObject> req = buildSearchRequest(r, objects, rtree);
            index.searchMapIndex(req, part);
            rtree = AbstractIndexPartCreator.packRtreeFile(rtree, nonpackRtree.getAbsolutePath(), packRtree.getAbsolutePath());
            TLongObjectHashMap<BinaryFileReference> treeHeader = new TLongObjectHashMap<BinaryFileReference>();
            long rootIndex = rtree.getFileHdr().getRootIndex();
            rtree.Node root = rtree.getReadNode(rootIndex);
            Rect rootBounds = calcBounds(root);
            if (rootBounds != null) {
                if (first) {
                    writer.writeMapEncodingRules(part.decodingRules);
                    first = false;
                }
                writer.startWriteMapLevelIndex(r.getMinZoom(), r.getMaxZoom(), rootBounds.getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY());
                IndexVectorMapCreator.writeBinaryMapTree(root, rootBounds, rtree, writer, treeHeader);
                writeBinaryMapBlock(root, rootBounds, rtree, writer, treeHeader, objects, r.getMapZoom(), false);
                writer.endWriteMapLevelIndex();
            }
        } finally {
            if (rtree != null) {
                RandomAccessFile file = rtree.getFileHdr().getFile();
                file.close();
            }
            nonpackRtree.delete();
            packRtree.delete();
            RTree.clearCache();
        }
    }
    writer.endWriteMapIndex();
}

Example 2 with BinaryFileReference

use of net.osmand.data.preparation.BinaryFileReference in project OsmAnd-tools by osmandapp.

the class IndexUploader method writeBinaryMapBlock.

public static void writeBinaryMapBlock(Node parent, Rect parentBounds, RTree r, BinaryMapIndexWriter writer, TLongObjectHashMap<BinaryFileReference> bounds, TLongObjectHashMap<BinaryMapDataObject> objects, MapZooms.MapZoomPair pair, boolean doNotSimplify) throws IOException, RTreeException {
    rtree.Element[] e = parent.getAllElements();
    MapDataBlock.Builder dataBlock = null;
    BinaryFileReference ref = bounds.get(parent.getNodeIndex());
    long baseId = 0;
    Map<String, Integer> tempStringTable = new LinkedHashMap<String, Integer>();
    for (int i = 0; i < parent.getTotalElements(); i++) {
        if (e[i].getElementType() == rtree.Node.LEAF_NODE) {
            long id = e[i].getPtr();
            if (objects.containsKey(id)) {
                long cid = id;
                BinaryMapDataObject mdo = objects.get(id);
                if (dataBlock == null) {
                    baseId = cid;
                    dataBlock = writer.createWriteMapDataBlock(baseId);
                    tempStringTable.clear();
                }
                int[] typeUse = mdo.getTypes();
                int[] addtypeUse = mdo.getAdditionalTypes();
                byte[] coordinates = new byte[8 * mdo.getPointsLength()];
                for (int t = 0; t < mdo.getPointsLength(); t++) {
                    Algorithms.putIntToBytes(coordinates, 8 * t, mdo.getPoint31XTile(t));
                    Algorithms.putIntToBytes(coordinates, 8 * t + 4, mdo.getPoint31YTile(t));
                }
                byte[] innerPolygonTypes = new byte[0];
                int[][] pip = mdo.getPolygonInnerCoordinates();
                if (pip != null && pip.length > 0) {
                    ByteArrayOutputStream bous = new ByteArrayOutputStream();
                    for (int s = 0; s < pip.length; s++) {
                        int[] st = pip[s];
                        for (int t = 0; t < st.length; t++) {
                            Algorithms.writeInt(bous, st[t]);
                        }
                        Algorithms.writeInt(bous, 0);
                        Algorithms.writeInt(bous, 0);
                    }
                    innerPolygonTypes = bous.toByteArray();
                }
                MapData mapData = writer.writeMapData(cid - baseId, parentBounds.getMinX(), parentBounds.getMinY(), mdo.isArea(), coordinates, innerPolygonTypes, typeUse, addtypeUse, null, mdo.getOrderedObjectNames(), tempStringTable, dataBlock, !doNotSimplify && pair.getMaxZoom() > 15);
                if (mapData != null) {
                    dataBlock.addDataObjects(mapData);
                }
            } else {
                // $NON-NLS-1$
                log.error("Something goes wrong with id = " + id);
            }
        }
    }
    if (dataBlock != null) {
        writer.writeMapDataBlock(dataBlock, tempStringTable, ref);
    }
    for (int i = 0; i < parent.getTotalElements(); i++) {
        if (e[i].getElementType() != rtree.Node.LEAF_NODE) {
            long ptr = e[i].getPtr();
            rtree.Node ns = r.getReadNode(ptr);
            writeBinaryMapBlock(ns, e[i].getRect(), r, writer, bounds, objects, pair, doNotSimplify);
        }
    }
}

Example 3 with BinaryFileReference

use of net.osmand.data.preparation.BinaryFileReference in project OsmAnd-tools by osmandapp.

the class IndexAddressCreator method writeCityBlockIndex.

private void writeCityBlockIndex(BinaryMapIndexWriter writer, int type, PreparedStatement streetstat, PreparedStatement waynodesStat, List<City> suburbs, List<City> cities, Map<String, City> postcodes, Map<String, List<MapObject>> namesIndex, Map<String, Integer> tagRules, IProgress progress) throws IOException, SQLException {
    List<BinaryFileReference> refs = new ArrayList<BinaryFileReference>();
    // 1. write cities
    writer.startCityBlockIndex(type);
    for (City c : cities) {
        refs.add(writer.writeCityHeader(c, c.getType().ordinal(), tagRules));
    }
    for (int i = 0; i < cities.size(); i++) {
        City city = cities.get(i);
        BinaryFileReference ref = refs.get(i);
        putNamedMapObject(namesIndex, city, ref.getStartPointer());
        if (type == CITIES_TYPE) {
            progress.progress(1);
        } else {
            if ((cities.size() - i) % 100 == 0) {
                progress.progress(1);
            }
        }
        Map<Street, List<Node>> streetNodes = new LinkedHashMap<Street, List<Node>>();
        List<City> listSuburbs = null;
        if (suburbs != null) {
            for (City suburb : suburbs) {
                if (suburb.getIsInValue().toLowerCase().contains(city.getName().toLowerCase())) {
                    if (listSuburbs == null) {
                        listSuburbs = new ArrayList<City>();
                    }
                    listSuburbs.add(suburb);
                }
            }
        }
        long time = System.currentTimeMillis();
        List<Street> streets = readStreetsBuildings(streetstat, city, waynodesStat, streetNodes, listSuburbs);
        long f = System.currentTimeMillis() - time;
        writer.writeCityIndex(city, streets, streetNodes, ref, tagRules);
        int bCount = 0;
        // register postcodes and name index
        for (Street s : streets) {
            putNamedMapObject(namesIndex, s, s.getFileOffset());
            for (Building b : s.getBuildings()) {
                bCount++;
                if (city.getPostcode() != null && b.getPostcode() == null) {
                    b.setPostcode(city.getPostcode());
                }
                if (b.getPostcode() != null) {
                    if (!postcodes.containsKey(b.getPostcode())) {
                        City p = City.createPostcode(b.getPostcode());
                        p.setLocation(b.getLocation().getLatitude(), b.getLocation().getLongitude());
                        postcodes.put(b.getPostcode(), p);
                    }
                    City post = postcodes.get(b.getPostcode());
                    Street newS = post.getStreetByName(s.getName());
                    if (newS == null) {
                        newS = new Street(post);
                        newS.copyNames(s);
                        newS.setLocation(s.getLocation().getLatitude(), s.getLocation().getLongitude());
                        // newS.getWayNodes().addAll(s.getWayNodes());
                        newS.setId(s.getId());
                        post.registerStreet(newS);
                    }
                    newS.addBuildingCheckById(b);
                }
            }
        }
        if (f > 500) {
            if (logMapDataWarn != null) {
                logMapDataWarn.info("! " + city.getName() + " ! " + f + " ms " + streets.size() + " streets " + bCount + " buildings");
            } else {
                log.info("! " + city.getName() + " ! " + f + " ms " + streets.size() + " streets " + bCount + " buildings");
            }
        }
    }
    writer.endCityBlockIndex();
}

Example 4 with BinaryFileReference

use of net.osmand.data.preparation.BinaryFileReference in project OsmAnd-tools by osmandapp.

the class IndexAddressCreator method writeBinaryAddressIndex.

public void writeBinaryAddressIndex(BinaryMapIndexWriter writer, String regionName, IProgress progress) throws IOException, SQLException {
    processPostcodes();
    cleanCityPart();
    streetDAO.close();
    closePreparedStatements(addressCityStat);
    mapConnection.commit();
    createDatabaseIndexes(mapConnection);
    mapConnection.commit();
    Map<String, City> postcodes = new TreeMap<String, City>();
    updatePostcodeBoundaries(progress, postcodes);
    mapConnection.commit();
    List<String> additionalTags = new ArrayList<String>();
    Map<String, Integer> tagRules = new HashMap<String, Integer>();
    int ind = 0;
    for (String lng : langAttributes) {
        additionalTags.add("name:" + lng);
        tagRules.put("name:" + lng, ind);
        ind++;
    }
    writer.startWriteAddressIndex(regionName, additionalTags);
    Map<CityType, List<City>> cities = readCities(mapConnection);
    PreparedStatement streetstat = // 
    mapConnection.prepareStatement(// $NON-NLS-1$
    "SELECT A.id, A.name, A.name_en, A.latitude, A.longitude, " + // $NON-NLS-1$
    "B.id, B.name, B.name_en, B.latitude, B.longitude, B.postcode, A.cityPart, " + " B.name2, B.name_en2, B.lat2, B.lon2, B.interval, B.interpolateType, A.cityPart == C.name as MainTown " + // $NON-NLS-1$
    "FROM street A LEFT JOIN building B ON B.street = A.id JOIN city C ON A.city = C.id " + // $NON-NLS-1$
    "WHERE A.city = ? ORDER BY MainTown DESC, A.name ASC");
    PreparedStatement waynodesStat = // $NON-NLS-1$
    mapConnection.prepareStatement("SELECT A.id, A.latitude, A.longitude FROM street_node A WHERE A.street = ? ");
    // collect suburbs with is in value
    List<City> suburbs = new ArrayList<City>();
    List<City> cityTowns = new ArrayList<City>();
    List<City> villages = new ArrayList<City>();
    for (CityType t : cities.keySet()) {
        if (t == CityType.CITY || t == CityType.TOWN) {
            cityTowns.addAll(cities.get(t));
        } else {
            villages.addAll(cities.get(t));
        }
        if (t == CityType.SUBURB) {
            for (City c : cities.get(t)) {
                if (c.getIsInValue() != null) {
                    suburbs.add(c);
                }
            }
        }
    }
    Map<String, List<MapObject>> namesIndex = new TreeMap<String, List<MapObject>>(Collator.getInstance());
    // $NON-NLS-1$
    progress.startTask(Messages.getString("IndexCreator.SERIALIZING_ADDRESS"), cityTowns.size() + villages.size() / 100 + 1);
    writeCityBlockIndex(writer, CITIES_TYPE, streetstat, waynodesStat, suburbs, cityTowns, postcodes, namesIndex, tagRules, progress);
    writeCityBlockIndex(writer, VILLAGES_TYPE, streetstat, waynodesStat, null, villages, postcodes, namesIndex, tagRules, progress);
    // write postcodes
    List<BinaryFileReference> refs = new ArrayList<BinaryFileReference>();
    writer.startCityBlockIndex(POSTCODES_TYPE);
    ArrayList<City> posts = new ArrayList<City>(postcodes.values());
    for (City s : posts) {
        refs.add(writer.writeCityHeader(s, -1, tagRules));
    }
    for (int i = 0; i < posts.size(); i++) {
        City postCode = posts.get(i);
        BinaryFileReference ref = refs.get(i);
        putNamedMapObject(namesIndex, postCode, ref.getStartPointer());
        ArrayList<Street> streets = new ArrayList<Street>(postCode.getStreets());
        Collections.sort(streets, new Comparator<Street>() {

            final net.osmand.Collator clt = OsmAndCollator.primaryCollator();

            @Override
            public int compare(Street o1, Street o2) {
                return clt.compare(o1.getName(), o2.getName());
            }
        });
        writer.writeCityIndex(postCode, streets, null, ref, tagRules);
    }
    writer.endCityBlockIndex();
    progress.finishTask();
    writer.writeAddressNameIndex(namesIndex);
    writer.endWriteAddressIndex();
    writer.flush();
    streetstat.close();
    if (waynodesStat != null) {
        waynodesStat.close();
    }
}

Example 5 with BinaryFileReference

use of net.osmand.data.preparation.BinaryFileReference in project OsmAnd-tools by osmandapp.

the class ObfFileInMemory method writeMapData.

private void writeMapData(BinaryMapIndexWriter writer, MapZoomPair mapZoomPair, TLongObjectHashMap<BinaryMapDataObject> objects, File fileToWrite, boolean doNotSimplify) throws IOException, RTreeException {
    File nonpackRtree = new File(fileToWrite.getParentFile(), "nonpack" + mapZoomPair.getMinZoom() + "." + fileToWrite.getName() + ".rtree");
    File packRtree = new File(fileToWrite.getParentFile(), "pack" + mapZoomPair.getMinZoom() + "." + fileToWrite.getName() + ".rtree");
    RTree rtree = null;
    try {
        rtree = new RTree(nonpackRtree.getAbsolutePath());
        for (long key : objects.keys()) {
            BinaryMapDataObject obj = objects.get(key);
            int minX = obj.getPoint31XTile(0);
            int maxX = obj.getPoint31XTile(0);
            int maxY = obj.getPoint31YTile(0);
            int minY = obj.getPoint31YTile(0);
            for (int i = 1; i < obj.getPointsLength(); i++) {
                minX = Math.min(minX, obj.getPoint31XTile(i));
                minY = Math.min(minY, obj.getPoint31YTile(i));
                maxX = Math.max(maxX, obj.getPoint31XTile(i));
                maxY = Math.max(maxY, obj.getPoint31YTile(i));
            }
            try {
                rtree.insert(new LeafElement(new Rect(minX, minY, maxX, maxY), obj.getId()));
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
        rtree = AbstractIndexPartCreator.packRtreeFile(rtree, nonpackRtree.getAbsolutePath(), packRtree.getAbsolutePath());
        TLongObjectHashMap<BinaryFileReference> treeHeader = new TLongObjectHashMap<BinaryFileReference>();
        long rootIndex = rtree.getFileHdr().getRootIndex();
        rtree.Node root = rtree.getReadNode(rootIndex);
        Rect rootBounds = IndexUploader.calcBounds(root);
        if (rootBounds != null) {
            writer.startWriteMapLevelIndex(mapZoomPair.getMinZoom(), mapZoomPair.getMaxZoom(), rootBounds.getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY());
            IndexVectorMapCreator.writeBinaryMapTree(root, rootBounds, rtree, writer, treeHeader);
            IndexUploader.writeBinaryMapBlock(root, rootBounds, rtree, writer, treeHeader, objects, mapZoomPair, doNotSimplify);
            writer.endWriteMapLevelIndex();
        }
    } finally {
        if (rtree != null) {
            RandomAccessFile file = rtree.getFileHdr().getFile();
            file.close();
        }
        nonpackRtree.delete();
        packRtree.delete();
        RTree.clearCache();
    }
}