Search in sources :

Example 6 with IndexCreatorSettings

use of net.osmand.obf.preparation.IndexCreatorSettings in project OsmAnd-tools by osmandapp.

the class TravelGuideCreatorMain method generateTravelGuide.

private void generateTravelGuide(String dir) throws SQLException, IOException, XmlPullParserException, InterruptedException {
    File directory = new File(dir);
    if (!directory.isDirectory()) {
        throw new RuntimeException("Supplied path is not a directory");
    }
    File[] files = directory.listFiles();
    File sqliteFile = new File(directory, TRAVEL_GUIDE_NAME + BINARY_WIKIVOYAGE_MAP_INDEX_EXT);
    Connection conn = DBDialect.SQLITE.getDatabaseConnection(sqliteFile.getCanonicalPath(), LOG);
    if (conn == null) {
        LOG.error("Couldn't establish the database connection");
        System.exit(1);
    }
    Map<String, List<File>> mapping = getFileMapping(files);
    WikivoyageDataGenerator dataGenerator = new WikivoyageDataGenerator();
    generateTravelSqlite(mapping, conn);
    dataGenerator.generateSearchTable(conn);
    createPopularArticlesTable(conn);
    conn.close();
    File osmFile = new File(directory, TRAVEL_GUIDE_NAME + OSM_GZ_EXT);
    WikivoyageGenOSM.genWikivoyageOsm(sqliteFile, osmFile, -1);
    IndexCreatorSettings settings = new IndexCreatorSettings();
    settings.indexPOI = true;
    IndexCreator ic = new IndexCreator(directory, settings);
    ic.setMapFileName(directory.getName() + BINARY_TRAVEL_GUIDE_MAP_INDEX_EXT);
    MapRenderingTypesEncoder types = new MapRenderingTypesEncoder(settings.renderingTypesFile, osmFile.getName());
    ic.generateIndexes(osmFile, new ConsoleProgressImplementation(), null, MapZooms.getDefault(), types, LOG);
    osmFile.delete();
    sqliteFile.delete();
    new File("regions.ocbf").delete();
}
Also used : IndexCreatorSettings(net.osmand.obf.preparation.IndexCreatorSettings) MapRenderingTypesEncoder(net.osmand.osm.MapRenderingTypesEncoder) Connection(java.sql.Connection) IndexCreator(net.osmand.obf.preparation.IndexCreator) File(java.io.File) ConsoleProgressImplementation(net.osmand.impl.ConsoleProgressImplementation)

Example 7 with IndexCreatorSettings

use of net.osmand.obf.preparation.IndexCreatorSettings in project OsmAnd-tools by osmandapp.

the class CombineSRTMIntoFile method process.

private static void process(BinaryMapDataObject country, List<BinaryMapDataObject> boundaries, String downloadName, File directoryWithSRTMFiles, File directoryWithTargetFiles, boolean dryRun, int limit, boolean feet) throws IOException, SQLException, InterruptedException, IllegalArgumentException, XmlPullParserException {
    final String suffix = "_" + IndexConstants.BINARY_MAP_VERSION + (feet ? IndexConstants.BINARY_SRTM_FEET_MAP_INDEX_EXT : IndexConstants.BINARY_SRTM_MAP_INDEX_EXT);
    String name = country.getName();
    String dwName = Algorithms.capitalizeFirstLetterAndLowercase(downloadName + suffix);
    final File targetFile = new File(directoryWithTargetFiles, dwName);
    if (targetFile.exists()) {
        System.out.println("Already processed " + name);
        return;
    }
    Set<String> srtmFileNames = new TreeSet<String>();
    QuadRect qr = new QuadRect(180, -90, -180, 90);
    MultipolygonBuilder bld = new MultipolygonBuilder();
    if (boundaries != null) {
        for (BinaryMapDataObject o : boundaries) {
            bld.addOuterWay(convertToWay(o));
            updateBbox(o, qr);
        }
    } else {
        bld.addOuterWay(convertToWay(country));
        updateBbox(country, qr);
    }
    Multipolygon polygon = bld.build();
    int rightLon = (int) Math.floor(qr.right);
    int leftLon = (int) Math.floor(qr.left);
    int bottomLat = (int) Math.floor(qr.bottom);
    int topLat = (int) Math.floor(qr.top);
    boolean onetile = leftLon == rightLon && bottomLat == topLat;
    for (int lon = leftLon; lon <= rightLon; lon++) {
        for (int lat = bottomLat; lat <= topLat; lat++) {
            boolean isOut = !polygon.containsPoint(lat + 0.5, lon + 0.5) && !onetile;
            if (isOut) {
                LatLon bl = new LatLon(lat, lon);
                LatLon br = new LatLon(lat, lon + 1);
                LatLon tr = new LatLon(lat + 1, lon + 1);
                LatLon tl = new LatLon(lat + 1, lon);
                for (Ring r : polygon.getOuterRings()) {
                    List<Node> border = r.getBorder();
                    Node prev = border.get(border.size() - 1);
                    for (int i = 0; i < border.size() && isOut; i++) {
                        Node n = border.get(i);
                        if (MapAlgorithms.linesIntersect(prev.getLatLon(), n.getLatLon(), tr, tl)) {
                            isOut = false;
                        } else if (MapAlgorithms.linesIntersect(prev.getLatLon(), n.getLatLon(), tr, br)) {
                            isOut = false;
                        } else if (MapAlgorithms.linesIntersect(prev.getLatLon(), n.getLatLon(), bl, tl)) {
                            isOut = false;
                        } else if (MapAlgorithms.linesIntersect(prev.getLatLon(), n.getLatLon(), br, bl)) {
                            isOut = false;
                        }
                        prev = n;
                    }
                    if (!isOut) {
                        break;
                    }
                }
            }
            if (!isOut) {
                final String filename = getFileName(lon, lat);
                srtmFileNames.add(filename);
            }
        }
    }
    System.out.println();
    System.out.println("-----------------------------");
    System.out.println("PROCESSING " + name + " lon [" + leftLon + " - " + rightLon + "] lat [" + bottomLat + " - " + topLat + "] TOTAL " + srtmFileNames.size() + " files " + srtmFileNames);
    if (dryRun) {
        return;
    }
    if (srtmFileNames.size() > limit) {
        System.out.println("\n\n!!!!!!!! WARNING BECAUSE LIMIT OF FILES EXCEEDED !!!!!!!!!\n\n");
        return;
    }
    File procFile = new File(directoryWithTargetFiles, dwName + ".proc");
    boolean locked = !procFile.createNewFile();
    if (locked) {
        System.out.println("\n\n!!!!!!!! WARNING FILE IS BEING PROCESSED !!!!!!!!!\n\n");
        return;
    }
    // final File work = new File(directoryWithTargetFiles, "work");
    // Map<File, String> mp = new HashMap<File, String>();
    long length = 0;
    List<File> files = new ArrayList<File>();
    for (String file : srtmFileNames) {
        final File fl = new File(directoryWithSRTMFiles, file + ".osm.bz2");
        if (!fl.exists()) {
            System.err.println("!! Missing " + name + " because " + file + " doesn't exist");
        } else {
            length += fl.length();
            files.add(fl);
        // File ttf = new File(fl.getParentFile(), Algorithms.capitalizeFirstLetterAndLowercase(file) + "_"+ name + ".obf");
        // mp.put(ttf, null);
        }
    }
    if (files.isEmpty()) {
        System.err.println("!!! WARNING " + name + " because no files are present to index !!!");
    } else {
        IndexCreatorSettings settings = new IndexCreatorSettings();
        settings.indexMap = true;
        settings.zoomWaySmoothness = 2;
        settings.boundary = polygon;
        IndexCreator ic = new IndexCreator(targetFile.getParentFile(), settings);
        // if (srtmFileNames.size() > NUMBER_OF_FILES_TO_PROCESS_ON_DISK || length > SIZE_GB_TO_COMBINE_INRAM) {
        // ic.setDialects(DBDialect.SQLITE, DBDialect.SQLITE);
        // System.out.println("SQLITE on disk is used.");
        // } else {
        ic.setDialects(DBDialect.SQLITE, DBDialect.SQLITE_IN_MEMORY);
        // System.out.println("SQLITE in memory used: be aware whole database is stored in memory.");
        // }
        ic.setRegionName(name + " contour lines");
        ic.setMapFileName(targetFile.getName());
        File nodesDB = new File(targetFile.getParentFile(), dwName + "." + IndexCreator.TEMP_NODES_DB);
        ic.setNodesDBFile(nodesDB);
        ic.generateIndexes(files.toArray(new File[files.size()]), new ConsoleProgressImplementation(1), null, MapZooms.parseZooms("11-12;13-"), new MapRenderingTypesEncoder(targetFile.getName()), log, true, false);
        nodesDB.delete();
        RTree.clearCache();
    }
    procFile.delete();
// if(length > Integer.MAX_VALUE) {
// System.err.println("!! Can't process " + name + " because too big");
// } else {
// BinaryInspector.combineParts(targetFile, mp);
// }
// for(String file : srtmFileNames) {
// final File fl = new File(work, file);
// fl.delete();
// }
}
Also used : IndexCreatorSettings(net.osmand.obf.preparation.IndexCreatorSettings) Node(net.osmand.osm.edit.Node) ArrayList(java.util.ArrayList) IndexCreator(net.osmand.obf.preparation.IndexCreator) QuadRect(net.osmand.data.QuadRect) ConsoleProgressImplementation(net.osmand.impl.ConsoleProgressImplementation) LatLon(net.osmand.data.LatLon) MapRenderingTypesEncoder(net.osmand.osm.MapRenderingTypesEncoder) BinaryMapDataObject(net.osmand.binary.BinaryMapDataObject) TreeSet(java.util.TreeSet) Ring(net.osmand.data.Ring) File(java.io.File) Multipolygon(net.osmand.data.Multipolygon) MultipolygonBuilder(net.osmand.data.MultipolygonBuilder)

Example 8 with IndexCreatorSettings

use of net.osmand.obf.preparation.IndexCreatorSettings in project OsmAnd-tools by osmandapp.

the class CountryOcbfGeneration method createFile.

private void createFile(CountryRegion global, Map<String, Set<TranslateEntity>> translates, Map<String, File> polygonFiles, String targetObf, String targetOsmXml) throws IOException, SQLException, InterruptedException, XmlPullParserException {
    File osm = new File(targetOsmXml);
    XmlSerializer serializer = new org.kxml2.io.KXmlSerializer();
    FileOutputStream fous = new FileOutputStream(osm);
    serializer.setOutput(fous, "UTF-8");
    serializer.startDocument("UTF-8", true);
    serializer.startTag(null, "osm");
    serializer.attribute(null, "version", "0.6");
    serializer.attribute(null, "generator", "OsmAnd");
    serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true);
    for (CountryRegion r : global.children) {
        r.parent = null;
        processRegion(r, translates, polygonFiles, targetObf, targetOsmXml, "", serializer);
    }
    serializer.endDocument();
    serializer.flush();
    fous.close();
    IndexCreatorSettings settings = new IndexCreatorSettings();
    settings.indexMap = true;
    settings.indexAddress = false;
    settings.indexPOI = false;
    settings.indexTransport = false;
    settings.indexRouting = false;
    // $NON-NLS-1$
    IndexCreator creator = new IndexCreator(new File(targetObf).getParentFile(), settings);
    creator.setMapFileName(new File(targetObf).getName());
    MapZooms zooms = MapZooms.parseZooms("5-6");
    creator.generateIndexes(osm, new ConsoleProgressImplementation(1), null, zooms, new MapRenderingTypesEncoder("regions"), log);
}
Also used : IndexCreatorSettings(net.osmand.obf.preparation.IndexCreatorSettings) MapRenderingTypesEncoder(net.osmand.osm.MapRenderingTypesEncoder) FileOutputStream(java.io.FileOutputStream) MapZooms(net.osmand.binary.MapZooms) IndexCreator(net.osmand.obf.preparation.IndexCreator) File(java.io.File) ConsoleProgressImplementation(net.osmand.impl.ConsoleProgressImplementation) XmlSerializer(org.xmlpull.v1.XmlSerializer)

Example 9 with IndexCreatorSettings

use of net.osmand.obf.preparation.IndexCreatorSettings in project OsmAnd-tools by osmandapp.

the class IndexBatchCreator method generateIndex.

protected void generateIndex(File file, String rName, RegionSpecificData rdata, Set<String> alreadyGeneratedFiles) {
    try {
        // be independent of previous results
        RTree.clearCache();
        String regionName = file.getName();
        log.warn("-------------------------------------------");
        log.warn("----------- Generate " + file.getName() + "\n\n\n");
        int i = file.getName().indexOf('.');
        if (i > -1) {
            regionName = Algorithms.capitalizeFirstLetterAndLowercase(file.getName().substring(0, i));
        }
        if (Algorithms.isEmpty(rName)) {
            rName = regionName;
        } else {
            rName = Algorithms.capitalizeFirstLetterAndLowercase(rName);
        }
        DBDialect osmDb = this.osmDbDialect;
        if (file.length() / 1024 / 1024 > INMEM_LIMIT && osmDb == DBDialect.SQLITE_IN_MEMORY) {
            log.warn("Switching SQLITE in memory dialect to SQLITE");
            osmDb = DBDialect.SQLITE;
        }
        final boolean indAddr = indexAddress && (rdata == null || rdata.indexAddress);
        final boolean indPoi = indexPOI && (rdata == null || rdata.indexPOI);
        final boolean indTransport = indexTransport && (rdata == null || rdata.indexTransport);
        final boolean indMap = indexMap && (rdata == null || rdata.indexMap);
        final boolean indRouting = indexRouting && (rdata == null || rdata.indexRouting);
        if (!indAddr && !indPoi && !indTransport && !indMap && !indRouting) {
            log.warn("! Skip country because nothing to index !");
            file.delete();
            return;
        }
        IndexCreatorSettings settings = new IndexCreatorSettings();
        settings.indexMap = indMap;
        settings.indexAddress = indAddr;
        settings.indexPOI = indPoi;
        settings.indexTransport = indTransport;
        settings.indexRouting = indRouting;
        if (zoomWaySmoothness != null) {
            settings.zoomWaySmoothness = zoomWaySmoothness;
        }
        boolean worldMaps = rName.toLowerCase().contains("world");
        if (worldMaps) {
            if (rName.toLowerCase().contains("basemap")) {
                return;
            }
            if (rName.toLowerCase().contains("seamarks")) {
                settings.keepOnlySeaObjects = true;
                settings.indexTransport = false;
                settings.indexAddress = false;
            }
        } else {
            if (srtmDir != null && (rdata == null || rdata.indexSRTM)) {
                settings.srtmDataFolder = srtmDir;
            }
        }
        IndexCreator indexCreator = new IndexCreator(workDir, settings);
        indexCreator.setDialects(osmDb, osmDb);
        indexCreator.setLastModifiedDate(file.lastModified());
        indexCreator.setRegionName(rName);
        String mapFileName = regionName + "_" + IndexConstants.BINARY_MAP_VERSION + IndexConstants.BINARY_MAP_INDEX_EXT;
        indexCreator.setMapFileName(mapFileName);
        try {
            alreadyGeneratedFiles.add(file.getName());
            Log warningsAboutMapData = null;
            File logFileName = new File(workDir, mapFileName + GEN_LOG_EXT);
            FileHandler fh = null;
            // configure log path
            try {
                FileOutputStream fout = new FileOutputStream(logFileName);
                fout.write((new Date() + "\n").getBytes());
                fout.write((MapCreatorVersion.APP_MAP_CREATOR_FULL_NAME + "\n").getBytes());
                fout.close();
                fh = new FileHandler(logFileName.getAbsolutePath(), 10 * 1000 * 1000, 1, true);
                fh.setFormatter(new SimpleFormatter());
                fh.setLevel(Level.ALL);
                Jdk14Logger jdk14Logger = new Jdk14Logger("tempLogger");
                jdk14Logger.getLogger().setLevel(Level.ALL);
                jdk14Logger.getLogger().setUseParentHandlers(false);
                jdk14Logger.getLogger().addHandler(fh);
                warningsAboutMapData = jdk14Logger;
            } catch (SecurityException e1) {
                e1.printStackTrace();
            } catch (IOException e1) {
                e1.printStackTrace();
            }
            if (fh != null) {
                LogManager.getLogManager().getLogger("").addHandler(fh);
            }
            try {
                indexCreator.generateIndexes(file, new ConsoleProgressImplementation(1), null, mapZooms, new MapRenderingTypesEncoder(renderingTypesFile, file.getName()), warningsAboutMapData);
            } finally {
                if (fh != null) {
                    fh.close();
                    LogManager.getLogManager().getLogger("").removeHandler(fh);
                }
            }
            File generated = new File(workDir, mapFileName);
            File dest = new File(indexDirFiles, generated.getName());
            if (!generated.renameTo(dest)) {
                FileOutputStream fout = new FileOutputStream(dest);
                FileInputStream fin = new FileInputStream(generated);
                Algorithms.streamCopy(fin, fout);
                fin.close();
                fout.close();
            }
            File copyLog = new File(indexDirFiles, logFileName.getName());
            FileOutputStream fout = new FileOutputStream(copyLog);
            FileInputStream fin = new FileInputStream(logFileName);
            Algorithms.streamCopy(fin, fout);
            fin.close();
            fout.close();
        // logFileName.renameTo(new File(indexDirFiles, logFileName.getName()));
        } catch (Exception e) {
            // $NON-NLS-1$
            log.error("Exception generating indexes for " + file.getName(), e);
        }
    } catch (OutOfMemoryError e) {
        System.gc();
        log.error("OutOfMemory", e);
    }
    System.gc();
}
Also used : IndexCreatorSettings(net.osmand.obf.preparation.IndexCreatorSettings) Jdk14Logger(org.apache.commons.logging.impl.Jdk14Logger) Log(org.apache.commons.logging.Log) SimpleFormatter(java.util.logging.SimpleFormatter) IndexCreator(net.osmand.obf.preparation.IndexCreator) IOException(java.io.IOException) ConsoleProgressImplementation(net.osmand.impl.ConsoleProgressImplementation) Date(java.util.Date) FileInputStream(java.io.FileInputStream) IOException(java.io.IOException) FileNotFoundException(java.io.FileNotFoundException) ParserConfigurationException(javax.xml.parsers.ParserConfigurationException) SAXException(org.xml.sax.SAXException) FileHandler(java.util.logging.FileHandler) DBDialect(net.osmand.obf.preparation.DBDialect) MapRenderingTypesEncoder(net.osmand.osm.MapRenderingTypesEncoder) FileOutputStream(java.io.FileOutputStream) File(java.io.File)

Example 10 with IndexCreatorSettings

use of net.osmand.obf.preparation.IndexCreatorSettings in project OsmAnd-tools by osmandapp.

the class WikipediaByCountryDivider method generateObf.

private static void generateObf(File osmGz, File obf) throws IOException, SQLException, InterruptedException, XmlPullParserException {
    // be independent of previous results
    RTree.clearCache();
    IndexCreatorSettings settings = new IndexCreatorSettings();
    settings.indexMap = false;
    settings.indexAddress = false;
    settings.indexPOI = true;
    settings.indexTransport = false;
    settings.indexRouting = false;
    settings.poiZipLongStrings = true;
    // $NON-NLS-1$
    IndexCreator creator = new IndexCreator(obf.getParentFile(), settings);
    new File(obf.getParentFile(), IndexCreator.TEMP_NODES_DB).delete();
    creator.setMapFileName(obf.getName());
    creator.generateIndexes(osmGz, new ConsoleProgressImplementation(1), null, MapZooms.getDefault(), new MapRenderingTypesEncoder(obf.getName()), log);
}
Also used : IndexCreatorSettings(net.osmand.obf.preparation.IndexCreatorSettings) MapRenderingTypesEncoder(net.osmand.osm.MapRenderingTypesEncoder) IndexCreator(net.osmand.obf.preparation.IndexCreator) File(java.io.File) ConsoleProgressImplementation(net.osmand.impl.ConsoleProgressImplementation)

Aggregations

IndexCreatorSettings (net.osmand.obf.preparation.IndexCreatorSettings)12 MapRenderingTypesEncoder (net.osmand.osm.MapRenderingTypesEncoder)10 File (java.io.File)9 IndexCreator (net.osmand.obf.preparation.IndexCreator)9 ConsoleProgressImplementation (net.osmand.impl.ConsoleProgressImplementation)7 FileOutputStream (java.io.FileOutputStream)3 ArrayList (java.util.ArrayList)3 FileInputStream (java.io.FileInputStream)2 IOException (java.io.IOException)2 SQLException (java.sql.SQLException)2 HashMap (java.util.HashMap)2 LinkedHashMap (java.util.LinkedHashMap)2 List (java.util.List)2 TreeSet (java.util.TreeSet)2 GPXFile (net.osmand.GPXUtilities.GPXFile)2 IProgress (net.osmand.IProgress)2 BinaryMapIndexReader (net.osmand.binary.BinaryMapIndexReader)2 Node (net.osmand.osm.edit.Node)2 TLongHashSet (gnu.trove.set.hash.TLongHashSet)1 FileNotFoundException (java.io.FileNotFoundException)1