Examples with IndexCreator net.osmand.obf.preparation.IndexCreator used on opensource projects

Example 6 with IndexCreator

use of net.osmand.obf.preparation.IndexCreator in project OsmAnd-tools by osmandapp.

the class CombineSRTMIntoFile method process.

private static void process(BinaryMapDataObject country, List<BinaryMapDataObject> boundaries, String downloadName, File directoryWithSRTMFiles, File directoryWithTargetFiles, boolean dryRun, int limit, boolean feet) throws IOException, SQLException, InterruptedException, IllegalArgumentException, XmlPullParserException {
    final String suffix = "_" + IndexConstants.BINARY_MAP_VERSION + (feet ? IndexConstants.BINARY_SRTM_FEET_MAP_INDEX_EXT : IndexConstants.BINARY_SRTM_MAP_INDEX_EXT);
    String name = country.getName();
    String dwName = Algorithms.capitalizeFirstLetterAndLowercase(downloadName + suffix);
    final File targetFile = new File(directoryWithTargetFiles, dwName);
    if (targetFile.exists()) {
        System.out.println("Already processed " + name);
        return;
    }
    Set<String> srtmFileNames = new TreeSet<String>();
    QuadRect qr = new QuadRect(180, -90, -180, 90);
    MultipolygonBuilder bld = new MultipolygonBuilder();
    if (boundaries != null) {
        for (BinaryMapDataObject o : boundaries) {
            bld.addOuterWay(convertToWay(o));
            updateBbox(o, qr);
        }
    } else {
        bld.addOuterWay(convertToWay(country));
        updateBbox(country, qr);
    }
    Multipolygon polygon = bld.build();
    int rightLon = (int) Math.floor(qr.right);
    int leftLon = (int) Math.floor(qr.left);
    int bottomLat = (int) Math.floor(qr.bottom);
    int topLat = (int) Math.floor(qr.top);
    boolean onetile = leftLon == rightLon && bottomLat == topLat;
    for (int lon = leftLon; lon <= rightLon; lon++) {
        for (int lat = bottomLat; lat <= topLat; lat++) {
            boolean isOut = !polygon.containsPoint(lat + 0.5, lon + 0.5) && !onetile;
            if (isOut) {
                LatLon bl = new LatLon(lat, lon);
                LatLon br = new LatLon(lat, lon + 1);
                LatLon tr = new LatLon(lat + 1, lon + 1);
                LatLon tl = new LatLon(lat + 1, lon);
                for (Ring r : polygon.getOuterRings()) {
                    List<Node> border = r.getBorder();
                    Node prev = border.get(border.size() - 1);
                    for (int i = 0; i < border.size() && isOut; i++) {
                        Node n = border.get(i);
                        if (MapAlgorithms.linesIntersect(prev.getLatLon(), n.getLatLon(), tr, tl)) {
                            isOut = false;
                        } else if (MapAlgorithms.linesIntersect(prev.getLatLon(), n.getLatLon(), tr, br)) {
                            isOut = false;
                        } else if (MapAlgorithms.linesIntersect(prev.getLatLon(), n.getLatLon(), bl, tl)) {
                            isOut = false;
                        } else if (MapAlgorithms.linesIntersect(prev.getLatLon(), n.getLatLon(), br, bl)) {
                            isOut = false;
                        }
                        prev = n;
                    }
                    if (!isOut) {
                        break;
                    }
                }
            }
            if (!isOut) {
                final String filename = getFileName(lon, lat);
                srtmFileNames.add(filename);
            }
        }
    }
    System.out.println();
    System.out.println("-----------------------------");
    System.out.println("PROCESSING " + name + " lon [" + leftLon + " - " + rightLon + "] lat [" + bottomLat + " - " + topLat + "] TOTAL " + srtmFileNames.size() + " files " + srtmFileNames);
    if (dryRun) {
        return;
    }
    if (srtmFileNames.size() > limit) {
        System.out.println("\n\n!!!!!!!! WARNING BECAUSE LIMIT OF FILES EXCEEDED !!!!!!!!!\n\n");
        return;
    }
    File procFile = new File(directoryWithTargetFiles, dwName + ".proc");
    boolean locked = !procFile.createNewFile();
    if (locked) {
        System.out.println("\n\n!!!!!!!! WARNING FILE IS BEING PROCESSED !!!!!!!!!\n\n");
        return;
    }
    // final File work = new File(directoryWithTargetFiles, "work");
    // Map<File, String> mp = new HashMap<File, String>();
    long length = 0;
    List<File> files = new ArrayList<File>();
    for (String file : srtmFileNames) {
        final File fl = new File(directoryWithSRTMFiles, file + ".osm.bz2");
        if (!fl.exists()) {
            System.err.println("!! Missing " + name + " because " + file + " doesn't exist");
        } else {
            length += fl.length();
            files.add(fl);
        // File ttf = new File(fl.getParentFile(), Algorithms.capitalizeFirstLetterAndLowercase(file) + "_"+ name + ".obf");
        // mp.put(ttf, null);
        }
    }
    if (files.isEmpty()) {
        System.err.println("!!! WARNING " + name + " because no files are present to index !!!");
    } else {
        IndexCreatorSettings settings = new IndexCreatorSettings();
        settings.indexMap = true;
        settings.zoomWaySmoothness = 2;
        settings.boundary = polygon;
        IndexCreator ic = new IndexCreator(targetFile.getParentFile(), settings);
        // if (srtmFileNames.size() > NUMBER_OF_FILES_TO_PROCESS_ON_DISK || length > SIZE_GB_TO_COMBINE_INRAM) {
        // ic.setDialects(DBDialect.SQLITE, DBDialect.SQLITE);
        // System.out.println("SQLITE on disk is used.");
        // } else {
        ic.setDialects(DBDialect.SQLITE, DBDialect.SQLITE_IN_MEMORY);
        // System.out.println("SQLITE in memory used: be aware whole database is stored in memory.");
        // }
        ic.setRegionName(name + " contour lines");
        ic.setMapFileName(targetFile.getName());
        File nodesDB = new File(targetFile.getParentFile(), dwName + "." + IndexCreator.TEMP_NODES_DB);
        ic.setNodesDBFile(nodesDB);
        ic.generateIndexes(files.toArray(new File[files.size()]), new ConsoleProgressImplementation(1), null, MapZooms.parseZooms("11-12;13-"), new MapRenderingTypesEncoder(targetFile.getName()), log, true, false);
        nodesDB.delete();
        RTree.clearCache();
    }
    procFile.delete();
// if(length > Integer.MAX_VALUE) {
// System.err.println("!! Can't process " + name + " because too big");
// } else {
// BinaryInspector.combineParts(targetFile, mp);
// }
// for(String file : srtmFileNames) {
// final File fl = new File(work, file);
// fl.delete();
// }
}

Example 7 with IndexCreator

use of net.osmand.obf.preparation.IndexCreator in project OsmAnd-tools by osmandapp.

the class CountryOcbfGeneration method createFile.

private void createFile(CountryRegion global, Map<String, Set<TranslateEntity>> translates, Map<String, File> polygonFiles, String targetObf, String targetOsmXml) throws IOException, SQLException, InterruptedException, XmlPullParserException {
    File osm = new File(targetOsmXml);
    XmlSerializer serializer = new org.kxml2.io.KXmlSerializer();
    FileOutputStream fous = new FileOutputStream(osm);
    serializer.setOutput(fous, "UTF-8");
    serializer.startDocument("UTF-8", true);
    serializer.startTag(null, "osm");
    serializer.attribute(null, "version", "0.6");
    serializer.attribute(null, "generator", "OsmAnd");
    serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true);
    for (CountryRegion r : global.children) {
        r.parent = null;
        processRegion(r, translates, polygonFiles, targetObf, targetOsmXml, "", serializer);
    }
    serializer.endDocument();
    serializer.flush();
    fous.close();
    IndexCreatorSettings settings = new IndexCreatorSettings();
    settings.indexMap = true;
    settings.indexAddress = false;
    settings.indexPOI = false;
    settings.indexTransport = false;
    settings.indexRouting = false;
    // $NON-NLS-1$
    IndexCreator creator = new IndexCreator(new File(targetObf).getParentFile(), settings);
    creator.setMapFileName(new File(targetObf).getName());
    MapZooms zooms = MapZooms.parseZooms("5-6");
    creator.generateIndexes(osm, new ConsoleProgressImplementation(1), null, zooms, new MapRenderingTypesEncoder("regions"), log);
}

Example 8 with IndexCreator

use of net.osmand.obf.preparation.IndexCreator in project OsmAnd-tools by osmandapp.

the class IndexBatchCreator method generateIndex.

protected void generateIndex(File file, String rName, RegionSpecificData rdata, Set<String> alreadyGeneratedFiles) {
    try {
        // be independent of previous results
        RTree.clearCache();
        String regionName = file.getName();
        log.warn("-------------------------------------------");
        log.warn("----------- Generate " + file.getName() + "\n\n\n");
        int i = file.getName().indexOf('.');
        if (i > -1) {
            regionName = Algorithms.capitalizeFirstLetterAndLowercase(file.getName().substring(0, i));
        }
        if (Algorithms.isEmpty(rName)) {
            rName = regionName;
        } else {
            rName = Algorithms.capitalizeFirstLetterAndLowercase(rName);
        }
        DBDialect osmDb = this.osmDbDialect;
        if (file.length() / 1024 / 1024 > INMEM_LIMIT && osmDb == DBDialect.SQLITE_IN_MEMORY) {
            log.warn("Switching SQLITE in memory dialect to SQLITE");
            osmDb = DBDialect.SQLITE;
        }
        final boolean indAddr = indexAddress && (rdata == null || rdata.indexAddress);
        final boolean indPoi = indexPOI && (rdata == null || rdata.indexPOI);
        final boolean indTransport = indexTransport && (rdata == null || rdata.indexTransport);
        final boolean indMap = indexMap && (rdata == null || rdata.indexMap);
        final boolean indRouting = indexRouting && (rdata == null || rdata.indexRouting);
        if (!indAddr && !indPoi && !indTransport && !indMap && !indRouting) {
            log.warn("! Skip country because nothing to index !");
            file.delete();
            return;
        }
        IndexCreatorSettings settings = new IndexCreatorSettings();
        settings.indexMap = indMap;
        settings.indexAddress = indAddr;
        settings.indexPOI = indPoi;
        settings.indexTransport = indTransport;
        settings.indexRouting = indRouting;
        if (zoomWaySmoothness != null) {
            settings.zoomWaySmoothness = zoomWaySmoothness;
        }
        boolean worldMaps = rName.toLowerCase().contains("world");
        if (worldMaps) {
            if (rName.toLowerCase().contains("basemap")) {
                return;
            }
            if (rName.toLowerCase().contains("seamarks")) {
                settings.keepOnlySeaObjects = true;
                settings.indexTransport = false;
                settings.indexAddress = false;
            }
        } else {
            if (srtmDir != null && (rdata == null || rdata.indexSRTM)) {
                settings.srtmDataFolder = srtmDir;
            }
        }
        IndexCreator indexCreator = new IndexCreator(workDir, settings);
        indexCreator.setDialects(osmDb, osmDb);
        indexCreator.setLastModifiedDate(file.lastModified());
        indexCreator.setRegionName(rName);
        String mapFileName = regionName + "_" + IndexConstants.BINARY_MAP_VERSION + IndexConstants.BINARY_MAP_INDEX_EXT;
        indexCreator.setMapFileName(mapFileName);
        try {
            alreadyGeneratedFiles.add(file.getName());
            Log warningsAboutMapData = null;
            File logFileName = new File(workDir, mapFileName + GEN_LOG_EXT);
            FileHandler fh = null;
            // configure log path
            try {
                FileOutputStream fout = new FileOutputStream(logFileName);
                fout.write((new Date() + "\n").getBytes());
                fout.write((MapCreatorVersion.APP_MAP_CREATOR_FULL_NAME + "\n").getBytes());
                fout.close();
                fh = new FileHandler(logFileName.getAbsolutePath(), 10 * 1000 * 1000, 1, true);
                fh.setFormatter(new SimpleFormatter());
                fh.setLevel(Level.ALL);
                Jdk14Logger jdk14Logger = new Jdk14Logger("tempLogger");
                jdk14Logger.getLogger().setLevel(Level.ALL);
                jdk14Logger.getLogger().setUseParentHandlers(false);
                jdk14Logger.getLogger().addHandler(fh);
                warningsAboutMapData = jdk14Logger;
            } catch (SecurityException e1) {
                e1.printStackTrace();
            } catch (IOException e1) {
                e1.printStackTrace();
            }
            if (fh != null) {
                LogManager.getLogManager().getLogger("").addHandler(fh);
            }
            try {
                indexCreator.generateIndexes(file, new ConsoleProgressImplementation(1), null, mapZooms, new MapRenderingTypesEncoder(renderingTypesFile, file.getName()), warningsAboutMapData);
            } finally {
                if (fh != null) {
                    fh.close();
                    LogManager.getLogManager().getLogger("").removeHandler(fh);
                }
            }
            File generated = new File(workDir, mapFileName);
            File dest = new File(indexDirFiles, generated.getName());
            if (!generated.renameTo(dest)) {
                FileOutputStream fout = new FileOutputStream(dest);
                FileInputStream fin = new FileInputStream(generated);
                Algorithms.streamCopy(fin, fout);
                fin.close();
                fout.close();
            }
            File copyLog = new File(indexDirFiles, logFileName.getName());
            FileOutputStream fout = new FileOutputStream(copyLog);
            FileInputStream fin = new FileInputStream(logFileName);
            Algorithms.streamCopy(fin, fout);
            fin.close();
            fout.close();
        // logFileName.renameTo(new File(indexDirFiles, logFileName.getName()));
        } catch (Exception e) {
            // $NON-NLS-1$
            log.error("Exception generating indexes for " + file.getName(), e);
        }
    } catch (OutOfMemoryError e) {
        System.gc();
        log.error("OutOfMemory", e);
    }
    System.gc();
}

Example 9 with IndexCreator

use of net.osmand.obf.preparation.IndexCreator in project OsmAnd-tools by osmandapp.

the class WikipediaByCountryDivider method generateObf.

private static void generateObf(File osmGz, File obf) throws IOException, SQLException, InterruptedException, XmlPullParserException {
    // be independent of previous results
    RTree.clearCache();
    IndexCreatorSettings settings = new IndexCreatorSettings();
    settings.indexMap = false;
    settings.indexAddress = false;
    settings.indexPOI = true;
    settings.indexTransport = false;
    settings.indexRouting = false;
    settings.poiZipLongStrings = true;
    // $NON-NLS-1$
    IndexCreator creator = new IndexCreator(obf.getParentFile(), settings);
    new File(obf.getParentFile(), IndexCreator.TEMP_NODES_DB).delete();
    creator.setMapFileName(obf.getName());
    creator.generateIndexes(osmGz, new ConsoleProgressImplementation(1), null, MapZooms.getDefault(), new MapRenderingTypesEncoder(obf.getName()), log);
}

Example 10 with IndexCreator

use of net.osmand.obf.preparation.IndexCreator in project OsmAnd-tools by osmandapp.

the class MainUtilities method generateObf.

private static void generateObf(List<String> subArgs, IndexCreatorSettings settings) throws IOException, SQLException, InterruptedException, XmlPullParserException {
    File fileToGen = new File(subArgs.get(0));
    IndexCreator ic = new IndexCreator(new File("."), settings);
    ic.setDialects(settings.processInRam ? DBDialect.SQLITE_IN_MEMORY : DBDialect.SQLITE, settings.processInRam ? DBDialect.SQLITE_IN_MEMORY : DBDialect.SQLITE);
    ic.setLastModifiedDate(fileToGen.lastModified());
    String regionName = fileToGen.getName();
    MapRenderingTypesEncoder types = new MapRenderingTypesEncoder(settings.renderingTypesFile, regionName);
    ic.generateIndexes(fileToGen, new ConsoleProgressImplementation(), null, MapZooms.getDefault(), types, log);
}