Examples with IndexCreator net.osmand.data.preparation.IndexCreator used on opensource projects

Example 6 with IndexCreator

use of net.osmand.data.preparation.IndexCreator in project OsmAnd-tools by osmandapp.

the class CombineSRTMIntoFile method process.

private static void process(BinaryMapDataObject country, List<BinaryMapDataObject> boundaries, String downloadName, File directoryWithSRTMFiles, File directoryWithTargetFiles, boolean dryRun) throws IOException, SQLException, InterruptedException, IllegalArgumentException, XmlPullParserException {
    final String suffix = "_" + IndexConstants.BINARY_MAP_VERSION + IndexConstants.BINARY_SRTM_MAP_INDEX_EXT;
    String name = country.getName();
    final File targetFile = new File(directoryWithTargetFiles, Algorithms.capitalizeFirstLetterAndLowercase(downloadName + suffix));
    if (targetFile.exists()) {
        System.out.println("Already processed " + name);
        return;
    }
    Set<String> srtmFileNames = new TreeSet<String>();
    QuadRect qr = new QuadRect(180, -90, -180, 90);
    MultipolygonBuilder bld = new MultipolygonBuilder();
    bld.addOuterWay(convertToWay(country));
    updateBbox(country, qr);
    if (boundaries != null) {
        for (BinaryMapDataObject o : boundaries) {
            bld.addOuterWay(convertToWay(o));
            updateBbox(o, qr);
        }
    }
    Multipolygon polygon = bld.build();
    System.out.println("RINGS OF MULTIPOLYGON ARE " + polygon.areRingsComplete());
    int rightLon = (int) Math.floor(qr.right);
    int leftLon = (int) Math.floor(qr.left);
    int bottomLat = (int) Math.floor(qr.bottom);
    int topLat = (int) Math.floor(qr.top);
    boolean onetile = leftLon == rightLon && bottomLat == topLat;
    for (int lon = leftLon; lon <= rightLon; lon++) {
        for (int lat = bottomLat; lat <= topLat; lat++) {
            boolean isOut = !polygon.containsPoint(lat + 0.5, lon + 0.5) && !onetile;
            if (isOut) {
                LatLon bl = new LatLon(lat, lon);
                LatLon br = new LatLon(lat, lon + 1);
                LatLon tr = new LatLon(lat + 1, lon + 1);
                LatLon tl = new LatLon(lat + 1, lon);
                for (Ring r : polygon.getOuterRings()) {
                    List<Node> border = r.getBorder();
                    Node prev = border.get(border.size() - 1);
                    for (int i = 0; i < border.size() && isOut; i++) {
                        Node n = border.get(i);
                        if (MapAlgorithms.linesIntersect(prev.getLatLon(), n.getLatLon(), tr, tl)) {
                            isOut = false;
                        } else if (MapAlgorithms.linesIntersect(prev.getLatLon(), n.getLatLon(), tr, br)) {
                            isOut = false;
                        } else if (MapAlgorithms.linesIntersect(prev.getLatLon(), n.getLatLon(), bl, tl)) {
                            isOut = false;
                        } else if (MapAlgorithms.linesIntersect(prev.getLatLon(), n.getLatLon(), br, bl)) {
                            isOut = false;
                        }
                        prev = n;
                    }
                    if (!isOut) {
                        break;
                    }
                }
            }
            if (!isOut) {
                final String filename = getFileName(lon, lat);
                srtmFileNames.add(filename);
            }
        }
    }
    System.out.println();
    System.out.println("PROCESSING " + name + " lon [" + leftLon + " - " + rightLon + "] lat [" + bottomLat + " - " + topLat + "] TOTAL " + srtmFileNames.size() + " files " + srtmFileNames);
    System.out.println("-----------------------------");
    if (dryRun) {
        return;
    }
    // final File work = new File(directoryWithTargetFiles, "work");
    // Map<File, String> mp = new HashMap<File, String>();
    // long length = 0;
    List<File> files = new ArrayList<File>();
    for (String file : srtmFileNames) {
        final File fl = new File(directoryWithSRTMFiles, file + ".osm.bz2");
        if (!fl.exists()) {
            System.err.println("!! Missing " + name + " because " + file + " doesn't exist");
        } else {
            files.add(fl);
        // File ttf = new File(fl.getParentFile(), Algorithms.capitalizeFirstLetterAndLowercase(file) + "_"+ name + ".obf");
        // mp.put(ttf, null);
        }
    }
    // be independent of previous results
    new File(targetFile.getParentFile(), IndexCreator.TEMP_NODES_DB).delete();
    RTree.clearCache();
    IndexCreator ic = new IndexCreator(targetFile.getParentFile());
    if (srtmFileNames.size() > 100) {
        ic.setDialects(DBDialect.SQLITE, DBDialect.SQLITE);
    } else {
        ic.setDialects(DBDialect.SQLITE_IN_MEMORY, DBDialect.SQLITE_IN_MEMORY);
    }
    ic.setIndexMap(true);
    ic.setRegionName(name + " contour lines");
    ic.setMapFileName(targetFile.getName());
    ic.setBoundary(polygon);
    ic.setZoomWaySmoothness(2);
    ic.generateIndexes(files.toArray(new File[files.size()]), new ConsoleProgressImplementation(1), null, MapZooms.parseZooms("11-12;13-"), new MapRenderingTypesEncoder(targetFile.getName()), log, true, false);
// if(length > Integer.MAX_VALUE) {
// System.err.println("!! Can't process " + name + " because too big");
// } else {
// BinaryInspector.combineParts(targetFile, mp);
// }
// for(String file : srtmFileNames) {
// final File fl = new File(work, file);
// fl.delete();
// }
}

Example 7 with IndexCreator

use of net.osmand.data.preparation.IndexCreator in project OsmAnd-tools by osmandapp.

the class IndexBatchCreator method generateIndex.

protected void generateIndex(File file, String rName, RegionSpecificData rdata, Set<String> alreadyGeneratedFiles) {
    try {
        // be independent of previous results
        RTree.clearCache();
        String regionName = file.getName();
        log.warn("-------------------------------------------");
        log.warn("----------- Generate " + file.getName() + "\n\n\n");
        int i = file.getName().indexOf('.');
        if (i > -1) {
            regionName = Algorithms.capitalizeFirstLetterAndLowercase(file.getName().substring(0, i));
        }
        if (Algorithms.isEmpty(rName)) {
            rName = regionName;
        } else {
            rName = Algorithms.capitalizeFirstLetterAndLowercase(rName);
        }
        DBDialect osmDb = this.osmDbDialect;
        if (file.length() / 1024 / 1024 > INMEM_LIMIT && osmDb == DBDialect.SQLITE_IN_MEMORY) {
            log.warn("Switching SQLITE in memory dialect to SQLITE");
            osmDb = DBDialect.SQLITE;
        }
        IndexCreator indexCreator = new IndexCreator(workDir);
        boolean worldMaps = rName.toLowerCase().contains("world");
        if (srtmDir != null && (rdata == null || rdata.indexSRTM) && !worldMaps) {
            indexCreator.setSRTMData(srtmDir);
        }
        indexCreator.setDialects(osmDb, osmDb);
        final boolean indAddr = indexAddress && (rdata == null || rdata.indexAddress);
        final boolean indPoi = indexPOI && (rdata == null || rdata.indexPOI);
        final boolean indTransport = indexTransport && (rdata == null || rdata.indexTransport);
        final boolean indMap = indexMap && (rdata == null || rdata.indexMap);
        final boolean indRouting = indexRouting && (rdata == null || rdata.indexRouting);
        if (!indAddr && !indPoi && !indTransport && !indMap && !indRouting) {
            log.warn("! Skip country because nothing to index !");
            file.delete();
            return;
        }
        indexCreator.setIndexAddress(indAddr);
        indexCreator.setIndexPOI(indPoi);
        indexCreator.setIndexTransport(indTransport);
        indexCreator.setIndexMap(indMap);
        indexCreator.setIndexRouting(indRouting);
        indexCreator.setLastModifiedDate(file.lastModified());
        indexCreator.setRegionName(rName);
        if (rdata != null && rdata.cityAdminLevel != null) {
            indexCreator.setCityAdminLevel(rdata.cityAdminLevel);
        }
        if (zoomWaySmoothness != null) {
            indexCreator.setZoomWaySmoothness(zoomWaySmoothness);
        }
        String mapFileName = regionName + "_" + IndexConstants.BINARY_MAP_VERSION + IndexConstants.BINARY_MAP_INDEX_EXT;
        indexCreator.setMapFileName(mapFileName);
        try {
            alreadyGeneratedFiles.add(file.getName());
            Log warningsAboutMapData = null;
            File logFileName = new File(workDir, mapFileName + GEN_LOG_EXT);
            FileHandler fh = null;
            // configure log path
            try {
                FileOutputStream fout = new FileOutputStream(logFileName);
                fout.write((new Date() + "\n").getBytes());
                fout.write((MapCreatorVersion.APP_MAP_CREATOR_FULL_NAME + "\n").getBytes());
                fout.close();
                fh = new FileHandler(logFileName.getAbsolutePath(), 10 * 1000 * 1000, 1, true);
                fh.setFormatter(new SimpleFormatter());
                fh.setLevel(Level.ALL);
                Jdk14Logger jdk14Logger = new Jdk14Logger("tempLogger");
                jdk14Logger.getLogger().setLevel(Level.ALL);
                jdk14Logger.getLogger().setUseParentHandlers(false);
                jdk14Logger.getLogger().addHandler(fh);
                warningsAboutMapData = jdk14Logger;
            } catch (SecurityException e1) {
                e1.printStackTrace();
            } catch (IOException e1) {
                e1.printStackTrace();
            }
            if (fh != null) {
                LogManager.getLogManager().getLogger("").addHandler(fh);
            }
            try {
                indexCreator.generateIndexes(file, new ConsoleProgressImplementation(1), null, mapZooms, new MapRenderingTypesEncoder(renderingTypesFile, file.getName()), warningsAboutMapData);
            } finally {
                if (fh != null) {
                    fh.close();
                    LogManager.getLogManager().getLogger("").removeHandler(fh);
                }
            }
            File generated = new File(workDir, mapFileName);
            File dest = new File(indexDirFiles, generated.getName());
            if (!generated.renameTo(dest)) {
                FileOutputStream fout = new FileOutputStream(dest);
                FileInputStream fin = new FileInputStream(generated);
                Algorithms.streamCopy(fin, fout);
                fin.close();
                fout.close();
            }
            File copyLog = new File(indexDirFiles, logFileName.getName());
            FileOutputStream fout = new FileOutputStream(copyLog);
            FileInputStream fin = new FileInputStream(logFileName);
            Algorithms.streamCopy(fin, fout);
            fin.close();
            fout.close();
        // logFileName.renameTo(new File(indexDirFiles, logFileName.getName()));
        } catch (Exception e) {
            // $NON-NLS-1$
            log.error("Exception generating indexes for " + file.getName(), e);
        }
    } catch (OutOfMemoryError e) {
        System.gc();
        log.error("OutOfMemory", e);
    }
    System.gc();
}

Example 8 with IndexCreator

use of net.osmand.data.preparation.IndexCreator in project OsmAnd-tools by osmandapp.

the class MainUtilities method main.

public static void main(String[] args) throws Exception {
    if (args.length == 0) {
        printSynopsys();
    } else if (args[0].equals("--test-osm-live-tag-removal")) {
        generateAllOsmLiveTests(new File(System.getProperty("repo.dir") + "/resources/test-resources/osm_live"), System.getProperty("maps.dir"), false);
    // String test = "2017_06_18-10_30_tagRemovalBug_01.xml";
    // String osmLivePath = System.getProperty("repo.dir")+"/resources/test-resources/osm_live/";
    // Algorithms.removeAllFiles(new File(osmLivePath, AugmentedDiffsInspector.DEFAULT_REGION));
    // AugmentedDiffsInspector.main(new String[] { osmLivePath + test, osmLivePath });
    // GenerateDailyObf.main(new String[] { osmLivePath });
    } else {
        String utl = args[0];
        List<String> subArgs = new ArrayList<String>(Arrays.asList(args).subList(1, args.length));
        String[] subArgsArray = subArgs.toArray(new String[args.length - 1]);
        if (utl.equals("check-ocean-tile")) {
            OceanTilesCreator.checkOceanTile(subArgsArray);
        } else if (utl.equals("compare")) {
            BinaryComparator.main(subArgsArray);
        } else if (utl.equals("merge-index")) {
            BinaryMerger.main(subArgsArray);
        } else if (utl.equals("generate-region-tags")) {
            GenerateRegionTags.main(subArgsArray);
        } else if (utl.equals("generate-ocean-tile-osm")) {
            OceanTilesCreator.createJOSMFile(subArgsArray);
        } else if (utl.equals("generate-java-style")) {
            RenderingRulesStoragePrinter.main(subArgsArray);
        } else if (utl.equals("explain-rendering-style")) {
            RenderingRulesStorage.main(subArgsArray);
        } else if (utl.equals("generate-obf-diff")) {
            ObfDiffGenerator.main(subArgsArray);
        } else if (utl.equals("generate-ocean-tile")) {
            OceanTilesCreator.createTilesFile(subArgsArray[0], subArgsArray.length > 1 ? subArgsArray[1] : null);
        } else if (utl.equals("test-routing")) {
            net.osmand.router.TestRouting.main(subArgsArray);
        } else if (utl.equals("generate-ocbf")) {
            CountryOcbfGeneration.main(subArgsArray);
        } else if (utl.equals("generate-obf")) {
            IndexCreator ic = new IndexCreator(new File("."));
            ic.setIndexMap(true);
            ic.setIndexPOI(true);
            ic.setIndexRouting(true);
            ic.setIndexTransport(true);
            ic.setIndexAddress(true);
            ic.setLastModifiedDate(new File(subArgsArray[0]).lastModified());
            generateObf(subArgsArray, ic);
        } else if (utl.equals("generate-obf-no-address")) {
            IndexCreator ic = new IndexCreator(new File("."));
            ic.setIndexMap(true);
            ic.setIndexPOI(true);
            ic.setIndexRouting(true);
            ic.setIndexTransport(true);
            ic.setLastModifiedDate(new File(subArgsArray[0]).lastModified());
            generateObf(subArgsArray, ic);
        } else if (utl.equals("generate-map")) {
            IndexCreator ic = new IndexCreator(new File("."));
            ic.setIndexMap(true);
            ic.setLastModifiedDate(new File(subArgsArray[0]).lastModified());
            generateObf(subArgsArray, ic);
        } else if (utl.equals("split-obf")) {
            ObfRegionSplitter.main(subArgsArray);
        } else if (utl.equals("merge-bulk-osmlive-day")) {
            ObfDiffMerger.mergeBulkOsmLiveDay(subArgsArray[0]);
        } else if (utl.equals("merge-bulk-osmlive-month")) {
            ObfDiffMerger.mergeBulkOsmLiveMonth(subArgsArray[0]);
        } else if (utl.equals("merge-flat-obf")) {
            ObfDiffMerger.main(subArgsArray);
        } else if (utl.equals("generate-address")) {
            IndexCreator ic = new IndexCreator(new File("."));
            ic.setIndexAddress(true);
            ic.setLastModifiedDate(new File(subArgsArray[0]).lastModified());
            generateObf(subArgsArray, ic);
        } else if (utl.equals("extract-roads-only")) {
            File mainFile = new File(subArgsArray[0]);
            IndexUploader.extractRoadOnlyFile(mainFile, new File(mainFile.getParentFile(), mainFile.getName().replace(IndexConstants.BINARY_MAP_INDEX_EXT, IndexConstants.BINARY_ROAD_MAP_INDEX_EXT)));
        } else if (utl.equals("generate-poi")) {
            IndexCreator ic = new IndexCreator(new File("."));
            ic.setIndexPOI(true);
            ic.setLastModifiedDate(new File(subArgsArray[0]).lastModified());
            generateObf(subArgsArray, ic);
        } else if (utl.equals("delete-unused-strings")) {
            ResourceDeleter.main(subArgsArray);
        } else if (utl.equals("merge-std-files")) {
            BinaryMerger.mergeStandardFiles(subArgsArray);
        } else if (utl.equals("generate-roads")) {
            IndexCreator ic = new IndexCreator(new File("."));
            ic.setIndexRouting(true);
            ic.setLastModifiedDate(new File(subArgsArray[0]).lastModified());
            generateObf(subArgsArray, ic);
        } else if (utl.contentEquals("generate-osmlive-tests")) {
            if (subArgsArray.length < 1) {
                System.out.println("Usage: <path_to_directory_with_resources_project> <optional_path_to_unpack_files>");
                return;
            }
            File testResources = new File(subArgsArray[0] + "/resources/test-resources/osm_live/");
            generateAllOsmLiveTests(testResources, subArgsArray.length > 1 ? subArgsArray[1] : null, false);
        } else if (utl.contentEquals("generate-from-overpass")) {
            if (subArgsArray.length < 3) {
                System.out.println("Usage: PATH_TO_OVERPASS PATH_TO_WORKING_DIR PATH_TO_REGIONS");
                return;
            }
            String[] argsToGenerateOsm = new String[] { subArgsArray[0], subArgsArray[1], subArgsArray[2] };
            AugmentedDiffsInspector.main(argsToGenerateOsm);
            String[] argsToGenerateObf = new String[] { subArgsArray[1] };
            GenerateDailyObf.main(argsToGenerateObf);
        } else {
            printSynopsys();
        }
    }
}