Examples with DBDialect net.osmand.obf.preparation.DBDialect used on opensource projects

Example 1 with DBDialect

use of net.osmand.obf.preparation.DBDialect in project OsmAnd-tools by osmandapp.

the class WikivoyageGenOSM method genWikivoyageOsm.

public static void genWikivoyageOsm(File wikivoyageFile, File outputFile, int LIMIT) throws SQLException, IOException {
    DBDialect dialect = DBDialect.SQLITE;
    Connection connection = (Connection) dialect.getDatabaseConnection(wikivoyageFile.getCanonicalPath(), log);
    Statement statement = connection.createStatement();
    // popular_articles : trip_id, popularity_index, order_index, population, title, lat, lon, lang
    // travel_articles:
    // population, country, region, city_type, osm_i,
    ResultSet rs = statement.executeQuery("select trip_id, title, lang, lat, lon, content_gz, " + "gpx_gz, image_title, banner_title, is_part_of, is_parent_of, aggregated_part_of, contents_json from travel_articles order by trip_id asc");
    int count = 0, totalArticles = 0, emptyLocation = 0, emptyContent = 0;
    CombinedWikivoyageArticle combinedArticle = new CombinedWikivoyageArticle();
    XmlSerializer serializer = null;
    OutputStream outputStream = null;
    if (outputFile != null) {
        outputStream = new FileOutputStream(outputFile);
        if (outputFile.getName().endsWith(".gz")) {
            outputStream = new GZIPOutputStream(outputStream);
        }
        serializer = PlatformUtil.newSerializer();
        serializer.setOutput(new OutputStreamWriter(outputStream));
        // $NON-NLS-1$
        serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true);
        // $NON-NLS-1$
        serializer.startDocument("UTF-8", true);
        // $NON-NLS-1$
        serializer.startTag(null, "osm");
        // $NON-NLS-1$ //$NON-NLS-2$
        serializer.attribute(null, "version", "0.6");
    }
    while (rs.next()) {
        int rind = 1;
        long tripId = rs.getLong(rind++);
        if (tripId != combinedArticle.tripId && combinedArticle.tripId != -1) {
            boolean res = combineAndSave(combinedArticle, serializer);
            if (res) {
                totalArticles++;
            }
            combinedArticle.clear();
        }
        combinedArticle.tripId = tripId;
        String title = rs.getString(rind++);
        String lang = rs.getString(rind++);
        double lat = rs.getDouble(rind++);
        double lon = rs.getDouble(rind++);
        // rind++;
        String content = Algorithms.gzipToString(rs.getBytes(rind++));
        GZIPInputStream bytesStream = new GZIPInputStream(new ByteArrayInputStream(rs.getBytes(rind++)));
        GPXFile gpxFile = GPXUtilities.loadGPXFile(bytesStream);
        String imageTitle = rs.getString(rind++);
        String bannerTitle = rs.getString(rind++);
        String isPartOf = rs.getString(rind++);
        String isParentOf = rs.getString(rind++);
        String isAggrPartOf = rs.getString(rind++);
        String contentJson = rs.getString(rind);
        combinedArticle.addArticle(lang, title, gpxFile, lat, lon, content, imageTitle, bannerTitle, isPartOf, isParentOf, isAggrPartOf, contentJson);
        if (gpxFile == null || gpxFile.isPointsEmpty()) {
            if (lat == 0 && lon == 0) {
                emptyLocation++;
            } else {
                emptyContent++;
            }
        }
        if (count >= LIMIT && LIMIT != -1) {
            break;
        }
        count++;
    }
    combineAndSave(combinedArticle, serializer);
    if (serializer != null) {
        serializer.endTag(null, "osm");
        serializer.flush();
        outputStream.close();
    }
    List<String> l = new ArrayList<String>(categories.keySet());
    Collections.sort(l, new Comparator<String>() {

        @Override
        public int compare(String s1, String s2) {
            return Integer.compare(categories.get(s1), categories.get(s2));
        }
    });
    int total = 0;
    for (String s : l) {
        total += categories.get(s);
    }
    for (String s : l) {
        int cnt = categories.get(s);
        System.out.println(String.format("%#.2f%% %s  %d %s", cnt * 100.0 / total, s, cnt, categoriesExample.get(s)));
    }
    System.out.println(String.format("Total saved articles: %d", totalArticles));
    System.out.println(String.format("Empty article: %d no points in article + %d no location page articles (total %d) ", emptyContent, emptyLocation, total));
}

Example 2 with DBDialect

use of net.osmand.obf.preparation.DBDialect in project OsmAnd-tools by osmandapp.

the class WikiDatabasePreparation method processWikidataRegions.

private static void processWikidataRegions(final String sqliteFileName) throws SQLException, IOException {
    File wikiDB = new File(sqliteFileName);
    log.info("Processing wikidata regions...");
    DBDialect dialect = DBDialect.SQLITE;
    Connection conn = dialect.getDatabaseConnection(wikiDB.getAbsolutePath(), log);
    OsmandRegions regions = new OsmandRegions();
    regions.prepareFile();
    regions.cacheAllCountries();
    PreparedStatement wikiRegionPrep = conn.prepareStatement("INSERT INTO wiki_region(id, regionName) VALUES(?, ? )");
    ResultSet rs = conn.createStatement().executeQuery("SELECT id, lat, lon from wiki_coords");
    int batch = 0;
    List<String> rgs = new ArrayList<String>();
    while (rs.next()) {
        rgs = regions.getRegionsToDownload(rs.getDouble(2), rs.getDouble(3), rgs);
        for (String reg : rgs) {
            wikiRegionPrep.setLong(1, rs.getLong(1));
            wikiRegionPrep.setString(2, reg);
            wikiRegionPrep.addBatch();
            if (batch++ > WikiDataHandler.BATCH_SIZE) {
                wikiRegionPrep.executeBatch();
                batch = 0;
            }
        }
    }
    wikiRegionPrep.executeBatch();
    rs.close();
    wikiRegionPrep.close();
    conn.close();
}

Example 3 with DBDialect

use of net.osmand.obf.preparation.DBDialect in project OsmAnd-tools by osmandapp.

the class IndexBatchCreator method generateIndex.

protected void generateIndex(File file, String rName, RegionSpecificData rdata, Set<String> alreadyGeneratedFiles) {
    try {
        // be independent of previous results
        RTree.clearCache();
        String regionName = file.getName();
        log.warn("-------------------------------------------");
        log.warn("----------- Generate " + file.getName() + "\n\n\n");
        int i = file.getName().indexOf('.');
        if (i > -1) {
            regionName = Algorithms.capitalizeFirstLetterAndLowercase(file.getName().substring(0, i));
        }
        if (Algorithms.isEmpty(rName)) {
            rName = regionName;
        } else {
            rName = Algorithms.capitalizeFirstLetterAndLowercase(rName);
        }
        DBDialect osmDb = this.osmDbDialect;
        if (file.length() / 1024 / 1024 > INMEM_LIMIT && osmDb == DBDialect.SQLITE_IN_MEMORY) {
            log.warn("Switching SQLITE in memory dialect to SQLITE");
            osmDb = DBDialect.SQLITE;
        }
        final boolean indAddr = indexAddress && (rdata == null || rdata.indexAddress);
        final boolean indPoi = indexPOI && (rdata == null || rdata.indexPOI);
        final boolean indTransport = indexTransport && (rdata == null || rdata.indexTransport);
        final boolean indMap = indexMap && (rdata == null || rdata.indexMap);
        final boolean indRouting = indexRouting && (rdata == null || rdata.indexRouting);
        if (!indAddr && !indPoi && !indTransport && !indMap && !indRouting) {
            log.warn("! Skip country because nothing to index !");
            file.delete();
            return;
        }
        IndexCreatorSettings settings = new IndexCreatorSettings();
        settings.indexMap = indMap;
        settings.indexAddress = indAddr;
        settings.indexPOI = indPoi;
        settings.indexTransport = indTransport;
        settings.indexRouting = indRouting;
        if (zoomWaySmoothness != null) {
            settings.zoomWaySmoothness = zoomWaySmoothness;
        }
        boolean worldMaps = rName.toLowerCase().contains("world");
        if (worldMaps) {
            if (rName.toLowerCase().contains("basemap")) {
                return;
            }
            if (rName.toLowerCase().contains("seamarks")) {
                settings.keepOnlySeaObjects = true;
                settings.indexTransport = false;
                settings.indexAddress = false;
            }
        } else {
            if (srtmDir != null && (rdata == null || rdata.indexSRTM)) {
                settings.srtmDataFolder = srtmDir;
            }
        }
        IndexCreator indexCreator = new IndexCreator(workDir, settings);
        indexCreator.setDialects(osmDb, osmDb);
        indexCreator.setLastModifiedDate(file.lastModified());
        indexCreator.setRegionName(rName);
        String mapFileName = regionName + "_" + IndexConstants.BINARY_MAP_VERSION + IndexConstants.BINARY_MAP_INDEX_EXT;
        indexCreator.setMapFileName(mapFileName);
        try {
            alreadyGeneratedFiles.add(file.getName());
            Log warningsAboutMapData = null;
            File logFileName = new File(workDir, mapFileName + GEN_LOG_EXT);
            FileHandler fh = null;
            // configure log path
            try {
                FileOutputStream fout = new FileOutputStream(logFileName);
                fout.write((new Date() + "\n").getBytes());
                fout.write((MapCreatorVersion.APP_MAP_CREATOR_FULL_NAME + "\n").getBytes());
                fout.close();
                fh = new FileHandler(logFileName.getAbsolutePath(), 10 * 1000 * 1000, 1, true);
                fh.setFormatter(new SimpleFormatter());
                fh.setLevel(Level.ALL);
                Jdk14Logger jdk14Logger = new Jdk14Logger("tempLogger");
                jdk14Logger.getLogger().setLevel(Level.ALL);
                jdk14Logger.getLogger().setUseParentHandlers(false);
                jdk14Logger.getLogger().addHandler(fh);
                warningsAboutMapData = jdk14Logger;
            } catch (SecurityException e1) {
                e1.printStackTrace();
            } catch (IOException e1) {
                e1.printStackTrace();
            }
            if (fh != null) {
                LogManager.getLogManager().getLogger("").addHandler(fh);
            }
            try {
                indexCreator.generateIndexes(file, new ConsoleProgressImplementation(1), null, mapZooms, new MapRenderingTypesEncoder(renderingTypesFile, file.getName()), warningsAboutMapData);
            } finally {
                if (fh != null) {
                    fh.close();
                    LogManager.getLogManager().getLogger("").removeHandler(fh);
                }
            }
            File generated = new File(workDir, mapFileName);
            File dest = new File(indexDirFiles, generated.getName());
            if (!generated.renameTo(dest)) {
                FileOutputStream fout = new FileOutputStream(dest);
                FileInputStream fin = new FileInputStream(generated);
                Algorithms.streamCopy(fin, fout);
                fin.close();
                fout.close();
            }
            File copyLog = new File(indexDirFiles, logFileName.getName());
            FileOutputStream fout = new FileOutputStream(copyLog);
            FileInputStream fin = new FileInputStream(logFileName);
            Algorithms.streamCopy(fin, fout);
            fin.close();
            fout.close();
        // logFileName.renameTo(new File(indexDirFiles, logFileName.getName()));
        } catch (Exception e) {
            // $NON-NLS-1$
            log.error("Exception generating indexes for " + file.getName(), e);
        }
    } catch (OutOfMemoryError e) {
        System.gc();
        log.error("OutOfMemory", e);
    }
    System.gc();
}

Example 4 with DBDialect

use of net.osmand.obf.preparation.DBDialect in project OsmAnd-tools by osmandapp.

the class WikivoyageDataGenerator method main.

public static void main(String[] args) throws SQLException, IOException {
    boolean uncompressed = false;
    File wikivoyageFile = new File(args[0]);
    if (!wikivoyageFile.exists()) {
        throw new IllegalArgumentException("Wikivoyage file doesn't exist: " + args[0]);
    }
    File citiesObfFile = null;
    File workingDir = wikivoyageFile.getParentFile();
    for (int i = 1; i < args.length; i++) {
        String val = args[i].substring(args[i].indexOf('=') + 1);
        if (args[i].startsWith("--uncompressed=")) {
            uncompressed = Boolean.parseBoolean(val);
        } else if (args[i].startsWith("--cities-obf=")) {
            citiesObfFile = new File(val);
        }
    }
    System.out.println("Process " + wikivoyageFile.getName() + " " + (uncompressed ? "uncompressed" : ""));
    DBDialect dialect = DBDialect.SQLITE;
    Connection conn = (Connection) dialect.getDatabaseConnection(wikivoyageFile.getAbsolutePath(), log);
    WikivoyageDataGenerator generator = new WikivoyageDataGenerator();
    generator.regions = new OsmandRegions();
    generator.regions.prepareFile();
    generator.regions.cacheAllCountries();
    conn.createStatement().execute("CREATE INDEX IF NOT EXISTS index_image_title ON travel_articles(image_title);");
    conn.createStatement().execute("CREATE INDEX IF NOT EXISTS index_banner_title ON travel_articles(banner_title);");
    printStep("Download/Copy proper headers for articles");
    generator.updateProperHeaderForArticles(conn, workingDir);
    printStep("Copy headers between lang");
    generator.copyImagesBetweenArticles(conn, "image_title");
    generator.copyImagesBetweenArticles(conn, "banner_title");
    printStep("Generate agg part of");
    generator.generateAggPartOf(conn);
    printStep("Generate is parent of");
    generator.generateIsParentOf(conn);
    printStep("Generate search table");
    generator.generateSearchTable(conn);
    if (citiesObfFile != null) {
        printStep("Add osm city data");
    }
    generator.addCitiesData(citiesObfFile, conn);
    printStep("Populate popular articles");
    generator.createPopularArticlesTable(conn);
    conn.createStatement().execute("DROP INDEX IF EXISTS index_image_title ");
    conn.createStatement().execute("DROP INDEX IF EXISTS index_banner_title ");
    conn.close();
}