Examples with DBDialect net.osmand.data.preparation.DBDialect used on opensource projects

Example 1 with DBDialect

use of net.osmand.data.preparation.DBDialect in project OsmAnd-tools by osmandapp.

the class SearchDBCreator method generateIdsIfMissing.

private static void generateIdsIfMissing(Connection conn, String workingDir) throws SQLException {
    long maxId = 0;
    DBDialect dialect = DBDialect.SQLITE;
    Connection langConn = (Connection) dialect.getDatabaseConnection(workingDir + "langlink.sqlite", log);
    PreparedStatement st = langConn.prepareStatement("SELECT MAX(id) FROM langlinks");
    ResultSet rs = st.executeQuery();
    while (rs.next()) {
        maxId = rs.getLong(1) + 1;
    }
    st.close();
    rs.close();
    langConn.close();
    if (maxId == 0) {
        return;
    }
    int batch = 0;
    PreparedStatement ps = conn.prepareStatement("SELECT title FROM wikivoyage_articles WHERE city_id = 0");
    PreparedStatement prep = conn.prepareStatement("UPDATE wikivoyage_articles SET city_id = ? WHERE title = ?");
    ResultSet res = ps.executeQuery();
    while (res.next()) {
        String title = res.getString("title");
        prep.setLong(1, maxId++);
        prep.setString(2, title);
        prep.addBatch();
        if (batch++ > 500) {
            prep.executeBatch();
            batch = 0;
        }
    }
    prep.addBatch();
    prep.executeBatch();
    prep.close();
    res.close();
}

Example 2 with DBDialect

use of net.osmand.data.preparation.DBDialect in project OsmAnd-tools by osmandapp.

the class SearchDBCreator method main.

public static void main(String[] args) throws SQLException {
    String pathTodb = "/home/paul/osmand/wikivoyage/full_wikivoyage.sqlite";
    if (args.length > 0) {
        pathTodb = args[0];
    }
    DBDialect dialect = DBDialect.SQLITE;
    Connection conn = (Connection) dialect.getDatabaseConnection(pathTodb, log);
    generateIdsIfMissing(conn, pathTodb.substring(0, pathTodb.lastIndexOf("/") + 1));
    conn.createStatement().execute("DROP TABLE IF EXISTS wikivoyage_search;");
    conn.createStatement().execute("CREATE TABLE wikivoyage_search(search_term text, city_id long, article_title text, lang text)");
    conn.createStatement().execute("CREATE INDEX IF NOT EXISTS index_search_term ON wikivoyage_search(search_term);");
    conn.createStatement().execute("CREATE INDEX IF NOT EXISTS index_search_city ON wikivoyage_search(city_id)");
    PreparedStatement ps = conn.prepareStatement("INSERT INTO wikivoyage_search VALUES (?, ?, ?, ?)");
    PreparedStatement data = conn.prepareStatement("SELECT title, city_id, lang FROM wikivoyage_articles");
    ResultSet rs = data.executeQuery();
    int batch = 0;
    while (rs.next()) {
        String title = rs.getString("title");
        String titleToSplit = title.replaceAll("[/\\)\\(-]", " ").replaceAll(" +", " ");
        long id = rs.getLong("city_id");
        for (String s : titleToSplit.split(" ")) {
            ps.setString(1, s.toLowerCase());
            ps.setLong(2, id);
            ps.setString(3, title);
            ps.setString(4, rs.getString("lang"));
            ps.addBatch();
            if (batch++ > 500) {
                ps.executeBatch();
                batch = 0;
            }
        }
    }
    ps.addBatch();
    ps.executeBatch();
    ps.close();
    data.close();
    rs.close();
    conn.close();
}

Example 3 with DBDialect

use of net.osmand.data.preparation.DBDialect in project OsmAnd-tools by osmandapp.

the class WikiVoyagePreparation method processLangLinks.

private static void processLangLinks(File langlinkFolder, File langlinkFile) throws IOException, SQLException {
    if (!langlinkFolder.isDirectory()) {
        System.err.println("Specified langlink folder is not a directory");
        System.exit(-1);
    }
    DBDialect dialect = DBDialect.SQLITE;
    Connection conn = (Connection) dialect.getDatabaseConnection(langlinkFile.getAbsolutePath(), log);
    conn.createStatement().execute("CREATE TABLE langlinks (id long NOT NULL DEFAULT 0, " + "title text UNIQUE NOT NULL DEFAULT '')");
    conn.createStatement().execute("CREATE INDEX IF NOT EXISTS index_title ON langlinks(title);");
    PreparedStatement prep = conn.prepareStatement("INSERT OR IGNORE INTO langlinks VALUES (?, ?)");
    int batch = 0;
    long maxId = 0;
    int langNum = 1;
    Map<Integer, Set<Long>> genIds = new HashMap<>();
    Map<Long, Long> currMapping = new HashMap<>();
    for (File f : langlinkFolder.listFiles()) {
        Set<Long> ids = new HashSet<>();
        InputStream fis = new FileInputStream(f);
        if (f.getName().endsWith("gz")) {
            fis = new GZIPInputStream(fis);
        }
        InputStreamReader read = new InputStreamReader(fis, "UTF-8");
        char[] cbuf = new char[1000];
        int cnt;
        boolean values = false;
        String buf = "";
        List<String> insValues = new ArrayList<String>();
        while ((cnt = read.read(cbuf)) >= 0) {
            String str = new String(cbuf, 0, cnt);
            buf += str;
            if (!values) {
                if (buf.contains("VALUES")) {
                    buf = buf.substring(buf.indexOf("VALUES") + "VALUES".length());
                    values = true;
                }
            } else {
                boolean openString = false;
                int word = -1;
                int last = 0;
                for (int k = 0; k < buf.length(); k++) {
                    if (openString) {
                        if (buf.charAt(k) == '\'' && (buf.charAt(k - 1) != '\\' || buf.charAt(k - 2) == '\\')) {
                            openString = false;
                        }
                    } else if (buf.charAt(k) == ',' && word == -1) {
                        continue;
                    } else if (buf.charAt(k) == '(') {
                        word = k;
                        insValues.clear();
                    } else if (buf.charAt(k) == ')' || buf.charAt(k) == ',') {
                        String vl = buf.substring(word + 1, k).trim();
                        if (vl.startsWith("'")) {
                            vl = vl.substring(1, vl.length() - 1);
                        }
                        insValues.add(vl);
                        if (buf.charAt(k) == ')') {
                            long id = Long.valueOf(insValues.get(0));
                            maxId = Math.max(maxId, id);
                            Long genId = currMapping.get(id);
                            if (genId == null) {
                                for (Set<Long> set : genIds.values()) {
                                    if (set.contains(id)) {
                                        genId = maxId++;
                                        currMapping.put(id, genId);
                                    }
                                }
                            }
                            ids.add(genId == null ? id : genId);
                            prep.setLong(1, genId == null ? id : genId);
                            prep.setString(2, insValues.get(2));
                            prep.addBatch();
                            if (batch++ > 500) {
                                prep.executeBatch();
                                batch = 0;
                            }
                            last = k + 1;
                            word = -1;
                        } else {
                            word = k;
                        }
                    } else if (buf.charAt(k) == '\'') {
                        openString = true;
                    }
                }
                buf = buf.substring(last);
            }
        }
        genIds.put(langNum, ids);
        currMapping.clear();
        langNum++;
        read.close();
    }
    prep.addBatch();
    prep.executeBatch();
    prep.close();
    conn.close();
}

Example 4 with DBDialect

use of net.osmand.data.preparation.DBDialect in project OsmAnd-tools by osmandapp.

the class IndexBatchCreator method generateIndex.

protected void generateIndex(File file, String rName, RegionSpecificData rdata, Set<String> alreadyGeneratedFiles) {
    try {
        // be independent of previous results
        RTree.clearCache();
        String regionName = file.getName();
        log.warn("-------------------------------------------");
        log.warn("----------- Generate " + file.getName() + "\n\n\n");
        int i = file.getName().indexOf('.');
        if (i > -1) {
            regionName = Algorithms.capitalizeFirstLetterAndLowercase(file.getName().substring(0, i));
        }
        if (Algorithms.isEmpty(rName)) {
            rName = regionName;
        } else {
            rName = Algorithms.capitalizeFirstLetterAndLowercase(rName);
        }
        DBDialect osmDb = this.osmDbDialect;
        if (file.length() / 1024 / 1024 > INMEM_LIMIT && osmDb == DBDialect.SQLITE_IN_MEMORY) {
            log.warn("Switching SQLITE in memory dialect to SQLITE");
            osmDb = DBDialect.SQLITE;
        }
        IndexCreator indexCreator = new IndexCreator(workDir);
        boolean worldMaps = rName.toLowerCase().contains("world");
        if (srtmDir != null && (rdata == null || rdata.indexSRTM) && !worldMaps) {
            indexCreator.setSRTMData(srtmDir);
        }
        indexCreator.setDialects(osmDb, osmDb);
        final boolean indAddr = indexAddress && (rdata == null || rdata.indexAddress);
        final boolean indPoi = indexPOI && (rdata == null || rdata.indexPOI);
        final boolean indTransport = indexTransport && (rdata == null || rdata.indexTransport);
        final boolean indMap = indexMap && (rdata == null || rdata.indexMap);
        final boolean indRouting = indexRouting && (rdata == null || rdata.indexRouting);
        if (!indAddr && !indPoi && !indTransport && !indMap && !indRouting) {
            log.warn("! Skip country because nothing to index !");
            file.delete();
            return;
        }
        indexCreator.setIndexAddress(indAddr);
        indexCreator.setIndexPOI(indPoi);
        indexCreator.setIndexTransport(indTransport);
        indexCreator.setIndexMap(indMap);
        indexCreator.setIndexRouting(indRouting);
        indexCreator.setLastModifiedDate(file.lastModified());
        indexCreator.setRegionName(rName);
        if (rdata != null && rdata.cityAdminLevel != null) {
            indexCreator.setCityAdminLevel(rdata.cityAdminLevel);
        }
        if (zoomWaySmoothness != null) {
            indexCreator.setZoomWaySmoothness(zoomWaySmoothness);
        }
        String mapFileName = regionName + "_" + IndexConstants.BINARY_MAP_VERSION + IndexConstants.BINARY_MAP_INDEX_EXT;
        indexCreator.setMapFileName(mapFileName);
        try {
            alreadyGeneratedFiles.add(file.getName());
            Log warningsAboutMapData = null;
            File logFileName = new File(workDir, mapFileName + GEN_LOG_EXT);
            FileHandler fh = null;
            // configure log path
            try {
                FileOutputStream fout = new FileOutputStream(logFileName);
                fout.write((new Date() + "\n").getBytes());
                fout.write((MapCreatorVersion.APP_MAP_CREATOR_FULL_NAME + "\n").getBytes());
                fout.close();
                fh = new FileHandler(logFileName.getAbsolutePath(), 10 * 1000 * 1000, 1, true);
                fh.setFormatter(new SimpleFormatter());
                fh.setLevel(Level.ALL);
                Jdk14Logger jdk14Logger = new Jdk14Logger("tempLogger");
                jdk14Logger.getLogger().setLevel(Level.ALL);
                jdk14Logger.getLogger().setUseParentHandlers(false);
                jdk14Logger.getLogger().addHandler(fh);
                warningsAboutMapData = jdk14Logger;
            } catch (SecurityException e1) {
                e1.printStackTrace();
            } catch (IOException e1) {
                e1.printStackTrace();
            }
            if (fh != null) {
                LogManager.getLogManager().getLogger("").addHandler(fh);
            }
            try {
                indexCreator.generateIndexes(file, new ConsoleProgressImplementation(1), null, mapZooms, new MapRenderingTypesEncoder(renderingTypesFile, file.getName()), warningsAboutMapData);
            } finally {
                if (fh != null) {
                    fh.close();
                    LogManager.getLogManager().getLogger("").removeHandler(fh);
                }
            }
            File generated = new File(workDir, mapFileName);
            File dest = new File(indexDirFiles, generated.getName());
            if (!generated.renameTo(dest)) {
                FileOutputStream fout = new FileOutputStream(dest);
                FileInputStream fin = new FileInputStream(generated);
                Algorithms.streamCopy(fin, fout);
                fin.close();
                fout.close();
            }
            File copyLog = new File(indexDirFiles, logFileName.getName());
            FileOutputStream fout = new FileOutputStream(copyLog);
            FileInputStream fin = new FileInputStream(logFileName);
            Algorithms.streamCopy(fin, fout);
            fin.close();
            fout.close();
        // logFileName.renameTo(new File(indexDirFiles, logFileName.getName()));
        } catch (Exception e) {
            // $NON-NLS-1$
            log.error("Exception generating indexes for " + file.getName(), e);
        }
    } catch (OutOfMemoryError e) {
        System.gc();
        log.error("OutOfMemory", e);
    }
    System.gc();
}