Examples with FormattedText biblemulticonverter.data.FormattedText used on opensource projects

Example 26 with FormattedText

use of biblemulticonverter.data.FormattedText in project BibleMultiConverter by schierlm.

the class Compact method doImport.

protected Bible doImport(BufferedReader br) throws IOException {
    String line = br.readLine();
    if (!line.startsWith(MAGIC))
        throw new IOException("Invalid header line: " + line);
    Bible result = new Bible(line.substring(MAGIC.length()));
    Book currentBook = null;
    Chapter currentChapter = null;
    int nextVerseNum = -1;
    while ((line = br.readLine()) != null) {
        if (line.length() == 0)
            throw new IOException("Empty line");
        char start = line.charAt(0);
        while (start == '+') {
            if (currentChapter == null)
                currentBook.getChapters().add(new Chapter());
            currentChapter = null;
            nextVerseNum = -1;
            line = line.substring(1);
            if (line.length() == 0)
                throw new IOException("Empty line");
            start = line.charAt(0);
        }
        if (start == '=') {
            String[] fields = line.substring(1).split("\t", -1);
            if (fields.length != 4)
                throw new IOException("Unsupported chapter heading: " + line);
            currentBook = new Book(fields[0], BookID.fromOsisId(fields[1]), fields[2], fields[3]);
            result.getBooks().add(currentBook);
            currentChapter = null;
            nextVerseNum = -1;
        } else if (start == '0' && line.startsWith("0 ")) {
            if (currentChapter == null) {
                currentChapter = new Chapter();
                currentBook.getChapters().add(currentChapter);
                nextVerseNum = 1;
            }
            if (currentChapter.getProlog() != null)
                throw new IOException("More than one prolog for the same chapter");
            FormattedText prolog = new FormattedText();
            parseCompact(prolog.getAppendVisitor(), line.substring(2));
            currentChapter.setProlog(prolog);
            prolog.finished();
        } else if (start >= '1' && start <= '9' && line.contains(" ")) {
            if (currentChapter == null) {
                currentChapter = new Chapter();
                currentBook.getChapters().add(currentChapter);
            }
            int pos = line.indexOf(' ');
            Verse v = new Verse(line.substring(0, pos));
            parseCompact(v.getAppendVisitor(), line.substring(pos + 1));
            v.finished();
            currentChapter.getVerses().add(v);
            Matcher m = Utils.compilePattern("([0-9]+).*").matcher(v.getNumber());
            if (!m.matches()) {
                throw new IllegalStateException();
            }
            nextVerseNum = Integer.parseInt(m.group(1)) + 1;
        } else if (start == ' ' || (start >= 'A' && start <= 'Z') || (start >= 'a' && start <= 'z')) {
            if (currentChapter == null) {
                currentChapter = new Chapter();
                currentBook.getChapters().add(currentChapter);
                nextVerseNum = 1;
            }
            if (start == ' ')
                line = line.substring(1);
            Verse v = new Verse("" + nextVerseNum);
            parseCompact(v.getAppendVisitor(), line);
            v.finished();
            currentChapter.getVerses().add(v);
            nextVerseNum++;
        } else {
            throw new IOException("Unsupported line: " + line);
        }
    }
    return result;
}

Example 27 with FormattedText

use of biblemulticonverter.data.FormattedText in project BibleMultiConverter by schierlm.

the class Diffable method doImport.

protected Bible doImport(BufferedReader br) throws IOException {
    String line = br.readLine();
    if (!line.startsWith(MAGIC))
        throw new IOException("Invalid header line: " + line);
    Bible result = new Bible(line.substring(MAGIC.length()));
    Map<String, Book> bookMap = new HashMap<String, Book>();
    while ((line = br.readLine()) != null) {
        line = line.trim();
        if (line.length() == 0 || line.startsWith("#"))
            continue;
        String[] parts = line.split(" ", 3);
        if (parts.length != 3)
            throw new IOException("Not enough fields: " + line);
        try {
            if (parts[1].equals("=")) {
                String[] fields = parts[2].split("\t");
                if (fields.length != 3)
                    throw new IOException("Malformed header line (not 3 fields): " + parts[3]);
                BookID id = BookID.fromOsisId(fields[0]);
                if (id == null)
                    throw new IOException("Unknown book ID: " + fields[0]);
                Book newBook = new Book(parts[0], id, fields[1], fields[2]);
                result.getBooks().add(newBook);
                Book oldBook = bookMap.get(parts[0]);
                if (oldBook != null) {
                    newBook.getChapters().addAll(oldBook.getChapters());
                    result.getBooks().remove(oldBook);
                }
                bookMap.put(parts[0], newBook);
                continue;
            }
            Book book = bookMap.get(parts[0]);
            if (book == null)
                throw new IOException("Unknown book prefix (header line missing?): " + parts[0]);
            if (parts[1].equals("->")) {
                if (!parts[2].equals("-")) {
                    new StrippedDiffable().renameBookInXref(result, parts[0], parts[2], false);
                    Book destBook = bookMap.get(parts[2]);
                    if (destBook == null)
                        throw new IOException("Unknown destination book (header line missing?): " + parts[2]);
                    destBook.getChapters().addAll(book.getChapters());
                }
                result.getBooks().remove(book);
                bookMap.remove(parts[0]);
                continue;
            } else if (parts[1].equals("^^")) {
                Book destBook = bookMap.get(parts[2]);
                if (destBook == null)
                    throw new IOException("Unknown destination book (header line missing?): " + parts[2]);
                result.getBooks().remove(book);
                result.getBooks().add(result.getBooks().indexOf(destBook), book);
                continue;
            }
            int chapterNumber;
            String verse;
            if (parts[1].contains(":")) {
                String[] chapVerse = parts[1].split(":", 2);
                chapterNumber = Integer.parseInt(chapVerse[0]);
                verse = chapVerse[1];
            } else {
                chapterNumber = Integer.parseInt(parts[1]);
                verse = null;
            }
            while (book.getChapters().size() < chapterNumber) {
                book.getChapters().add(new Chapter());
            }
            Chapter chapter = book.getChapters().get(chapterNumber - 1);
            FormattedText target;
            if (verse == null) {
                if (chapter.getProlog() == null)
                    chapter.setProlog(new FormattedText());
                target = chapter.getProlog();
            } else {
                int idx = chapter.getVerseIndex(verse);
                if (idx == -1) {
                    Verse v = new Verse(verse);
                    chapter.getVerses().add(v);
                    target = v;
                } else {
                    target = chapter.getVerses().get(idx);
                }
            }
            parseDiffable(target.getAppendVisitor(), parts[2]);
        } catch (Exception ex) {
            throw new IOException("Error while parsing line: " + line, ex);
        }
    }
    for (Book book : result.getBooks()) {
        for (Chapter chapter : book.getChapters()) {
            if (chapter.getProlog() != null)
                chapter.getProlog().finished();
            for (Verse v : chapter.getVerses()) v.finished();
        }
    }
    return result;
}

Example 28 with FormattedText

use of biblemulticonverter.data.FormattedText in project BibleMultiConverter by schierlm.

the class HaggaiXML method parseChapterObject.

private void parseChapterObject(Object e3, Chapter chapter, Map<BookID, String> abbrMap, List<Headline> headlineBuffer) throws IOException {
    if (e3 instanceof CAPTION) {
        CAPTION caption = (CAPTION) e3;
        Headline h = new Headline(9);
        if (parseContent(h.getAppendVisitor(), caption.getContent(), abbrMap)) {
            h.trimWhitespace();
            h.finished();
            headlineBuffer.add(h);
        }
    } else if (e3 instanceof REMARK) {
        REMARK remark = (REMARK) e3;
        Verse v = chapter.getVerses().get(chapter.getVerses().size() - 1);
        if (remark.getContent().size() != 1)
            return;
        String remarkText = normalize((String) remark.getContent().get(0), true).trim();
        v.getAppendVisitor().visitFootnote().visitText(remarkText);
    } else if (e3 instanceof PROLOG) {
        PROLOG prolog = (PROLOG) e3;
        if (chapter.getProlog() != null)
            return;
        FormattedText prologText = new FormattedText();
        if (parseContent(prologText.getAppendVisitor(), prolog.getContent(), abbrMap)) {
            prologText.trimWhitespace();
            prologText.finished();
            chapter.setProlog(prologText);
        }
    } else if (e3 instanceof PARAGRAPH) {
        PARAGRAPH para = (PARAGRAPH) e3;
        for (Object ee3 : para.getCAPTIONOrPROLOGOrREMARK()) {
            parseChapterObject(ee3, chapter, abbrMap, headlineBuffer);
        }
        chapter.getVerses().get(chapter.getVerses().size() - 1).getAppendVisitor().visitLineBreak(LineBreakKind.PARAGRAPH);
    } else if (e3 instanceof JAXBElement<?>) {
        JAXBElement<?> je = (JAXBElement<?>) e3;
        if (je.getName().getLocalPart().equals("VERSE") && je.getValue() instanceof VERSE) {
            VERSE vers = (VERSE) je.getValue();
            int vnumber = vers.getVnumber().intValue();
            if (vnumber == 0)
                vnumber = chapter.getVerses().size() + 1;
            String verseNumber = "" + vnumber;
            while (chapter.getVerseIndex(verseNumber) != -1) {
                String infix = "";
                for (char ch = 'a'; ch <= 'z'; ch++) {
                    if (chapter.getVerseIndex(vnumber + infix + ch) == -1) {
                        verseNumber = vnumber + infix + ch;
                        break;
                    }
                }
                infix += "-";
            }
            Verse verse = new Verse(verseNumber);
            Visitor<RuntimeException> visitor = verse.getAppendVisitor();
            boolean contentFound = false;
            if (headlineBuffer.size() > 0) {
                for (Headline h : headlineBuffer) {
                    h.accept(visitor.visitHeadline(h.getDepth()));
                }
                headlineBuffer.clear();
                contentFound = true;
            }
            contentFound |= parseContent(visitor, vers.getContent(), abbrMap);
            if (contentFound) {
                verse.trimWhitespace();
                chapter.getVerses().add(verse);
            }
        } else {
            throw new IOException(je.getName() + "/" + je.getDeclaredType());
        }
    } else {
        throw new IOException(e3.getClass().toString());
    }
}

Example 29 with FormattedText

use of biblemulticonverter.data.FormattedText in project BibleMultiConverter by schierlm.

the class MyBibleZone method doImport.

@Override
public Bible doImport(File inputFile) throws Exception {
    SqlJetDb db = SqlJetDb.open(inputFile, false);
    SqlJetDb footnoteDB = null;
    File footnoteFile = new File(inputFile.getParentFile(), inputFile.getName().replace(".SQLite3", ".commentaries.SQLite3"));
    if (inputFile.getName().endsWith(".SQLite3") && footnoteFile.exists()) {
        footnoteDB = SqlJetDb.open(footnoteFile, false);
        if (!footnoteDB.getTable("commentaries").getIndexesNames().contains("commentaries_index")) {
            footnoteDB.close();
            footnoteDB = SqlJetDb.open(footnoteFile, true);
            checkIndex(footnoteDB, "commentaries", "commentaries_index", "CREATE INDEX commentaries_index on commentaries(book_number, chapter_number_from, verse_number_from)");
        }
        footnoteDB.beginTransaction(SqlJetTransactionMode.READ_ONLY);
    }
    if (!db.getTable("verses").getIndexesNames().contains("versesIndex") || (db.getSchema().getTable("stories") != null && !db.getTable("stories").getIndexesNames().contains("stories_index"))) {
        db.close();
        db = SqlJetDb.open(inputFile, true);
        checkIndex(db, "verses", "verses_index", "CREATE UNIQUE INDEX verses_index on verses (book_number, chapter, verse)");
        if (db.getSchema().getTable("stories") != null)
            if (db.getSchema().getTable("stories").getColumn("order_if_several") == null)
                checkIndex(db, "stories", "stories_index", "CREATE UNIQUE INDEX stories_index on stories(book_number, chapter, verse)");
            else
                checkIndex(db, "stories", "stories_index", "CREATE UNIQUE INDEX stories_index on stories(book_number, chapter, verse, order_if_several)");
    }
    db.beginTransaction(SqlJetTransactionMode.READ_ONLY);
    String bibleName = null;
    MetadataBook mb = new MetadataBook();
    ISqlJetCursor cursor = db.getTable("info").open();
    while (!cursor.eof()) {
        String fn = cursor.getString("name");
        String fv = cursor.getString("value");
        if (fn.equals("description")) {
            bibleName = fv;
        } else if (!fv.isEmpty()) {
            fv = fv.replaceAll("[\r\n]+", "\n").replaceAll(" *\n *", "\n").replaceAll("\n$", "");
            try {
                mb.setValue("MyBible.zone@" + fn.replace('_', '.'), fv);
            } catch (IllegalArgumentException ex) {
                System.out.println("WARNING: Skipping malformed metadata property " + fn);
            }
        }
        cursor.next();
    }
    cursor.close();
    if (bibleName == null) {
        System.out.println("WARNING: No bible name in info table");
        bibleName = inputFile.getName();
    }
    Bible result = new Bible(bibleName.trim());
    if (!mb.getKeys().isEmpty()) {
        mb.finished();
        result.getBooks().add(mb.getBook());
    }
    Map<Integer, Book> bookIDMap = new HashMap<>();
    cursor = db.getTable("books").open();
    while (!cursor.eof()) {
        int num = (int) cursor.getInteger("book_number");
        String col = cursor.getString("book_color");
        String shortName = cursor.getString("short_name").trim().replace(" ", "").replaceAll("[^A-Z0-9a-zäöü]++", "");
        if (!shortName.isEmpty())
            shortName = shortName.substring(0, 1).toUpperCase() + shortName.substring(1);
        String longName = cursor.getString("long_name").trim();
        BookID bid = null;
        for (MyBibleZoneBook bi : BOOK_INFO) {
            if (bi.bookNumber == num) {
                bid = bi.bookID;
                if (!col.equals(bi.bookColor))
                    System.out.println("WARNING: Book " + bid.getOsisID() + " uses color " + col + " and not " + bi.bookColor);
            }
        }
        if (bid == null) {
            System.out.println("WARNING: Book number " + num + " unknown; skipping: " + shortName + "/" + longName);
            // generate dummy entry not stored in result object
            bookIDMap.put(num, new Book("Xxx", BookID.BOOK_Gen, "X", "X"));
        } else {
            if (shortName.length() < 2)
                shortName = bid.getOsisID().replaceAll("[^A-Z0-9a-zäöü]++", "");
            Book bk = new Book(shortName, bid, longName, longName);
            result.getBooks().add(bk);
            bookIDMap.put(num, bk);
        }
        cursor.next();
    }
    cursor.close();
    if (db.getSchema().getTable("introductions") != null) {
        cursor = db.getTable("introductions").open();
        while (!cursor.eof()) {
            int num = (int) cursor.getInteger("book_number");
            String intro = cursor.getString("introduction");
            Book bk;
            if (num == 0) {
                bk = new Book("Intro", BookID.INTRODUCTION, "_Introduction_", "_Introduction_");
                if (!result.getBooks().isEmpty() && result.getBooks().get(0).getId().equals(BookID.METADATA)) {
                    result.getBooks().add(1, bk);
                } else {
                    result.getBooks().add(0, bk);
                }
            } else {
                bk = bookIDMap.get(num);
            }
            if (bk == null) {
                System.out.println("WARNING: Skipping introduction for nonexisting book " + num);
            } else {
                FormattedText ft = new FormattedText();
                convertFromHTML(intro, ft.getAppendVisitor());
                ft.finished();
                if (bk.getChapters().isEmpty())
                    bk.getChapters().add(new Chapter());
                bk.getChapters().get(0).setProlog(ft);
            }
            cursor.next();
        }
        cursor.close();
    }
    cursor = db.getTable("verses").order("verses_index");
    while (!cursor.eof()) {
        int b = (int) cursor.getInteger("book_number");
        int c = (int) cursor.getInteger("chapter");
        int v = (int) cursor.getInteger("verse");
        String text = cursor.getString("text");
        if (text == null)
            text = "";
        text = text.trim();
        if (!text.isEmpty()) {
            Book bk = bookIDMap.get(b);
            if (bk == null) {
                System.out.println("WARNING: Verse for unknown book " + b + " skipped");
            } else {
                while (bk.getChapters().size() < c) bk.getChapters().add(new Chapter());
                Chapter ch = bk.getChapters().get(c - 1);
                Verse vv = new Verse("" + v);
                try {
                    String rest = convertFromVerse(text, vv.getAppendVisitor(), footnoteDB, new int[] { b, c, v });
                    if (!rest.isEmpty()) {
                        System.out.println("WARNING: Treating tags as plaintext: " + rest);
                        vv.getAppendVisitor().visitText(rest.replace('\t', ' ').replaceAll("  +", " "));
                    }
                } catch (RuntimeException ex) {
                    throw new RuntimeException(text, ex);
                }
                ch.getVerses().add(vv);
                vv.finished();
            }
        }
        cursor.next();
    }
    cursor.close();
    if (db.getSchema().getTable("stories") != null) {
        cursor = db.getTable("stories").order("stories_index");
        Map<Verse, List<FormattedText.Headline>> subheadings = new HashMap<>();
        Map<Verse, Chapter> subheadingChapters = new HashMap<>();
        while (!cursor.eof()) {
            int b = (int) cursor.getInteger("book_number");
            int c = (int) cursor.getInteger("chapter");
            int v = (int) cursor.getInteger("verse");
            String title = cursor.getString("title").trim();
            Book bk = bookIDMap.get(b);
            if (bk == null) {
                System.out.println("WARNING: Subheading for unknown book " + b + " skipped");
            } else if (bk.getChapters().size() < c) {
                System.out.println("WARNING: Subheading for unknown chapter " + b + " " + c + " skipped");
            } else {
                Chapter ch = bk.getChapters().get(c - 1);
                Verse vv = null;
                for (Verse vvv : ch.getVerses()) {
                    if (vvv.getNumber().equals("" + v))
                        vv = vvv;
                }
                if (vv == null) {
                    System.out.println("WARNING: Subheading for unknown verse " + b + " " + c + ":" + v + " skipped");
                } else {
                    List<FormattedText.Headline> hls = subheadings.get(vv);
                    if (hls == null) {
                        hls = new ArrayList<>();
                        subheadings.put(vv, hls);
                        subheadingChapters.put(vv, ch);
                    }
                    Headline hl = new Headline(1);
                    while (title.contains("<x>")) {
                        int pos = title.indexOf("<x>");
                        hl.getAppendVisitor().visitText(title.substring(0, pos));
                        title = title.substring(pos + 3);
                        pos = title.indexOf("</x>");
                        if (pos == -1)
                            System.out.println("WARNING: Unclosed cross reference: " + title);
                        else {
                            String ref = title.substring(0, pos);
                            title = title.substring(pos + 4);
                            hl.getAppendVisitor().visitFormattingInstruction(FormattingInstructionKind.BOLD).visitText(ref);
                        }
                    }
                    hl.getAppendVisitor().visitText(title);
                    hl.finished();
                    hls.add(hl);
                }
            }
            cursor.next();
        }
        cursor.close();
        for (Verse vv : subheadings.keySet()) {
            Chapter cc = subheadingChapters.get(vv);
            Verse vnew = new Verse(vv.getNumber());
            for (Headline hl : subheadings.get(vv)) {
                hl.accept(vnew.getAppendVisitor().visitHeadline(hl.getDepth()));
            }
            vv.accept(vnew.getAppendVisitor());
            vnew.finished();
            int pos = cc.getVerses().indexOf(vv);
            cc.getVerses().set(pos, vnew);
        }
    }
    if (footnoteDB != null) {
        footnoteDB.commit();
        footnoteDB.close();
    }
    db.commit();
    db.close();
    return result;
}

Example 30 with FormattedText

use of biblemulticonverter.data.FormattedText in project BibleMultiConverter by schierlm.

the class MyBibleZone method doExport.

@Override
public void doExport(Bible bible, String... exportArgs) throws Exception {
    String outfile = exportArgs[0];
    if (!outfile.endsWith(".SQLite3"))
        outfile += ".SQLite3";
    boolean hasFootnotes = false, hasStrongs = false;
    for (Book bk : bible.getBooks()) {
        for (Chapter ch : bk.getChapters()) {
            for (Verse vv : ch.getVerses()) {
                String elementTypes = vv.getElementTypes(Integer.MAX_VALUE);
                if (elementTypes.contains("f")) {
                    hasFootnotes = true;
                }
                if (elementTypes.contains("g")) {
                    hasStrongs = true;
                }
            }
        }
    }
    new File(outfile).delete();
    SqlJetDb db = SqlJetDb.open(new File(outfile), true);
    db.getOptions().setAutovacuum(true);
    db.beginTransaction(SqlJetTransactionMode.WRITE);
    db.getOptions().setUserVersion(0);
    db.createTable("CREATE TABLE info (name TEXT, value TEXT)");
    db.createTable("CREATE TABLE books (book_number NUMERIC, book_color TEXT, short_name TEXT, long_name TEXT)");
    db.createTable("CREATE TABLE introductions (book_number NUMERIC, introduction TEXT)");
    db.createIndex("CREATE UNIQUE INDEX introductions_index on introductions(book_number)");
    db.createTable("CREATE TABLE verses (book_number INTEGER, chapter INTEGER, verse INTEGER, text TEXT)");
    db.createIndex("CREATE UNIQUE INDEX verses_index on verses (book_number, chapter, verse)");
    db.createTable("CREATE TABLE stories (book_number NUMERIC, chapter NUMERIC, verse NUMERIC, order_if_several NUMERIC, title TEXT)");
    db.createIndex("CREATE UNIQUE INDEX stories_index on stories(book_number, chapter, verse, order_if_several)");
    Map<String, String> infoValues = new LinkedHashMap<>();
    MetadataBook mb = bible.getMetadataBook();
    if (mb == null)
        mb = new MetadataBook();
    infoValues.put("language", "xx");
    infoValues.put("description", bible.getName());
    infoValues.put("detailed_info", "");
    infoValues.put("russian_numbering", "false");
    infoValues.put("chapter_string", "Chapter");
    infoValues.put("introduction_string", "Introduction");
    infoValues.put("strong_numbers", hasStrongs ? "true" : "false");
    infoValues.put("right_to_left", "false");
    infoValues.put("digits0-9", "0123456789");
    infoValues.put("swaps_non_localized_words_in_mixed_language_line", "false");
    infoValues.put("localized_book_abbreviations", "false");
    infoValues.put("font_scale", "1.0");
    infoValues.put("contains_accents", "true");
    for (String mbkey : mb.getKeys()) {
        if (mbkey.startsWith("MyBible.zone@")) {
            infoValues.put(mbkey.substring(13).replace('.', '_'), mb.getValue(mbkey));
        } else {
            infoValues.put("detailed_info", infoValues.get("detailed_info") + "\r\n<br><b>" + mbkey + ":</b>" + mb.getValue(mbkey));
        }
    }
    String bibleIntro = null, singleFootnoteMarker = null, singleXrefMarker = null;
    if (exportArgs.length > 1) {
        Properties props = new Properties();
        FileInputStream in = new FileInputStream(exportArgs[1]);
        props.load(in);
        in.close();
        bibleIntro = (String) props.remove("__INTRODUCTION__");
        singleFootnoteMarker = (String) props.remove("__FOOTNOTE_MARKER__");
        singleXrefMarker = (String) props.remove("__XREF_MARKER__");
        for (Object key : props.keySet()) {
            String template = props.getProperty(key.toString());
            template = template.replace("${name}", bible.getName());
            for (String mbkey : mb.getKeys()) template = template.replace("${" + mbkey + "}", mb.getValue(mbkey));
            infoValues.put(key.toString(), template);
        }
    }
    ISqlJetTable infoTable = db.getTable("info");
    ISqlJetTable booksTable = db.getTable("books");
    ISqlJetTable introductionsTable = db.getTable("introductions");
    ISqlJetTable versesTable = db.getTable("verses");
    ISqlJetTable storiesTable = db.getTable("stories");
    for (Map.Entry<String, String> entry : infoValues.entrySet()) {
        infoTable.insert(entry.getKey(), entry.getValue());
    }
    SqlJetDb cdb = null;
    ISqlJetTable footnotesTable = null;
    if (hasFootnotes) {
        String commentaryfile = outfile.replace(".SQLite3", ".commentaries.SQLite3");
        new File(commentaryfile).delete();
        cdb = SqlJetDb.open(new File(commentaryfile), true);
        cdb.getOptions().setAutovacuum(true);
        cdb.beginTransaction(SqlJetTransactionMode.WRITE);
        cdb.getOptions().setUserVersion(0);
        cdb.createTable("CREATE TABLE info (name TEXT, value TEXT)");
        cdb.createTable("CREATE TABLE commentaries (book_number NUMERIC, chapter_number_from NUMERIC, verse_number_from NUMERIC, chapter_number_to NUMERIC, verse_number_to NUMERIC, marker TEXT, text TEXT )");
        cdb.createIndex("CREATE INDEX commentaries_index on commentaries(book_number, chapter_number_from, verse_number_from)");
        ISqlJetTable cInfoTable = cdb.getTable("info");
        for (String key : Arrays.asList("language", "description", "russian_numbering")) {
            cInfoTable.insert(key, infoValues.get(key));
        }
        cInfoTable.insert("is_footnotes", "true");
        footnotesTable = cdb.getTable("commentaries");
    }
    final Set<String> unsupportedFeatures = new HashSet<>();
    FormattedText introProlog = null;
    for (Book bk : bible.getBooks()) {
        if (bk.getId() == BookID.INTRODUCTION || bk.getId() == BookID.INTRODUCTION_OT || bk.getId() == BookID.INTRODUCTION_NT || bk.getId() == BookID.APPENDIX) {
            if (introProlog == null)
                introProlog = new FormattedText();
            introProlog.getAppendVisitor().visitHeadline(1).visitText(bk.getLongName());
            bk.getChapters().get(0).getProlog().accept(introProlog.getAppendVisitor());
            continue;
        }
        MyBibleZoneBook info = null;
        for (MyBibleZoneBook bi : BOOK_INFO) {
            if (bi.bookID == bk.getId())
                info = bi;
        }
        if (info == null) {
            System.out.println("WARNING: Skipping unsupported book " + bk.getId());
            continue;
        }
        booksTable.insert(info.bookNumber, info.bookColor, bk.getAbbr(), bk.getShortName());
        FormattedText prologs = null;
        for (int cn = 1; cn <= bk.getChapters().size(); cn++) {
            Chapter ch = bk.getChapters().get(cn - 1);
            if (ch.getProlog() != null) {
                if (prologs == null)
                    prologs = new FormattedText();
                prologs.getAppendVisitor().visitHeadline(1).visitText(cn == 1 ? bk.getLongName() : "" + cn);
                ch.getProlog().accept(prologs.getAppendVisitor());
            }
            int vn = 0;
            for (VirtualVerse vv : ch.createVirtualVerses()) {
                vn++;
                while (vn < vv.getNumber()) versesTable.insert(info.bookNumber, cn, vn++, "");
                if (vn != vv.getNumber())
                    throw new RuntimeException(vn + " != " + vv.getNumber());
                for (int hl = 0; hl < vv.getHeadlines().size(); hl++) {
                    final StringBuilder sb = new StringBuilder();
                    final Map<StringBuilder, String> xrefTags = new HashMap<>();
                    vv.getHeadlines().get(hl).accept(new VisitorAdapter<RuntimeException>(null) {

                        @Override
                        protected Visitor<RuntimeException> wrapChildVisitor(Visitor<RuntimeException> childVisitor) throws RuntimeException {
                            return this;
                        }

                        @Override
                        protected void beforeVisit() throws RuntimeException {
                            unsupportedFeatures.add("markup in headline");
                        }

                        @Override
                        public void visitText(String text) throws RuntimeException {
                            sb.append(text.replace('<', '〈').replace('>', '〉'));
                        }

                        @Override
                        public Visitor<RuntimeException> visitFootnote() throws RuntimeException {
                            // handle this separately; we do not like
                            // footnote text inside the headline!
                            unsupportedFeatures.add("footnote in headline");
                            return new VisitorAdapter<RuntimeException>(null) {

                                @Override
                                protected Visitor<RuntimeException> wrapChildVisitor(Visitor<RuntimeException> childVisitor) throws RuntimeException {
                                    return this;
                                }

                                @Override
                                public Visitor<RuntimeException> visitCrossReference(String bookAbbr, BookID book, int firstChapter, String firstVerse, int lastChapter, String lastVerse) throws RuntimeException {
                                    if (!BOOK_NUMBERS.containsKey(book))
                                        return null;
                                    final StringBuilder innerBuilder = new StringBuilder();
                                    String endVerse = firstChapter != lastChapter ? "-" + lastChapter + ":" + lastVerse : !firstVerse.equals(lastVerse) ? "-" + lastVerse : "";
                                    xrefTags.put(innerBuilder, "<x>" + BOOK_NUMBERS.get(book) + " " + firstChapter + ":" + firstVerse + endVerse + "</x>");
                                    return new VisitorAdapter<RuntimeException>(null) {

                                        @Override
                                        protected void beforeVisit() throws RuntimeException {
                                            throw new RuntimeException("Unsupported content inside headline xref");
                                        }

                                        @Override
                                        public void visitText(String text) throws RuntimeException {
                                            innerBuilder.append(text.replace('<', '〈').replace('>', '〉'));
                                        }
                                    };
                                }
                            };
                        }

                        @Override
                        public Visitor<RuntimeException> visitExtraAttribute(ExtraAttributePriority prio, String category, String key, String value) throws RuntimeException {
                            unsupportedFeatures.add("extra atrribute in headline");
                            return prio.handleVisitor(category, this);
                        }
                    });
                    String headline = sb.toString();
                    for (Map.Entry<StringBuilder, String> xrefTag : xrefTags.entrySet()) {
                        headline = headline.replace(xrefTag.getKey().toString(), xrefTag.getValue());
                    }
                    storiesTable.insert(info.bookNumber, cn, vn, hl, headline);
                }
                StringBuilder vb = new StringBuilder();
                Map<String, MyBibleHTMLVisitor> footnotes = new HashMap<>();
                MyBibleVerseVisitor mbvv = new MyBibleVerseVisitor(vb, footnotes, unsupportedFeatures);
                for (Verse v : vv.getVerses()) {
                    if (!v.getNumber().equals("" + vv.getNumber())) {
                        vb.append(" <e>(" + v.getNumber() + ")</e> ");
                    }
                    mbvv.reset();
                    v.accept(mbvv);
                }
                if (singleXrefMarker != null || singleFootnoteMarker != null) {
                    String singleXref = null, singleFootnote = null;
                    for (Map.Entry<String, MyBibleHTMLVisitor> fn : footnotes.entrySet()) {
                        if (!fn.getKey().matches("\\[[0-9]+\\]"))
                            continue;
                        if (fn.getValue().getResult().startsWith(FormattedText.XREF_MARKER) && singleXrefMarker != null) {
                            if (singleXref == null) {
                                singleXref = fn.getKey();
                            } else {
                                System.out.println("WARNING: More than one XREF footnote in verse " + info.bookID + " " + cn + ":" + vn);
                                singleXref = "-";
                            }
                        } else if (singleFootnoteMarker != null) {
                            if (singleFootnote == null) {
                                singleFootnote = fn.getKey();
                            } else {
                                System.out.println("WARNING: More than one normal footnote in verse " + info.bookID + " " + cn + ":" + vn);
                                singleFootnote = "-";
                            }
                        }
                    }
                    if (singleXref != null && !singleXref.equals("-")) {
                        MyBibleHTMLVisitor xfn = footnotes.remove(singleXref);
                        if (xfn == null)
                            throw new RuntimeException();
                        footnotes.put(singleXrefMarker, xfn);
                        String verse = vb.toString();
                        vb.setLength(0);
                        vb.append(verse.replace("<f>" + singleXref + "</f>", "<f>" + singleXrefMarker + "</f>"));
                    }
                    if (singleFootnote != null && !singleFootnote.equals("-")) {
                        MyBibleHTMLVisitor sfn = footnotes.remove(singleFootnote);
                        if (sfn == null)
                            throw new RuntimeException();
                        footnotes.put(singleFootnoteMarker, sfn);
                        String verse = vb.toString();
                        vb.setLength(0);
                        vb.append(verse.replace("<f>" + singleFootnote + "</f>", "<f>" + singleFootnoteMarker + "</f>"));
                    }
                }
                for (Map.Entry<String, MyBibleHTMLVisitor> fn : footnotes.entrySet()) {
                    footnotesTable.insert(info.bookNumber, cn, vn, cn, vn, fn.getKey(), fn.getValue().getResult());
                }
                versesTable.insert(info.bookNumber, cn, vn, vb.toString().trim());
            }
        }
        if (prologs != null) {
            MyBibleHTMLVisitor v = new MyBibleHTMLVisitor(unsupportedFeatures, "in introduction");
            prologs.accept(v);
            introductionsTable.insert(info.bookNumber, v.getResult());
        }
    }
    if (bibleIntro != null) {
        introductionsTable.insert(0, bibleIntro);
    } else if (introProlog != null) {
        MyBibleHTMLVisitor v = new MyBibleHTMLVisitor(unsupportedFeatures, "in introduction");
        introProlog.accept(v);
        introductionsTable.insert(0, v.getResult());
    }
    if (!unsupportedFeatures.isEmpty()) {
        System.out.println("WARNING: Skipped unsupported features: " + unsupportedFeatures);
    }
    db.commit();
    db.close();
    if (cdb != null) {
        cdb.commit();
        cdb.close();
    }
}