Examples with IMPORT com.ichi2.async.CollectionTask.TASK_TYPE.IMPORT used on opensource projects

Example 1 with IMPORT

use of com.ichi2.async.CollectionTask.TASK_TYPE.IMPORT in project AnkiChinaAndroid by ankichinateam.

the class ImportTest method testAnki2Mediadupes.

@Test
public void testAnki2Mediadupes() throws IOException, JSONException, ImportExportException {
    List<String> expected;
    List<String> actual;
    // add a note that references a sound
    Note n = testCol.newNote();
    n.setField(0, "[sound:foo.mp3]");
    long mid = n.model().getLong("id");
    testCol.addNote(n);
    // add that sound to the media folder
    FileOutputStream os;
    os = new FileOutputStream(new File(testCol.getMedia().dir(), "foo.mp3"), false);
    os.write("foo".getBytes());
    os.close();
    testCol.close();
    // it should be imported correctly into an empty deck
    Collection empty = Shared.getEmptyCol(InstrumentationRegistry.getInstrumentation().getTargetContext());
    Importer imp = new Anki2Importer(empty, testCol.getPath());
    imp.run();
    expected = Collections.singletonList("foo.mp3");
    actual = Arrays.asList(new File(empty.getMedia().dir()).list());
    actual.retainAll(expected);
    assertEquals(expected.size(), actual.size());
    // and importing again will not duplicate, as the file content matches
    empty.remCards(empty.getDb().queryLongList("select id from cards"));
    imp = new Anki2Importer(empty, testCol.getPath());
    imp.run();
    expected = Collections.singletonList("foo.mp3");
    actual = Arrays.asList(new File(empty.getMedia().dir()).list());
    actual.retainAll(expected);
    assertEquals(expected.size(), actual.size());
    n = empty.getNote(empty.getDb().queryLongScalar("select id from notes"));
    assertTrue(n.getFields()[0].contains("foo.mp3"));
    // if the local file content is different, and import should trigger a rename
    empty.remCards(empty.getDb().queryLongList("select id from cards"));
    os = new FileOutputStream(new File(empty.getMedia().dir(), "foo.mp3"), false);
    os.write("bar".getBytes());
    os.close();
    imp = new Anki2Importer(empty, testCol.getPath());
    imp.run();
    expected = Arrays.asList("foo.mp3", String.format("foo_%s.mp3", mid));
    actual = Arrays.asList(new File(empty.getMedia().dir()).list());
    actual.retainAll(expected);
    assertEquals(expected.size(), actual.size());
    n = empty.getNote(empty.getDb().queryLongScalar("select id from notes"));
    assertTrue(n.getFields()[0].contains("_"));
    // if the localized media file already exists, we rewrite the note and media
    empty.remCards(empty.getDb().queryLongList("select id from cards"));
    os = new FileOutputStream(new File(empty.getMedia().dir(), "foo.mp3"));
    os.write("bar".getBytes());
    os.close();
    imp = new Anki2Importer(empty, testCol.getPath());
    imp.run();
    expected = Arrays.asList("foo.mp3", String.format("foo_%s.mp3", mid));
    actual = Arrays.asList(new File(empty.getMedia().dir()).list());
    actual.retainAll(expected);
    assertEquals(expected.size(), actual.size());
    n = empty.getNote(empty.getDb().queryLongScalar("select id from notes"));
    assertTrue(n.getFields()[0].contains("_"));
    empty.close();
}

Example 2 with IMPORT

use of com.ichi2.async.CollectionTask.TASK_TYPE.IMPORT in project AnkiChinaAndroid by ankichinateam.

the class ImportTest method testCsv.

@Test
@SdkSuppress(minSdkVersion = Build.VERSION_CODES.O)
public void testCsv() throws IOException {
    String file = Shared.getTestFilePath(InstrumentationRegistry.getInstrumentation().getTargetContext(), "text-2fields.txt");
    TextImporter i = new TextImporter(testCol, file);
    i.initMapping();
    i.run();
    if (TestEnvironment.isDisplayingDefaultEnglishStrings()) {
        assertThat(i.getLog(), contains("‘多すぎる too many fields’ had 3 fields, expected 2", "‘not, enough, fields’ had 1 fields, expected 2", "Appeared twice in file: 飲む", "Empty first field:  to play", "5 notes added, 0 notes updated, 0 notes unchanged."));
    } else {
        assertThat(i.getLog(), hasSize(5));
    }
    assertEquals(5, i.getTotal());
    // if we run the import again, it should update instead
    i.run();
    if (TestEnvironment.isDisplayingDefaultEnglishStrings()) {
        assertThat(i.getLog(), contains("‘多すぎる too many fields’ had 3 fields, expected 2", "‘not, enough, fields’ had 1 fields, expected 2", "Appeared twice in file: 飲む", "Empty first field:  to play", "0 notes added, 0 notes updated, 5 notes unchanged.", "First field matched: 食べる", "First field matched: 飲む", "First field matched: テスト", "First field matched: to eat", "First field matched: 遊ぶ"));
    } else {
        assertThat(i.getLog(), hasSize(10));
    }
    assertEquals(5, i.getTotal());
    // but importing should not clobber tags if they're unmapped
    Note n = testCol.getNote(testCol.getDb().queryLongScalar("select id from notes"));
    n.addTag("test");
    n.flush();
    i.run();
    n.load();
    assertThat(n.getTags(), contains("test"));
    assertThat(n.getTags(), hasSize(1));
    // if add-only mode, count will be 0
    i.setImportMode(NoteImporter.ImportMode.IGNORE_MODE);
    i.run();
    assertEquals(0, i.getTotal());
    // and if dupes mode, will reimport everything
    assertEquals(5, testCol.cardCount());
    i.setImportMode(NoteImporter.ImportMode.ADD_MODE);
    i.run();
    // includes repeated field
    assertEquals(6, i.getTotal());
    assertEquals(11, testCol.cardCount());
}

Example 3 with IMPORT

use of com.ichi2.async.CollectionTask.TASK_TYPE.IMPORT in project AnkiChinaAndroid by ankichinateam.

the class IntentHandler method handleFileImport.

private void handleFileImport(Intent intent, Intent reloadIntent, String action) {
    Timber.i("Handling file import");
    ImportResult importResult = ImportUtils.handleFileImport(this, intent);
    // Start DeckPicker if we correctly processed ACTION_VIEW
    if (importResult.isSuccess()) {
        Timber.d("onCreate() import successful");
        reloadIntent.setAction(action);
        // reloadIntent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP);
        startActivity(reloadIntent);
        AnkiActivity.finishActivityWithFade(this);
    } else {
        Timber.i("File import failed");
        // Don't import the file if it didn't load properly or doesn't have apkg extension
        ImportUtils.showImportUnsuccessfulDialog(this, importResult.getHumanReadableMessage(), true);
    }
}

Example 4 with IMPORT

use of com.ichi2.async.CollectionTask.TASK_TYPE.IMPORT in project AnkiChinaAndroid by ankichinateam.

the class CollectionTask method doInBackgroundImportReplace.

private TaskData doInBackgroundImportReplace(TaskData param) {
    Timber.d("doInBackgroundImportReplace");
    String path = param.getString();
    Resources res = AnkiDroidApp.getInstance().getBaseContext().getResources();
    // extract the deck from the zip file
    String colPath = CollectionHelper.getCollectionPath(mContext);
    File dir = new File(new File(colPath).getParentFile(), "tmpzip");
    if (dir.exists()) {
        BackupManager.removeDir(dir);
    }
    // from anki2.py
    String colname = "collection.anki21";
    ZipFile zip;
    try {
        zip = new ZipFile(new File(path));
    } catch (IOException e) {
        Timber.e(e, "doInBackgroundImportReplace - Error while unzipping");
        AnkiDroidApp.sendExceptionReport(e, "doInBackgroundImportReplace0");
        return new TaskData(false);
    }
    try {
        // v2 scheduler?
        if (zip.getEntry(colname) == null) {
            colname = CollectionHelper.COLLECTION_FILENAME;
        }
        Utils.unzipFiles(zip, dir.getAbsolutePath(), new String[] { colname, "media" }, null);
    } catch (IOException e) {
        AnkiDroidApp.sendExceptionReport(e, "doInBackgroundImportReplace - unzip");
        return new TaskData(false);
    }
    String colFile = new File(dir, colname).getAbsolutePath();
    if (!(new File(colFile)).exists()) {
        return new TaskData(false);
    }
    Collection tmpCol = null;
    try {
        tmpCol = Storage.Collection(mContext, colFile);
        if (!tmpCol.validCollection()) {
            tmpCol.close();
            return new TaskData(false);
        }
    } catch (Exception e) {
        Timber.e("Error opening new collection file... probably it's invalid");
        try {
            tmpCol.close();
        } catch (Exception e2) {
        // do nothing
        }
        AnkiDroidApp.sendExceptionReport(e, "doInBackgroundImportReplace - open col");
        return new TaskData(false);
    } finally {
        if (tmpCol != null) {
            tmpCol.close();
        }
    }
    publishProgress(new TaskData(res.getString(R.string.importing_collection)));
    if (hasValidCol()) {
        // unload collection and trigger a backup
        Time time = CollectionHelper.getInstance().getTimeSafe(mContext);
        CollectionHelper.getInstance().closeCollection(true, "Importing new collection");
        CollectionHelper.getInstance().lockCollection();
        BackupManager.performBackupInBackground(colPath, true, time);
    }
    // overwrite collection
    File f = new File(colFile);
    if (!f.renameTo(new File(colPath))) {
        // Exit early if this didn't work
        return new TaskData(false);
    }
    int addedCount = -1;
    try {
        CollectionHelper.getInstance().unlockCollection();
        // because users don't have a backup of media, it's safer to import new
        // data and rely on them running a media db check to get rid of any
        // unwanted media. in the future we might also want to duplicate this step
        // import media
        HashMap<String, String> nameToNum = new HashMap<>();
        HashMap<String, String> numToName = new HashMap<>();
        File mediaMapFile = new File(dir.getAbsolutePath(), "media");
        if (mediaMapFile.exists()) {
            JsonReader jr = new JsonReader(new FileReader(mediaMapFile));
            jr.beginObject();
            String name;
            String num;
            while (jr.hasNext()) {
                num = jr.nextName();
                name = jr.nextString();
                nameToNum.put(name, num);
                numToName.put(num, name);
            }
            jr.endObject();
            jr.close();
        }
        String mediaDir = Media.getCollectionMediaPath(colPath);
        int total = nameToNum.size();
        int i = 0;
        for (Map.Entry<String, String> entry : nameToNum.entrySet()) {
            String file = entry.getKey();
            String c = entry.getValue();
            File of = new File(mediaDir, file);
            if (!of.exists()) {
                Utils.unzipFiles(zip, mediaDir, new String[] { c }, numToName);
            }
            ++i;
            publishProgress(new TaskData(res.getString(R.string.import_media_count, (i + 1) * 100 / total)));
        }
        zip.close();
        // delete tmp dir
        BackupManager.removeDir(dir);
        return new TaskData(true);
    } catch (RuntimeException e) {
        Timber.e(e, "doInBackgroundImportReplace - RuntimeException");
        AnkiDroidApp.sendExceptionReport(e, "doInBackgroundImportReplace1");
        return new TaskData(false);
    } catch (FileNotFoundException e) {
        Timber.e(e, "doInBackgroundImportReplace - FileNotFoundException");
        AnkiDroidApp.sendExceptionReport(e, "doInBackgroundImportReplace2");
        return new TaskData(false);
    } catch (IOException e) {
        Timber.e(e, "doInBackgroundImportReplace - IOException");
        AnkiDroidApp.sendExceptionReport(e, "doInBackgroundImportReplace3");
        return new TaskData(false);
    }
}

Example 5 with IMPORT

use of com.ichi2.async.CollectionTask.TASK_TYPE.IMPORT in project AnkiChinaAndroid by ankichinateam.

the class Anki2Importer method _importCards.

private void _importCards() {
    if (mMustResetLearning) {
        try {
            mSrc.changeSchedulerVer(2);
        } catch (ConfirmModSchemaException e) {
            throw new RuntimeException("Changing the scheduler of an import should not cause schema modification", e);
        }
    }
    // build map of guid -> (ord -> cid) and used id cache
    mCards = new HashMap<>();
    Map<Long, Boolean> existing = new HashMap<>();
    Cursor cur = null;
    try {
        cur = mDst.getDb().getDatabase().query("select f.guid, c.ord, c.id from cards c, notes f " + "where c.nid = f.id", null);
        while (cur.moveToNext()) {
            String guid = cur.getString(0);
            int ord = cur.getInt(1);
            long cid = cur.getLong(2);
            existing.put(cid, true);
            if (mCards.containsKey(guid)) {
                mCards.get(guid).put(ord, cid);
            } else {
                Map<Integer, Long> map = new HashMap<>();
                map.put(ord, cid);
                mCards.put(guid, map);
            }
        }
    } finally {
        if (cur != null) {
            cur.close();
        }
    }
    // loop through src
    List<Object[]> cards = new ArrayList<>();
    int totalCardCount = 0;
    final int thresExecCards = 1000;
    List<Object[]> revlog = new ArrayList<>();
    int totalRevlogCount = 0;
    final int thresExecRevlog = 1000;
    int usn = mDst.usn();
    long aheadBy = mSrc.getSched().getToday() - mDst.getSched().getToday();
    try {
        mDst.getDb().getDatabase().beginTransaction();
        cur = mSrc.getDb().getDatabase().query("select f.guid, f.mid, c.* from cards c, notes f " + "where c.nid = f.id", null);
        // Counters for progress updates
        int total = cur.getCount();
        boolean largeCollection = total > 200;
        int onePercent = total / 100;
        int i = 0;
        while (cur.moveToNext()) {
            Object[] card = new Object[] { cur.getString(0), cur.getLong(1), cur.getLong(2), cur.getLong(3), cur.getLong(4), cur.getInt(5), cur.getLong(6), cur.getInt(7), cur.getInt(8), cur.getInt(9), cur.getLong(10), cur.getLong(11), cur.getLong(12), cur.getInt(13), cur.getInt(14), cur.getInt(15), cur.getLong(16), cur.getLong(17), cur.getInt(18), cur.getString(19) };
            String guid = (String) card[0];
            if (mChangedGuids.containsKey(guid)) {
                guid = mChangedGuids.get(guid);
            }
            if (mIgnoredGuids.containsKey(guid)) {
                continue;
            }
            // does the card's note exist in dst col?
            if (!mNotes.containsKey(guid)) {
                continue;
            }
            Object[] dnid = mNotes.get(guid);
            // does the card already exist in the dst col?
            int ord = (Integer) card[5];
            if (mCards.containsKey(guid) && mCards.get(guid).containsKey(ord)) {
                // fixme: in future, could update if newer mod time
                continue;
            }
            // doesn't exist. strip off note info, and save src id for later
            Object[] oc = card;
            card = new Object[oc.length - 2];
            System.arraycopy(oc, 2, card, 0, card.length);
            long scid = (Long) card[0];
            // ensure the card id is unique
            while (existing.containsKey(card[0])) {
                card[0] = (Long) card[0] + 999;
            }
            existing.put((Long) card[0], true);
            // update cid, nid, etc
            card[1] = mNotes.get(guid)[0];
            card[2] = _did((Long) card[2]);
            if (mTopID < 0) {
                mTopID = (long) card[2];
            }
            card[4] = mCol.getTime().intTime();
            card[5] = usn;
            // review cards have a due date relative to collection
            if ((Integer) card[7] == 2 || (Integer) card[7] == 3 || (Integer) card[6] == 2) {
                card[8] = (Long) card[8] - aheadBy;
            }
            // odue needs updating too
            if (((Long) card[14]).longValue() != 0) {
                card[14] = (Long) card[14] - aheadBy;
            }
            // if odid true, convert card from filtered to normal
            if ((Long) card[15] != 0) {
                // odid
                card[15] = 0;
                // odue
                card[8] = card[14];
                card[14] = 0;
                // queue
                if ((Integer) card[6] == 1) {
                    // type
                    card[7] = 0;
                } else {
                    card[7] = card[6];
                }
                // type
                if ((Integer) card[6] == 1) {
                    card[6] = 0;
                }
            }
            cards.add(card);
            // we need to import revlog, rewriting card ids and bumping usn
            try (Cursor cur2 = mSrc.getDb().getDatabase().query("select * from revlog where cid = " + scid, null)) {
                while (cur2.moveToNext()) {
                    Object[] rev = new Object[] { cur2.getLong(0), cur2.getLong(1), cur2.getInt(2), cur2.getInt(3), cur2.getLong(4), cur2.getLong(5), cur2.getLong(6), cur2.getLong(7), cur2.getInt(8) };
                    rev[1] = card[0];
                    rev[2] = mDst.usn();
                    revlog.add(rev);
                }
            }
            i++;
            // apply card changes partially
            if (cards.size() >= thresExecCards) {
                totalCardCount += cards.size();
                insertCards(cards);
                cards.clear();
                Timber.d("add cards: %d", totalCardCount);
            }
            // apply revlog changes partially
            if (revlog.size() >= thresExecRevlog) {
                totalRevlogCount += revlog.size();
                insertRevlog(revlog);
                revlog.clear();
                Timber.d("add revlog: %d", totalRevlogCount);
            }
            if (total != 0 && (!largeCollection || i % onePercent == 0)) {
                publishProgress(100, i * 100 / total, 0);
            }
        }
        publishProgress(100, 100, 0);
        // count total values
        totalCardCount += cards.size();
        totalRevlogCount += revlog.size();
        Timber.d("add cards total:  %d", totalCardCount);
        Timber.d("add revlog total: %d", totalRevlogCount);
        // apply (for last chunk)
        insertCards(cards);
        cards.clear();
        insertRevlog(revlog);
        revlog.clear();
        mLog.add(getRes().getString(R.string.import_complete_count, totalCardCount));
        mDst.getDb().getDatabase().setTransactionSuccessful();
    } finally {
        if (cur != null) {
            cur.close();
        }
        if (mDst.getDb().getDatabase().inTransaction()) {
            try {
                mDst.getDb().getDatabase().endTransaction();
            } catch (Exception e) {
                Timber.w(e);
            }
        }
    }
}