Examples with UploadedFileInfo net.osmand.plus.backup.BackupDbHelper.UploadedFileInfo used on opensource projects

Example 1 with UploadedFileInfo

use of net.osmand.plus.backup.BackupDbHelper.UploadedFileInfo in project Osmand by osmandapp.

the class BackupImporter method getRemoteItems.

@NonNull
private List<SettingsItem> getRemoteItems(@NonNull List<RemoteFile> remoteFiles, boolean readItems) throws IllegalArgumentException, IOException {
    if (remoteFiles.isEmpty()) {
        return Collections.emptyList();
    }
    List<SettingsItem> items = new ArrayList<>();
    try {
        OperationLog operationLog = new OperationLog("getRemoteItems", BackupHelper.DEBUG);
        operationLog.startOperation();
        JSONObject json = new JSONObject();
        JSONArray itemsJson = new JSONArray();
        json.put("items", itemsJson);
        Map<File, RemoteFile> remoteInfoFilesMap = new HashMap<>();
        Map<String, RemoteFile> remoteItemFilesMap = new HashMap<>();
        List<RemoteFile> remoteInfoFiles = new ArrayList<>();
        Set<String> remoteInfoNames = new HashSet<>();
        List<RemoteFile> noInfoRemoteItemFiles = new ArrayList<>();
        OsmandApplication app = backupHelper.getApp();
        File tempDir = FileUtils.getTempDir(app);
        List<RemoteFile> uniqueRemoteFiles = new ArrayList<>();
        Set<String> uniqueFileIds = new TreeSet<>();
        for (RemoteFile rf : remoteFiles) {
            String fileId = rf.getTypeNamePath();
            if (uniqueFileIds.add(fileId) && !rf.isDeleted()) {
                uniqueRemoteFiles.add(rf);
            }
        }
        operationLog.log("build uniqueRemoteFiles");
        Map<String, UploadedFileInfo> infoMap = backupHelper.getDbHelper().getUploadedFileInfoMap();
        BackupInfo backupInfo = backupHelper.getBackup().getBackupInfo();
        List<RemoteFile> filesToDelete = backupInfo != null ? backupInfo.filesToDelete : Collections.emptyList();
        for (RemoteFile remoteFile : uniqueRemoteFiles) {
            String fileName = remoteFile.getTypeNamePath();
            if (fileName.endsWith(INFO_EXT)) {
                boolean delete = false;
                String origFileName = remoteFile.getName().substring(0, remoteFile.getName().length() - INFO_EXT.length());
                for (RemoteFile file : filesToDelete) {
                    if (file.getName().equals(origFileName)) {
                        delete = true;
                        break;
                    }
                }
                UploadedFileInfo fileInfo = infoMap.get(remoteFile.getType() + "___" + origFileName);
                long uploadTime = fileInfo != null ? fileInfo.getUploadTime() : 0;
                if (readItems && (uploadTime != remoteFile.getClienttimems() || delete)) {
                    remoteInfoFilesMap.put(new File(tempDir, fileName), remoteFile);
                }
                String itemFileName = fileName.substring(0, fileName.length() - INFO_EXT.length());
                remoteInfoNames.add(itemFileName);
                remoteInfoFiles.add(remoteFile);
            } else if (!remoteItemFilesMap.containsKey(fileName)) {
                remoteItemFilesMap.put(fileName, remoteFile);
            }
        }
        operationLog.log("build maps");
        for (Entry<String, RemoteFile> remoteFileEntry : remoteItemFilesMap.entrySet()) {
            String itemFileName = remoteFileEntry.getKey();
            RemoteFile remoteFile = remoteFileEntry.getValue();
            boolean hasInfo = false;
            for (String remoteInfoName : remoteInfoNames) {
                if (itemFileName.equals(remoteInfoName) || itemFileName.startsWith(remoteInfoName + "/")) {
                    hasInfo = true;
                    break;
                }
            }
            if (!hasInfo && !remoteFile.isRecordedVoiceFile()) {
                noInfoRemoteItemFiles.add(remoteFile);
            }
        }
        operationLog.log("build noInfoRemoteItemFiles");
        if (readItems) {
            generateItemsJson(itemsJson, remoteInfoFilesMap, noInfoRemoteItemFiles);
        } else {
            generateItemsJson(itemsJson, remoteInfoFiles, noInfoRemoteItemFiles);
        }
        operationLog.log("generateItemsJson");
        SettingsItemsFactory itemsFactory = new SettingsItemsFactory(app, json);
        operationLog.log("create setting items");
        List<SettingsItem> settingsItemList = itemsFactory.getItems();
        if (settingsItemList.isEmpty()) {
            return Collections.emptyList();
        }
        updateFilesInfo(remoteItemFilesMap, settingsItemList);
        items.addAll(settingsItemList);
        operationLog.log("updateFilesInfo");
        if (readItems) {
            Map<RemoteFile, SettingsItemReader<? extends SettingsItem>> remoteFilesForRead = new HashMap<>();
            for (SettingsItem item : settingsItemList) {
                if (item.shouldReadOnCollecting()) {
                    List<RemoteFile> foundRemoteFiles = getItemRemoteFiles(item, remoteItemFilesMap);
                    for (RemoteFile remoteFile : foundRemoteFiles) {
                        SettingsItemReader<? extends SettingsItem> reader = item.getReader();
                        if (reader != null) {
                            remoteFilesForRead.put(remoteFile, reader);
                        }
                    }
                }
            }
            Map<File, RemoteFile> remoteFilesForDownload = new HashMap<>();
            for (RemoteFile remoteFile : remoteFilesForRead.keySet()) {
                String fileName = remoteFile.getTypeNamePath();
                remoteFilesForDownload.put(new File(tempDir, fileName), remoteFile);
            }
            if (!remoteFilesForDownload.isEmpty()) {
                downloadAndReadItemFiles(remoteFilesForRead, remoteFilesForDownload);
            }
            operationLog.log("readItems");
        }
        operationLog.finishOperation();
    } catch (IllegalArgumentException e) {
        throw new IllegalArgumentException("Error reading items", e);
    } catch (JSONException e) {
        throw new IllegalArgumentException("Error parsing items", e);
    } catch (IOException e) {
        throw new IOException(e);
    }
    return items;
}

Example 2 with UploadedFileInfo

use of net.osmand.plus.backup.BackupDbHelper.UploadedFileInfo in project Osmand by osmandapp.

the class BackupHelper method generateBackupInfo.

@SuppressLint("StaticFieldLeak")
void generateBackupInfo(@NonNull final Map<String, LocalFile> localFiles, @NonNull final Map<String, RemoteFile> uniqueRemoteFiles, @NonNull final Map<String, RemoteFile> deletedRemoteFiles, @Nullable final OnGenerateBackupInfoListener listener) {
    OperationLog operationLog = new OperationLog("generateBackupInfo", DEBUG, 200);
    operationLog.startOperation();
    AsyncTask<Void, Void, BackupInfo> task = new AsyncTask<Void, Void, BackupInfo>() {

        @Override
        protected BackupInfo doInBackground(Void... voids) {
            BackupInfo info = new BackupInfo();
            /*
				operationLog.log("=== localFiles ===");
				for (LocalFile localFile : localFiles.values()) {
					operationLog.log(localFile.toString());
				}
				operationLog.log("=== localFiles ===");
				operationLog.log("=== uniqueRemoteFiles ===");
				for (RemoteFile remoteFile : uniqueRemoteFiles.values()) {
					operationLog.log(remoteFile.toString());
				}
				operationLog.log("=== uniqueRemoteFiles ===");
				operationLog.log("=== deletedRemoteFiles ===");
				for (RemoteFile remoteFile : deletedRemoteFiles.values()) {
					operationLog.log(remoteFile.toString());
				}
				operationLog.log("=== deletedRemoteFiles ===");
				*/
            List<RemoteFile> remoteFiles = new ArrayList<>(uniqueRemoteFiles.values());
            remoteFiles.addAll(deletedRemoteFiles.values());
            for (RemoteFile remoteFile : remoteFiles) {
                ExportSettingsType exportType = ExportSettingsType.getExportSettingsTypeForRemoteFile(remoteFile);
                if (exportType == null || !ExportSettingsType.isTypeEnabled(exportType) || remoteFile.isRecordedVoiceFile()) {
                    continue;
                }
                LocalFile localFile = localFiles.get(remoteFile.getTypeNamePath());
                if (localFile != null) {
                    long remoteUploadTime = remoteFile.getClienttimems();
                    long localUploadTime = localFile.uploadTime;
                    if (remoteFile.isDeleted()) {
                        info.localFilesToDelete.add(localFile);
                    } else if (remoteUploadTime == localUploadTime) {
                        if (localUploadTime < localFile.localModifiedTime) {
                            info.filesToUpload.add(localFile);
                            info.filesToDownload.add(remoteFile);
                        }
                    } else {
                        info.filesToMerge.add(new Pair<>(localFile, remoteFile));
                        info.filesToDownload.add(remoteFile);
                    }
                    long localFileSize = localFile.file == null ? 0 : localFile.file.length();
                    long remoteFileSize = remoteFile.getFilesize();
                    if (remoteFileSize > 0 && localFileSize > 0 && localFileSize != remoteFileSize && !info.filesToDownload.contains(remoteFile)) {
                        info.filesToDownload.add(remoteFile);
                    }
                }
                if (localFile == null && !remoteFile.isDeleted()) {
                    UploadedFileInfo fileInfo = dbHelper.getUploadedFileInfo(remoteFile.getType(), remoteFile.getName());
                    // suggest to remove only if file exists in db
                    if (fileInfo != null) {
                        info.filesToDelete.add(remoteFile);
                    }
                    info.filesToDownload.add(remoteFile);
                }
            }
            for (LocalFile localFile : localFiles.values()) {
                ExportSettingsType exportType = localFile.item != null ? ExportSettingsType.getExportSettingsTypeForItem(localFile.item) : null;
                if (exportType == null || !ExportSettingsType.isTypeEnabled(exportType)) {
                    continue;
                }
                boolean hasRemoteFile = uniqueRemoteFiles.containsKey(localFile.getTypeFileName());
                if (!hasRemoteFile) {
                    boolean isEmpty = localFile.item instanceof CollectionSettingsItem<?> && ((CollectionSettingsItem<?>) localFile.item).isEmpty();
                    if (!isEmpty) {
                        info.filesToUpload.add(localFile);
                    }
                }
            }
            info.createItemCollections(app);
            operationLog.log("=== filesToUpload ===");
            for (LocalFile localFile : info.filesToUpload) {
                operationLog.log(localFile.toString());
            }
            operationLog.log("=== filesToUpload ===");
            operationLog.log("=== filesToDownload ===");
            for (RemoteFile remoteFile : info.filesToDownload) {
                operationLog.log(remoteFile.toString());
            }
            operationLog.log("=== filesToDownload ===");
            operationLog.log("=== filesToDelete ===");
            for (RemoteFile remoteFile : info.filesToDelete) {
                operationLog.log(remoteFile.toString());
            }
            operationLog.log("=== filesToDelete ===");
            operationLog.log("=== filesToMerge ===");
            for (Pair<LocalFile, RemoteFile> filePair : info.filesToMerge) {
                operationLog.log("LOCAL=" + filePair.first.toString() + " REMOTE=" + filePair.second.toString());
            }
            operationLog.log("=== filesToMerge ===");
            return info;
        }

        @Override
        protected void onPostExecute(BackupInfo backupInfo) {
            operationLog.finishOperation(backupInfo.toString());
            if (listener != null) {
                listener.onBackupInfoGenerated(backupInfo, null);
            }
        }
    };
    task.executeOnExecutor(executor);
}

Example 3 with UploadedFileInfo

use of net.osmand.plus.backup.BackupDbHelper.UploadedFileInfo in project Osmand by osmandapp.

the class BackupHelper method collectLocalFiles.

@SuppressLint("StaticFieldLeak")
void collectLocalFiles(@Nullable final OnCollectLocalFilesListener listener) {
    OperationLog operationLog = new OperationLog("collectLocalFiles", DEBUG);
    operationLog.startOperation();
    AsyncTask<Void, LocalFile, List<LocalFile>> task = new AsyncTask<Void, LocalFile, List<LocalFile>>() {

        BackupDbHelper dbHelper;

        SQLiteConnection db;

        Map<String, UploadedFileInfo> infos;

        @Override
        protected void onPreExecute() {
            dbHelper = app.getBackupHelper().getDbHelper();
            db = dbHelper.openConnection(true);
        }

        @Override
        protected List<LocalFile> doInBackground(Void... voids) {
            List<LocalFile> result = new ArrayList<>();
            infos = dbHelper.getUploadedFileInfoMap();
            List<SettingsItem> localItems = getLocalItems();
            operationLog.log("getLocalItems");
            for (SettingsItem item : localItems) {
                String fileName = BackupHelper.getItemFileName(item);
                if (item instanceof FileSettingsItem) {
                    FileSettingsItem fileItem = (FileSettingsItem) item;
                    File file = fileItem.getFile();
                    if (file.isDirectory()) {
                        if (item instanceof GpxSettingsItem) {
                            continue;
                        } else if (fileItem.getSubtype() == FileSubtype.VOICE) {
                            File jsFile = new File(file, file.getName() + "_" + IndexConstants.TTSVOICE_INDEX_EXT_JS);
                            if (jsFile.exists()) {
                                fileName = jsFile.getPath().replace(app.getAppPath(null).getPath() + "/", "");
                                createLocalFile(result, item, fileName, jsFile, jsFile.lastModified());
                                continue;
                            }
                        } else if (fileItem.getSubtype() == FileSubtype.TTS_VOICE) {
                            String langName = file.getName().replace(IndexConstants.VOICE_PROVIDER_SUFFIX, "");
                            File jsFile = new File(file, langName + "_" + IndexConstants.TTSVOICE_INDEX_EXT_JS);
                            if (jsFile.exists()) {
                                fileName = jsFile.getPath().replace(app.getAppPath(null).getPath() + "/", "");
                                createLocalFile(result, item, fileName, jsFile, jsFile.lastModified());
                                continue;
                            }
                        } else if (fileItem.getSubtype() == FileSubtype.TILES_MAP) {
                            continue;
                        }
                        List<File> dirs = new ArrayList<>();
                        dirs.add(file);
                        Algorithms.collectDirs(file, dirs);
                        operationLog.log("collectDirs " + file.getName() + " BEGIN");
                        for (File dir : dirs) {
                            File[] files = dir.listFiles();
                            if (files != null && files.length > 0) {
                                for (File f : files) {
                                    if (!f.isDirectory()) {
                                        fileName = f.getPath().replace(app.getAppPath(null).getPath() + "/", "");
                                        createLocalFile(result, item, fileName, f, f.lastModified());
                                    }
                                }
                            }
                        }
                        operationLog.log("collectDirs " + file.getName() + " END");
                    } else if (fileItem.getSubtype() == FileSubtype.TILES_MAP) {
                        if (file.getName().endsWith(SQLiteTileSource.EXT)) {
                            createLocalFile(result, item, fileName, file, file.lastModified());
                        }
                    } else {
                        createLocalFile(result, item, fileName, file, file.lastModified());
                    }
                } else {
                    createLocalFile(result, item, fileName, null, item.getLastModifiedTime());
                }
            }
            return result;
        }

        private void createLocalFile(@NonNull List<LocalFile> result, @NonNull SettingsItem item, @NonNull String fileName, @Nullable File file, long lastModifiedTime) {
            LocalFile localFile = new LocalFile();
            localFile.file = file;
            localFile.item = item;
            localFile.fileName = fileName;
            localFile.localModifiedTime = lastModifiedTime;
            if (infos != null) {
                UploadedFileInfo fileInfo = infos.get(item.getType().name() + "___" + fileName);
                if (fileInfo != null) {
                    localFile.uploadTime = fileInfo.getUploadTime();
                    String lastMd5 = fileInfo.getMd5Digest();
                    boolean needM5Digest = item instanceof StreamSettingsItem && ((StreamSettingsItem) item).needMd5Digest() && localFile.uploadTime < lastModifiedTime && !Algorithms.isEmpty(lastMd5);
                    if (needM5Digest && file != null && file.exists()) {
                        FileInputStream is = null;
                        try {
                            is = new FileInputStream(file);
                            String md5 = new String(Hex.encodeHex(DigestUtils.md5(is)));
                            if (md5.equals(lastMd5)) {
                                item.setLocalModifiedTime(localFile.uploadTime);
                                localFile.localModifiedTime = localFile.uploadTime;
                            }
                        } catch (IOException e) {
                            LOG.error(e.getMessage(), e);
                        } finally {
                            Algorithms.closeStream(is);
                        }
                    }
                }
            }
            result.add(localFile);
            publishProgress(localFile);
        }

        private List<SettingsItem> getLocalItems() {
            List<ExportSettingsType> types = ExportSettingsType.getEnabledTypes();
            Iterator<ExportSettingsType> it = types.iterator();
            while (it.hasNext()) {
                ExportSettingsType type = it.next();
                if (!getBackupTypePref(type).get()) {
                    it.remove();
                }
            }
            return app.getFileSettingsHelper().getFilteredSettingsItems(types, true, true);
        }

        @Override
        protected void onProgressUpdate(LocalFile... localFiles) {
            if (listener != null) {
                listener.onFileCollected(localFiles[0]);
            }
        }

        @Override
        protected void onPostExecute(List<LocalFile> localFiles) {
            if (db != null) {
                db.close();
            }
            operationLog.finishOperation(" Files=" + localFiles.size());
            if (listener != null) {
                listener.onFilesCollected(localFiles);
            }
        }
    };
    task.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
}

Example 4 with UploadedFileInfo

use of net.osmand.plus.backup.BackupDbHelper.UploadedFileInfo in project Osmand by osmandapp.

the class ItemViewHolder method setupItemView.

protected void setupItemView(@NonNull String exportKey, @NonNull SettingsItem item, boolean deleteItem) {
    OsmandApplication app = getApplication();
    String publicName = item.getPublicName(app);
    if (item instanceof FileSettingsItem) {
        FileSettingsItem settingsItem = (FileSettingsItem) item;
        if (settingsItem.getSubtype() == FileSubtype.VOICE) {
            publicName += " (" + app.getString(R.string.shared_string_recorded) + ")";
        } else if (settingsItem.getSubtype() == FileSubtype.TTS_VOICE) {
            publicName += " (" + app.getString(R.string.tts_title) + ")";
        }
    }
    title.setText(publicName);
    String fileName = BackupHelper.getItemFileName(item);
    String summary = app.getString(R.string.last_backup);
    UploadedFileInfo info = app.getBackupHelper().getDbHelper().getUploadedFileInfo(item.getType().name(), fileName);
    if (info != null) {
        String time = OsmAndFormatter.getFormattedPassedTime(app, info.getUploadTime(), app.getString(R.string.shared_string_never));
        description.setText(app.getString(R.string.ltr_or_rtl_combine_via_colon, summary, time));
    } else {
        description.setText(app.getString(R.string.ltr_or_rtl_combine_via_colon, summary, app.getString(R.string.shared_string_never)));
    }
    icon.setImageDrawable(getItemIcon(item));
    NetworkSettingsHelper settingsHelper = app.getNetworkSettingsHelper();
    ImportBackupTask importTask = settingsHelper.getImportTask(exportKey);
    ExportBackupTask exportTask = settingsHelper.getExportTask(exportKey);
    if (exportTask == null && importTask == null) {
        AndroidUiHelper.updateVisibility(secondIcon, deleteItem);
        AndroidUiHelper.updateVisibility(progressBar, false);
        secondIcon.setImageDrawable(getContentIcon(deleteItem ? R.drawable.ic_action_delete_dark : R.drawable.ic_action_cloud_done));
    } else {
        ItemProgressInfo progressInfo = null;
        if (exportTask != null) {
            progressInfo = exportTask.getItemProgressInfo(item.getType().name(), fileName);
        }
        if (importTask != null) {
            progressInfo = importTask.getItemProgressInfo(item.getType().name(), fileName);
        }
        setupProgress(progressInfo);
    }
    itemView.setTag(item);
}