Examples with StoryBookLearningEvent ai.elimu.model.analytics.StoryBookLearningEvent used on opensource projects

Example 1 with StoryBookLearningEvent

use of ai.elimu.model.analytics.StoryBookLearningEvent in project webapp by elimu-ai.

the class CsvAnalyticsExtractionHelper method getStoryBookLearningEventsFromCsvBackup.

/**
 * For information on how the CSV files were generated, see {@link StoryBookLearningEventCsvExportController#handleRequest}.
 * <p />
 * Also see {@link StoryBookLearningEventsRestController#handleUploadCsvRequest}
 */
public static List<StoryBookLearningEvent> getStoryBookLearningEventsFromCsvBackup(File csvFile, ApplicationDao applicationDao, StoryBookDao storyBookDao) {
    logger.info("getStoryBookLearningEventsFromCsvBackup");
    List<StoryBookLearningEvent> storyBookLearningEvents = new ArrayList<>();
    Path csvFilePath = Paths.get(csvFile.toURI());
    logger.info("csvFilePath: " + csvFilePath);
    try {
        Reader reader = Files.newBufferedReader(csvFilePath);
        CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader(// The Room database ID
        "id", "time", "android_id", "package_name", "storybook_id", "storybook_title", "learning_event_type").withSkipHeaderRecord();
        CSVParser csvParser = new CSVParser(reader, csvFormat);
        for (CSVRecord csvRecord : csvParser) {
            logger.info("csvRecord: " + csvRecord);
            // Convert from CSV to Java (see similar code in StoryBookLearningEventsRestController)
            StoryBookLearningEvent storyBookLearningEvent = new StoryBookLearningEvent();
            long timeInMillis = Long.valueOf(csvRecord.get("time"));
            Calendar time = Calendar.getInstance();
            time.setTimeInMillis(timeInMillis);
            storyBookLearningEvent.setTime(time);
            String androidId = csvRecord.get("android_id");
            storyBookLearningEvent.setAndroidId(androidId);
            String packageName = csvRecord.get("package_name");
            storyBookLearningEvent.setPackageName(packageName);
            Application application = applicationDao.readByPackageName(packageName);
            logger.info("application: " + application);
            storyBookLearningEvent.setApplication(application);
            Long storyBookId = Long.valueOf(csvRecord.get("storybook_id"));
            logger.info("storyBookId: " + storyBookId);
            storyBookLearningEvent.setStoryBookId(storyBookId);
            String storyBookTitle = csvRecord.get("storybook_title");
            logger.info("storyBookTitle: \"" + storyBookTitle + "\"");
            storyBookLearningEvent.setStoryBookTitle(storyBookTitle);
            StoryBook storyBook = storyBookDao.readByTitle(storyBookTitle);
            logger.info("storyBook: " + storyBook);
            storyBookLearningEvent.setStoryBook(storyBook);
            LearningEventType learningEventType = LearningEventType.valueOf(csvRecord.get("learning_event_type"));
            logger.info("learningEventType: " + learningEventType);
            storyBookLearningEvent.setLearningEventType(learningEventType);
            storyBookLearningEvents.add(storyBookLearningEvent);
        }
    } catch (IOException ex) {
        logger.error(ex);
    }
    return storyBookLearningEvents;
}

Example 2 with StoryBookLearningEvent

use of ai.elimu.model.analytics.StoryBookLearningEvent in project webapp by elimu-ai.

the class DbContentImportHelper method performDatabaseContentImport.

/**
 * Extracts educational content from the CSV files in {@code src/main/resources/db/content_TEST/<Language>/} and
 * stores it in the database.
 *
 * @param environment The environment from which to import the database content.
 * @param language The language to use during the import.
 * @param webApplicationContext Context needed to access DAOs.
 */
public synchronized void performDatabaseContentImport(Environment environment, Language language, WebApplicationContext webApplicationContext) {
    logger.info("performDatabaseContentImport");
    logger.info("environment: " + environment + ", language: " + language);
    if (!((environment == Environment.TEST) || (environment == Environment.PROD))) {
        throw new IllegalArgumentException("Database content can only be imported from the TEST environment or from the PROD environment");
    }
    String contentDirectoryPath = "db" + File.separator + "content_" + environment + File.separator + language.toString().toLowerCase();
    logger.info("contentDirectoryPath: \"" + contentDirectoryPath + "\"");
    URL contentDirectoryURL = getClass().getClassLoader().getResource(contentDirectoryPath);
    logger.info("contentDirectoryURL: " + contentDirectoryURL);
    if (contentDirectoryURL == null) {
        logger.warn("The content directory was not found. Aborting content import.");
        return;
    }
    File contentDirectory = new File(contentDirectoryURL.getPath());
    logger.info("contentDirectory: " + contentDirectory);
    contributorDao = (ContributorDao) webApplicationContext.getBean("contributorDao");
    Contributor contributor = new Contributor();
    contributor.setEmail("dev@elimu.ai");
    contributor.setFirstName("Dev");
    contributor.setLastName("Contributor");
    contributor.setRoles(new HashSet<>(Arrays.asList(Role.CONTRIBUTOR, Role.EDITOR, Role.ANALYST, Role.ADMIN)));
    contributor.setRegistrationTime(Calendar.getInstance());
    contributorDao.create(contributor);
    // Extract and import Letters from CSV file in src/main/resources/
    File lettersCsvFile = new File(contentDirectory, "letters.csv");
    List<Letter> letters = CsvContentExtractionHelper.getLettersFromCsvBackup(lettersCsvFile, soundDao);
    logger.info("letters.size(): " + letters.size());
    letterDao = (LetterDao) webApplicationContext.getBean("letterDao");
    letterContributionEventDao = (LetterContributionEventDao) webApplicationContext.getBean("letterContributionEventDao");
    for (Letter letter : letters) {
        letterDao.create(letter);
        LetterContributionEvent letterContributionEvent = new LetterContributionEvent();
        letterContributionEvent.setContributor(contributor);
        letterContributionEvent.setLetter(letter);
        letterContributionEvent.setRevisionNumber(1);
        letterContributionEvent.setTime(Calendar.getInstance());
        letterContributionEvent.setTimeSpentMs((long) (Math.random() * 10) * 60000L);
        letterContributionEvent.setPlatform(Platform.WEBAPP);
        letterContributionEventDao.create(letterContributionEvent);
    }
    // Extract and import Sounds from CSV file in src/main/resources/
    File soundsCsvFile = new File(contentDirectory, "sounds.csv");
    List<Sound> sounds = CsvContentExtractionHelper.getSoundsFromCsvBackup(soundsCsvFile);
    logger.info("sounds.size(): " + sounds.size());
    soundDao = (SoundDao) webApplicationContext.getBean("soundDao");
    for (Sound sound : sounds) {
        soundDao.create(sound);
    }
    // Extract and import letter-sound correspondences in src/main/resources/
    File letterToAllophioneMappingsCsvFile = new File(contentDirectory, "letter-sound-correspondences.csv");
    List<LetterSoundCorrespondence> letterSoundCorrespondences = CsvContentExtractionHelper.getLetterSoundCorrespondencesFromCsvBackup(letterToAllophioneMappingsCsvFile, letterDao, soundDao, letterSoundCorrespondenceDao);
    logger.info("letterSoundCorrespondences.size(): " + letterSoundCorrespondences.size());
    letterSoundCorrespondenceDao = (LetterSoundCorrespondenceDao) webApplicationContext.getBean("letterSoundCorrespondenceDao");
    letterSoundCorrespondenceContributionEventDao = (LetterSoundCorrespondenceContributionEventDao) webApplicationContext.getBean("letterSoundCorrespondenceContributionEventDao");
    for (LetterSoundCorrespondence letterSoundCorrespondence : letterSoundCorrespondences) {
        letterSoundCorrespondenceDao.create(letterSoundCorrespondence);
        LetterSoundCorrespondenceContributionEvent letterSoundCorrespondenceContributionEvent = new LetterSoundCorrespondenceContributionEvent();
        letterSoundCorrespondenceContributionEvent.setContributor(contributor);
        letterSoundCorrespondenceContributionEvent.setLetterSoundCorrespondence(letterSoundCorrespondence);
        letterSoundCorrespondenceContributionEvent.setRevisionNumber(1);
        letterSoundCorrespondenceContributionEvent.setTime(Calendar.getInstance());
        letterSoundCorrespondenceContributionEvent.setTimeSpentMs((long) (Math.random() * 10) * 60000L);
        letterSoundCorrespondenceContributionEvent.setPlatform(Platform.WEBAPP);
        letterSoundCorrespondenceContributionEventDao.create(letterSoundCorrespondenceContributionEvent);
    }
    // Extract and import Words from CSV file in src/main/resources/
    File wordsCsvFile = new File(contentDirectory, "words.csv");
    List<Word> words = CsvContentExtractionHelper.getWordsFromCsvBackup(wordsCsvFile, letterDao, soundDao, letterSoundCorrespondenceDao, wordDao);
    logger.info("words.size(): " + words.size());
    wordDao = (WordDao) webApplicationContext.getBean("wordDao");
    wordContributionEventDao = (WordContributionEventDao) webApplicationContext.getBean("wordContributionEventDao");
    for (Word word : words) {
        wordDao.create(word);
        WordContributionEvent wordContributionEvent = new WordContributionEvent();
        wordContributionEvent.setContributor(contributor);
        wordContributionEvent.setWord(word);
        wordContributionEvent.setRevisionNumber(1);
        wordContributionEvent.setTime(Calendar.getInstance());
        wordContributionEvent.setTimeSpentMs((long) (Math.random() * 10) * 60000L);
        wordContributionEvent.setPlatform(Platform.WEBAPP);
        wordContributionEventDao.create(wordContributionEvent);
    }
    // Extract and import Numbers from CSV file in src/main/resources/
    File numbersCsvFile = new File(contentDirectory, "numbers.csv");
    List<Number> numbers = CsvContentExtractionHelper.getNumbersFromCsvBackup(numbersCsvFile, wordDao);
    logger.info("numbers.size(): " + numbers.size());
    numberDao = (NumberDao) webApplicationContext.getBean("numberDao");
    numberContributionEventDao = (NumberContributionEventDao) webApplicationContext.getBean("numberContributionEventDao");
    for (Number number : numbers) {
        numberDao.create(number);
        NumberContributionEvent numberContributionEvent = new NumberContributionEvent();
        numberContributionEvent.setContributor(contributor);
        numberContributionEvent.setNumber(number);
        numberContributionEvent.setRevisionNumber(1);
        numberContributionEvent.setTime(Calendar.getInstance());
        numberContributionEvent.setTimeSpentMs((long) (Math.random() * 10) * 60000L);
        numberContributionEvent.setPlatform(Platform.WEBAPP);
        numberContributionEventDao.create(numberContributionEvent);
    }
    // Extract and import Syllables from CSV file in src/main/resources/
    // TODO
    // Extract and import Emojis from CSV file in src/main/resources/
    File emojisCsvFile = new File(contentDirectory, "emojis.csv");
    List<Emoji> emojis = CsvContentExtractionHelper.getEmojisFromCsvBackup(emojisCsvFile, wordDao);
    logger.info("emojis.size(): " + emojis.size());
    emojiDao = (EmojiDao) webApplicationContext.getBean("emojiDao");
    for (Emoji emoji : emojis) {
        emojiDao.create(emoji);
    }
    // Extract and import Images from CSV file in src/main/resources/
    // TODO
    // Extract and import Audios from CSV file in src/main/resources/
    // TODO
    // Extract and import StoryBooks from CSV file in src/main/resources/
    File storyBooksCsvFile = new File(contentDirectory, "storybooks.csv");
    List<StoryBookGson> storyBookGsons = CsvContentExtractionHelper.getStoryBooksFromCsvBackup(storyBooksCsvFile);
    logger.info("storyBookGsons.size(): " + storyBookGsons.size());
    storyBookDao = (StoryBookDao) webApplicationContext.getBean("storyBookDao");
    storyBookChapterDao = (StoryBookChapterDao) webApplicationContext.getBean("storyBookChapterDao");
    storyBookParagraphDao = (StoryBookParagraphDao) webApplicationContext.getBean("storyBookParagraphDao");
    storyBookContributionEventDao = (StoryBookContributionEventDao) webApplicationContext.getBean("storyBookContributionEventDao");
    for (StoryBookGson storyBookGson : storyBookGsons) {
        // Convert from GSON to JPA
        StoryBook storyBook = new StoryBook();
        storyBook.setTitle(storyBookGson.getTitle());
        storyBook.setDescription(storyBookGson.getDescription());
        // TODO: storyBook.setContentLicense();
        // TODO: storyBook.setAttributionUrl();
        storyBook.setReadingLevel(storyBookGson.getReadingLevel());
        storyBookDao.create(storyBook);
        for (StoryBookChapterGson storyBookChapterGson : storyBookGson.getStoryBookChapters()) {
            // Convert from GSON to JPA
            StoryBookChapter storyBookChapter = new StoryBookChapter();
            storyBookChapter.setStoryBook(storyBook);
            storyBookChapter.setSortOrder(storyBookChapterGson.getSortOrder());
            // TODO: storyBookChapter.setImage();
            storyBookChapterDao.create(storyBookChapter);
            for (StoryBookParagraphGson storyBookParagraphGson : storyBookChapterGson.getStoryBookParagraphs()) {
                // Convert from GSON to JPA
                StoryBookParagraph storyBookParagraph = new StoryBookParagraph();
                storyBookParagraph.setStoryBookChapter(storyBookChapter);
                storyBookParagraph.setSortOrder(storyBookParagraphGson.getSortOrder());
                storyBookParagraph.setOriginalText(storyBookParagraphGson.getOriginalText());
                List<String> wordsInOriginalText = WordExtractionHelper.getWords(storyBookParagraph.getOriginalText(), language);
                logger.info("wordsInOriginalText.size(): " + wordsInOriginalText.size());
                List<Word> paragraphWords = new ArrayList<>();
                logger.info("paragraphWords.size(): " + paragraphWords.size());
                for (String wordInOriginalText : wordsInOriginalText) {
                    logger.info("wordInOriginalText: \"" + wordInOriginalText + "\"");
                    wordInOriginalText = wordInOriginalText.toLowerCase();
                    logger.info("wordInOriginalText (lower-case): \"" + wordInOriginalText + "\"");
                    Word word = wordDao.readByText(wordInOriginalText);
                    logger.info("word: " + word);
                    paragraphWords.add(word);
                }
                storyBookParagraph.setWords(paragraphWords);
                storyBookParagraphDao.create(storyBookParagraph);
            }
        }
        StoryBookContributionEvent storyBookContributionEvent = new StoryBookContributionEvent();
        storyBookContributionEvent.setContributor(contributor);
        storyBookContributionEvent.setStoryBook(storyBook);
        storyBookContributionEvent.setRevisionNumber(1);
        storyBookContributionEvent.setTime(Calendar.getInstance());
        storyBookContributionEvent.setTimeSpentMs((long) (Math.random() * 10) * 60000L);
        storyBookContributionEvent.setPlatform(Platform.WEBAPP);
        storyBookContributionEventDao.create(storyBookContributionEvent);
    }
    // Extract and import Videos from CSV file in src/main/resources/
    // TODO
    String analyticsDirectoryPath = "db" + File.separator + "analytics_" + environment + File.separator + language.toString().toLowerCase();
    logger.info("analyticsDirectoryPath: \"" + analyticsDirectoryPath + "\"");
    URL analyticsDirectoryURL = getClass().getClassLoader().getResource(analyticsDirectoryPath);
    logger.info("analyticsDirectoryURL: " + analyticsDirectoryURL);
    if (analyticsDirectoryURL == null) {
        logger.warn("The analytics directory was not found. Aborting analytics import.");
        return;
    }
    File analyticsDirectory = new File(analyticsDirectoryURL.getPath());
    logger.info("analyticsDirectory: " + analyticsDirectory);
    // Extract and import LetterLearningEvents from CSV file in src/main/resources/
    // TODO
    // Extract and import WordLearningEvents from CSV file in src/main/resources/
    // TODO
    // Extract and import StoryBookLearningEvents from CSV file in src/main/resources/
    File storyBookLearningEventsCsvFile = new File(analyticsDirectory, "storybook-learning-events.csv");
    applicationDao = (ApplicationDao) webApplicationContext.getBean("applicationDao");
    List<StoryBookLearningEvent> storyBookLearningEvents = CsvAnalyticsExtractionHelper.getStoryBookLearningEventsFromCsvBackup(storyBookLearningEventsCsvFile, applicationDao, storyBookDao);
    logger.info("storyBookLearningEvents.size(): " + storyBookLearningEvents.size());
    storyBookLearningEventDao = (StoryBookLearningEventDao) webApplicationContext.getBean("storyBookLearningEventDao");
    for (StoryBookLearningEvent storyBookLearningEvent : storyBookLearningEvents) {
        storyBookLearningEventDao.create(storyBookLearningEvent);
    }
    logger.info("Content import complete");
}

Example 3 with StoryBookLearningEvent

use of ai.elimu.model.analytics.StoryBookLearningEvent in project webapp by elimu-ai.

the class StoryBookLearningEventCsvExportController method handleRequest.

@RequestMapping(value = "/storybook-learning-events.csv", method = RequestMethod.GET)
public void handleRequest(HttpServletResponse response, OutputStream outputStream) throws IOException {
    logger.info("handleRequest");
    List<StoryBookLearningEvent> storyBookLearningEvents = storyBookLearningEventDao.readAll();
    logger.info("storyBookLearningEvents.size(): " + storyBookLearningEvents.size());
    CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader(// The Room database ID
    "id", "time", "android_id", "package_name", "storybook_id", "storybook_title", "learning_event_type");
    StringWriter stringWriter = new StringWriter();
    CSVPrinter csvPrinter = new CSVPrinter(stringWriter, csvFormat);
    for (StoryBookLearningEvent storyBookLearningEvent : storyBookLearningEvents) {
        logger.info("storyBookLearningEvent.getId(): " + storyBookLearningEvent.getId());
        csvPrinter.printRecord(storyBookLearningEvent.getId(), storyBookLearningEvent.getTime().getTimeInMillis(), storyBookLearningEvent.getAndroidId(), storyBookLearningEvent.getApplication().getPackageName(), storyBookLearningEvent.getStoryBook().getId(), storyBookLearningEvent.getStoryBookTitle(), storyBookLearningEvent.getLearningEventType());
        csvPrinter.flush();
    }
    String csvFileContent = stringWriter.toString();
    response.setContentType("text/csv");
    byte[] bytes = csvFileContent.getBytes();
    response.setContentLength(bytes.length);
    try {
        outputStream.write(bytes);
        outputStream.flush();
        outputStream.close();
    } catch (IOException ex) {
        logger.error(ex);
    }
}

Example 4 with StoryBookLearningEvent

use of ai.elimu.model.analytics.StoryBookLearningEvent in project webapp by elimu-ai.

the class StoryBookLearningEventsRestController method handleUploadCsvRequest.

@RequestMapping(value = "/csv", method = RequestMethod.POST)
public String handleUploadCsvRequest(@RequestParam("file") MultipartFile multipartFile, HttpServletResponse response) {
    logger.info("handleUploadCsvRequest");
    String name = multipartFile.getName();
    logger.info("name: " + name);
    // Expected format: "7161a85a0e4751cd_storybook-learning-events_2020-04-23.csv"
    String originalFilename = multipartFile.getOriginalFilename();
    logger.info("originalFilename: " + originalFilename);
    // TODO: Send notification to the #📊-data-collection channel in Discord
    // Hide parts of the Android ID, e.g. "7161***51cd_word-learning-events_2020-04-23.csv"
    String anonymizedOriginalFilename = originalFilename.substring(0, 4) + "***" + originalFilename.substring(12);
    DiscordHelper.sendChannelMessage("Received dataset: `" + anonymizedOriginalFilename + "`", null, null, null, null);
    String androidIdExtractedFromFilename = AnalyticsHelper.extractAndroidIdFromCsvFilename(originalFilename);
    logger.info("androidIdExtractedFromFilename: \"" + androidIdExtractedFromFilename + "\"");
    String contentType = multipartFile.getContentType();
    logger.info("contentType: " + contentType);
    JSONObject jsonObject = new JSONObject();
    try {
        byte[] bytes = multipartFile.getBytes();
        logger.info("bytes.length: " + bytes.length);
        // Store a backup of the original CSV file on the filesystem (in case it will be needed for debugging)
        File elimuAiDir = new File(System.getProperty("user.home"), ".elimu-ai");
        File languageDir = new File(elimuAiDir, "lang-" + Language.valueOf(ConfigHelper.getProperty("content.language")));
        File analyticsDir = new File(languageDir, "analytics");
        File androidIdDir = new File(analyticsDir, "android-id_" + androidIdExtractedFromFilename);
        File storyBookLearningEventsDir = new File(androidIdDir, "storybook-learning-events");
        storyBookLearningEventsDir.mkdirs();
        File csvFile = new File(storyBookLearningEventsDir, originalFilename);
        logger.info("Storing CSV file at " + csvFile);
        multipartFile.transferTo(csvFile);
        // Iterate each row in the CSV file
        Path csvFilePath = Paths.get(csvFile.toURI());
        logger.info("csvFilePath: " + csvFilePath);
        Reader reader = Files.newBufferedReader(csvFilePath);
        CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader(// The Room database ID
        "id", "time", "android_id", "package_name", "storybook_id", "storybook_title", "learning_event_type").withSkipHeaderRecord();
        CSVParser csvParser = new CSVParser(reader, csvFormat);
        for (CSVRecord csvRecord : csvParser) {
            logger.info("csvRecord: " + csvRecord);
            // Convert from CSV to Java
            StoryBookLearningEvent storyBookLearningEvent = new StoryBookLearningEvent();
            long timeInMillis = Long.valueOf(csvRecord.get("time"));
            Calendar time = Calendar.getInstance();
            time.setTimeInMillis(timeInMillis);
            storyBookLearningEvent.setTime(time);
            String androidId = csvRecord.get("android_id");
            storyBookLearningEvent.setAndroidId(androidId);
            String packageName = csvRecord.get("package_name");
            storyBookLearningEvent.setPackageName(packageName);
            Application application = applicationDao.readByPackageName(packageName);
            logger.info("application: " + application);
            storyBookLearningEvent.setApplication(application);
            Long storyBookId = Long.valueOf(csvRecord.get("storybook_id"));
            storyBookLearningEvent.setStoryBookId(storyBookId);
            String storyBookTitle = csvRecord.get("storybook_title");
            storyBookLearningEvent.setStoryBookTitle(storyBookTitle);
            StoryBook storyBook = storyBookDao.read(storyBookId);
            logger.info("storyBook: " + storyBook);
            storyBookLearningEvent.setStoryBook(storyBook);
            LearningEventType learningEventType = LearningEventType.valueOf(csvRecord.get("learning_event_type"));
            logger.info("learningEventType: " + learningEventType);
            storyBookLearningEvent.setLearningEventType(learningEventType);
            // Check if the event has already been stored in the database
            StoryBookLearningEvent existingStoryBookLearningEvent = storyBookLearningEventDao.read(time, androidId, application, storyBook);
            logger.info("existingStoryBookLearningEvent: " + existingStoryBookLearningEvent);
            if (existingStoryBookLearningEvent == null) {
                // Store the event in the database
                storyBookLearningEventDao.create(storyBookLearningEvent);
                logger.info("Stored StoryBookLearningEvent in database with ID " + storyBookLearningEvent.getId());
                jsonObject.put("result", "success");
                jsonObject.put("successMessage", "The StoryBookLearningEvent was stored in the database");
            } else {
                // Return error message saying that the event has already been uploaded
                logger.warn("The event has already been stored in the database");
                jsonObject.put("result", "error");
                jsonObject.put("errorMessage", "The event has already been stored in the database");
                response.setStatus(HttpStatus.CONFLICT.value());
            }
        }
    } catch (Exception ex) {
        logger.error(ex);
        jsonObject.put("result", "error");
        jsonObject.put("errorMessage", ex.getMessage());
        response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
    }
    String jsonResponse = jsonObject.toString();
    logger.info("jsonResponse: " + jsonResponse);
    return jsonResponse;
}