use of ai.elimu.model.v2.gson.content.StoryBookChapterGson in project webapp by elimu-ai.
the class CsvContentExtractionHelper method getStoryBooksFromCsvBackup.
/**
* For information on how the CSV files were generated, see {@link StoryBookCsvExportController#handleRequest}.
* <p />
* Also see {@link #getStoryBookChaptersFromCsvBackup}
*/
public static List<StoryBookGson> getStoryBooksFromCsvBackup(File csvFile) {
logger.info("getStoryBooksFromCsvBackup");
List<StoryBookGson> storyBookGsons = new ArrayList<>();
Path csvFilePath = Paths.get(csvFile.toURI());
logger.info("csvFilePath: " + csvFilePath);
try {
Reader reader = Files.newBufferedReader(csvFilePath);
CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader("id", "title", "description", "content_license", "attribution_url", "reading_level", "cover_image_id", "chapters").withSkipHeaderRecord();
CSVParser csvParser = new CSVParser(reader, csvFormat);
for (CSVRecord csvRecord : csvParser) {
logger.info("csvRecord: " + csvRecord);
// Convert from CSV to GSON
StoryBookGson storyBookGson = new StoryBookGson();
String title = csvRecord.get("title");
storyBookGson.setTitle(title);
String description = csvRecord.get("description");
storyBookGson.setDescription(description);
if (StringUtils.isNotBlank(csvRecord.get("content_license"))) {
ContentLicense contentLicense = ContentLicense.valueOf(csvRecord.get("content_license"));
// storyBookGson.setContentLicense(contentLicense);
}
String attributionUrl = csvRecord.get("attribution_url");
if (StringUtils.isNotBlank(csvRecord.get("reading_level"))) {
ReadingLevel readingLevel = ReadingLevel.valueOf(csvRecord.get("reading_level"));
storyBookGson.setReadingLevel(readingLevel);
}
if (StringUtils.isNotBlank(csvRecord.get("cover_image_id"))) {
Long coverImageId = Long.valueOf(csvRecord.get("cover_image_id"));
// storyBookGson.setCoverImage();
}
List<StoryBookChapterGson> storyBookChapterGsons = new ArrayList<>();
JSONArray chaptersJsonArray = new JSONArray(csvRecord.get("chapters"));
logger.info("chaptersJsonArray: " + chaptersJsonArray);
for (int i = 0; i < chaptersJsonArray.length(); i++) {
JSONObject chapterJsonObject = chaptersJsonArray.getJSONObject(i);
logger.info("chapterJsonObject: " + chapterJsonObject);
StoryBookChapterGson storyBookChapterGson = new StoryBookChapterGson();
storyBookChapterGson.setSortOrder(chapterJsonObject.getInt("sortOrder"));
List<StoryBookParagraphGson> storyBookParagraphGsons = new ArrayList<>();
JSONArray paragraphsJsonArray = chapterJsonObject.getJSONArray("storyBookParagraphs");
logger.info("paragraphsJsonArray: " + paragraphsJsonArray);
for (int j = 0; j < paragraphsJsonArray.length(); j++) {
JSONObject paragraphJsonObject = paragraphsJsonArray.getJSONObject(j);
logger.info("paragraphJsonObject: " + paragraphJsonObject);
StoryBookParagraphGson storyBookParagraphGson = new StoryBookParagraphGson();
storyBookParagraphGson.setSortOrder(paragraphJsonObject.getInt("sortOrder"));
storyBookParagraphGson.setOriginalText(paragraphJsonObject.getString("originalText"));
// TODO: setWords
storyBookParagraphGsons.add(storyBookParagraphGson);
}
storyBookChapterGson.setStoryBookParagraphs(storyBookParagraphGsons);
storyBookChapterGsons.add(storyBookChapterGson);
}
storyBookGson.setStoryBookChapters(storyBookChapterGsons);
storyBookGsons.add(storyBookGson);
}
} catch (IOException ex) {
logger.error(ex);
}
return storyBookGsons;
}
use of ai.elimu.model.v2.gson.content.StoryBookChapterGson in project webapp by elimu-ai.
the class JpaToGsonConverter method getStoryBookChapterGson.
public static StoryBookChapterGson getStoryBookChapterGson(StoryBookChapter storyBookChapter) {
if (storyBookChapter == null) {
return null;
} else {
StoryBookChapterGson storyBookChapterGson = new StoryBookChapterGson();
// BaseEntity
storyBookChapterGson.setId(storyBookChapter.getId());
// StoryBookChapter
storyBookChapterGson.setSortOrder(storyBookChapter.getSortOrder());
if (storyBookChapter.getImage() != null) {
ImageGson imageGson = new ImageGson();
imageGson.setId(storyBookChapter.getImage().getId());
storyBookChapterGson.setImage(imageGson);
}
return storyBookChapterGson;
}
}
use of ai.elimu.model.v2.gson.content.StoryBookChapterGson in project webapp by elimu-ai.
the class StoryBookCsvExportController method handleRequest.
@RequestMapping(value = "/storybooks.csv", method = RequestMethod.GET)
public void handleRequest(HttpServletResponse response, OutputStream outputStream) throws IOException {
logger.info("handleRequest");
List<StoryBook> storyBooks = storyBookDao.readAllOrdered();
logger.info("storyBooks.size(): " + storyBooks.size());
CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader("id", "title", "description", "content_license", "attribution_url", "reading_level", "cover_image_id", "chapters");
StringWriter stringWriter = new StringWriter();
CSVPrinter csvPrinter = new CSVPrinter(stringWriter, csvFormat);
for (StoryBook storyBook : storyBooks) {
logger.info("storyBook.getTitle(): \"" + storyBook.getTitle() + "\"");
Long coverImageId = null;
if (storyBook.getCoverImage() != null) {
coverImageId = storyBook.getCoverImage().getId();
}
// Store chapters as JSON objects
JSONArray chaptersJsonArray = new JSONArray();
List<StoryBookChapter> storyBookChapters = storyBookChapterDao.readAll(storyBook);
logger.info("storyBookChapters.size(): " + storyBookChapters.size());
for (StoryBookChapter storyBookChapter : storyBookChapters) {
logger.info("storyBookChapter.getId(): " + storyBookChapter.getId());
StoryBookChapterGson storyBookChapterGson = JpaToGsonConverter.getStoryBookChapterGson(storyBookChapter);
// TODO: move this code block to JpaToGsonConverter?
if (storyBookChapterGson.getImage() != null) {
ImageGson imageGsonWithIdOnly = new ImageGson();
imageGsonWithIdOnly.setId(storyBookChapterGson.getImage().getId());
storyBookChapterGson.setImage(imageGsonWithIdOnly);
}
// Store paragraphs as JSON objects
List<StoryBookParagraphGson> storyBookParagraphs = new ArrayList<>();
logger.info("storyBookParagraphs.size(): " + storyBookParagraphs.size());
for (StoryBookParagraph storyBookParagraph : storyBookParagraphDao.readAll(storyBookChapter)) {
logger.info("storyBookParagraph.getId(): " + storyBookParagraph.getId());
StoryBookParagraphGson storyBookParagraphGson = JpaToGsonConverter.getStoryBookParagraphGson(storyBookParagraph);
storyBookParagraphGson.setWords(null);
storyBookParagraphs.add(storyBookParagraphGson);
}
storyBookChapterGson.setStoryBookParagraphs(storyBookParagraphs);
String json = new Gson().toJson(storyBookChapterGson);
JSONObject jsonObject = new JSONObject(json);
logger.info("jsonObject: " + jsonObject);
chaptersJsonArray.put(jsonObject);
}
logger.info("chaptersJsonArray: " + chaptersJsonArray);
csvPrinter.printRecord(storyBook.getId(), storyBook.getTitle(), storyBook.getDescription(), storyBook.getContentLicense(), storyBook.getAttributionUrl(), storyBook.getReadingLevel(), coverImageId, chaptersJsonArray);
csvPrinter.flush();
}
String csvFileContent = stringWriter.toString();
response.setContentType("text/csv");
byte[] bytes = csvFileContent.getBytes();
response.setContentLength(bytes.length);
try {
outputStream.write(bytes);
outputStream.flush();
outputStream.close();
} catch (IOException ex) {
logger.error(ex);
}
}
use of ai.elimu.model.v2.gson.content.StoryBookChapterGson in project webapp by elimu-ai.
the class DbContentImportHelper method performDatabaseContentImport.
/**
* Extracts educational content from the CSV files in {@code src/main/resources/db/content_TEST/<Language>/} and
* stores it in the database.
*
* @param environment The environment from which to import the database content.
* @param language The language to use during the import.
* @param webApplicationContext Context needed to access DAOs.
*/
public synchronized void performDatabaseContentImport(Environment environment, Language language, WebApplicationContext webApplicationContext) {
logger.info("performDatabaseContentImport");
logger.info("environment: " + environment + ", language: " + language);
if (!((environment == Environment.TEST) || (environment == Environment.PROD))) {
throw new IllegalArgumentException("Database content can only be imported from the TEST environment or from the PROD environment");
}
String contentDirectoryPath = "db" + File.separator + "content_" + environment + File.separator + language.toString().toLowerCase();
logger.info("contentDirectoryPath: \"" + contentDirectoryPath + "\"");
URL contentDirectoryURL = getClass().getClassLoader().getResource(contentDirectoryPath);
logger.info("contentDirectoryURL: " + contentDirectoryURL);
if (contentDirectoryURL == null) {
logger.warn("The content directory was not found. Aborting content import.");
return;
}
File contentDirectory = new File(contentDirectoryURL.getPath());
logger.info("contentDirectory: " + contentDirectory);
contributorDao = (ContributorDao) webApplicationContext.getBean("contributorDao");
Contributor contributor = new Contributor();
contributor.setEmail("dev@elimu.ai");
contributor.setFirstName("Dev");
contributor.setLastName("Contributor");
contributor.setRoles(new HashSet<>(Arrays.asList(Role.CONTRIBUTOR, Role.EDITOR, Role.ANALYST, Role.ADMIN)));
contributor.setRegistrationTime(Calendar.getInstance());
contributorDao.create(contributor);
// Extract and import Letters from CSV file in src/main/resources/
File lettersCsvFile = new File(contentDirectory, "letters.csv");
List<Letter> letters = CsvContentExtractionHelper.getLettersFromCsvBackup(lettersCsvFile, soundDao);
logger.info("letters.size(): " + letters.size());
letterDao = (LetterDao) webApplicationContext.getBean("letterDao");
letterContributionEventDao = (LetterContributionEventDao) webApplicationContext.getBean("letterContributionEventDao");
for (Letter letter : letters) {
letterDao.create(letter);
LetterContributionEvent letterContributionEvent = new LetterContributionEvent();
letterContributionEvent.setContributor(contributor);
letterContributionEvent.setLetter(letter);
letterContributionEvent.setRevisionNumber(1);
letterContributionEvent.setTime(Calendar.getInstance());
letterContributionEvent.setTimeSpentMs((long) (Math.random() * 10) * 60000L);
letterContributionEvent.setPlatform(Platform.WEBAPP);
letterContributionEventDao.create(letterContributionEvent);
}
// Extract and import Sounds from CSV file in src/main/resources/
File soundsCsvFile = new File(contentDirectory, "sounds.csv");
List<Sound> sounds = CsvContentExtractionHelper.getSoundsFromCsvBackup(soundsCsvFile);
logger.info("sounds.size(): " + sounds.size());
soundDao = (SoundDao) webApplicationContext.getBean("soundDao");
for (Sound sound : sounds) {
soundDao.create(sound);
}
// Extract and import letter-sound correspondences in src/main/resources/
File letterToAllophioneMappingsCsvFile = new File(contentDirectory, "letter-sound-correspondences.csv");
List<LetterSoundCorrespondence> letterSoundCorrespondences = CsvContentExtractionHelper.getLetterSoundCorrespondencesFromCsvBackup(letterToAllophioneMappingsCsvFile, letterDao, soundDao, letterSoundCorrespondenceDao);
logger.info("letterSoundCorrespondences.size(): " + letterSoundCorrespondences.size());
letterSoundCorrespondenceDao = (LetterSoundCorrespondenceDao) webApplicationContext.getBean("letterSoundCorrespondenceDao");
letterSoundCorrespondenceContributionEventDao = (LetterSoundCorrespondenceContributionEventDao) webApplicationContext.getBean("letterSoundCorrespondenceContributionEventDao");
for (LetterSoundCorrespondence letterSoundCorrespondence : letterSoundCorrespondences) {
letterSoundCorrespondenceDao.create(letterSoundCorrespondence);
LetterSoundCorrespondenceContributionEvent letterSoundCorrespondenceContributionEvent = new LetterSoundCorrespondenceContributionEvent();
letterSoundCorrespondenceContributionEvent.setContributor(contributor);
letterSoundCorrespondenceContributionEvent.setLetterSoundCorrespondence(letterSoundCorrespondence);
letterSoundCorrespondenceContributionEvent.setRevisionNumber(1);
letterSoundCorrespondenceContributionEvent.setTime(Calendar.getInstance());
letterSoundCorrespondenceContributionEvent.setTimeSpentMs((long) (Math.random() * 10) * 60000L);
letterSoundCorrespondenceContributionEvent.setPlatform(Platform.WEBAPP);
letterSoundCorrespondenceContributionEventDao.create(letterSoundCorrespondenceContributionEvent);
}
// Extract and import Words from CSV file in src/main/resources/
File wordsCsvFile = new File(contentDirectory, "words.csv");
List<Word> words = CsvContentExtractionHelper.getWordsFromCsvBackup(wordsCsvFile, letterDao, soundDao, letterSoundCorrespondenceDao, wordDao);
logger.info("words.size(): " + words.size());
wordDao = (WordDao) webApplicationContext.getBean("wordDao");
wordContributionEventDao = (WordContributionEventDao) webApplicationContext.getBean("wordContributionEventDao");
for (Word word : words) {
wordDao.create(word);
WordContributionEvent wordContributionEvent = new WordContributionEvent();
wordContributionEvent.setContributor(contributor);
wordContributionEvent.setWord(word);
wordContributionEvent.setRevisionNumber(1);
wordContributionEvent.setTime(Calendar.getInstance());
wordContributionEvent.setTimeSpentMs((long) (Math.random() * 10) * 60000L);
wordContributionEvent.setPlatform(Platform.WEBAPP);
wordContributionEventDao.create(wordContributionEvent);
}
// Extract and import Numbers from CSV file in src/main/resources/
File numbersCsvFile = new File(contentDirectory, "numbers.csv");
List<Number> numbers = CsvContentExtractionHelper.getNumbersFromCsvBackup(numbersCsvFile, wordDao);
logger.info("numbers.size(): " + numbers.size());
numberDao = (NumberDao) webApplicationContext.getBean("numberDao");
numberContributionEventDao = (NumberContributionEventDao) webApplicationContext.getBean("numberContributionEventDao");
for (Number number : numbers) {
numberDao.create(number);
NumberContributionEvent numberContributionEvent = new NumberContributionEvent();
numberContributionEvent.setContributor(contributor);
numberContributionEvent.setNumber(number);
numberContributionEvent.setRevisionNumber(1);
numberContributionEvent.setTime(Calendar.getInstance());
numberContributionEvent.setTimeSpentMs((long) (Math.random() * 10) * 60000L);
numberContributionEvent.setPlatform(Platform.WEBAPP);
numberContributionEventDao.create(numberContributionEvent);
}
// Extract and import Syllables from CSV file in src/main/resources/
// TODO
// Extract and import Emojis from CSV file in src/main/resources/
File emojisCsvFile = new File(contentDirectory, "emojis.csv");
List<Emoji> emojis = CsvContentExtractionHelper.getEmojisFromCsvBackup(emojisCsvFile, wordDao);
logger.info("emojis.size(): " + emojis.size());
emojiDao = (EmojiDao) webApplicationContext.getBean("emojiDao");
for (Emoji emoji : emojis) {
emojiDao.create(emoji);
}
// Extract and import Images from CSV file in src/main/resources/
// TODO
// Extract and import Audios from CSV file in src/main/resources/
// TODO
// Extract and import StoryBooks from CSV file in src/main/resources/
File storyBooksCsvFile = new File(contentDirectory, "storybooks.csv");
List<StoryBookGson> storyBookGsons = CsvContentExtractionHelper.getStoryBooksFromCsvBackup(storyBooksCsvFile);
logger.info("storyBookGsons.size(): " + storyBookGsons.size());
storyBookDao = (StoryBookDao) webApplicationContext.getBean("storyBookDao");
storyBookChapterDao = (StoryBookChapterDao) webApplicationContext.getBean("storyBookChapterDao");
storyBookParagraphDao = (StoryBookParagraphDao) webApplicationContext.getBean("storyBookParagraphDao");
storyBookContributionEventDao = (StoryBookContributionEventDao) webApplicationContext.getBean("storyBookContributionEventDao");
for (StoryBookGson storyBookGson : storyBookGsons) {
// Convert from GSON to JPA
StoryBook storyBook = new StoryBook();
storyBook.setTitle(storyBookGson.getTitle());
storyBook.setDescription(storyBookGson.getDescription());
// TODO: storyBook.setContentLicense();
// TODO: storyBook.setAttributionUrl();
storyBook.setReadingLevel(storyBookGson.getReadingLevel());
storyBookDao.create(storyBook);
for (StoryBookChapterGson storyBookChapterGson : storyBookGson.getStoryBookChapters()) {
// Convert from GSON to JPA
StoryBookChapter storyBookChapter = new StoryBookChapter();
storyBookChapter.setStoryBook(storyBook);
storyBookChapter.setSortOrder(storyBookChapterGson.getSortOrder());
// TODO: storyBookChapter.setImage();
storyBookChapterDao.create(storyBookChapter);
for (StoryBookParagraphGson storyBookParagraphGson : storyBookChapterGson.getStoryBookParagraphs()) {
// Convert from GSON to JPA
StoryBookParagraph storyBookParagraph = new StoryBookParagraph();
storyBookParagraph.setStoryBookChapter(storyBookChapter);
storyBookParagraph.setSortOrder(storyBookParagraphGson.getSortOrder());
storyBookParagraph.setOriginalText(storyBookParagraphGson.getOriginalText());
List<String> wordsInOriginalText = WordExtractionHelper.getWords(storyBookParagraph.getOriginalText(), language);
logger.info("wordsInOriginalText.size(): " + wordsInOriginalText.size());
List<Word> paragraphWords = new ArrayList<>();
logger.info("paragraphWords.size(): " + paragraphWords.size());
for (String wordInOriginalText : wordsInOriginalText) {
logger.info("wordInOriginalText: \"" + wordInOriginalText + "\"");
wordInOriginalText = wordInOriginalText.toLowerCase();
logger.info("wordInOriginalText (lower-case): \"" + wordInOriginalText + "\"");
Word word = wordDao.readByText(wordInOriginalText);
logger.info("word: " + word);
paragraphWords.add(word);
}
storyBookParagraph.setWords(paragraphWords);
storyBookParagraphDao.create(storyBookParagraph);
}
}
StoryBookContributionEvent storyBookContributionEvent = new StoryBookContributionEvent();
storyBookContributionEvent.setContributor(contributor);
storyBookContributionEvent.setStoryBook(storyBook);
storyBookContributionEvent.setRevisionNumber(1);
storyBookContributionEvent.setTime(Calendar.getInstance());
storyBookContributionEvent.setTimeSpentMs((long) (Math.random() * 10) * 60000L);
storyBookContributionEvent.setPlatform(Platform.WEBAPP);
storyBookContributionEventDao.create(storyBookContributionEvent);
}
// Extract and import Videos from CSV file in src/main/resources/
// TODO
String analyticsDirectoryPath = "db" + File.separator + "analytics_" + environment + File.separator + language.toString().toLowerCase();
logger.info("analyticsDirectoryPath: \"" + analyticsDirectoryPath + "\"");
URL analyticsDirectoryURL = getClass().getClassLoader().getResource(analyticsDirectoryPath);
logger.info("analyticsDirectoryURL: " + analyticsDirectoryURL);
if (analyticsDirectoryURL == null) {
logger.warn("The analytics directory was not found. Aborting analytics import.");
return;
}
File analyticsDirectory = new File(analyticsDirectoryURL.getPath());
logger.info("analyticsDirectory: " + analyticsDirectory);
// Extract and import LetterLearningEvents from CSV file in src/main/resources/
// TODO
// Extract and import WordLearningEvents from CSV file in src/main/resources/
// TODO
// Extract and import StoryBookLearningEvents from CSV file in src/main/resources/
File storyBookLearningEventsCsvFile = new File(analyticsDirectory, "storybook-learning-events.csv");
applicationDao = (ApplicationDao) webApplicationContext.getBean("applicationDao");
List<StoryBookLearningEvent> storyBookLearningEvents = CsvAnalyticsExtractionHelper.getStoryBookLearningEventsFromCsvBackup(storyBookLearningEventsCsvFile, applicationDao, storyBookDao);
logger.info("storyBookLearningEvents.size(): " + storyBookLearningEvents.size());
storyBookLearningEventDao = (StoryBookLearningEventDao) webApplicationContext.getBean("storyBookLearningEventDao");
for (StoryBookLearningEvent storyBookLearningEvent : storyBookLearningEvents) {
storyBookLearningEventDao.create(storyBookLearningEvent);
}
logger.info("Content import complete");
}
use of ai.elimu.model.v2.gson.content.StoryBookChapterGson in project webapp by elimu-ai.
the class StoryBooksJsonService method getStoryBooksJSONArray.
@Cacheable("storyBooks")
public JSONArray getStoryBooksJSONArray() {
logger.info("getStoryBooksJSONArray");
Date dateStart = new Date();
JSONArray storyBooksJsonArray = new JSONArray();
for (StoryBook storyBook : storyBookDao.readAllOrdered()) {
StoryBookGson storyBookGson = JpaToGsonConverter.getStoryBookGson(storyBook);
// Add chapters
List<StoryBookChapterGson> storyBookChapterGsons = new ArrayList<>();
for (StoryBookChapter storyBookChapter : storyBookChapterDao.readAll(storyBook)) {
StoryBookChapterGson storyBookChapterGson = JpaToGsonConverter.getStoryBookChapterGson(storyBookChapter);
// Add paragraphs
List<StoryBookParagraphGson> storyBookParagraphGsons = new ArrayList<>();
for (StoryBookParagraph storyBookParagraph : storyBookParagraphDao.readAll(storyBookChapter)) {
StoryBookParagraphGson storyBookParagraphGson = JpaToGsonConverter.getStoryBookParagraphGson(storyBookParagraph);
storyBookParagraphGsons.add(storyBookParagraphGson);
}
storyBookChapterGson.setStoryBookParagraphs(storyBookParagraphGsons);
storyBookChapterGsons.add(storyBookChapterGson);
}
storyBookGson.setStoryBookChapters(storyBookChapterGsons);
String json = new Gson().toJson(storyBookGson);
storyBooksJsonArray.put(new JSONObject(json));
}
Date dateEnd = new Date();
logger.info("getStoryBooksJSONArray duration: " + (dateEnd.getTime() - dateStart.getTime()) + " ms");
return storyBooksJsonArray;
}
Aggregations