use of ai.elimu.model.content.Word in project webapp by elimu-ai.
the class LetterSoundCorrespondenceUsageCountScheduler method execute.
// At 06:15 every day
@Scheduled(cron = "00 15 06 * * *")
public synchronized void execute() {
logger.info("execute");
logger.info("Calculating usage count for LetterSoundCorrespondences");
// <id, usageCount>
Map<Long, Integer> letterSoundCorrespondenceFrequencyMap = new HashMap<>();
List<Word> words = wordDao.readAll();
logger.info("words.size(): " + words.size());
for (Word word : words) {
logger.info("word.getText(): " + word.getText());
for (LetterSoundCorrespondence letterSoundCorrespondence : word.getLetterSoundCorrespondences()) {
letterSoundCorrespondenceFrequencyMap.put(letterSoundCorrespondence.getId(), letterSoundCorrespondenceFrequencyMap.getOrDefault(letterSoundCorrespondence.getId(), 0) + word.getUsageCount());
}
}
// Update the values previously stored in the database
for (LetterSoundCorrespondence letterSoundCorrespondence : letterSoundCorrespondenceDao.readAll()) {
logger.info("letterSoundCorrespondence.getId(): " + letterSoundCorrespondence.getId());
logger.info("letterSoundCorrespondence Letters: \"" + letterSoundCorrespondence.getLetters().stream().map(Letter::getText).collect(Collectors.joining()) + "\"");
logger.info("letterSoundCorrespondence Sounds: /" + letterSoundCorrespondence.getSounds().stream().map(Sound::getValueIpa).collect(Collectors.joining()) + "/");
logger.info("letterSoundCorrespondence.getUsageCount() (before update): " + letterSoundCorrespondence.getUsageCount());
int newUsageCount = 0;
if (letterSoundCorrespondenceFrequencyMap.containsKey(letterSoundCorrespondence.getId())) {
newUsageCount = letterSoundCorrespondenceFrequencyMap.get(letterSoundCorrespondence.getId());
}
logger.info("newUsageCount: " + newUsageCount);
letterSoundCorrespondence.setUsageCount(newUsageCount);
letterSoundCorrespondenceDao.update(letterSoundCorrespondence);
logger.info("letterSoundCorrespondence.getUsageCount() (after update): " + letterSoundCorrespondence.getUsageCount());
}
logger.info("execute complete");
}
use of ai.elimu.model.content.Word in project webapp by elimu-ai.
the class WordLearningEventsRestController method handleUploadCsvRequest.
@RequestMapping(value = "/csv", method = RequestMethod.POST)
public String handleUploadCsvRequest(@RequestParam("file") MultipartFile multipartFile, HttpServletResponse response) {
logger.info("handleUploadCsvRequest");
String name = multipartFile.getName();
logger.info("name: " + name);
// Expected format: "7161a85a0e4751cd_word-learning-events_2020-04-23.csv"
String originalFilename = multipartFile.getOriginalFilename();
logger.info("originalFilename: " + originalFilename);
// TODO: Send notification to the #📊-data-collection channel in Discord
// Hide parts of the Android ID, e.g. "7161***51cd_word-learning-events_2020-04-23.csv"
String anonymizedOriginalFilename = originalFilename.substring(0, 4) + "***" + originalFilename.substring(12);
DiscordHelper.sendChannelMessage("Received dataset: `" + anonymizedOriginalFilename + "`", null, null, null, null);
String androidIdExtractedFromFilename = AnalyticsHelper.extractAndroidIdFromCsvFilename(originalFilename);
logger.info("androidIdExtractedFromFilename: \"" + androidIdExtractedFromFilename + "\"");
String contentType = multipartFile.getContentType();
logger.info("contentType: " + contentType);
JSONObject jsonObject = new JSONObject();
try {
byte[] bytes = multipartFile.getBytes();
logger.info("bytes.length: " + bytes.length);
// Store a backup of the original CSV file on the filesystem (in case it will be needed for debugging)
File elimuAiDir = new File(System.getProperty("user.home"), ".elimu-ai");
File languageDir = new File(elimuAiDir, "lang-" + Language.valueOf(ConfigHelper.getProperty("content.language")));
File analyticsDir = new File(languageDir, "analytics");
File androidIdDir = new File(analyticsDir, "android-id_" + androidIdExtractedFromFilename);
File wordLearningEventsDir = new File(androidIdDir, "word-learning-events");
wordLearningEventsDir.mkdirs();
File csvFile = new File(wordLearningEventsDir, originalFilename);
logger.info("Storing CSV file at " + csvFile);
multipartFile.transferTo(csvFile);
// Iterate each row in the CSV file
Path csvFilePath = Paths.get(csvFile.toURI());
logger.info("csvFilePath: " + csvFilePath);
Reader reader = Files.newBufferedReader(csvFilePath);
CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader(// The Room database ID
"id", "time", "android_id", "package_name", "word_id", "word_text", "learning_event_type").withSkipHeaderRecord();
CSVParser csvParser = new CSVParser(reader, csvFormat);
for (CSVRecord csvRecord : csvParser) {
logger.info("csvRecord: " + csvRecord);
// Convert from CSV to Java
WordLearningEvent wordLearningEvent = new WordLearningEvent();
long timeInMillis = Long.valueOf(csvRecord.get("time"));
Calendar time = Calendar.getInstance();
time.setTimeInMillis(timeInMillis);
wordLearningEvent.setTime(time);
String androidId = csvRecord.get("android_id");
wordLearningEvent.setAndroidId(androidId);
String packageName = csvRecord.get("package_name");
wordLearningEvent.setPackageName(packageName);
Application application = applicationDao.readByPackageName(packageName);
logger.info("application: " + application);
if (application == null) {
// Return error message saying that the reporting Application has not yet been added
logger.warn("An Application with package name " + packageName + " was not found");
jsonObject.put("result", "error");
jsonObject.put("errorMessage", "An Application with package name " + packageName + " was not found");
response.setStatus(HttpStatus.UNPROCESSABLE_ENTITY.value());
break;
}
wordLearningEvent.setApplication(application);
Long wordId = Long.valueOf(csvRecord.get("word_id"));
Word word = wordDao.read(wordId);
logger.info("word: " + word);
wordLearningEvent.setWord(word);
if (word == null) {
// Return error message saying that the Word ID was not found
logger.warn("A Word with ID " + wordId + " was not found");
jsonObject.put("result", "error");
jsonObject.put("errorMessage", "A Word with ID " + wordId + " was not found");
response.setStatus(HttpStatus.UNPROCESSABLE_ENTITY.value());
break;
}
String wordText = csvRecord.get("word_text");
wordLearningEvent.setWordText(wordText);
LearningEventType learningEventType = LearningEventType.valueOf(csvRecord.get("learning_event_type"));
wordLearningEvent.setLearningEventType(learningEventType);
// Check if the event has already been stored in the database
WordLearningEvent existingWordLearningEvent = wordLearningEventDao.read(time, androidId, application, word);
logger.info("existingWordLearningEvent: " + existingWordLearningEvent);
if (existingWordLearningEvent == null) {
// Store the event in the database
wordLearningEventDao.create(wordLearningEvent);
logger.info("Stored WordLearningEvent in database with ID " + wordLearningEvent.getId());
jsonObject.put("result", "success");
jsonObject.put("successMessage", "The WordLearningEvent was stored in the database");
} else {
// Return error message saying that the event has already been uploaded
logger.warn("The event has already been stored in the database");
jsonObject.put("result", "error");
jsonObject.put("errorMessage", "The event has already been stored in the database");
response.setStatus(HttpStatus.CONFLICT.value());
}
}
} catch (Exception ex) {
logger.error(ex);
jsonObject.put("result", "error");
jsonObject.put("errorMessage", ex.getMessage());
response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
}
String jsonResponse = jsonObject.toString();
logger.info("jsonResponse: " + jsonResponse);
return jsonResponse;
}
use of ai.elimu.model.content.Word in project webapp by elimu-ai.
the class AudioContributionsRestController method getWordsPendingRecording.
/**
* Get {@link Word}s pending {@link Audio} recording for the current {@link Contributor}.
*/
@RequestMapping(value = "/words", method = RequestMethod.GET)
public String getWordsPendingRecording(HttpServletRequest request, HttpServletResponse response) {
logger.info("getWordsPendingRecording");
JSONObject jsonObject = new JSONObject();
// Lookup the Contributor by ID
String providerIdGoogle = request.getHeader("providerIdGoogle");
logger.info("providerIdGoogle: " + providerIdGoogle);
if (StringUtils.isBlank(providerIdGoogle)) {
jsonObject.put("result", "error");
jsonObject.put("errorMessage", "Missing providerIdGoogle");
response.setStatus(HttpStatus.BAD_REQUEST.value());
String jsonResponse = jsonObject.toString();
logger.info("jsonResponse: " + jsonResponse);
return jsonResponse;
}
Contributor contributor = contributorDao.readByProviderIdGoogle(providerIdGoogle);
logger.info("contributor: " + contributor);
if (contributor == null) {
jsonObject.put("result", "error");
jsonObject.put("errorMessage", "The Contributor was not found.");
response.setStatus(HttpStatus.UNPROCESSABLE_ENTITY.value());
String jsonResponse = jsonObject.toString();
logger.info("jsonResponse: " + jsonResponse);
return jsonResponse;
}
// Get the IDs of Words that have already been recorded by the Contributor
List<AudioContributionEvent> audioContributionEvents = audioContributionEventDao.readAll(contributor);
logger.info("audioContributionEvents.size(): " + audioContributionEvents.size());
HashMap<Long, Void> idsOfRecordedWordsHashMap = new HashMap<>();
for (AudioContributionEvent audioContributionEvent : audioContributionEvents) {
Audio audio = audioContributionEvent.getAudio();
Word word = audio.getWord();
if (word != null) {
idsOfRecordedWordsHashMap.put(word.getId(), null);
}
}
logger.info("idsOfRecordedWordsHashMap.size(): " + idsOfRecordedWordsHashMap.size());
// For each Word, check if the Contributor has already contributed a
// corresponding Audio recording. If not, add it to the list of pending recordings.
List<Word> wordsPendingAudioRecording = new ArrayList<>();
for (Word word : wordDao.readAllOrderedByUsage()) {
if (!idsOfRecordedWordsHashMap.containsKey(word.getId())) {
wordsPendingAudioRecording.add(word);
}
}
logger.info("wordsPendingAudioRecording.size(): " + wordsPendingAudioRecording.size());
// Convert to JSON
JSONArray wordsJsonArray = new JSONArray();
for (Word word : wordsPendingAudioRecording) {
WordGson wordGson = JpaToGsonConverter.getWordGson(word);
String json = new Gson().toJson(wordGson);
wordsJsonArray.put(new JSONObject(json));
}
String jsonResponse = wordsJsonArray.toString();
logger.info("jsonResponse: " + jsonResponse);
return jsonResponse;
}
use of ai.elimu.model.content.Word in project webapp by elimu-ai.
the class WordContributionRestController method postWordContribution.
/**
* Handles the creation of a new {@link Word} & the corresponding {@link WordContributionEvent}.
*
* @param requestBody JSON should contain fields required for the creation of a {@link WordContributionEventGson}
* object.
*/
@RequestMapping(value = "/word", method = RequestMethod.POST)
public String postWordContribution(HttpServletRequest request, HttpServletResponse response, @RequestBody String requestBody) {
logger.info("postWordContribution");
// Validate the Contributor.
JSONObject jsonObject = new JSONObject();
String providerIdGoogle = request.getHeader("providerIdGoogle");
logger.info("providerIdGoogle: " + providerIdGoogle);
if (StringUtils.isBlank(providerIdGoogle)) {
jsonObject.put("result", "error");
jsonObject.put("errorMessage", "Missing providerIdGoogle");
response.setStatus(HttpStatus.BAD_REQUEST.value());
String jsonResponse = jsonObject.toString();
logger.info("jsonResponse: " + jsonResponse);
return jsonResponse;
}
// Lookup the Contributor by ID
Contributor contributor = contributorDao.readByProviderIdGoogle(providerIdGoogle);
logger.info("contributor: " + contributor);
if (contributor == null) {
jsonObject.put("result", "error");
jsonObject.put("errorMessage", "The Contributor was not found.");
response.setStatus(HttpStatus.UNPROCESSABLE_ENTITY.value());
String jsonResponse = jsonObject.toString();
logger.info("jsonResponse: " + jsonResponse);
return jsonResponse;
}
logger.info("requestBody: " + requestBody);
// Convert the request body to a WordContributionEventGson
WordContributionEventGson wordContributionEventGson = new Gson().fromJson(requestBody, WordContributionEventGson.class);
logger.info("wordContributionEventGson: " + wordContributionEventGson);
// Extract the WordGson from the WordContributionEventGson
WordGson wordGson = wordContributionEventGson.getWord();
logger.info("wordGson: " + wordGson);
// Check if the word is already existing.
Word existingWord = wordDao.readByText(wordGson.getText().toLowerCase());
if (existingWord != null) {
jsonObject.put("result", "error");
jsonObject.put("errorMessage", "NonUnique");
response.setStatus(HttpStatus.CONFLICT.value());
String jsonResponse = jsonObject.toString();
logger.info("jsonResponse: " + jsonResponse);
return jsonResponse;
}
try {
// Convert the WordGson to Word POJO.
Word word = new Word();
word.setWordType(wordGson.getWordType());
word.setText(wordGson.getText().toLowerCase());
List<LetterSoundCorrespondenceGson> letterSoundCorrespondencesGsons = wordGson.getLetterSoundCorrespondences();
List<LetterSoundCorrespondence> letterSoundCorrespondences = new ArrayList<>();
for (LetterSoundCorrespondenceGson letterSoundCorrespondenceGson : letterSoundCorrespondencesGsons) {
LetterSoundCorrespondence letterSoundCorrespondence = letterSoundCorrespondenceDao.read(letterSoundCorrespondenceGson.getId());
letterSoundCorrespondences.add(letterSoundCorrespondence);
}
word.setLetterSoundCorrespondences(letterSoundCorrespondences);
wordDao.create(word);
WordContributionEvent wordContributionEvent = new WordContributionEvent();
wordContributionEvent.setContributor(contributor);
wordContributionEvent.setTime(wordContributionEventGson.getTime());
wordContributionEvent.setWord(word);
wordContributionEvent.setRevisionNumber(word.getRevisionNumber());
wordContributionEvent.setComment(StringUtils.abbreviate(wordContributionEventGson.getComment(), 1000));
wordContributionEvent.setTimeSpentMs(System.currentTimeMillis() - wordContributionEvent.getTime().getTimeInMillis());
// TODO: wordContributionEvent.setTimeSpentMs(wordContributionEventGson.getTimeSpentMs());
// refer to: https://github.com/elimu-ai/webapp/pull/1289#discussion_r642024541
wordContributionEvent.setPlatform(Platform.CROWDSOURCE_APP);
wordContributionEventDao.create(wordContributionEvent);
String contentUrl = "http://" + EnvironmentContextLoaderListener.PROPERTIES.getProperty("content.language").toLowerCase() + ".elimu.ai/content/word/edit/" + word.getId();
DiscordHelper.sendChannelMessage("Word created: " + contentUrl, "\"" + wordContributionEvent.getWord().getText() + "\"", "Comment: \"" + wordContributionEvent.getComment() + "\"", null, null);
response.setStatus(HttpStatus.CREATED.value());
} catch (Exception ex) {
logger.error(ex);
jsonObject.put("result", "error");
jsonObject.put("errorMessage", ex.getMessage());
response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
}
String jsonResponse = jsonObject.toString();
logger.info("jsonResponse: " + jsonResponse);
return jsonResponse;
}
use of ai.elimu.model.content.Word in project webapp by elimu-ai.
the class ParagraphWordScheduler method execute.
// Every hour
@Scheduled(cron = "00 00 * * * *")
public synchronized void execute() {
logger.info("execute");
Language language = Language.valueOf(ConfigHelper.getProperty("content.language"));
List<StoryBookParagraph> storyBookParagraphs = storyBookParagraphDao.readAll();
logger.info("storyBookParagraphs.size(): " + storyBookParagraphs.size());
for (StoryBookParagraph storyBookParagraph : storyBookParagraphs) {
List<String> wordsInOriginalText = WordExtractionHelper.getWords(storyBookParagraph.getOriginalText(), language);
logger.info("wordsInOriginalText.size(): " + wordsInOriginalText.size());
// Look for matches of existing Words in the paragraph's original text
List<Word> words = new ArrayList<>();
for (String wordInOriginalText : wordsInOriginalText) {
logger.info("wordInOriginalText: \"" + wordInOriginalText + "\"");
wordInOriginalText = wordInOriginalText.toLowerCase();
logger.info("wordInOriginalText (lower-case): \"" + wordInOriginalText + "\"");
Word word = wordDao.readByText(wordInOriginalText);
logger.info("word: " + word);
words.add(word);
}
logger.info("words.size(): " + words.size());
storyBookParagraph.setWords(words);
// Update the paragraph's list of Words in the database
storyBookParagraphDao.update(storyBookParagraph);
}
// Refresh REST API cache
storyBooksJsonService.refreshStoryBooksJSONArray();
logger.info("execute complete");
}
Aggregations