Examples with CSVRecord org.apache.commons.csv.CSVRecord used on opensource projects

Example 96 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project webapp by elimu-ai.

the class LetterAssessmentEventsRestController method handleUploadCsvRequest.

@RequestMapping(value = "/csv", method = RequestMethod.POST)
public String handleUploadCsvRequest(@RequestParam("file") MultipartFile multipartFile, HttpServletResponse response) {
    logger.info("handleUploadCsvRequest");
    String name = multipartFile.getName();
    logger.info("name: " + name);
    // Expected format: "7161a85a0e4751cd_letter-assessment-events_2020-04-23.csv"
    String originalFilename = multipartFile.getOriginalFilename();
    logger.info("originalFilename: " + originalFilename);
    // TODO: Send notification to the #📊-data-collection channel in Discord
    // Hide parts of the Android ID, e.g. "7161***51cd_word-learning-events_2020-04-23.csv"
    String anonymizedOriginalFilename = originalFilename.substring(0, 4) + "***" + originalFilename.substring(12);
    DiscordHelper.sendChannelMessage("Received dataset: `" + anonymizedOriginalFilename + "`", null, null, null, null);
    String androidIdExtractedFromFilename = AnalyticsHelper.extractAndroidIdFromCsvFilename(originalFilename);
    logger.info("androidIdExtractedFromFilename: \"" + androidIdExtractedFromFilename + "\"");
    String contentType = multipartFile.getContentType();
    logger.info("contentType: " + contentType);
    JSONObject jsonObject = new JSONObject();
    try {
        byte[] bytes = multipartFile.getBytes();
        logger.info("bytes.length: " + bytes.length);
        // Store a backup of the original CSV file on the filesystem (in case it will be needed for debugging)
        File elimuAiDir = new File(System.getProperty("user.home"), ".elimu-ai");
        File languageDir = new File(elimuAiDir, "lang-" + Language.valueOf(ConfigHelper.getProperty("content.language")));
        File analyticsDir = new File(languageDir, "analytics");
        File androidIdDir = new File(analyticsDir, "android-id_" + androidIdExtractedFromFilename);
        File letterAssessmentEventsDir = new File(androidIdDir, "letter-assessment-events");
        letterAssessmentEventsDir.mkdirs();
        File csvFile = new File(letterAssessmentEventsDir, originalFilename);
        logger.info("Storing CSV file at " + csvFile);
        multipartFile.transferTo(csvFile);
        // Iterate each row in the CSV file
        Path csvFilePath = Paths.get(csvFile.toURI());
        logger.info("csvFilePath: " + csvFilePath);
        Reader reader = Files.newBufferedReader(csvFilePath);
        CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader(// The Room database ID
        "id", "time", "android_id", "package_name", "letter_id", "letter_text", "mastery_score", "time_spent_ms").withSkipHeaderRecord();
        CSVParser csvParser = new CSVParser(reader, csvFormat);
        for (CSVRecord csvRecord : csvParser) {
            logger.info("csvRecord: " + csvRecord);
        // Convert from CSV to Java
        // TODO
        }
    } catch (Exception ex) {
        logger.error(ex);
        jsonObject.put("result", "error");
        jsonObject.put("errorMessage", ex.getMessage());
        response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
    }
    String jsonResponse = jsonObject.toString();
    logger.info("jsonResponse: " + jsonResponse);
    return jsonResponse;
}

Example 97 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project webapp by elimu-ai.

the class StoryBookLearningEventsRestController method handleUploadCsvRequest.

@RequestMapping(value = "/csv", method = RequestMethod.POST)
public String handleUploadCsvRequest(@RequestParam("file") MultipartFile multipartFile, HttpServletResponse response) {
    logger.info("handleUploadCsvRequest");
    String name = multipartFile.getName();
    logger.info("name: " + name);
    // Expected format: "7161a85a0e4751cd_storybook-learning-events_2020-04-23.csv"
    String originalFilename = multipartFile.getOriginalFilename();
    logger.info("originalFilename: " + originalFilename);
    // TODO: Send notification to the #📊-data-collection channel in Discord
    // Hide parts of the Android ID, e.g. "7161***51cd_word-learning-events_2020-04-23.csv"
    String anonymizedOriginalFilename = originalFilename.substring(0, 4) + "***" + originalFilename.substring(12);
    DiscordHelper.sendChannelMessage("Received dataset: `" + anonymizedOriginalFilename + "`", null, null, null, null);
    String androidIdExtractedFromFilename = AnalyticsHelper.extractAndroidIdFromCsvFilename(originalFilename);
    logger.info("androidIdExtractedFromFilename: \"" + androidIdExtractedFromFilename + "\"");
    String contentType = multipartFile.getContentType();
    logger.info("contentType: " + contentType);
    JSONObject jsonObject = new JSONObject();
    try {
        byte[] bytes = multipartFile.getBytes();
        logger.info("bytes.length: " + bytes.length);
        // Store a backup of the original CSV file on the filesystem (in case it will be needed for debugging)
        File elimuAiDir = new File(System.getProperty("user.home"), ".elimu-ai");
        File languageDir = new File(elimuAiDir, "lang-" + Language.valueOf(ConfigHelper.getProperty("content.language")));
        File analyticsDir = new File(languageDir, "analytics");
        File androidIdDir = new File(analyticsDir, "android-id_" + androidIdExtractedFromFilename);
        File storyBookLearningEventsDir = new File(androidIdDir, "storybook-learning-events");
        storyBookLearningEventsDir.mkdirs();
        File csvFile = new File(storyBookLearningEventsDir, originalFilename);
        logger.info("Storing CSV file at " + csvFile);
        multipartFile.transferTo(csvFile);
        // Iterate each row in the CSV file
        Path csvFilePath = Paths.get(csvFile.toURI());
        logger.info("csvFilePath: " + csvFilePath);
        Reader reader = Files.newBufferedReader(csvFilePath);
        CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader(// The Room database ID
        "id", "time", "android_id", "package_name", "storybook_id", "storybook_title", "learning_event_type").withSkipHeaderRecord();
        CSVParser csvParser = new CSVParser(reader, csvFormat);
        for (CSVRecord csvRecord : csvParser) {
            logger.info("csvRecord: " + csvRecord);
            // Convert from CSV to Java
            StoryBookLearningEvent storyBookLearningEvent = new StoryBookLearningEvent();
            long timeInMillis = Long.valueOf(csvRecord.get("time"));
            Calendar time = Calendar.getInstance();
            time.setTimeInMillis(timeInMillis);
            storyBookLearningEvent.setTime(time);
            String androidId = csvRecord.get("android_id");
            storyBookLearningEvent.setAndroidId(androidId);
            String packageName = csvRecord.get("package_name");
            storyBookLearningEvent.setPackageName(packageName);
            Application application = applicationDao.readByPackageName(packageName);
            logger.info("application: " + application);
            storyBookLearningEvent.setApplication(application);
            Long storyBookId = Long.valueOf(csvRecord.get("storybook_id"));
            storyBookLearningEvent.setStoryBookId(storyBookId);
            String storyBookTitle = csvRecord.get("storybook_title");
            storyBookLearningEvent.setStoryBookTitle(storyBookTitle);
            StoryBook storyBook = storyBookDao.read(storyBookId);
            logger.info("storyBook: " + storyBook);
            storyBookLearningEvent.setStoryBook(storyBook);
            LearningEventType learningEventType = LearningEventType.valueOf(csvRecord.get("learning_event_type"));
            logger.info("learningEventType: " + learningEventType);
            storyBookLearningEvent.setLearningEventType(learningEventType);
            // Check if the event has already been stored in the database
            StoryBookLearningEvent existingStoryBookLearningEvent = storyBookLearningEventDao.read(time, androidId, application, storyBook);
            logger.info("existingStoryBookLearningEvent: " + existingStoryBookLearningEvent);
            if (existingStoryBookLearningEvent == null) {
                // Store the event in the database
                storyBookLearningEventDao.create(storyBookLearningEvent);
                logger.info("Stored StoryBookLearningEvent in database with ID " + storyBookLearningEvent.getId());
                jsonObject.put("result", "success");
                jsonObject.put("successMessage", "The StoryBookLearningEvent was stored in the database");
            } else {
                // Return error message saying that the event has already been uploaded
                logger.warn("The event has already been stored in the database");
                jsonObject.put("result", "error");
                jsonObject.put("errorMessage", "The event has already been stored in the database");
                response.setStatus(HttpStatus.CONFLICT.value());
            }
        }
    } catch (Exception ex) {
        logger.error(ex);
        jsonObject.put("result", "error");
        jsonObject.put("errorMessage", ex.getMessage());
        response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
    }
    String jsonResponse = jsonObject.toString();
    logger.info("jsonResponse: " + jsonResponse);
    return jsonResponse;
}

Example 98 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project webapp by elimu-ai.

the class WordLearningEventsRestController method handleUploadCsvRequest.

@RequestMapping(value = "/csv", method = RequestMethod.POST)
public String handleUploadCsvRequest(@RequestParam("file") MultipartFile multipartFile, HttpServletResponse response) {
    logger.info("handleUploadCsvRequest");
    String name = multipartFile.getName();
    logger.info("name: " + name);
    // Expected format: "7161a85a0e4751cd_word-learning-events_2020-04-23.csv"
    String originalFilename = multipartFile.getOriginalFilename();
    logger.info("originalFilename: " + originalFilename);
    // TODO: Send notification to the #📊-data-collection channel in Discord
    // Hide parts of the Android ID, e.g. "7161***51cd_word-learning-events_2020-04-23.csv"
    String anonymizedOriginalFilename = originalFilename.substring(0, 4) + "***" + originalFilename.substring(12);
    DiscordHelper.sendChannelMessage("Received dataset: `" + anonymizedOriginalFilename + "`", null, null, null, null);
    String androidIdExtractedFromFilename = AnalyticsHelper.extractAndroidIdFromCsvFilename(originalFilename);
    logger.info("androidIdExtractedFromFilename: \"" + androidIdExtractedFromFilename + "\"");
    String contentType = multipartFile.getContentType();
    logger.info("contentType: " + contentType);
    JSONObject jsonObject = new JSONObject();
    try {
        byte[] bytes = multipartFile.getBytes();
        logger.info("bytes.length: " + bytes.length);
        // Store a backup of the original CSV file on the filesystem (in case it will be needed for debugging)
        File elimuAiDir = new File(System.getProperty("user.home"), ".elimu-ai");
        File languageDir = new File(elimuAiDir, "lang-" + Language.valueOf(ConfigHelper.getProperty("content.language")));
        File analyticsDir = new File(languageDir, "analytics");
        File androidIdDir = new File(analyticsDir, "android-id_" + androidIdExtractedFromFilename);
        File wordLearningEventsDir = new File(androidIdDir, "word-learning-events");
        wordLearningEventsDir.mkdirs();
        File csvFile = new File(wordLearningEventsDir, originalFilename);
        logger.info("Storing CSV file at " + csvFile);
        multipartFile.transferTo(csvFile);
        // Iterate each row in the CSV file
        Path csvFilePath = Paths.get(csvFile.toURI());
        logger.info("csvFilePath: " + csvFilePath);
        Reader reader = Files.newBufferedReader(csvFilePath);
        CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader(// The Room database ID
        "id", "time", "android_id", "package_name", "word_id", "word_text", "learning_event_type").withSkipHeaderRecord();
        CSVParser csvParser = new CSVParser(reader, csvFormat);
        for (CSVRecord csvRecord : csvParser) {
            logger.info("csvRecord: " + csvRecord);
            // Convert from CSV to Java
            WordLearningEvent wordLearningEvent = new WordLearningEvent();
            long timeInMillis = Long.valueOf(csvRecord.get("time"));
            Calendar time = Calendar.getInstance();
            time.setTimeInMillis(timeInMillis);
            wordLearningEvent.setTime(time);
            String androidId = csvRecord.get("android_id");
            wordLearningEvent.setAndroidId(androidId);
            String packageName = csvRecord.get("package_name");
            wordLearningEvent.setPackageName(packageName);
            Application application = applicationDao.readByPackageName(packageName);
            logger.info("application: " + application);
            if (application == null) {
                // Return error message saying that the reporting Application has not yet been added
                logger.warn("An Application with package name " + packageName + " was not found");
                jsonObject.put("result", "error");
                jsonObject.put("errorMessage", "An Application with package name " + packageName + " was not found");
                response.setStatus(HttpStatus.UNPROCESSABLE_ENTITY.value());
                break;
            }
            wordLearningEvent.setApplication(application);
            Long wordId = Long.valueOf(csvRecord.get("word_id"));
            Word word = wordDao.read(wordId);
            logger.info("word: " + word);
            wordLearningEvent.setWord(word);
            if (word == null) {
                // Return error message saying that the Word ID was not found
                logger.warn("A Word with ID " + wordId + " was not found");
                jsonObject.put("result", "error");
                jsonObject.put("errorMessage", "A Word with ID " + wordId + " was not found");
                response.setStatus(HttpStatus.UNPROCESSABLE_ENTITY.value());
                break;
            }
            String wordText = csvRecord.get("word_text");
            wordLearningEvent.setWordText(wordText);
            LearningEventType learningEventType = LearningEventType.valueOf(csvRecord.get("learning_event_type"));
            wordLearningEvent.setLearningEventType(learningEventType);
            // Check if the event has already been stored in the database
            WordLearningEvent existingWordLearningEvent = wordLearningEventDao.read(time, androidId, application, word);
            logger.info("existingWordLearningEvent: " + existingWordLearningEvent);
            if (existingWordLearningEvent == null) {
                // Store the event in the database
                wordLearningEventDao.create(wordLearningEvent);
                logger.info("Stored WordLearningEvent in database with ID " + wordLearningEvent.getId());
                jsonObject.put("result", "success");
                jsonObject.put("successMessage", "The WordLearningEvent was stored in the database");
            } else {
                // Return error message saying that the event has already been uploaded
                logger.warn("The event has already been stored in the database");
                jsonObject.put("result", "error");
                jsonObject.put("errorMessage", "The event has already been stored in the database");
                response.setStatus(HttpStatus.CONFLICT.value());
            }
        }
    } catch (Exception ex) {
        logger.error(ex);
        jsonObject.put("result", "error");
        jsonObject.put("errorMessage", ex.getMessage());
        response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
    }
    String jsonResponse = jsonObject.toString();
    logger.info("jsonResponse: " + jsonResponse);
    return jsonResponse;
}

Example 99 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project webapp by elimu-ai.

the class WordAssessmentEventsRestController method handleUploadCsvRequest.

@RequestMapping(value = "/csv", method = RequestMethod.POST)
public String handleUploadCsvRequest(@RequestParam("file") MultipartFile multipartFile, HttpServletResponse response) {
    logger.info("handleUploadCsvRequest");
    String name = multipartFile.getName();
    logger.info("name: " + name);
    // Expected format: "7161a85a0e4751cd_word-assessment-events_2020-04-23.csv"
    String originalFilename = multipartFile.getOriginalFilename();
    logger.info("originalFilename: " + originalFilename);
    // TODO: Send notification to the #📊-data-collection channel in Discord
    // Hide parts of the Android ID, e.g. "7161***51cd_word-learning-events_2020-04-23.csv"
    String anonymizedOriginalFilename = originalFilename.substring(0, 4) + "***" + originalFilename.substring(12);
    DiscordHelper.sendChannelMessage("Received dataset: `" + anonymizedOriginalFilename + "`", null, null, null, null);
    String androidIdExtractedFromFilename = AnalyticsHelper.extractAndroidIdFromCsvFilename(originalFilename);
    logger.info("androidIdExtractedFromFilename: \"" + androidIdExtractedFromFilename + "\"");
    String contentType = multipartFile.getContentType();
    logger.info("contentType: " + contentType);
    JSONObject jsonObject = new JSONObject();
    try {
        byte[] bytes = multipartFile.getBytes();
        logger.info("bytes.length: " + bytes.length);
        // Store a backup of the original CSV file on the filesystem (in case it will be needed for debugging)
        File elimuAiDir = new File(System.getProperty("user.home"), ".elimu-ai");
        File languageDir = new File(elimuAiDir, "lang-" + Language.valueOf(ConfigHelper.getProperty("content.language")));
        File analyticsDir = new File(languageDir, "analytics");
        File androidIdDir = new File(analyticsDir, "android-id_" + androidIdExtractedFromFilename);
        File wordAssessmentEventsDir = new File(androidIdDir, "word-assessment-events");
        wordAssessmentEventsDir.mkdirs();
        File csvFile = new File(wordAssessmentEventsDir, originalFilename);
        logger.info("Storing CSV file at " + csvFile);
        multipartFile.transferTo(csvFile);
        // Iterate each row in the CSV file
        Path csvFilePath = Paths.get(csvFile.toURI());
        logger.info("csvFilePath: " + csvFilePath);
        Reader reader = Files.newBufferedReader(csvFilePath);
        CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader(// The Room database ID
        "id", "time", "android_id", "package_name", "word_id", "word_text", "mastery_score", "time_spent_ms").withSkipHeaderRecord();
        CSVParser csvParser = new CSVParser(reader, csvFormat);
        for (CSVRecord csvRecord : csvParser) {
            logger.info("csvRecord: " + csvRecord);
        // Convert from CSV to Java
        // TODO
        }
    } catch (Exception ex) {
        logger.error(ex);
        jsonObject.put("result", "error");
        jsonObject.put("errorMessage", ex.getMessage());
        response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
    }
    String jsonResponse = jsonObject.toString();
    logger.info("jsonResponse: " + jsonResponse);
    return jsonResponse;
}

Example 100 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project webapp by elimu-ai.

the class CsvAnalyticsExtractionHelper method getStoryBookLearningEventsFromCsvBackup.

/**
 * For information on how the CSV files were generated, see {@link StoryBookLearningEventCsvExportController#handleRequest}.
 * <p />
 * Also see {@link StoryBookLearningEventsRestController#handleUploadCsvRequest}
 */
public static List<StoryBookLearningEvent> getStoryBookLearningEventsFromCsvBackup(File csvFile, ApplicationDao applicationDao, StoryBookDao storyBookDao) {
    logger.info("getStoryBookLearningEventsFromCsvBackup");
    List<StoryBookLearningEvent> storyBookLearningEvents = new ArrayList<>();
    Path csvFilePath = Paths.get(csvFile.toURI());
    logger.info("csvFilePath: " + csvFilePath);
    try {
        Reader reader = Files.newBufferedReader(csvFilePath);
        CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader(// The Room database ID
        "id", "time", "android_id", "package_name", "storybook_id", "storybook_title", "learning_event_type").withSkipHeaderRecord();
        CSVParser csvParser = new CSVParser(reader, csvFormat);
        for (CSVRecord csvRecord : csvParser) {
            logger.info("csvRecord: " + csvRecord);
            // Convert from CSV to Java (see similar code in StoryBookLearningEventsRestController)
            StoryBookLearningEvent storyBookLearningEvent = new StoryBookLearningEvent();
            long timeInMillis = Long.valueOf(csvRecord.get("time"));
            Calendar time = Calendar.getInstance();
            time.setTimeInMillis(timeInMillis);
            storyBookLearningEvent.setTime(time);
            String androidId = csvRecord.get("android_id");
            storyBookLearningEvent.setAndroidId(androidId);
            String packageName = csvRecord.get("package_name");
            storyBookLearningEvent.setPackageName(packageName);
            Application application = applicationDao.readByPackageName(packageName);
            logger.info("application: " + application);
            storyBookLearningEvent.setApplication(application);
            Long storyBookId = Long.valueOf(csvRecord.get("storybook_id"));
            logger.info("storyBookId: " + storyBookId);
            storyBookLearningEvent.setStoryBookId(storyBookId);
            String storyBookTitle = csvRecord.get("storybook_title");
            logger.info("storyBookTitle: \"" + storyBookTitle + "\"");
            storyBookLearningEvent.setStoryBookTitle(storyBookTitle);
            StoryBook storyBook = storyBookDao.readByTitle(storyBookTitle);
            logger.info("storyBook: " + storyBook);
            storyBookLearningEvent.setStoryBook(storyBook);
            LearningEventType learningEventType = LearningEventType.valueOf(csvRecord.get("learning_event_type"));
            logger.info("learningEventType: " + learningEventType);
            storyBookLearningEvent.setLearningEventType(learningEventType);
            storyBookLearningEvents.add(storyBookLearningEvent);
        }
    } catch (IOException ex) {
        logger.error(ex);
    }
    return storyBookLearningEvents;
}