Examples with SummaryStatsBean org.akaza.openclinica.bean.submit.crfdata.SummaryStatsBean used on opensource projects

Example 1 with SummaryStatsBean

use of org.akaza.openclinica.bean.submit.crfdata.SummaryStatsBean in project OpenClinica by OpenClinica.

the class ImportCRFDataServlet method processRequest.

@Override
public void processRequest() throws Exception {
    resetPanel();
    panel.setStudyInfoShown(false);
    panel.setOrderedData(true);
    FormProcessor fp = new FormProcessor(request);
    // checks which module the requests are from
    String module = fp.getString(MODULE);
    // keep the module in the session
    session.setAttribute(MODULE, module);
    String action = request.getParameter("action");
    CRFVersionBean version = (CRFVersionBean) session.getAttribute("version");
    File xsdFile = new File(SpringServletAccess.getPropertiesDir(context) + "ODM1-3-0.xsd");
    File xsdFile2 = new File(SpringServletAccess.getPropertiesDir(context) + "ODM1-2-1.xsd");
    if (StringUtil.isBlank(action)) {
        logger.info("action is blank");
        request.setAttribute("version", version);
        forwardPage(Page.IMPORT_CRF_DATA);
    }
    if ("confirm".equalsIgnoreCase(action)) {
        String dir = SQLInitServlet.getField("filePath");
        if (!new File(dir).exists()) {
            logger.info("The filePath in datainfo.properties is invalid " + dir);
            addPageMessage(respage.getString("filepath_you_defined_not_seem_valid"));
            forwardPage(Page.IMPORT_CRF_DATA);
        }
        // All the uploaded files will be saved in filePath/crf/original/
        String theDir = dir + "crf" + File.separator + "original" + File.separator;
        if (!new File(theDir).isDirectory()) {
            new File(theDir).mkdirs();
            logger.info("Made the directory " + theDir);
        }
        // MultipartRequest multi = new MultipartRequest(request, theDir, 50 * 1024 * 1024);
        File f = null;
        try {
            f = uploadFile(theDir, version);
        } catch (Exception e) {
            logger.warn("*** Found exception during file upload***");
            e.printStackTrace();
        }
        if (f == null) {
            forwardPage(Page.IMPORT_CRF_DATA);
        }
        // TODO
        // validation steps
        // 1. valid xml - validated by file uploader below
        // LocalConfiguration config = LocalConfiguration.getInstance();
        // config.getProperties().setProperty(
        // "org.exolab.castor.parser.namespaces",
        // "true");
        // config
        // .getProperties()
        // .setProperty("org.exolab.castor.sax.features",
        // "http://xml.org/sax/features/validation,
        // http://apache.org/xml/features/validation/schema,
        // http://apache.org/xml/features/validation/schema-full-checking");
        // // above sets to validate against namespace
        Mapping myMap = new Mapping();
        // @pgawade 18-April-2011 Fix for issue 8394
        String ODM_MAPPING_DIRPath = CoreResources.ODM_MAPPING_DIR;
        myMap.loadMapping(ODM_MAPPING_DIRPath + File.separator + "cd_odm_mapping.xml");
        Unmarshaller um1 = new Unmarshaller(myMap);
        // um1.addNamespaceToPackageMapping("http://www.openclinica.org/ns/odm_ext_v130/v3.1", "OpenClinica");
        // um1.addNamespaceToPackageMapping("http://www.cdisc.org/ns/odm/v1.3"
        // ,
        // "ODMContainer");
        boolean fail = false;
        ODMContainer odmContainer = new ODMContainer();
        session.removeAttribute("odmContainer");
        try {
            // schemaValidator.validateAgainstSchema(f, xsdFile);
            // utf-8 compliance, tbh 06/2009
            InputStreamReader isr = new InputStreamReader(new FileInputStream(f), "UTF-8");
            odmContainer = (ODMContainer) um1.unmarshal(isr);
            logger.debug("Found crf data container for study oid: " + odmContainer.getCrfDataPostImportContainer().getStudyOID());
            logger.debug("found length of subject list: " + odmContainer.getCrfDataPostImportContainer().getSubjectData().size());
            // 2. validates against ODM 1.3
            // check it all below, throw an exception and route to a
            // different
            // page if not working
            // TODO this block of code needs the xerces serializer in order
            // to
            // work
            // StringWriter myWriter = new StringWriter();
            // Marshaller m1 = new Marshaller(myWriter);
            // 
            // m1.setProperty("org.exolab.castor.parser.namespaces",
            // "true");
            // m1
            // .setProperty("org.exolab.castor.sax.features",
            // "http://xml.org/sax/features/validation,
            // http://apache.org/xml/features/validation/schema,
            // http://apache.org/xml/features/validation/schema-full-checking
            // ");
            // 
            // m1.setMapping(myMap);
            // m1.setNamespaceMapping("",
            // "http://www.cdisc.org/ns/odm/v1.3");
            // m1.setSchemaLocation("http://www.cdisc.org/ns/odm/v1.3
            // ODM1-3.xsd");
            // m1.marshal(odmContainer);
            // if you havent thrown it, you wont throw it here
            addPageMessage(respage.getString("passed_xml_validation"));
        } catch (Exception me1) {
            me1.printStackTrace();
            // expanding it to all exceptions, but hoping to catch Marshal
            // Exception or SAX Exceptions
            logger.info("found exception with xml transform");
            // 
            logger.info("trying 1.2.1");
            try {
                schemaValidator.validateAgainstSchema(f, xsdFile2);
                // for backwards compatibility, we also try to validate vs
                // 1.2.1 ODM 06/2008
                InputStreamReader isr = new InputStreamReader(new FileInputStream(f), "UTF-8");
                odmContainer = (ODMContainer) um1.unmarshal(isr);
            } catch (Exception me2) {
                // not sure if we want to report me2
                MessageFormat mf = new MessageFormat("");
                mf.applyPattern(respage.getString("your_xml_is_not_well_formed"));
                Object[] arguments = { me1.getMessage() };
                addPageMessage(mf.format(arguments));
                // 
                // addPageMessage("Your XML is not well-formed, and does not
                // comply with the ODM 1.3 Schema. Please check it, and try
                // again. It returned the message: "
                // + me1.getMessage());
                // me1.printStackTrace();
                forwardPage(Page.IMPORT_CRF_DATA);
            // you can't really wait to forward because then you throw
            // NPEs
            // in the next few parts of the code
            }
        }
        // TODO need to output further here
        // 2.a. is the study the same one that the user is in right now?
        // 3. validates against study metadata
        // 3.a. is that study subject in that study?
        // 3.b. is that study event def in that study?
        // 3.c. is that site in that study?
        // 3.d. is that crf version in that study event def?
        // 3.e. are those item groups in that crf version?
        // 3.f. are those items in that item group?
        List<String> errors = getImportCRFDataService().validateStudyMetadata(odmContainer, ub.getActiveStudyId());
        if (errors != null) {
            // add to session
            // forward to another page
            logger.info(errors.toString());
            for (String error : errors) {
                addPageMessage(error);
            }
            if (errors.size() > 0) {
                // fail = true;
                forwardPage(Page.IMPORT_CRF_DATA);
            } else {
                addPageMessage(respage.getString("passed_study_check"));
                addPageMessage(respage.getString("passed_oid_metadata_check"));
            }
        }
        logger.debug("passed error check");
        // TODO ADD many validation steps before we get to the
        // session-setting below
        // 4. is the event in the correct status to accept data import?
        // -- scheduled, data entry started, completed
        // (and the event should already be created)
        // (and the event should be independent, ie not affected by other
        // events)
        Boolean eventCRFStatusesValid = getImportCRFDataService().eventCRFStatusesValid(odmContainer, ub);
        ImportCRFInfoContainer importCrfInfo = new ImportCRFInfoContainer(odmContainer, sm.getDataSource());
        // The eventCRFBeans list omits EventCRFs that don't match UpsertOn rules. If EventCRF did not exist and
        // doesn't match upsert, it won't be created.
        List<EventCRFBean> eventCRFBeans = getImportCRFDataService().fetchEventCRFBeans(odmContainer, ub);
        List<DisplayItemBeanWrapper> displayItemBeanWrappers = new ArrayList<DisplayItemBeanWrapper>();
        HashMap<String, String> totalValidationErrors = new HashMap<String, String>();
        HashMap<String, String> hardValidationErrors = new HashMap<String, String>();
        // The following map is used for setting the EventCRF status post import.
        HashMap<Integer, String> importedCRFStatuses = getImportCRFDataService().fetchEventCRFStatuses(odmContainer);
        // method in the ImportCRFDataService is modified for this fix.
        if (eventCRFBeans == null) {
            fail = true;
            addPageMessage(respage.getString("no_event_status_matching"));
        } else {
            ArrayList<Integer> permittedEventCRFIds = new ArrayList<Integer>();
            logger.info("found a list of eventCRFBeans: " + eventCRFBeans.toString());
            // List<DisplayItemBeanWrapper> displayItemBeanWrappers = new ArrayList<DisplayItemBeanWrapper>();
            // HashMap<String, String> totalValidationErrors = new
            // HashMap<String, String>();
            // HashMap<String, String> hardValidationErrors = new
            // HashMap<String, String>();
            logger.debug("found event crfs " + eventCRFBeans.size());
            // -- does the event already exist? if not, fail
            if (!eventCRFBeans.isEmpty()) {
                for (EventCRFBean eventCRFBean : eventCRFBeans) {
                    DataEntryStage dataEntryStage = eventCRFBean.getStage();
                    Status eventCRFStatus = eventCRFBean.getStatus();
                    logger.info("Event CRF Bean: id " + eventCRFBean.getId() + ", data entry stage " + dataEntryStage.getName() + ", status " + eventCRFStatus.getName());
                    if (eventCRFStatus.equals(Status.AVAILABLE) || dataEntryStage.equals(DataEntryStage.INITIAL_DATA_ENTRY) || dataEntryStage.equals(DataEntryStage.INITIAL_DATA_ENTRY_COMPLETE) || dataEntryStage.equals(DataEntryStage.DOUBLE_DATA_ENTRY_COMPLETE) || dataEntryStage.equals(DataEntryStage.DOUBLE_DATA_ENTRY)) {
                        // actually want the negative
                        // was status == available and the stage questions, but
                        // when you are at 'data entry complete' your status is
                        // set to 'unavailable'.
                        // >> tbh 09/2008
                        // HOWEVER, when one event crf is removed and the rest
                        // are good, what happens???
                        // need to create a list and inform that one is blocked
                        // and the rest are not...
                        // 
                        permittedEventCRFIds.add(new Integer(eventCRFBean.getId()));
                    } else {
                    // fail = true;
                    // addPageMessage(respage.getString(
                    // "the_event_crf_not_correct_status"));
                    // forwardPage(Page.IMPORT_CRF_DATA);
                    }
                }
                // did we exclude all the event CRFs? if not, pass, else fail
                if (eventCRFBeans.size() >= permittedEventCRFIds.size()) {
                    addPageMessage(respage.getString("passed_event_crf_status_check"));
                } else {
                    fail = true;
                    addPageMessage(respage.getString("the_event_crf_not_correct_status"));
                }
                try {
                    List<DisplayItemBeanWrapper> tempDisplayItemBeanWrappers = new ArrayList<DisplayItemBeanWrapper>();
                    tempDisplayItemBeanWrappers = getImportCRFDataService().lookupValidationErrors(request, odmContainer, ub, totalValidationErrors, hardValidationErrors, permittedEventCRFIds);
                    logger.debug("generated display item bean wrappers " + tempDisplayItemBeanWrappers.size());
                    logger.debug("size of total validation errors: " + totalValidationErrors.size());
                    displayItemBeanWrappers.addAll(tempDisplayItemBeanWrappers);
                } catch (NullPointerException npe1) {
                    // what if you have 2 event crfs but the third is a fake?
                    fail = true;
                    logger.debug("threw a NPE after calling lookup validation errors");
                    System.out.println(ExceptionUtils.getStackTrace(npe1));
                    addPageMessage(respage.getString("an_error_was_thrown_while_validation_errors"));
                // npe1.printStackTrace();
                } catch (OpenClinicaException oce1) {
                    fail = true;
                    logger.debug("threw an OCE after calling lookup validation errors " + oce1.getOpenClinicaMessage());
                    addPageMessage(oce1.getOpenClinicaMessage());
                }
            } else if (!eventCRFStatusesValid) {
                fail = true;
                addPageMessage(respage.getString("the_event_crf_not_correct_status"));
            } else {
                fail = true;
                addPageMessage(respage.getString("no_event_crfs_matching_the_xml_metadata"));
            }
        // for (HashMap<String, String> crfData : importedData) {
        // DisplayItemBeanWrapper displayItemBeanWrapper =
        // testing(request,
        // crfData);
        // displayItemBeanWrappers.add(displayItemBeanWrapper);
        // errors = displayItemBeanWrapper.getValidationErrors();
        // 
        // }
        }
        if (fail) {
            logger.debug("failed here - forwarding...");
            forwardPage(Page.IMPORT_CRF_DATA);
        } else {
            addPageMessage(respage.getString("passing_crf_edit_checks"));
            session.setAttribute("odmContainer", odmContainer);
            session.setAttribute("importedData", displayItemBeanWrappers);
            session.setAttribute("validationErrors", totalValidationErrors);
            session.setAttribute("hardValidationErrors", hardValidationErrors);
            session.setAttribute("importedCRFStatuses", importedCRFStatuses);
            session.setAttribute("importCrfInfo", importCrfInfo);
            // above are updated 'statically' by the method that originally
            // generated the wrappers; soon the only thing we will use
            // wrappers for is the 'overwrite' flag
            logger.debug("+++ content of total validation errors: " + totalValidationErrors.toString());
            SummaryStatsBean ssBean = getImportCRFDataService().generateSummaryStatsBean(odmContainer, displayItemBeanWrappers, importCrfInfo);
            session.setAttribute("summaryStats", ssBean);
            // will have to set hard edit checks here as well
            session.setAttribute("subjectData", odmContainer.getCrfDataPostImportContainer().getSubjectData());
            forwardPage(Page.VERIFY_IMPORT_SERVLET);
        }
    // }
    }
}

Example 2 with SummaryStatsBean

use of org.akaza.openclinica.bean.submit.crfdata.SummaryStatsBean in project OpenClinica by OpenClinica.

the class ImportCRFDataService method generateSummaryStatsBean.

public SummaryStatsBean generateSummaryStatsBean(ODMContainer odmContainer, List<DisplayItemBeanWrapper> wrappers, ImportCRFInfoContainer importCrfInfo) {
    int countSubjects = 0;
    int countEventCRFs = 0;
    int discNotesGenerated = 0;
    for (DisplayItemBeanWrapper wr : wrappers) {
        HashMap validations = wr.getValidationErrors();
        discNotesGenerated += validations.size();
    }
    ArrayList<SubjectDataBean> subjectDataBeans = odmContainer.getCrfDataPostImportContainer().getSubjectData();
    countSubjects += subjectDataBeans.size();
    for (SubjectDataBean subjectDataBean : subjectDataBeans) {
        ArrayList<StudyEventDataBean> studyEventDataBeans = subjectDataBean.getStudyEventData();
        for (StudyEventDataBean studyEventDataBean : studyEventDataBeans) {
            ArrayList<FormDataBean> formDataBeans = studyEventDataBean.getFormData();
            // this would be the place to add more stats
            for (FormDataBean formDataBean : formDataBeans) {
                countEventCRFs += 1;
            }
        }
    }
    SummaryStatsBean ssBean = new SummaryStatsBean();
    ssBean.setDiscNoteCount(discNotesGenerated);
    ssBean.setEventCrfCount(countEventCRFs);
    ssBean.setStudySubjectCount(countSubjects);
    ssBean.setSkippedCrfCount(importCrfInfo.getCountSkippedEventCrfs());
    return ssBean;
}

Example 3 with SummaryStatsBean

use of org.akaza.openclinica.bean.submit.crfdata.SummaryStatsBean in project OpenClinica by OpenClinica.

the class ImportSpringJob method processData.

/*
     * processData, a method which should take in all XML files, check to see if they were imported previously, ? insert
     * them into the database if not, and return a message which will go to audit and to the end user.
     */
private ArrayList<String> processData(File[] dest, DataSource dataSource, ResourceBundle respage, ResourceBundle resword, UserAccountBean ub, StudyBean studyBean, File destDirectory, TriggerBean triggerBean, RuleSetServiceInterface ruleSetService) throws Exception {
    StringBuffer msg = new StringBuffer();
    StringBuffer auditMsg = new StringBuffer();
    Mapping myMap = new Mapping();
    String propertiesPath = CoreResources.PROPERTIES_DIR;
    new File(propertiesPath + File.separator + "ODM1-3-0.xsd");
    File xsdFile2 = new File(propertiesPath + File.separator + "ODM1-2-1.xsd");
    // @pgawade 18-April-2011 Fix for issue 8394
    String ODM_MAPPING_DIR_path = CoreResources.ODM_MAPPING_DIR;
    myMap.loadMapping(ODM_MAPPING_DIR_path + File.separator + "cd_odm_mapping.xml");
    Unmarshaller um1 = new Unmarshaller(myMap);
    ODMContainer odmContainer = new ODMContainer();
    // File("log.txt")));
    for (File f : dest) {
        // >> tbh
        boolean fail = false;
        // all whitespace, one or more times
        String regex = "\\s+";
        // replace with underscores
        String replacement = "_";
        String pattern = "yyyy" + File.separator + "MM" + File.separator + "dd" + File.separator + "HHmmssSSS" + File.separator;
        SimpleDateFormat sdfDir = new SimpleDateFormat(pattern);
        String generalFileDir = sdfDir.format(new java.util.Date());
        File logDestDirectory = new File(destDirectory + File.separator + generalFileDir + f.getName().replaceAll(regex, replacement) + ".log.txt");
        if (!logDestDirectory.isDirectory()) {
            logger.debug("creating new dir: " + logDestDirectory.getAbsolutePath());
            logDestDirectory.mkdirs();
        }
        File newFile = new File(logDestDirectory, "log.txt");
        // FileOutputStream out = new FileOutputStream(new
        // File(logDestDirectory, "log.txt"));
        // BufferedWriter out = null;
        // wrap the below in a try-catch?
        BufferedWriter out = new BufferedWriter(new FileWriter(newFile));
        // << tbh 06/2010
        if (f != null) {
            String firstLine = "<P>" + f.getName() + ": ";
            msg.append(firstLine);
            out.write(firstLine);
            auditMsg.append(firstLine);
        } else {
            msg.append("<P>" + respage.getString("unreadable_file") + ": ");
            out.write("<P>" + respage.getString("unreadable_file") + ": ");
            auditMsg.append("<P>" + respage.getString("unreadable_file") + ": ");
        }
        try {
            // schemaValidator.validateAgainstSchema(f, xsdFile);
            odmContainer = (ODMContainer) um1.unmarshal(new FileReader(f));
            logger.debug("Found crf data container for study oid: " + odmContainer.getCrfDataPostImportContainer().getStudyOID());
            logger.debug("found length of subject list: " + odmContainer.getCrfDataPostImportContainer().getSubjectData().size());
        } catch (Exception me1) {
            // fail against one, try another
            try {
                schemaValidator.validateAgainstSchema(f, xsdFile2);
                // for backwards compatibility, we also try to validate vs
                // 1.2.1 ODM 06/2008
                odmContainer = (ODMContainer) um1.unmarshal(new FileReader(f));
            } catch (Exception me2) {
                // not sure if we want to report me2
                MessageFormat mf = new MessageFormat("");
                mf.applyPattern(respage.getString("your_xml_is_not_well_formed"));
                Object[] arguments = { me1.getMessage() };
                msg.append(mf.format(arguments) + "<br/>");
                auditMsg.append(mf.format(arguments) + "<br/>");
                // break here with an exception
                logger.error("found an error with XML: " + msg.toString());
                // continue looping
                continue;
            }
        }
        // next: check, then import
        List<String> errors = getImportCRFDataService(dataSource).validateStudyMetadata(odmContainer, studyBean.getId());
        // the user could be in any study ...
        if (errors != null) {
            if (errors.size() > 0) {
                out.write("<P>Errors:<br/>");
                for (String error : errors) {
                    out.write(error + "<br/>");
                }
                out.write("</P>");
                // fail = true;
                // forwardPage(Page.IMPORT_CRF_DATA);
                // break here with an exception
                // throw new Exception("Your XML in the file " + f.getName()
                // + " was well formed, but generated metadata errors: " +
                // errors.toString());
                // msg.append("Your XML in the file " + f.getName() +
                // " was well formed, but generated metadata errors: " +
                // errors.toString());
                MessageFormat mf = new MessageFormat("");
                mf.applyPattern(respage.getString("your_xml_in_the_file"));
                Object[] arguments = { f.getName(), errors.size() };
                auditMsg.append(mf.format(arguments) + "<br/>");
                msg.append(mf.format(arguments) + "<br/>");
                auditMsg.append("You can see the log file <a href='" + SQLInitServlet.getField("sysURL.base") + "ViewLogMessage?n=" + generalFileDir + f.getName() + "&tn=" + triggerBean.getName() + "&gn=1'>here</a>.<br/>");
                msg.append("You can see the log file <a href='" + SQLInitServlet.getField("sysURL.base") + "ViewLogMessage?n=" + generalFileDir + f.getName() + "&tn=" + triggerBean.getName() + "&gn=1'>here</a>.<br/>");
                // auditMsg.append("Your XML in the file " + f.getName() +
                // " was well formed, but generated " + errors.size() +
                // " metadata errors." + "<br/>");
                out.close();
                continue;
            } else {
                msg.append(respage.getString("passed_study_check") + "<br/>");
                msg.append(respage.getString("passed_oid_metadata_check") + "<br/>");
                auditMsg.append(respage.getString("passed_study_check") + "<br/>");
                auditMsg.append(respage.getString("passed_oid_metadata_check") + "<br/>");
            }
        }
        ImportCRFInfoContainer importCrfInfo = new ImportCRFInfoContainer(odmContainer, dataSource);
        // validation errors, the same as in the ImportCRFDataServlet. DRY?
        List<EventCRFBean> eventCRFBeans = getImportCRFDataService(dataSource).fetchEventCRFBeans(odmContainer, ub);
        ArrayList<Integer> permittedEventCRFIds = new ArrayList<Integer>();
        Boolean eventCRFStatusesValid = getImportCRFDataService(dataSource).eventCRFStatusesValid(odmContainer, ub);
        List<DisplayItemBeanWrapper> displayItemBeanWrappers = new ArrayList<DisplayItemBeanWrapper>();
        HashMap<String, String> totalValidationErrors = new HashMap<String, String>();
        HashMap<String, String> hardValidationErrors = new HashMap<String, String>();
        // The following map is used for setting the EventCRF status post import.
        HashMap<Integer, String> importedCRFStatuses = getImportCRFDataService(dataSource).fetchEventCRFStatuses(odmContainer);
        // -- does the event already exist? if not, fail
        if (eventCRFBeans == null) {
            fail = true;
            msg.append(respage.getString("no_event_status_matching"));
            out.write(respage.getString("no_event_status_matching"));
            out.close();
            continue;
        } else if (!eventCRFBeans.isEmpty()) {
            logger.debug("found a list of eventCRFBeans: " + eventCRFBeans.toString());
            for (EventCRFBean eventCRFBean : eventCRFBeans) {
                DataEntryStage dataEntryStage = eventCRFBean.getStage();
                Status eventCRFStatus = eventCRFBean.getStatus();
                logger.debug("Event CRF Bean: id " + eventCRFBean.getId() + ", data entry stage " + dataEntryStage.getName() + ", status " + eventCRFStatus.getName());
                if (eventCRFStatus.equals(Status.AVAILABLE) || dataEntryStage.equals(DataEntryStage.INITIAL_DATA_ENTRY) || dataEntryStage.equals(DataEntryStage.INITIAL_DATA_ENTRY_COMPLETE) || dataEntryStage.equals(DataEntryStage.DOUBLE_DATA_ENTRY_COMPLETE) || dataEntryStage.equals(DataEntryStage.DOUBLE_DATA_ENTRY)) {
                    permittedEventCRFIds.add(new Integer(eventCRFBean.getId()));
                } else {
                    // break out here with an exception
                    // throw new
                    // Exception("Your listed Event CRF in the file " +
                    // f.getName() +
                    // " does not exist, or has already been locked for import."
                    // );
                    MessageFormat mf = new MessageFormat("");
                    mf.applyPattern(respage.getString("your_listed_crf_in_the_file"));
                    Object[] arguments = { f.getName() };
                    msg.append(mf.format(arguments) + "<br/>");
                    auditMsg.append(mf.format(arguments) + "<br/>");
                    out.write(mf.format(arguments) + "<br/>");
                    out.close();
                    continue;
                }
            }
            if (eventCRFBeans.size() >= permittedEventCRFIds.size()) {
                msg.append(respage.getString("passed_event_crf_status_check") + "<br/>");
                auditMsg.append(respage.getString("passed_event_crf_status_check") + "<br/>");
            } else {
                fail = true;
                msg.append(respage.getString("the_event_crf_not_correct_status") + "<br/>");
                auditMsg.append(respage.getString("the_event_crf_not_correct_status") + "<br/>");
            }
            // create a 'fake' request to generate the validation errors
            // here, tbh 05/2009
            MockHttpServletRequest request = new MockHttpServletRequest();
            // Locale locale = new Locale("en-US");
            request.addPreferredLocale(locale);
            try {
                List<DisplayItemBeanWrapper> tempDisplayItemBeanWrappers = new ArrayList<DisplayItemBeanWrapper>();
                tempDisplayItemBeanWrappers = getImportCRFDataService(dataSource).lookupValidationErrors(request, odmContainer, ub, totalValidationErrors, hardValidationErrors, permittedEventCRFIds);
                logger.debug("size of total validation errors: " + totalValidationErrors.size());
                displayItemBeanWrappers.addAll(tempDisplayItemBeanWrappers);
            } catch (NullPointerException npe1) {
                // what if you have 2 event crfs but the third is a fake?
                npe1.printStackTrace();
                fail = true;
                logger.debug("threw a NPE after calling lookup validation errors");
                msg.append(respage.getString("an_error_was_thrown_while_validation_errors") + "<br/>");
                auditMsg.append(respage.getString("an_error_was_thrown_while_validation_errors") + "<br/>");
                out.write(respage.getString("an_error_was_thrown_while_validation_errors") + "<br/>");
                logger.debug("=== threw the null pointer, import ===");
            } catch (OpenClinicaException oce1) {
                fail = true;
                logger.error("threw an OCE after calling lookup validation errors " + oce1.getOpenClinicaMessage());
                msg.append(oce1.getOpenClinicaMessage() + "<br/>");
                // auditMsg.append(oce1.getOpenClinicaMessage() + "<br/>");
                out.write(oce1.getOpenClinicaMessage() + "<br/>");
            }
        } else if (!eventCRFStatusesValid) {
            fail = true;
            msg.append(respage.getString("the_event_crf_not_correct_status"));
            out.write(respage.getString("the_event_crf_not_correct_status"));
            out.close();
            continue;
        } else {
            // fail = true;
            // break here with an exception
            msg.append(respage.getString("no_event_crfs_matching_the_xml_metadata") + "<br/>");
            // auditMsg.append(respage.getString("no_event_crfs_matching_the_xml_metadata")
            // + "<br/>");
            out.write(respage.getString("no_event_crfs_matching_the_xml_metadata") + "<br/>");
            // throw new Exception(msg.toString());
            out.close();
            continue;
        }
        ArrayList<SubjectDataBean> subjectData = odmContainer.getCrfDataPostImportContainer().getSubjectData();
        if (!hardValidationErrors.isEmpty()) {
            String messageHardVals = triggerService.generateHardValidationErrorMessage(subjectData, hardValidationErrors, false);
            // byte[] messageHardValsBytes = messageHardVals.getBytes();
            out.write(messageHardVals);
            msg.append(respage.getString("file_generated_hard_validation_error"));
            // here we create a file and append the data, tbh 06/2010
            fail = true;
        } else {
            if (!totalValidationErrors.isEmpty()) {
                String totalValErrors = triggerService.generateHardValidationErrorMessage(subjectData, totalValidationErrors, false);
                out.write(totalValErrors);
            // here we also append data to the file, tbh 06/2010
            }
            String validMsgs = triggerService.generateValidMessage(subjectData, totalValidationErrors);
            out.write(validMsgs);
        // third place to append data to the file? tbh 06/2010
        }
        // << tbh 05/2010, bug #5110, leave off the detailed reports
        out.close();
        if (fail) {
            // forwardPage(Page.IMPORT_CRF_DATA);
            // break here with an exception
            // throw new Exception("Problems encountered with file " +
            // f.getName() + ": " + msg.toString());
            MessageFormat mf = new MessageFormat("");
            mf.applyPattern(respage.getString("problems_encountered_with_file"));
            Object[] arguments = { f.getName(), msg.toString() };
            msg = new StringBuffer(mf.format(arguments) + "<br/>");
            out.close();
            auditMsg.append("You can see the log file <a href='" + SQLInitServlet.getField("sysURL.base") + "ViewLogMessage?n=" + generalFileDir + f.getName() + "&tn=" + triggerBean.getName() + "&gn=1'>here</a>.<br/>");
            msg.append("You can see the log file <a href='" + SQLInitServlet.getField("sysURL.base") + "ViewLogMessage?n=" + generalFileDir + f.getName() + "&tn=" + triggerBean.getName() + "&gn=1'>here</a>.<br/>");
            // ": " + msg.toString() + "<br/>");
            continue;
        } else {
            msg.append(respage.getString("passing_crf_edit_checks") + "<br/>");
            auditMsg.append(respage.getString("passing_crf_edit_checks") + "<br/>");
            // session.setAttribute("importedData",
            // displayItemBeanWrappers);
            // session.setAttribute("validationErrors",
            // totalValidationErrors);
            // session.setAttribute("hardValidationErrors",
            // hardValidationErrors);
            // above are to be sent to the user, but what kind of message
            // can we make of them here?
            // if hard validation errors are present, we only generate one
            // table
            // otherwise, we generate the other two: validation errors and
            // valid data
            logger.debug("found total validation errors: " + totalValidationErrors.size());
            SummaryStatsBean ssBean = getImportCRFDataService(dataSource).generateSummaryStatsBean(odmContainer, displayItemBeanWrappers, importCrfInfo);
            // msg.append("===+");
            // the above is a special key that we will use to split the
            // message into two parts
            // a shorter version for the audit and
            // a longer version for the email
            msg.append(triggerService.generateSummaryStatsMessage(ssBean, respage) + "<br/>");
            // session.setAttribute("summaryStats", ssBean);
            // will have to set hard edit checks here as well
            // session.setAttribute("subjectData",
            // ArrayList<SubjectDataBean> subjectData =
            // odmContainer.getCrfDataPostImportContainer().getSubjectData();
            // forwardPage(Page.VERIFY_IMPORT_SERVLET);
            // instead of forwarding, go ahead and save it all, sending a
            // message at the end
            msg.append(triggerService.generateSkippedCRFMessage(importCrfInfo, resword) + "<br/>");
            // setup ruleSets to run if applicable
            List<ImportDataRuleRunnerContainer> containers = this.ruleRunSetup(dataSource, studyBean, ub, ruleSetService, odmContainer);
            CrfBusinessLogicHelper crfBusinessLogicHelper = new CrfBusinessLogicHelper(dataSource);
            for (DisplayItemBeanWrapper wrapper : displayItemBeanWrappers) {
                boolean resetSDV = false;
                int eventCrfBeanId = -1;
                EventCRFBean eventCrfBean = new EventCRFBean();
                logger.debug("right before we check to make sure it is savable: " + wrapper.isSavable());
                if (wrapper.isSavable()) {
                    ArrayList<Integer> eventCrfInts = new ArrayList<Integer>();
                    logger.debug("wrapper problems found : " + wrapper.getValidationErrors().toString());
                    itemDataDao.setFormatDates(false);
                    for (DisplayItemBean displayItemBean : wrapper.getDisplayItemBeans()) {
                        eventCrfBeanId = displayItemBean.getData().getEventCRFId();
                        eventCrfBean = (EventCRFBean) eventCrfDao.findByPK(eventCrfBeanId);
                        logger.debug("found value here: " + displayItemBean.getData().getValue());
                        logger.debug("found status here: " + eventCrfBean.getStatus().getName());
                        ItemDataBean itemDataBean = new ItemDataBean();
                        itemDataBean = itemDataDao.findByItemIdAndEventCRFIdAndOrdinal(displayItemBean.getItem().getId(), eventCrfBean.getId(), displayItemBean.getData().getOrdinal());
                        if (wrapper.isOverwrite() && itemDataBean.getStatus() != null) {
                            logger.debug("just tried to find item data bean on item name " + displayItemBean.getItem().getName());
                            if (!itemDataBean.getValue().equals(displayItemBean.getData().getValue()))
                                resetSDV = true;
                            itemDataBean.setUpdatedDate(new Date());
                            itemDataBean.setUpdater(ub);
                            itemDataBean.setValue(displayItemBean.getData().getValue());
                            // set status?
                            itemDataDao.update(itemDataBean);
                            logger.debug("updated: " + itemDataBean.getItemId());
                            // need to set pk here in order to create dn
                            displayItemBean.getData().setId(itemDataBean.getId());
                        } else {
                            resetSDV = true;
                            itemDataDao.create(displayItemBean.getData());
                            logger.debug("created: " + displayItemBean.getData().getItemId());
                            ItemDataBean itemDataBean2 = itemDataDao.findByItemIdAndEventCRFIdAndOrdinal(displayItemBean.getItem().getId(), eventCrfBean.getId(), displayItemBean.getData().getOrdinal());
                            logger.debug("found: id " + itemDataBean2.getId() + " name " + itemDataBean2.getName());
                            displayItemBean.getData().setId(itemDataBean2.getId());
                        }
                        ItemDAO idao = new ItemDAO(dataSource);
                        ItemBean ibean = (ItemBean) idao.findByPK(displayItemBean.getData().getItemId());
                        logger.debug("*** checking for validation errors: " + ibean.getName());
                        String itemOid = displayItemBean.getItem().getOid() + "_" + wrapper.getStudyEventRepeatKey() + "_" + displayItemBean.getData().getOrdinal() + "_" + wrapper.getStudySubjectOid();
                        if (wrapper.getValidationErrors().containsKey(itemOid)) {
                            ArrayList messageList = (ArrayList) wrapper.getValidationErrors().get(itemOid);
                            for (int iter = 0; iter < messageList.size(); iter++) {
                                String message = (String) messageList.get(iter);
                                DiscrepancyNoteBean parentDn = createDiscrepancyNote(ibean, message, eventCrfBean, displayItemBean, null, ub, dataSource, studyBean);
                                createDiscrepancyNote(ibean, message, eventCrfBean, displayItemBean, parentDn.getId(), ub, dataSource, studyBean);
                                logger.debug("*** created disc note with message: " + message);
                            // displayItemBean);
                            }
                        }
                        // Update CRF status
                        if (!eventCrfInts.contains(new Integer(eventCrfBean.getId()))) {
                            String eventCRFStatus = importedCRFStatuses.get(new Integer(eventCrfBean.getId()));
                            if (eventCRFStatus != null && eventCRFStatus.equals(DataEntryStage.INITIAL_DATA_ENTRY.getName()) && eventCrfBean.getStatus().isAvailable()) {
                                crfBusinessLogicHelper.markCRFStarted(eventCrfBean, ub);
                            } else {
                                crfBusinessLogicHelper.markCRFComplete(eventCrfBean, ub);
                            }
                            logger.debug("*** just updated event crf bean: " + eventCrfBean.getId());
                            eventCrfInts.add(new Integer(eventCrfBean.getId()));
                        }
                    }
                    itemDataDao.setFormatDates(true);
                    // Reset the SDV status if item data has been changed or added
                    if (eventCrfBean != null && resetSDV)
                        eventCrfDao.setSDVStatus(false, ub.getId(), eventCrfBean.getId());
                }
            }
            // msg.append("===+");
            msg.append(respage.getString("data_has_been_successfully_import") + "<br/>");
            auditMsg.append(respage.getString("data_has_been_successfully_import") + "<br/>");
            // MessageFormat mf = new MessageFormat("");
            String linkMessage = respage.getString("you_can_review_the_data") + SQLInitServlet.getField("sysURL.base") + respage.getString("you_can_review_the_data_2") + SQLInitServlet.getField("sysURL.base") + respage.getString("you_can_review_the_data_3") + generalFileDir + f.getName() + "&tn=" + triggerBean.getFullName() + "&gn=1" + respage.getString("you_can_review_the_data_4") + "<br/>";
            // mf.applyPattern(respage.getString("you_can_review_the_data"));
            // Object[] arguments = {
            // SQLInitServlet.getField("sysURL.base"),
            // SQLInitServlet.getField("sysURL.base"), f.getName() };
            msg.append(linkMessage);
            auditMsg.append(linkMessage);
            // was here but is now moved up, tbh
            // String finalLine =
            // "<p>You can review the entered data <a href='" +
            // SQLInitServlet.getField("sysURL.base") +
            // "ListStudySubjects'>here</a>.";
            // >> tbh additional message
            // "you can review the validation messages here" <-- where
            // 'here' is a link to view an external file
            // i.e. /ViewExternal?n=file_name.txt
            // << tbh 06/2010
            // msg.append(finalLine);
            // auditMsg.append(finalLine);
            auditMsg.append(this.runRules(studyBean, ub, containers, ruleSetService, ExecutionMode.SAVE));
        }
    }
    // end for loop
    // is the writer still not closed? try to close it
    ArrayList<String> retList = new ArrayList<String>();
    retList.add(msg.toString());
    retList.add(auditMsg.toString());
    // msg.toString();
    return retList;
}