Search in sources :

Example 1 with DataReaderService

use of com.axelor.apps.tool.reader.DataReaderService in project axelor-open-suite by axelor.

the class DataImportServiceImpl method process.

private List<CSVInput> process(DataReaderService reader, AdvancedImport advancedImport) throws AxelorException, IOException, ClassNotFoundException {
    String[] sheets = reader.getSheetNames();
    boolean isConfig = advancedImport.getIsConfigInFile();
    int linesToIgnore = advancedImport.getNbOfFirstLineIgnore();
    boolean isTabConfig = advancedImport.getIsFileTabConfigAdded();
    List<CSVInput> inputList = new ArrayList<CSVInput>();
    validatorService.sortFileTabList(advancedImport.getFileTabList());
    for (FileTab fileTab : advancedImport.getFileTabList()) {
        if (!Arrays.stream(sheets).anyMatch(sheet -> sheet.equals(fileTab.getName()))) {
            continue;
        }
        this.initializeVariables();
        String fileName = createDataFileName(fileTab);
        csvInput = this.createCSVInput(fileTab, fileName);
        ifList = new ArrayList<String>();
        try (CSVWriter csvWriter = new CSVWriter(new FileWriter(new File(dataDir, fileName)), CSV_SEPRATOR)) {
            int totalLines = reader.getTotalLines(fileTab.getName());
            if (totalLines == 0) {
                continue;
            }
            Mapper mapper = advancedImportService.getMapper(fileTab.getMetaModel().getFullName());
            List<String[]> allLines = new ArrayList<String[]>();
            int startIndex = isConfig ? 1 : linesToIgnore;
            String[] row = reader.read(fileTab.getName(), startIndex, 0);
            String[] headers = this.createHeader(row, fileTab, isConfig, mapper);
            allLines.add(headers);
            int tabConfigRowCount = 0;
            if (isTabConfig) {
                String[] objectRow = reader.read(fileTab.getName(), 0, 0);
                tabConfigRowCount = advancedImportService.getTabConfigRowCount(fileTab.getName(), reader, totalLines, objectRow);
            }
            startIndex = isConfig ? tabConfigRowCount + 3 : fileTab.getAdvancedImport().getIsHeader() ? linesToIgnore + 1 : linesToIgnore;
            for (int line = startIndex; line < totalLines; line++) {
                String[] dataRow = reader.read(fileTab.getName(), line, row.length);
                if (dataRow == null || Arrays.stream(dataRow).allMatch(StringUtils::isBlank)) {
                    continue;
                }
                String[] data = this.createData(dataRow, fileTab, isConfig, mapper);
                allLines.add(data);
            }
            csvWriter.writeAll(allLines);
            csvWriter.flush();
        }
        inputList.add(csvInput);
        importContext.put("ifConditions" + fileTab.getId(), ifList);
        importContext.put("jsonContextValues" + fileTab.getId(), createJsonContext(fileTab));
        importContext.put("actionsToApply" + fileTab.getId(), fileTab.getActions());
        XStream stream = XStreamUtils.createXStream();
        stream.processAnnotations(CSVConfig.class);
        LOG.debug("CSV Config created :" + "\n" + stream.toXML(csvInput));
    }
    return inputList;
}
Also used : XStream(com.thoughtworks.xstream.XStream) Arrays(java.util.Arrays) Date(java.util.Date) Inject(com.google.inject.Inject) CSVInput(com.axelor.data.csv.CSVInput) LoggerFactory(org.slf4j.LoggerFactory) Mapper(com.axelor.db.mapper.Mapper) JsonContext(com.axelor.rpc.JsonContext) FileField(com.axelor.apps.base.db.FileField) ByteArrayInputStream(java.io.ByteArrayInputStream) Map(java.util.Map) Inflector(com.axelor.common.Inflector) ZipEntry(java.util.zip.ZipEntry) TranslationService(com.axelor.apps.tool.service.TranslationService) ZipException(java.util.zip.ZipException) MethodHandles(java.lang.invoke.MethodHandles) CSVImporter(com.axelor.data.csv.CSVImporter) StringUtils(com.axelor.common.StringUtils) DataAdapter(com.axelor.data.adapter.DataAdapter) MetaModel(com.axelor.meta.db.MetaModel) List(java.util.List) CSVBind(com.axelor.data.csv.CSVBind) MetaSelectRepository(com.axelor.meta.db.repo.MetaSelectRepository) Entry(java.util.Map.Entry) FileTab(com.axelor.apps.base.db.FileTab) AdvancedImport(com.axelor.apps.base.db.AdvancedImport) ZipInputStream(java.util.zip.ZipInputStream) MetaSelectItem(com.axelor.meta.db.MetaSelectItem) Property(com.axelor.db.mapper.Property) SimpleDateFormat(java.text.SimpleDateFormat) HashMap(java.util.HashMap) ImporterListener(com.axelor.apps.base.service.imports.listener.ImporterListener) ArrayList(java.util.ArrayList) Strings(com.google.common.base.Strings) AxelorException(com.axelor.exception.AxelorException) Files(com.google.common.io.Files) CollectionUtils(org.apache.commons.collections.CollectionUtils) XStreamUtils(com.axelor.data.XStreamUtils) MetaSelectItemRepository(com.axelor.meta.db.repo.MetaSelectItemRepository) DataReaderFactory(com.axelor.apps.tool.reader.DataReaderFactory) MetaFiles(com.axelor.meta.MetaFiles) Logger(org.slf4j.Logger) Model(com.axelor.db.Model) FileWriter(java.io.FileWriter) TraceBackService(com.axelor.exception.service.TraceBackService) FileOutputStream(java.io.FileOutputStream) CSVWriter(com.opencsv.CSVWriter) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) JavaTimeAdapter(com.axelor.data.adapter.JavaTimeAdapter) FileInputStream(java.io.FileInputStream) File(java.io.File) DataReaderService(com.axelor.apps.tool.reader.DataReaderService) MetaFile(com.axelor.meta.db.MetaFile) FileFieldRepository(com.axelor.apps.base.db.repo.FileFieldRepository) CSVConfig(com.axelor.data.csv.CSVConfig) MetaSelect(com.axelor.meta.db.MetaSelect) Context(com.axelor.rpc.Context) XStream(com.thoughtworks.xstream.XStream) FileWriter(java.io.FileWriter) ArrayList(java.util.ArrayList) CSVInput(com.axelor.data.csv.CSVInput) CSVWriter(com.opencsv.CSVWriter) Mapper(com.axelor.db.mapper.Mapper) FileTab(com.axelor.apps.base.db.FileTab) File(java.io.File) MetaFile(com.axelor.meta.db.MetaFile)

Example 2 with DataReaderService

use of com.axelor.apps.tool.reader.DataReaderService in project axelor-open-suite by axelor.

the class DmnImportServiceImpl method importDmnTable.

@Override
public void importDmnTable(MetaFile dataFile, WkfDmnModel dmnModel) throws AxelorException {
    String extension = Files.getFileExtension(dataFile.getFileName());
    if (extension == null || (!extension.equals("xlsx") && !extension.equals("xls"))) {
        throw new AxelorException(TraceBackRepository.CATEGORY_CONFIGURATION_ERROR, IExceptionMessage.INVALID_IMPORT_FILE);
    }
    DataReaderService reader = dataReaderFactory.getDataReader(extension);
    reader.initialize(dataFile, null);
    this.process(reader, dmnModel);
}
Also used : AxelorException(com.axelor.exception.AxelorException) DataReaderService(com.axelor.apps.tool.reader.DataReaderService)

Example 3 with DataReaderService

use of com.axelor.apps.tool.reader.DataReaderService in project axelor-open-suite by axelor.

the class AdvancedImportServiceImpl method apply.

@Override
public boolean apply(AdvancedImport advancedImport) throws AxelorException, ClassNotFoundException {
    if (advancedImport.getImportFile() == null) {
        throw new AxelorException(TraceBackRepository.CATEGORY_CONFIGURATION_ERROR, I18n.get(IExceptionMessage.ADVANCED_IMPORT_NO_IMPORT_FILE));
    }
    String extension = Files.getFileExtension(advancedImport.getImportFile().getFileName());
    if (extension == null || (!extension.equals("xlsx") && !extension.equals("xls") && !extension.equals("csv"))) {
        return false;
    }
    DataReaderService reader = dataReaderFactory.getDataReader(extension);
    reader.initialize(advancedImport.getImportFile(), advancedImport.getFileSeparator());
    return this.process(reader, advancedImport);
}
Also used : AxelorException(com.axelor.exception.AxelorException) DataReaderService(com.axelor.apps.tool.reader.DataReaderService)

Example 4 with DataReaderService

use of com.axelor.apps.tool.reader.DataReaderService in project axelor-open-suite by axelor.

the class ValidatorService method validate.

public boolean validate(AdvancedImport advancedImport) throws AxelorException, IOException, ClassNotFoundException {
    if (advancedImport.getImportFile() == null) {
        throw new AxelorException(TraceBackRepository.CATEGORY_CONFIGURATION_ERROR, I18n.get(IExceptionMessage.ADVANCED_IMPORT_NO_IMPORT_FILE));
    }
    if (advancedImport.getAttachment() != null && !Files.getFileExtension(advancedImport.getAttachment().getFileName()).equals("zip")) {
        throw new AxelorException(TraceBackRepository.CATEGORY_CONFIGURATION_ERROR, I18n.get(IExceptionMessage.ADVANCED_IMPORT_ATTACHMENT_FORMAT));
    }
    String extension = Files.getFileExtension(advancedImport.getImportFile().getFileName());
    if (extension == null || (!extension.equals("xlsx") && !extension.equals("xls") && !extension.equals("csv"))) {
        throw new AxelorException(TraceBackRepository.CATEGORY_CONFIGURATION_ERROR, I18n.get(IExceptionMessage.ADVANCED_IMPORT_FILE_FORMAT_INVALID));
    }
    DataReaderService reader = dataReaderFactory.getDataReader(extension);
    reader.initialize(advancedImport.getImportFile(), advancedImport.getFileSeparator());
    return validate(reader, advancedImport);
}
Also used : AxelorException(com.axelor.exception.AxelorException) DataReaderService(com.axelor.apps.tool.reader.DataReaderService)

Example 5 with DataReaderService

use of com.axelor.apps.tool.reader.DataReaderService in project axelor-open-suite by axelor.

the class ValidatorService method validate.

@Transactional(rollbackOn = { AxelorException.class, Exception.class })
public boolean validate(DataReaderService reader, AdvancedImport advancedImport) throws IOException, ClassNotFoundException, AxelorException {
    boolean isLog = false;
    String[] sheets = reader.getSheetNames();
    this.validateTab(sheets, advancedImport);
    boolean isConfig = advancedImport.getIsConfigInFile();
    boolean isTabConfig = advancedImport.getIsFileTabConfigAdded();
    sortFileTabList(advancedImport.getFileTabList());
    for (FileTab fileTab : advancedImport.getFileTabList()) {
        if (!Arrays.stream(sheets).anyMatch(sheet -> sheet.equals(fileTab.getName()))) {
            continue;
        }
        fieldMap = new HashMap<>();
        titleMap = new HashMap<>();
        String sheet = fileTab.getName();
        logService.initialize(sheet);
        this.validateModel(fileTab);
        int tabConfigRowCount = 0;
        int totalLines = reader.getTotalLines(fileTab.getName());
        if (isConfig) {
            String[] objectRow = reader.read(sheet, 0, 0);
            if (isTabConfig) {
                tabConfigRowCount = advancedImportService.getTabConfigRowCount(sheet, reader, totalLines, objectRow);
            }
            this.validateObject(objectRow, fileTab, isTabConfig);
        }
        this.validateSearch(fileTab);
        this.validateObjectRequiredFields(fileTab);
        this.validateFieldAndData(reader, sheet, fileTab, isConfig, isTabConfig, tabConfigRowCount);
        this.validateActions(fileTab);
        if (fileTab.getValidationLog() != null) {
            fileTab.setValidationLog(null);
        }
        if (logService.isLogGenerated()) {
            logService.write();
            logService.close();
            File logFile = logService.getLogFile();
            fileTab.setValidationLog(metaFiles.upload(new FileInputStream(logFile), sheet + "_err.xlsx"));
            logFile.delete();
            isLog = true;
        } else {
            createCustomObjectSet(fileTab.getClass().getName(), fileTab.getMetaModel().getFullName(), 0);
            createCustomButton(fileTab.getClass().getName(), fileTab.getMetaModel().getFullName(), 1);
        }
        fileTabRepo.save(fileTab);
    }
    return isLog;
}
Also used : MetaJsonFieldRepository(com.axelor.meta.db.repo.MetaJsonFieldRepository) Arrays(java.util.Arrays) Inject(com.google.inject.Inject) ZonedDateTime(java.time.ZonedDateTime) LocalDateTime(java.time.LocalDateTime) Property(com.axelor.db.mapper.Property) HashMap(java.util.HashMap) Mapper(com.axelor.db.mapper.Mapper) StringUtils(org.apache.commons.lang3.StringUtils) Transactional(com.google.inject.persist.Transactional) ArrayList(java.util.ArrayList) FileField(com.axelor.apps.base.db.FileField) Strings(com.google.common.base.Strings) BigDecimal(java.math.BigDecimal) AxelorException(com.axelor.exception.AxelorException) Files(com.google.common.io.Files) CollectionUtils(org.apache.commons.collections.CollectionUtils) Map(java.util.Map) LocalTime(java.time.LocalTime) I18n(com.axelor.i18n.I18n) BigInteger(java.math.BigInteger) Inflector(com.axelor.common.Inflector) MetaJsonField(com.axelor.meta.db.MetaJsonField) DataReaderFactory(com.axelor.apps.tool.reader.DataReaderFactory) MetaFiles(com.axelor.meta.MetaFiles) Model(com.axelor.db.Model) TraceBackRepository(com.axelor.exception.db.repo.TraceBackRepository) MetaField(com.axelor.meta.db.MetaField) BufferedWriter(java.io.BufferedWriter) TraceBackService(com.axelor.exception.service.TraceBackService) IOException(java.io.IOException) FileInputStream(java.io.FileInputStream) Collectors(java.util.stream.Collectors) FileTabRepository(com.axelor.apps.base.db.repo.FileTabRepository) File(java.io.File) DataReaderService(com.axelor.apps.tool.reader.DataReaderService) DateTimeParseException(java.time.format.DateTimeParseException) List(java.util.List) FileFieldRepository(com.axelor.apps.base.db.repo.FileFieldRepository) IExceptionMessage(com.axelor.apps.base.exceptions.IExceptionMessage) LocalDate(java.time.LocalDate) DateTimeFormatter(java.time.format.DateTimeFormatter) FileTab(com.axelor.apps.base.db.FileTab) AdvancedImport(com.axelor.apps.base.db.AdvancedImport) FileTab(com.axelor.apps.base.db.FileTab) File(java.io.File) FileInputStream(java.io.FileInputStream) Transactional(com.google.inject.persist.Transactional)

Aggregations

DataReaderService (com.axelor.apps.tool.reader.DataReaderService)6 AxelorException (com.axelor.exception.AxelorException)5 AdvancedImport (com.axelor.apps.base.db.AdvancedImport)2 FileField (com.axelor.apps.base.db.FileField)2 FileTab (com.axelor.apps.base.db.FileTab)2 FileFieldRepository (com.axelor.apps.base.db.repo.FileFieldRepository)2 DataReaderFactory (com.axelor.apps.tool.reader.DataReaderFactory)2 Inflector (com.axelor.common.Inflector)2 DataAdapter (com.axelor.data.adapter.DataAdapter)2 CSVInput (com.axelor.data.csv.CSVInput)2 Model (com.axelor.db.Model)2 Mapper (com.axelor.db.mapper.Mapper)2 Property (com.axelor.db.mapper.Property)2 TraceBackService (com.axelor.exception.service.TraceBackService)2 MetaFiles (com.axelor.meta.MetaFiles)2 MetaFile (com.axelor.meta.db.MetaFile)2 Strings (com.google.common.base.Strings)2 Files (com.google.common.io.Files)2 Inject (com.google.inject.Inject)2 File (java.io.File)2