Examples with CSVConfig com.axelor.data.csv.CSVConfig used on opensource projects

Example 1 with CSVConfig

use of com.axelor.data.csv.CSVConfig in project axelor-open-suite by axelor.

the class DataImportServiceImpl method importData.

private MetaFile importData(List<CSVInput> inputs) throws IOException {
    if (CollectionUtils.isEmpty(inputs)) {
        return null;
    }
    CSVConfig config = new CSVConfig();
    config.setInputs(inputs);
    if (!CollectionUtils.isEmpty(adapterMap.values())) {
        config.getAdapters().addAll(adapterMap.values());
    }
    CSVImporter importer = new CSVImporter(config, dataDir.getAbsolutePath());
    ImporterListener listener = new ImporterListener("importData");
    importer.addListener(listener);
    importer.setContext(importContext);
    importer.run();
    if (!listener.isImported()) {
        MetaFile logFile = this.createImportLogFile(listener);
        return logFile;
    }
    return null;
}

Example 2 with CSVConfig

use of com.axelor.data.csv.CSVConfig in project axelor-open-suite by axelor.

the class AppServiceImpl method refreshApp.

@Override
public void refreshApp() throws IOException {
    File dataDir = Files.createTempDirectory(null).toFile();
    File imgDir = new File(dataDir, "img");
    imgDir.mkdir();
    CSVConfig csvConfig = new CSVConfig();
    csvConfig.setInputs(new ArrayList<>());
    List<MetaModel> metaModels = metaModelRepo.all().filter("self.name != 'App' and self.name like 'App%' and self.packageName =  ?1", App.class.getPackage().getName()).fetch();
    final List<String> appFieldTargetList = Stream.of(JPA.fields(App.class)).filter(p -> p.getType() == PropertyType.ONE_TO_ONE).filter(p -> p.getName().startsWith("app")).map(Property::getTarget).map(Class::getName).collect(Collectors.toList());
    log.debug("Total app models: {}", metaModels.size());
    for (MetaModel metaModel : metaModels) {
        if (!appFieldTargetList.contains(metaModel.getFullName())) {
            log.debug("Not a App class : {}", metaModel.getName());
            continue;
        }
        Class<?> klass;
        try {
            klass = Class.forName(metaModel.getFullName());
        } catch (ClassNotFoundException e) {
            continue;
        }
        Object obj = null;
        Query query = JPA.em().createQuery("SELECT id FROM " + metaModel.getName());
        try {
            obj = query.setMaxResults(1).getSingleResult();
        } catch (Exception ex) {
        }
        if (obj != null) {
            continue;
        }
        log.debug("App without app record: {}", metaModel.getName());
        String csvName = "base_" + inflector.camelize(klass.getSimpleName(), true) + ".csv";
        String pngName = inflector.dasherize(klass.getSimpleName()) + ".png";
        CSVInput input = new CSVInput();
        input.setFileName(csvName);
        input.setTypeName(App.class.getName());
        input.setCallable("com.axelor.csv.script.ImportApp:importApp");
        input.setSearch("self.code =:code");
        input.setSeparator(';');
        csvConfig.getInputs().add(input);
        CSVInput appInput = new CSVInput();
        appInput.setFileName(csvName);
        appInput.setTypeName(klass.getName());
        appInput.setSearch("self.app.code =:code");
        appInput.setSeparator(';');
        CSVBind appBind = new CSVBind();
        appBind.setColumn("code");
        appBind.setField("app");
        appBind.setSearch("self.code = :code");
        appInput.getBindings().add(appBind);
        csvConfig.getInputs().add(appInput);
        InputStream stream = klass.getResourceAsStream("/data-init/input/" + csvName);
        copyStream(stream, new File(dataDir, csvName));
        stream = klass.getResourceAsStream("/data-init/input/img/" + pngName);
        copyStream(stream, new File(imgDir, pngName));
    }
    if (!csvConfig.getInputs().isEmpty()) {
        CSVImporter importer = new CSVImporter(csvConfig, dataDir.getAbsolutePath());
        importer.run();
    }
}

Example 3 with CSVConfig

use of com.axelor.data.csv.CSVConfig in project axelor-open-suite by axelor.

the class DataBackupCreateService method create.

/* Generate csv Files for each individual MetaModel and single config file */
public DataBackup create(DataBackup dataBackup) throws IOException {
    File tempDir = Files.createTempDirectory(null).toFile();
    String tempDirectoryPath = tempDir.getAbsolutePath();
    int fetchLimit = dataBackup.getFetchLimit();
    int errorsCount = 0;
    fileNameList = new ArrayList<>();
    List<MetaModel> metaModelList = getMetaModels();
    LinkedList<CSVInput> simpleCsvs = new LinkedList<>();
    LinkedList<CSVInput> refernceCsvs = new LinkedList<>();
    LinkedList<CSVInput> notNullReferenceCsvs = new LinkedList<>();
    Map<String, List<String>> subClassesMap = getSubClassesMap();
    if (dataBackup.getCheckAllErrorFirst()) {
        dataBackup.setFetchLimit(1);
        errorsCount = checkErrors(dataBackup, metaModelList, tempDirectoryPath, subClassesMap);
        dataBackup.setFetchLimit(fetchLimit);
        fileNameList.clear();
    }
    if (errorsCount == 0) {
        for (MetaModel metaModel : metaModelList) {
            try {
                List<String> subClasses = subClassesMap.get(metaModel.getFullName());
                long totalRecord = getMetaModelDataCount(metaModel, subClasses);
                if (totalRecord > 0) {
                    LOG.debug("Exporting Model : " + metaModel.getFullName());
                    notNullReferenceFlag = false;
                    referenceFlag = false;
                    CSVWriter csvWriter = new CSVWriter(new FileWriter(new File(tempDirectoryPath, metaModel.getName() + ".csv")), SEPARATOR, QUOTE_CHAR);
                    CSVInput csvInput = writeCSVData(metaModel, csvWriter, dataBackup, totalRecord, subClasses, tempDirectoryPath);
                    csvWriter.close();
                    if (notNullReferenceFlag) {
                        notNullReferenceCsvs.add(csvInput);
                    } else if (referenceFlag) {
                        refernceCsvs.add(csvInput);
                        CSVInput temcsv = new CSVInput();
                        temcsv.setFileName(csvInput.getFileName());
                        temcsv.setTypeName(csvInput.getTypeName());
                        if (dataBackup.getIsRelativeDate()) {
                            temcsv.setBindings(new ArrayList<>());
                            getCsvInputForDateorDateTime(metaModel, temcsv);
                        }
                        if (AutoImportModelMap.containsKey(csvInput.getTypeName())) {
                            temcsv.setSearch(AutoImportModelMap.get(csvInput.getTypeName()).toString());
                        }
                        if (Class.forName(metaModel.getFullName()).getSuperclass() == App.class) {
                            temcsv.setSearch("self.code = :code");
                        }
                        if (!AutoImportModelMap.containsKey(csvInput.getTypeName()) && !((Class.forName(metaModel.getFullName()).getSuperclass()).equals(App.class))) {
                            temcsv.setSearch("self.importId = :importId");
                        }
                        simpleCsvs.add(temcsv);
                    } else {
                        simpleCsvs.add(csvInput);
                    }
                    fileNameList.add(metaModel.getName() + ".csv");
                }
            } catch (ClassNotFoundException | IOException e) {
                TraceBackService.trace(e, DataBackupService.class.getName());
            } catch (Exception e) {
                JPA.em().getTransaction().rollback();
                if (!dataBackup.getCheckAllErrorFirst()) {
                    sb.append("\nError occured while processing model : " + metaModel.getFullName() + "\n");
                    sb.append(e.getMessage() + "\n");
                }
                JPA.em().getTransaction().begin();
                dataBackup = Beans.get(DataBackupRepository.class).find(dataBackup.getId());
                errorsCount++;
            }
        }
        CSVConfig csvConfig = new CSVConfig();
        csvConfig.setInputs(simpleCsvs);
        csvConfig.getInputs().addAll(notNullReferenceCsvs);
        csvConfig.getInputs().addAll(refernceCsvs);
        csvConfig.getInputs().addAll(notNullReferenceCsvs);
        generateConfig(tempDirectoryPath, csvConfig);
        fileNameList.add(DataBackupServiceImpl.CONFIG_FILE_NAME);
    }
    try {
        if (!Strings.isNullOrEmpty(sb.toString())) {
            DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmSS");
            String logFileName = "DataBackupLog_" + LocalDateTime.now().format(formatter) + ".log";
            File file = new File(tempDir.getAbsolutePath(), logFileName);
            PrintWriter pw = new PrintWriter(file);
            pw.write(sb.toString());
            pw.close();
            if (file != null) {
                dataBackup.setLogMetaFile(metaFiles.upload(file));
            }
        }
        if (errorsCount == 0) {
            File zippedFile = generateZIP(tempDirectoryPath, fileNameList);
            dataBackup.setBackupMetaFile(metaFiles.upload(zippedFile));
        } else {
            dataBackup.setStatusSelect(DataBackupRepository.DATA_BACKUP_STATUS_ERROR);
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return dataBackup;
}