Examples with CsvToBeanBuilder com.opencsv.bean.CsvToBeanBuilder used on opensource projects

Example 6 with CsvToBeanBuilder

use of com.opencsv.bean.CsvToBeanBuilder in project Java-Tutorial by gpcodervn.

the class ParseCsvToObjectUsingAnnotation method main.

public static void main(String[] args) throws IOException {
    String csvFile = "data/data.csv";
    try (Reader reader = new FileReader(csvFile)) {
        CsvToBean<CountryAnn> csvToBean = new CsvToBeanBuilder<CountryAnn>(reader).withType(CountryAnn.class).withSkipLines(// skip header
        1).withIgnoreLeadingWhiteSpace(true).build();
        List<CountryAnn> contries = csvToBean.parse();
        for (CountryAnn contry : contries) {
            System.out.println(contry);
        }
    }
}

Example 7 with CsvToBeanBuilder

use of com.opencsv.bean.CsvToBeanBuilder in project cassandre-trading-bot by cassandre-tech.

the class StrategiesAutoConfiguration method loadTickersFromFiles.

/**
 * Load tickers in database.
 */
private void loadTickersFromFiles() {
    // Deleting everything before import.
    importedTickerRepository.deleteAllInBatch();
    // Getting the list of files to import and insert them in database.
    logger.info("Importing tickers...");
    AtomicLong counter = new AtomicLong(0);
    getFilesToLoad("classpath*:tickers-to-import*csv").stream().filter(resource -> resource.getFilename() != null).peek(resource -> logger.info("Importing tickers from {}", resource.getFilename())).forEach(resource -> {
        try {
            // Insert the tickers in database.
            new CsvToBeanBuilder<ImportedTicker>(Files.newBufferedReader(resource.getFile().toPath())).withType(ImportedTicker.class).withIgnoreLeadingWhiteSpace(true).build().parse().forEach(importedTicker -> {
                logger.debug("Importing ticker {}", importedTicker);
                importedTicker.setUid(counter.incrementAndGet());
                importedTickerRepository.save(importedTicker);
            });
        } catch (IOException e) {
            logger.error("Impossible to load imported tickers: {}", e.getMessage());
        }
    });
    logger.info("{} tickers imported", importedTickerRepository.count());
}

Example 8 with CsvToBeanBuilder

use of com.opencsv.bean.CsvToBeanBuilder in project streamliner by phdata.

the class GenerateStateCommand method build.

public static void build(String tableExclude, String tableInclude, String columnInclude, String columnExclude, String tableNameRemove, String columnNameRemove, String outputDir, String sourceSystemStateDirectory, String tableCsvPath, String columnCsvPath) {
    validateOutputDirectory(outputDir);
    validateSourceStateDirectory(sourceSystemStateDirectory);
    List<CSVTables> csvTables;
    List<CSVColumns> csvColumns;
    try {
        csvTables = new CsvToBeanBuilder(new FileReader(tableCsvPath)).withType(CSVTables.class).build().parse();
        csvColumns = new CsvToBeanBuilder(new FileReader(columnCsvPath)).withType(CSVColumns.class).build().parse();
    } catch (FileNotFoundException e) {
        throw new RuntimeException("File not found.", e);
    }
    Configuration sourceSchemaFile = StreamlinerUtil.createConfig(sourceSystemStateDirectory, null, "--source-state-directory");
    // log table count and any missing tables.
    logTablesDetail(csvTables, sourceSchemaFile.getTables());
    /*
    Assuming at a time either tableExclude or tableInclude will be applied.
    This will exlude or include the tables based on regex provided.
    */
    if (tableExclude != null) {
        excludeTables(csvTables, tableExclude);
    } else if (tableInclude != null) {
        includeTables(csvTables, tableInclude);
    }
    // This will remove the regex from table name.
    if (tableNameRemove != null) {
        removeTableName(csvTables, tableNameRemove);
    }
    Map<String, TableDefinition> sourceTableDefMap = createTableDefMap(sourceSchemaFile.getTables());
    TemplateContext context = new TemplateContext();
    // schema crawler sorts by lower case name so we ensure that behavior here
    csvTables.sort(Comparator.comparing(csvTable -> csvTable.getTableName().toLowerCase()));
    List<TableDefinition> tableDef = csvTables.stream().map(csvTable -> {
        TableDefinition sourceTableDef = sourceTableDefMap.get(csvTable.getTableName());
        // printed.
        if (sourceTableDef == null) {
            context.addError(String.format("Table '%s' not found in Source schema. Unable to get the Primary key for this table.", csvTable.getTableName()));
        }
        Map<String, ColumnDefinition> sourceColDefMap = null;
        if (sourceTableDef != null) {
            sourceColDefMap = createColumnDefMap(sourceTableDef.getColumns());
        }
        Map<String, ColumnDefinition> finalSourceColDefMap = sourceColDefMap;
        // iterate csvColumns and filter all the columns related to a table.
        List<CSVColumns> csvTableCols = csvColumns.stream().filter(csvColumn -> csvColumn.getTableName().equals(csvTable.getTableName())).collect(Collectors.toList());
        // columns are sorted based on ordinal position
        csvTableCols.sort(Comparator.comparing(column -> Integer.parseInt(column.getOrdinalPosition())));
        // log columns count and any missing columns.
        if (finalSourceColDefMap != null) {
            logColumnsDetail(csvTable, sourceTableDef, csvTableCols);
        }
        /*
                  Assuming at a time either columnExclude or columnInclude will be applied.
                  This will exlude or include the columns based on regex provided.
                  */
        if (columnExclude != null) {
            excludeColumns(csvTable, csvTableCols, columnExclude);
        } else if (columnInclude != null) {
            includeColumns(csvTable, csvTableCols, columnInclude);
        }
        // This will remove the regex from column name.
        if (columnNameRemove != null) {
            removeColumnName(csvTable, csvTableCols, columnNameRemove);
        }
        /* In case snowflake table does not exist in source schema, existing snowflake dataType, precision, scale and empty primary key is stored in state file.
                   * In case some columns are missing from source schema, existing snowflake dataType, precision, scale is stored in state file.
                   * Else values are picked from source schema file.*/
        List<ColumnDefinition> d = csvTableCols.stream().map(csvCol -> {
            String dataType = finalSourceColDefMap != null ? finalSourceColDefMap.get(csvCol.getColumnName()) == null ? csvCol.getDataType() : finalSourceColDefMap.get(csvCol.getColumnName()).getDataType() : csvCol.getDataType();
            Integer precision = finalSourceColDefMap != null ? finalSourceColDefMap.get(csvCol.getColumnName()) == null ? Integer.parseInt(csvCol.getPrecision()) : finalSourceColDefMap.get(csvCol.getColumnName()).getPrecision() : Integer.parseInt(csvCol.getPrecision());
            Integer scale = finalSourceColDefMap != null ? finalSourceColDefMap.get(csvCol.getColumnName()) == null ? Integer.parseInt(csvCol.getScale()) : finalSourceColDefMap.get(csvCol.getColumnName()).getScale() : Integer.parseInt(csvCol.getScale());
            return new ColumnDefinition(csvCol.getColumnName(), csvCol.getColumnName(), dataType, csvCol.getComment(), precision, scale, csvCol.getIsNullable().equals("YES"));
        }).collect(Collectors.toList());
        return new SnowflakeTable(csvTable.getTableName(), csvTable.getTableName(), csvTable.getTableComment(), sourceTableDef != null ? sourceTableDef.getPrimaryKeys() : new ArrayList<>(), null, null, null, new FileFormat(csvTable.getTableName(), "PARQUET"), d);
    }).collect(Collectors.toList());
    tableDef.forEach(table -> {
        List<TableDefinition> tableList = new ArrayList<>();
        tableList.add(table);
        Configuration conf = new Configuration(tableList);
        StreamlinerUtil.writeYamlFile(conf, String.format("%s/%s.yml", outputDir, table.getSourceName()));
    });
    if (context.hasErrors()) {
        List<String> errors = context.getErrors();
        String msg = String.format("There are %d errors which require investigation.", errors.size());
        errors.forEach(log::error);
        log.warn(msg);
    }
}

Example 9 with CsvToBeanBuilder

use of com.opencsv.bean.CsvToBeanBuilder in project C206_C125 by chrislima-inatel.

the class CsvUtils method readGameCsv.

public static List<Game> readGameCsv(Path csvFilePath) {
    List<Game> gameList = new ArrayList<>();
    try {
        Reader reader = Files.newBufferedReader(csvFilePath);
        CsvToBean<Game> csvToBean = new CsvToBeanBuilder(reader).withType(Game.class).withIgnoreLeadingWhiteSpace(true).build();
        gameList = csvToBean.parse();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return gameList;
}

Example 10 with CsvToBeanBuilder

use of com.opencsv.bean.CsvToBeanBuilder in project incubator-inlong by apache.

the class CommonDBServerController method upload.

/**
 * Batch import - upload import config
 */
@RequestMapping(value = "/upload", method = RequestMethod.POST)
@ApiOperation(value = "Batch Import - upload config")
public String upload(@RequestParam("file") MultipartFile file) throws Exception {
    Preconditions.checkFalse(file.isEmpty(), "File cannot be empty");
    String fileName = file.getOriginalFilename();
    LOGGER.info("The name of the uploaded file is: " + fileName);
    String filePath = ClassUtils.getDefaultClassLoader().getResource("").getPath();
    File fileDir = new File(filePath, "common_server");
    if (!fileDir.exists()) {
        fileDir.mkdirs();
    }
    String username = LoginUserUtils.getLoginUserDetail().getUserName();
    String time = new SimpleDateFormat("yyyyMMdd-HHmmss").format(new Date());
    fileName = username + "_" + time + "_" + fileName;
    Path path = Paths.get(fileDir.getAbsolutePath(), fileName);
    if (Files.exists(path)) {
        throw new BusinessException(ErrorCodeEnum.COMMON_FILE_UPLOAD_FAIL, "The file [" + fileName + "] already exists, please try again later");
    }
    int count = 0;
    try {
        // Save files
        file.transferTo(path.toFile());
        try (BufferedReader br = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
            ColumnPositionMappingStrategy strategy = new ColumnPositionMappingStrategy();
            strategy.setType(CommonDbServerInfo.class);
            String[] fields = { "connectionName", "dbType", "dbServerIp", "port", "dbName", "username", "password", "hasSelect", "hasInsert", "hasUpdate", "hasDelete", "inCharges", "isRegionId", "dbDescription", "backupDbServerIp", "backupDbPort", "visiblePerson", "visibleGroup" };
            strategy.setColumnMapping(fields);
            CsvToBean csvToBean = new CsvToBeanBuilder(br).withType(CommonDbServerInfo.class).withMappingStrategy(strategy).withIgnoreLeadingWhiteSpace(true).withSeparator(',').build();
            List<CommonDbServerInfo> dbServerInfos = csvToBean.parse();
            if (dbServerInfos.size() > 10000) {
                return "Failed, the number of data exceeds the upper limit [10000]";
            }
            StringBuilder sb = new StringBuilder();
            sb.append("csv format, The fields are connectionName, dbType, dbServerIp, port, dbName, " + "username, password, hasSelect, hasInsert, hasUpdate, hasDelete, inCharges, " + "isRegionId, dbDescription, backupDbServerIp, backupDbPort, visiblePerson, visibleGroup\n");
            sb.append("Incorrect data check\n");
            // check
            int i = 1;
            boolean passed = true;
            for (CommonDbServerInfo entry : dbServerInfos) {
                if (!SmallTools.ipCheck(entry.getDbServerIp())) {
                    sb.append(i + " column, dbServerIp=[" + entry.getDbServerIp() + "]Incorrect check\n");
                    passed = false;
                }
                if (!SmallTools.portCheck(entry.getPort())) {
                    sb.append(i + " column, port=[" + entry.getPort() + "]Incorrect check\n");
                    passed = false;
                }
                i++;
            }
            if (!passed) {
                return sb.toString();
            }
            for (CommonDbServerInfo entry : dbServerInfos) {
                commonDBServerService.save(entry);
                // System.out.println("entry = "+entry);
                count++;
            }
        }
        return "Overall success, a total of 2 " + count + "are written.";
    } catch (IllegalStateException | IOException e) {
        e.printStackTrace();
    }
    return "Overall failure, a total of 2 " + count + "are written.";
}