Examples with CsvListReader org.supercsv.io.CsvListReader used on opensource projects

Example 1 with CsvListReader

use of org.supercsv.io.CsvListReader in project Xponents by OpenSextant.

the class TextUtils method initLOCLanguageData.

/**
     * This is Libray of Congress data for language IDs. This is offered as a
     * tool to help downstream language ID and enrich metadata when tagging data
     * from particular countries.
     *
     * Reference: http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
     *
     * @throws java.io.IOException
     *             if resource file is not found
     */
public static void initLOCLanguageData() throws java.io.IOException {
    //
    // DATA FILE: http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
    java.io.InputStream io = TextUtils.class.getResourceAsStream("/ISO-639-2_utf-8.txt");
    java.io.Reader featIO = new InputStreamReader(io, "UTF-8");
    CsvListReader langReader = new CsvListReader(featIO, new CsvPreference.Builder('"', '|', "\n").build());
    CellProcessor[] cells = { new Optional(), new Optional(), new Optional(), new Optional(), new NotNull() };
    List<Object> lang = null;
    /*
         * ISO3,XX,ISO2,NAME,NAME_FR
         */
    while ((lang = langReader.read(cells)) != null) {
        //
        String names = (String) lang.get(3);
        if (isBlank(names)) {
            continue;
        }
        if ("NAME".equals(names)) {
            continue;
        }
        List<String> namelist = TextUtils.string2list(names, ";");
        String iso3 = (String) lang.get(0);
        if (iso3.startsWith("#")) {
            continue;
        }
        String iso2 = (String) lang.get(2);
        Language l = new Language(iso3, iso2, namelist.get(0));
        addLanguage(l);
    }
    langReader.close();
    // Popular languages that go by other codes.
    // ISO languages as listed by LOC are listed with Bibliographic vs.
    // Terminological codes.
    // FRE vs. FRA are subtle difference for French, but important if you
    // cannot find French by lang ID.
    //
    // Fully override French and Trad Chinese:
    Language fr = new Language("fra", "fr", "French");
    addLanguage(fr, true);
    Language zhtw = new Language("zh-tw", "zt", "Chinese/Taiwain");
    addLanguage(zhtw, true);
    // Delicately insert more common names and codes as well as locales
    // here.
    Language zh = new Language("zho", "zh", "Chinese");
    languageMapISO639.put("zho", zh);
    Language zhcn = new Language("chi", "zh", "Chinese");
    languageMapISO639.put("zh-cn", zhcn);
    Language fas = new Language("per", "fa", "Farsi");
    languageMapISO639.put("farsi", fas);
    // Locales of English -- are still "English"
    Language en1 = new Language("eng", "en", "English");
    languageMapISO639.put("en-gb", en1);
    languageMapISO639.put("en-us", en1);
    languageMapISO639.put("en-au", en1);
}

Example 2 with CsvListReader

use of org.supercsv.io.CsvListReader in project adempiere by adempiere.

the class CSVFactory method read.

public Collection<TestableMRP> read(InputStream in) throws Exception {
    ArrayList<TestableMRP> tests = new ArrayList<TestableMRP>();
    //
    reader = new CsvListReader(new InputStreamReader(in), CsvPreference.STANDARD_PREFERENCE);
    String[] header = reader.getCSVHeader(true);
    System.out.println("HEADER: " + MRPUtil.toString(header));
    //
    List<String> line;
    int last_lineNo = -1;
    boolean isPlanningLine = true;
    TestableMRP mrpTest = null;
    try {
        while ((line = reader.read()) != null) {
            if (last_lineNo == -1 || last_lineNo + 1 < reader.getLineNumber()) {
                isPlanningLine = true;
                if (mrpTest != null) {
                    tests.add(mrpTest);
                }
                mrpTest = new TestableMRP();
            }
            if (isPlanningLine) {
                readProductPlanning(mrpTest, header, line);
                isPlanningLine = false;
            } else {
                readMRPLine(mrpTest, header, line);
            }
            //
            last_lineNo = reader.getLineNumber();
        }
    } catch (Exception e) {
        throw new RuntimeException("Error on line " + reader.getLineNumber() + ": " + e.getLocalizedMessage(), e);
    }
    if (mrpTest != null) {
        tests.add(mrpTest);
    }
    //
    return tests;
}

Example 3 with CsvListReader

use of org.supercsv.io.CsvListReader in project adempiere by adempiere.

the class CSVFactory method read.

public Collection<MMScenario> read(InputStream in) throws Exception {
    ArrayList<MMScenario> tests = new ArrayList<MMScenario>();
    //
    reader = new CsvListReader(new InputStreamReader(in), CsvPreference.STANDARD_PREFERENCE);
    String[] header = getCSVHeader();
    //
    List<String> line;
    int last_lineNo = -1;
    MMScenario scenario = null;
    try {
        while ((line = reader.read()) != null) {
            if (last_lineNo == -1 || last_lineNo + 1 < reader.getLineNumber()) {
                if (scenario != null) {
                    tests.add(scenario);
                }
                scenario = new MMScenario("junit-test-line_" + (new DecimalFormat("000").format(reader.getLineNumber())));
            }
            readDocument(scenario, header, line);
            last_lineNo = reader.getLineNumber();
        }
    } catch (Exception e) {
        throw new RuntimeException("Error on line " + reader.getLineNumber() + ": " + e.getLocalizedMessage(), e);
    }
    if (scenario != null) {
        tests.add(scenario);
    }
    //
    return tests;
}

Example 4 with CsvListReader

use of org.supercsv.io.CsvListReader in project adempiere by adempiere.

the class CSVFactory method read.

public Collection<MMScenario> read(InputStream in) throws Exception {
    ArrayList<MMScenario> tests = new ArrayList<MMScenario>();
    //
    reader = new CsvListReader(new InputStreamReader(in), CsvPreference.STANDARD_PREFERENCE);
    String[] header = getCSVHeader();
    //
    List<String> line;
    int last_lineNo = -1;
    MMScenario scenario = null;
    try {
        while ((line = reader.read()) != null) {
            if (last_lineNo == -1 || last_lineNo + 1 < reader.getLineNumber()) {
                if (scenario != null) {
                    tests.add(scenario);
                }
                scenario = new MMScenario("junit-test-line_" + (new DecimalFormat("000").format(reader.getLineNumber())));
            }
            readDocument(scenario, header, line);
            last_lineNo = reader.getLineNumber();
        }
    } catch (Exception e) {
        throw new RuntimeException("Error on line " + reader.getLineNumber() + ": " + e.getLocalizedMessage(), e);
    }
    if (scenario != null) {
        tests.add(scenario);
    }
    //
    return tests;
}

Example 5 with CsvListReader

use of org.supercsv.io.CsvListReader in project voltdb by VoltDB.

the class CSVLoader method main.

/**
     * csvloader main. (main is directly used by tests as well be sure to reset statics that you need to start over)
     *
     * @param args
     * @throws IOException
     * @throws InterruptedException
     *
     */
public static void main(String[] args) throws IOException, InterruptedException {
    start = System.currentTimeMillis();
    long insertTimeStart = start;
    long insertTimeEnd;
    final CSVConfig cfg = new CSVConfig();
    cfg.parse(CSVLoader.class.getName(), args);
    config = cfg;
    if (config.noquotechar) {
        config.quotechar = '';
    }
    configuration();
    final Tokenizer tokenizer;
    ICsvListReader listReader = null;
    try {
        if (CSVLoader.standin) {
            tokenizer = new Tokenizer(new BufferedReader(new InputStreamReader(System.in)), csvPreference, config.strictquotes, config.escape, config.columnsizelimit, config.skip, config.header);
            listReader = new CsvListReader(tokenizer, csvPreference);
        } else {
            tokenizer = new Tokenizer(new FileReader(config.file), csvPreference, config.strictquotes, config.escape, config.columnsizelimit, config.skip, config.header);
            listReader = new CsvListReader(tokenizer, csvPreference);
        }
    } catch (FileNotFoundException e) {
        m_log.error("CSV file '" + config.file + "' could not be found.");
        System.exit(-1);
    }
    // Split server list
    final String[] serverlist = config.servers.split(",");
    // If we need to prompt the user for a password, do so.
    config.password = CLIConfig.readPasswordIfNeeded(config.user, config.password, "Enter password: ");
    // Create connection
    final ClientConfig c_config = new ClientConfig(config.user, config.password, null);
    if (config.ssl != null && !config.ssl.trim().isEmpty()) {
        c_config.setTrustStoreConfigFromPropertyFile(config.ssl);
        c_config.enableSSL();
    }
    // Set procedure all to infinite
    c_config.setProcedureCallTimeout(0);
    Client csvClient = null;
    try {
        csvClient = CSVLoader.getClient(c_config, serverlist, config.port);
    } catch (Exception e) {
        m_log.error("Error connecting to the servers: " + config.servers);
        System.exit(-1);
    }
    assert (csvClient != null);
    try {
        long readerTime;
        long insertCount;
        long ackCount;
        long rowsQueued;
        final CSVLoader errHandler = new CSVLoader();
        final CSVDataLoader dataLoader;
        errHandler.launchErrorFlushProcessor();
        if (config.useSuppliedProcedure) {
            dataLoader = new CSVTupleDataLoader((ClientImpl) csvClient, config.procedure, errHandler);
        } else {
            dataLoader = new CSVBulkDataLoader((ClientImpl) csvClient, config.table, config.batch, config.update, errHandler);
        }
        CSVFileReader.initializeReader(cfg, csvClient, listReader);
        CSVFileReader csvReader = new CSVFileReader(dataLoader, errHandler);
        Thread readerThread = new Thread(csvReader);
        readerThread.setName("CSVFileReader");
        readerThread.setDaemon(true);
        //Wait for reader to finish.
        readerThread.start();
        readerThread.join();
        insertTimeEnd = System.currentTimeMillis();
        csvClient.close();
        errHandler.waitForErrorFlushComplete();
        readerTime = (csvReader.m_parsingTime) / 1000000;
        insertCount = dataLoader.getProcessedRows();
        ackCount = insertCount - dataLoader.getFailedRows();
        rowsQueued = CSVFileReader.m_totalRowCount.get();
        //Close the reader.
        try {
            listReader.close();
        } catch (Exception ex) {
            m_log.error("Error closing reader: " + ex);
        } finally {
            m_log.debug("Rows Queued by Reader: " + rowsQueued);
        }
        if (errHandler.hasReachedErrorLimit()) {
            m_log.warn("The number of failed rows exceeds the configured maximum failed rows: " + config.maxerrors);
        }
        m_log.debug("Parsing CSV file took " + readerTime + " milliseconds.");
        m_log.debug("Inserting Data took " + ((insertTimeEnd - insertTimeStart) - readerTime) + " milliseconds.");
        m_log.info("Read " + insertCount + " rows from file and successfully inserted " + ackCount + " rows (final)");
        errHandler.produceFiles(ackCount, insertCount);
        close_cleanup();
        //In test junit mode we let it continue for reuse
        if (!CSVLoader.testMode) {
            System.exit(errHandler.m_errorInfo.isEmpty() ? 0 : -1);
        }
    } catch (Exception ex) {
        m_log.error("Exception Happened while loading CSV data: " + ex);
        System.exit(1);
    }
}