Examples with Tokenizer org.supercsv_voltpatches.tokenizer.Tokenizer used on opensource projects

Example 1 with Tokenizer

use of org.supercsv_voltpatches.tokenizer.Tokenizer in project voltdb by VoltDB.

the class CSVLoader method main.

/**
     * csvloader main. (main is directly used by tests as well be sure to reset statics that you need to start over)
     *
     * @param args
     * @throws IOException
     * @throws InterruptedException
     *
     */
public static void main(String[] args) throws IOException, InterruptedException {
    start = System.currentTimeMillis();
    long insertTimeStart = start;
    long insertTimeEnd;
    final CSVConfig cfg = new CSVConfig();
    cfg.parse(CSVLoader.class.getName(), args);
    config = cfg;
    if (config.noquotechar) {
        config.quotechar = '';
    }
    configuration();
    final Tokenizer tokenizer;
    ICsvListReader listReader = null;
    try {
        if (CSVLoader.standin) {
            tokenizer = new Tokenizer(new BufferedReader(new InputStreamReader(System.in)), csvPreference, config.strictquotes, config.escape, config.columnsizelimit, config.skip, config.header);
            listReader = new CsvListReader(tokenizer, csvPreference);
        } else {
            tokenizer = new Tokenizer(new FileReader(config.file), csvPreference, config.strictquotes, config.escape, config.columnsizelimit, config.skip, config.header);
            listReader = new CsvListReader(tokenizer, csvPreference);
        }
    } catch (FileNotFoundException e) {
        m_log.error("CSV file '" + config.file + "' could not be found.");
        System.exit(-1);
    }
    // Split server list
    final String[] serverlist = config.servers.split(",");
    // If we need to prompt the user for a password, do so.
    config.password = CLIConfig.readPasswordIfNeeded(config.user, config.password, "Enter password: ");
    // Create connection
    final ClientConfig c_config = new ClientConfig(config.user, config.password, null);
    if (config.ssl != null && !config.ssl.trim().isEmpty()) {
        c_config.setTrustStoreConfigFromPropertyFile(config.ssl);
        c_config.enableSSL();
    }
    // Set procedure all to infinite
    c_config.setProcedureCallTimeout(0);
    Client csvClient = null;
    try {
        csvClient = CSVLoader.getClient(c_config, serverlist, config.port);
    } catch (Exception e) {
        m_log.error("Error connecting to the servers: " + config.servers);
        System.exit(-1);
    }
    assert (csvClient != null);
    try {
        long readerTime;
        long insertCount;
        long ackCount;
        long rowsQueued;
        final CSVLoader errHandler = new CSVLoader();
        final CSVDataLoader dataLoader;
        errHandler.launchErrorFlushProcessor();
        if (config.useSuppliedProcedure) {
            dataLoader = new CSVTupleDataLoader((ClientImpl) csvClient, config.procedure, errHandler);
        } else {
            dataLoader = new CSVBulkDataLoader((ClientImpl) csvClient, config.table, config.batch, config.update, errHandler);
        }
        CSVFileReader.initializeReader(cfg, csvClient, listReader);
        CSVFileReader csvReader = new CSVFileReader(dataLoader, errHandler);
        Thread readerThread = new Thread(csvReader);
        readerThread.setName("CSVFileReader");
        readerThread.setDaemon(true);
        //Wait for reader to finish.
        readerThread.start();
        readerThread.join();
        insertTimeEnd = System.currentTimeMillis();
        csvClient.close();
        errHandler.waitForErrorFlushComplete();
        readerTime = (csvReader.m_parsingTime) / 1000000;
        insertCount = dataLoader.getProcessedRows();
        ackCount = insertCount - dataLoader.getFailedRows();
        rowsQueued = CSVFileReader.m_totalRowCount.get();
        //Close the reader.
        try {
            listReader.close();
        } catch (Exception ex) {
            m_log.error("Error closing reader: " + ex);
        } finally {
            m_log.debug("Rows Queued by Reader: " + rowsQueued);
        }
        if (errHandler.hasReachedErrorLimit()) {
            m_log.warn("The number of failed rows exceeds the configured maximum failed rows: " + config.maxerrors);
        }
        m_log.debug("Parsing CSV file took " + readerTime + " milliseconds.");
        m_log.debug("Inserting Data took " + ((insertTimeEnd - insertTimeStart) - readerTime) + " milliseconds.");
        m_log.info("Read " + insertCount + " rows from file and successfully inserted " + ackCount + " rows (final)");
        errHandler.produceFiles(ackCount, insertCount);
        close_cleanup();
        //In test junit mode we let it continue for reuse
        if (!CSVLoader.testMode) {
            System.exit(errHandler.m_errorInfo.isEmpty() ? 0 : -1);
        }
    } catch (Exception ex) {
        m_log.error("Exception Happened while loading CSV data: " + ex);
        System.exit(1);
    }
}