Examples with CSVRecord org.apache.commons.csv.CSVRecord used on opensource projects

Example 1 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project pinot by linkedin.

the class CSVRecordReader method next.

@Override
public GenericRow next(GenericRow row) {
    CSVRecord record = _iterator.next();
    for (final FieldSpec fieldSpec : _schema.getAllFieldSpecs()) {
        String column = fieldSpec.getName();
        String token = getValueForColumn(record, column);
        Object value = null;
        if (token == null || token.isEmpty()) {
            incrementNullCountFor(fieldSpec.getName());
        }
        if (fieldSpec.isSingleValueField()) {
            value = RecordReaderUtils.convertToDataType(token, fieldSpec.getDataType());
        } else {
            String[] tokens = (token != null) ? StringUtils.split(token, _delimiterString) : null;
            value = RecordReaderUtils.convertToDataTypeArray(tokens, fieldSpec.getDataType());
        }
        row.putField(column, value);
    }
    return row;
}

Example 2 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project phoenix by apache.

the class CSVCommonsLoaderIT method testTDVCommonsUpsert.

@Test
public void testTDVCommonsUpsert() throws Exception {
    CSVParser parser = null;
    PhoenixConnection conn = null;
    try {
        String stockTableName = generateUniqueName();
        // Create table
        String statements = "CREATE TABLE IF NOT EXISTS " + stockTableName + "(SYMBOL VARCHAR NOT NULL PRIMARY KEY, COMPANY VARCHAR);";
        conn = DriverManager.getConnection(getUrl()).unwrap(PhoenixConnection.class);
        PhoenixRuntime.executeStatements(conn, new StringReader(statements), null);
        // Upsert TDV file
        CSVCommonsLoader csvUtil = new CSVCommonsLoader(conn, stockTableName, Collections.<String>emptyList(), true, '\t', '"', null, CSVCommonsLoader.DEFAULT_ARRAY_ELEMENT_SEPARATOR);
        csvUtil.upsert(new StringReader(STOCK_TDV_VALUES_WITH_HEADER));
        // Compare Phoenix ResultSet with CSV file content
        PreparedStatement statement = conn.prepareStatement("SELECT SYMBOL, COMPANY FROM " + stockTableName);
        ResultSet phoenixResultSet = statement.executeQuery();
        parser = new CSVParser(new StringReader(STOCK_TDV_VALUES_WITH_HEADER), csvUtil.getFormat());
        for (CSVRecord record : parser) {
            assertTrue(phoenixResultSet.next());
            int i = 0;
            for (String value : record) {
                assertEquals(value, phoenixResultSet.getString(i + 1));
                i++;
            }
        }
        assertFalse(phoenixResultSet.next());
    } finally {
        if (parser != null)
            parser.close();
        if (conn != null)
            conn.close();
    }
}

Example 3 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project phoenix by apache.

the class CSVCommonsLoaderIT method testCSVUpsertWithBogusColumn.

@Test
public void testCSVUpsertWithBogusColumn() throws Exception {
    CSVParser parser = null;
    PhoenixConnection conn = null;
    try {
        String stockTableName = generateUniqueName();
        // Create table
        String statements = "CREATE TABLE IF NOT EXISTS " + stockTableName + "(SYMBOL VARCHAR NOT NULL PRIMARY KEY, COMPANY VARCHAR);";
        conn = DriverManager.getConnection(getUrl()).unwrap(PhoenixConnection.class);
        PhoenixRuntime.executeStatements(conn, new StringReader(statements), null);
        // Upsert CSV file, not strict
        CSVCommonsLoader csvUtil = new CSVCommonsLoader(conn, stockTableName, Arrays.asList(STOCK_COLUMNS_WITH_BOGUS), false);
        csvUtil.upsert(new StringReader(STOCK_CSV_VALUES));
        // Compare Phoenix ResultSet with CSV file content
        PreparedStatement statement = conn.prepareStatement("SELECT SYMBOL, COMPANY FROM " + stockTableName);
        ResultSet phoenixResultSet = statement.executeQuery();
        parser = new CSVParser(new StringReader(STOCK_CSV_VALUES), csvUtil.getFormat());
        for (CSVRecord record : parser) {
            assertTrue(phoenixResultSet.next());
            assertEquals(record.get(0), phoenixResultSet.getString(1));
            assertNull(phoenixResultSet.getString(2));
        }
        assertFalse(phoenixResultSet.next());
    } finally {
        if (parser != null)
            parser.close();
        if (conn != null)
            conn.close();
    }
}

Example 4 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project phoenix by apache.

the class CSVCommonsLoaderIT method testCSVCommonsUpsert.

@Test
public void testCSVCommonsUpsert() throws Exception {
    CSVParser parser = null;
    PhoenixConnection conn = null;
    try {
        String stockTableName = generateUniqueName();
        // Create table
        String statements = "CREATE TABLE IF NOT EXISTS " + stockTableName + "(SYMBOL VARCHAR NOT NULL PRIMARY KEY, COMPANY VARCHAR);";
        conn = DriverManager.getConnection(getUrl()).unwrap(PhoenixConnection.class);
        PhoenixRuntime.executeStatements(conn, new StringReader(statements), null);
        // Upsert CSV file
        CSVCommonsLoader csvUtil = new CSVCommonsLoader(conn, stockTableName, Collections.<String>emptyList(), true);
        csvUtil.upsert(new StringReader(STOCK_CSV_VALUES_WITH_HEADER));
        // Compare Phoenix ResultSet with CSV file content
        PreparedStatement statement = conn.prepareStatement("SELECT SYMBOL, COMPANY FROM " + stockTableName);
        ResultSet phoenixResultSet = statement.executeQuery();
        parser = new CSVParser(new StringReader(STOCK_CSV_VALUES_WITH_HEADER), csvUtil.getFormat());
        for (CSVRecord record : parser) {
            assertTrue(phoenixResultSet.next());
            int i = 0;
            for (String value : record) {
                assertEquals(value, phoenixResultSet.getString(i + 1));
                i++;
            }
        }
        assertFalse(phoenixResultSet.next());
    } finally {
        if (parser != null)
            parser.close();
        if (conn != null)
            conn.close();
    }
}

Example 5 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project phoenix by apache.

the class CSVCommonsLoaderIT method testCSVUpsertWithColumns.

@Test
public void testCSVUpsertWithColumns() throws Exception {
    CSVParser parser = null;
    PhoenixConnection conn = null;
    try {
        String stockTableName = generateUniqueName();
        // Create table
        String statements = "CREATE TABLE IF NOT EXISTS " + stockTableName + "(SYMBOL VARCHAR NOT NULL PRIMARY KEY, COMPANY VARCHAR);";
        conn = DriverManager.getConnection(getUrl()).unwrap(PhoenixConnection.class);
        PhoenixRuntime.executeStatements(conn, new StringReader(statements), null);
        // Upsert CSV file
        CSVCommonsLoader csvUtil = new CSVCommonsLoader(conn, stockTableName, Arrays.<String>asList(STOCK_COLUMNS), true);
        // no header
        csvUtil.upsert(new StringReader(STOCK_CSV_VALUES));
        // Compare Phoenix ResultSet with CSV file content
        PreparedStatement statement = conn.prepareStatement("SELECT SYMBOL, COMPANY FROM " + stockTableName);
        ResultSet phoenixResultSet = statement.executeQuery();
        parser = new CSVParser(new StringReader(STOCK_CSV_VALUES), csvUtil.getFormat());
        for (CSVRecord record : parser) {
            assertTrue(phoenixResultSet.next());
            int i = 0;
            for (String value : record) {
                assertEquals(value, phoenixResultSet.getString(i + 1));
                i++;
            }
        }
        assertFalse(phoenixResultSet.next());
    } finally {
        if (parser != null)
            parser.close();
        if (conn != null)
            conn.close();
    }
}