Examples with CSVReader au.com.bytecode.opencsv.CSVReader used on opensource projects

Example 6 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project Android-IMSI-Catcher-Detector by CellularPrivacy.

the class RealmHelper method populateDBeImport.

/**
     * Parses the downloaded CSV from OpenCellID and uses it to populate "Import" table.
     * <p/>
     * b)  Unfortunately there are 2 important missing items in the OCID CSV file:
     * - "time_first"
     * - "time_last"
     * <p/>
     * c)  In addition the OCID data often contain unexplained negative values for one or both of:
     * - "samples"
     * - "range"
     * <p/>
     * d) The difference between "Cellid" and "cid", is that "cellid" is the "Long CID",
     * consisting of RNC and a multiplier:
     * Long CID = 65536 * RNC + CID
     * See FAQ.
     * <p/>
     * ========================================================================
     * For details on available OpenCellID API DB values, see:
     * http://wiki.opencellid.org/wiki/API
     * http://wiki.opencellid.org/wiki/FAQ#Long_CellID_vs._short_Cell_ID
     * ========================================================================
     * # head -2 opencellid.csv
     * lat,lon,mcc,mnc,lac,cellid,averageSignalStrength,range,samples,changeable,radio,rnc,cid,psc,tac,pci,sid,nid,bid
     * <p/>
     * 0 lat                      TEXT
     * 1 lon                      TEXT
     * 2 mcc                      INTEGER
     * 3 mnc                      INTEGER
     * 4 lac                      INTEGER
     * 5 cellid                   INTEGER     (Long CID) = 65536 * RNC + CID
     * 6 averageSignalStrength    INTEGER     (rx_power)
     * 7 range                    INTEGER     (accu)
     * 8 samples                  INTEGER
     * 9 changeable               INTEGER     (isGPSexact)
     * 10 radio                   TEXT        (RAT)
     * 11 rnc                     INTEGER
     * 12 cid                     INTEGER     CID (Short)= "Long CID" mod 65536
     * 13 psc                     INTEGER
     * --------- vvv  See OCID API  vvv ---------
     * 14 tac                     -
     * 15 pci                     -
     * 16 sid                     -
     * 17 nid                     -
     * 18 bid                     -
     * <p/>
     * 54.63376,25.160243,246,3,20,1294,0,-1,1,1,GSM,,,,,,,,
     * ========================================================================
     */
public boolean populateDBeImport(Realm realm) {
    // This was not finding the file on a Samsung S5
    // String fileName = Environment.getExternalStorageDirectory()+ "/AIMSICD/OpenCellID/opencellid.csv";
    String fileName = mContext.getExternalFilesDir(null) + File.separator + "OpenCellID/opencellid.csv";
    File file = new File(fileName);
    try {
        if (file.exists()) {
            CSVReader csvReader = new CSVReader(new FileReader(file));
            List<String[]> csvCellID = new ArrayList<>();
            String[] next;
            while ((next = csvReader.readNext()) != null) {
                csvCellID.add(next);
            }
            if (!csvCellID.isEmpty()) {
                int lines = csvCellID.size();
                log.info("UpdateOpenCellID: OCID CSV size (lines): " + lines);
                int rowCounter;
                for (rowCounter = 1; rowCounter < lines; rowCounter++) {
                    addCSVRecord(realm, csvCellID.get(rowCounter));
                }
                log.debug("PopulateDBeImport(): inserted " + rowCounter + " cells.");
            }
        } else {
            log.error("Opencellid.csv file does not exist!");
        }
        return true;
    } catch (Exception e) {
        log.error("Error parsing OpenCellID data: " + e.getMessage());
        return false;
    } finally {
        try {
            // wait 1 second to allow user to see progress bar.
            Thread.sleep(1000);
        } catch (InterruptedException ex) {
            Thread.currentThread().interrupt();
        }
    }
}

Example 7 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project Android-IMSI-Catcher-Detector by CellularPrivacy.

the class ImportTask method doInBackground.

/**
     * Imports data from cell_towers.csv
     * <p/>
     * <blockquote>
     * opencellid.csv layout:
     * lat,lon,mcc,mnc,lac,cellid,averageSignalStrength,range,samples,changeable,radio,rnc,cid,psc,
     * tac,pci,sid,nid,bid
     * <p/>
     * example:
     * 52.201454,21.065345,260,2,58140,42042781,-59,1234,3,1,UMTS,641,34205,,,,
     * <p/>
     * cell_towers.csv layout:
     * radio,mcc,net,area,cell,unit,lon,lat,range,samples,changeable,created,updated,averageSignal
     * 0 radio
     * 1 mcc
     * 2 net (mnc)
     * 3 area (lac)
     * 4 cell (long)
     * 5 unit
     * 6 lon
     * 7 lat
     * 8 range
     * 9 samples
     * 10 changeable
     * 11 created
     * 12 updated
     * 13 averageSignal
     * <p/>
     * example:
     * UMTS,260,2,58140,42042781,,21.03006,52.207811,21,2,1,1379428153,1458591497,-92
     * </blockquote>
     */
@Override
protected String doInBackground(String... commandString) {
    try {
        @Cleanup Realm realm = Realm.getDefaultInstance();
        Long elapsedSeconds = System.currentTimeMillis() / 1000;
        // Prepare filtering values
        final String mccFilter = String.valueOf(mobileCountryCode);
        final String mncFilter = String.valueOf(mobileNetworkCode);
        long progress = 0;
        long failedRecords = 0;
        CSVReader csvReader = null;
        try {
            String[] next;
            csvReader = new CSVReader(createFileReader());
            // skip header
            csvReader.readNext();
            String[] opencellid_csv = new String[14];
            while ((next = csvReader.readNext()) != null) {
                if (next.length < 14) {
                    log.warn("Not enough values in string: " + Arrays.toString(next));
                    ++failedRecords;
                    continue;
                }
                if (!next[1].equals(mccFilter) || !next[2].equals(mncFilter)) {
                    continue;
                }
                if (next[6].isEmpty() || next[7].isEmpty()) {
                    continue;
                }
                GeoLocation location = GeoLocation.fromDegrees(Double.parseDouble(next[7]), Double.parseDouble(next[6]));
                if (location.distanceTo(currentLocation, EARTH_RADIUS) > locationRadius) {
                    continue;
                }
                try {
                    // set non-existent range, avgSignal, etc to "0" so they
                    // will be possibly filtered by checkDBe
                    // lat
                    opencellid_csv[0] = next[7];
                    // lon
                    opencellid_csv[1] = next[6];
                    // mcc
                    opencellid_csv[2] = next[1];
                    // mnc
                    opencellid_csv[3] = next[2];
                    // lac
                    opencellid_csv[4] = next[3];
                    // cellid, long
                    opencellid_csv[5] = next[4];
                    // averageSignalStrength
                    opencellid_csv[6] = stringOrZero(next[13]);
                    // range
                    opencellid_csv[7] = stringOrZero(next[8]);
                    // samples
                    opencellid_csv[8] = stringOrZero(next[9]);
                    // changeable
                    opencellid_csv[9] = stringOrZero(next[10]);
                    // radio
                    opencellid_csv[10] = next[0];
                    // rnc, not used
                    opencellid_csv[11] = null;
                    // cid, not used
                    opencellid_csv[12] = null;
                    // psc, not present
                    opencellid_csv[13] = null;
                    Date dateCreated = dateOrNow(next[11]);
                    Date dateUpdated = dateOrNow(next[12]);
                    mDbAdapter.addCSVRecord(realm, opencellid_csv, dateCreated, dateUpdated);
                    ++progress;
                } catch (NumberFormatException e) {
                    log.warn("Problem parsing a record: " + Arrays.toString(opencellid_csv), e);
                    ++failedRecords;
                }
                if ((progress % 100) == 0) {
                    log.debug("Imported records for now: " + String.valueOf(progress));
                // do not know progress because determining line count in gzipped
                // multi-gigabyte file is slow
                //publishProgress((int) progress, (int) totalRecords);
                }
                if ((progress % 1000) == 0) {
                    try {
                        // wait 1 second to allow user to see progress bar.
                        Thread.sleep(1000);
                    } catch (InterruptedException ex) {
                        Thread.currentThread().interrupt();
                    }
                }
            }
        } finally {
            if (csvReader != null) {
                csvReader.close();
            }
        }
        elapsedSeconds = (System.currentTimeMillis() / 1000) - elapsedSeconds;
        log.debug("Importing took " + String.valueOf(elapsedSeconds) + " seconds");
        log.debug("Imported records: " + String.valueOf(progress));
        log.debug("Failed records: " + String.valueOf(failedRecords));
        return "Successful";
    } catch (IOException e) {
        log.warn("Problem reading data from CSV", e);
        return null;
    }
}

Example 8 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project spatial-portal by AtlasOfLivingAustralia.

the class AddFacetController method buildLegend.

void buildLegend() {
    try {
        StringBuilder slist = new StringBuilder();
        if (query != null) {
            if (StringConstants.GRID.equals(colourmode)) {
                slist.append("name,red,green,blue,count");
                for (int i = 0; i < 600; i += 100) {
                    slist.append(">");
                    slist.append("\n").append(i).append(",").append(LegendObject.getRGB(Legend.getLinearColour(i, 0, 500, 0xFFFFFF00, 0xFFFF0000))).append(",");
                }
            } else {
                slist.append(query.getLegend(colourmode).getTable());
            }
        } else {
            return;
        }
        try {
            legendLines = new CSVReader(new StringReader(slist.toString())).readAll();
            //reset selection when legendLines is rebuilt
            uncheckAll();
        } catch (Exception e) {
            LOGGER.error("failed to read legend list as csv", e);
        }
        legendLines.remove(0);
        String h = "";
        facet = null;
        divContinous.setVisible(false);
        //test for range (user upload)
        if (legendLines.size() > 1) {
            String first = legendLines.get(0)[0];
            if (first == null || first.length() == 0 || first.startsWith(StringConstants.UNKNOWN)) {
                first = legendLines.get(1)[0];
            }
            if (!checkmarks && query.getLegend(colourmode) != null && query.getLegend(colourmode).getNumericLegend() != null) {
                setupForNumericalList(h);
            //test for manual range (solr query)
            } else if (StringConstants.OCCURRENCE_YEAR.equals(colourmode)) {
                setupForBiocacheNumber(h, colourmode, true);
            } else if (StringConstants.OCCURRENCE_YEAR_DECADE.equals(colourmode) || StringConstants.DECADE.equals(colourmode)) {
                setupForBiocacheDecade();
            } else if (StringConstants.COORDINATE_UNCERTAINTY.equals(colourmode) || StringConstants.UNCERTAINTY.equals(colourmode)) {
                setupForBiocacheNumber(h, colourmode, false);
            } else if (StringConstants.MONTH.equals(colourmode)) {
                setupForBiocacheMonth();
            }
        }
        /* apply something to line onclick in lb */
        legend.setItemRenderer(new ListitemRenderer() {

            @Override
            public void render(Listitem li, Object data, int itemIdx) {
                String[] ss = (String[]) data;
                if (ss == null) {
                    return;
                }
                Checkbox cb = null;
                if (checkmarks) {
                    cb = new Checkbox();
                    cb.addEventListener("onCheck", new EventListener() {

                        @Override
                        public void onEvent(Event event) throws Exception {
                            btnOk.setDisabled(false);
                            rebuildSelectedList();
                        }
                    });
                    cb.setVisible(!cbContinousRange.isChecked());
                }
                Listcell lc;
                lc = new Listcell();
                if (cb != null) {
                    cb.setParent(lc);
                    //Do not display checkboxes for facets that are not simple
                    cb.setVisible(!disableselection && !ss[0].endsWith(" more"));
                }
                lc.setParent(li);
                if (readonly) {
                    lc = new Listcell(actualToDisplayLabel(ss[0]));
                } else {
                    lc = new Listcell("group " + ss[0]);
                }
                lc.setParent(li);
                if (cb != null) {
                    cb.setChecked(selectedList.contains(lc.getLabel()));
                }
                int red = 0, green = 0, blue = 0;
                try {
                    red = Integer.parseInt(ss[1]);
                    green = Integer.parseInt(ss[2]);
                    blue = Integer.parseInt(ss[3]);
                } catch (NumberFormatException e) {
                    LOGGER.error("error parsing colours : " + ss[0], e);
                }
                lc = new Listcell("   ");
                lc.setStyle("background-color: rgb(" + red + "," + green + "," + blue + "); color: rgb(" + red + "," + green + "," + blue + ")");
                lc.setParent(li);
                //count
                try {
                    lhFourthColumn.setVisible(true);
                    lc = new Listcell(ss[4]);
                    lc.setParent(li);
                } catch (Exception e) {
                    lhFourthColumn.setVisible(false);
                }
            }
        });
        legend.setModel(new SimpleListModel(legendLines));
    } catch (Exception e) {
        LOGGER.error("error rendering legend", e);
    }
}

Example 9 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project spatial-portal by AtlasOfLivingAustralia.

the class BiocacheQuery method endemicSpeciesList.

public String endemicSpeciesList() {
    if (endemicSpeciesList != null) {
        return endemicSpeciesList;
    }
    if (CommonData.getSettings().containsKey("endemic.sp.method") && CommonData.getSettings().getProperty("endemic.sp.method").equals("true")) {
        String speciesList = speciesList();
        //can get species list counts as "kosher:true" or "kosher:*" only
        Map speciesCounts;
        if (getGeospatialKosher()[1]) {
            //[1] is 'include kosher:false'
            speciesCounts = CommonData.getSpeciesListCounts(false);
        } else {
            speciesCounts = CommonData.getSpeciesListCountsKosher(false);
        }
        StringBuilder sb = new StringBuilder();
        int speciesCol = 0;
        int countCol = 11;
        try {
            CSVReader csv = new CSVReader(new StringReader(speciesList));
            String[] row;
            int currentPos = 0;
            int nextPos = speciesList.indexOf('\n', currentPos + 1);
            //header
            sb.append(speciesList.substring(currentPos, nextPos));
            //header
            csv.readNext();
            while ((row = csv.readNext()) != null) {
                //add if species is not present elsewhere
                Long c = (Long) speciesCounts.get(row[speciesCol]);
                if (c != null && c <= Long.parseLong(row[countCol])) {
                    if (nextPos > speciesList.length()) {
                        nextPos = speciesList.length();
                    }
                    sb.append(speciesList.substring(currentPos, nextPos));
                } else if (c == null) {
                    LOGGER.error("failed to find species_guid: " + row[speciesCol] + " in CommonData.getSpeciesListCounts()");
                }
                currentPos = nextPos;
                nextPos = speciesList.indexOf('\n', currentPos + 1);
            }
        } catch (Exception e) {
            LOGGER.error("failed generating endemic species list", e);
        }
        endemicSpeciesList = sb.toString();
    } else {
        forMapping = true;
        if (paramId == null)
            makeParamId();
        HttpClient client = new HttpClient();
        String url = biocacheServer + ENDEMIC_LIST + paramId + "?facets=names_and_lsid";
        LOGGER.debug(url);
        GetMethod get = new GetMethod(url);
        try {
            client.executeMethod(get);
            JSONParser jp = new JSONParser();
            JSONArray ja = (JSONArray) jp.parse(get.getResponseBodyAsString());
            //extract endemic matches from the species list
            String speciesList = speciesList();
            StringBuilder sb = new StringBuilder();
            int idx = speciesList.indexOf('\n');
            if (idx > 0) {
                sb.append(speciesList.substring(0, idx));
            }
            for (int j = 0; j < ja.size(); j++) {
                JSONObject jo = (JSONObject) ja.get(j);
                if (jo.containsKey("label")) {
                    idx = speciesList.indexOf("\n" + jo.get("label") + ",");
                    if (idx > 0) {
                        int lineEnd = speciesList.indexOf('\n', idx + 1);
                        if (lineEnd < 0)
                            lineEnd = speciesList.length();
                        sb.append(speciesList.substring(idx, lineEnd));
                    }
                }
            }
            endemicSpeciesList = sb.toString();
        } catch (Exception e) {
            LOGGER.error("error getting endemic species result", e);
        }
    }
    return endemicSpeciesList;
}

Example 10 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project spatial-portal by AtlasOfLivingAustralia.

the class Sampling method getDownloadData.

static List<String[]> getDownloadData(String downloadUrl, int numberOfPoints) {
    List<String[]> output = new ArrayList<String[]>();
    try {
        HttpClient client = new HttpClient();
        GetMethod get = new GetMethod(downloadUrl);
        get.addRequestHeader(StringConstants.CONTENT_TYPE, "application/zip");
        client.executeMethod(get);
        try {
            ZipInputStream zip = new ZipInputStream(get.getResponseBodyAsStream());
            zip.getNextEntry();
            CSVReader csv = new CSVReader(new InputStreamReader(zip));
            //read first line
            String[] line = csv.readNext();
            //setup
            for (int i = 2; i < line.length; i++) {
                output.add(new String[numberOfPoints]);
            }
            int row = 0;
            while ((line = csv.readNext()) != null && row < numberOfPoints) {
                for (int i = 2; i - 2 < output.size() && i < line.length; i++) {
                    output.get(i - 2)[row] = line[i];
                }
                row = row + 1;
            }
            zip.close();
        } catch (Exception e) {
            LOGGER.error("failed to read zipped stream from: " + downloadUrl, e);
        }
    } catch (Exception e) {
        LOGGER.error("error getting response from url: " + downloadUrl, e);
    }
    return output;
}