Examples with DataBean com.localidata.process.bean.DataBean used on opensource projects

Example 6 with DataBean

use of com.localidata.process.bean.DataBean in project local-data-aragopedia by aragonopendata.

the class GenerateData method readXlsxFile.

private void readXlsxFile(File file, ConfigBean configBean) {
    log.debug("Init readXlsxFile");
    InputStream inp = null;
    Workbook wb = null;
    try {
        inp = new FileInputStream(file);
        wb = WorkbookFactory.create(inp);
    } catch (FileNotFoundException e) {
        log.error(e.getMessage());
    } catch (InvalidFormatException e) {
        log.error(e.getMessage());
    } catch (IOException e) {
        log.error(e.getMessage());
    }
    Sheet sheet = wb.getSheetAt(0);
    Row rowName = sheet.getRow(0);
    Row rowNameNormalized = sheet.getRow(1);
    Row rowNormalization = sheet.getRow(2);
    Row rowDimMesure = sheet.getRow(3);
    Row rowType = sheet.getRow(4);
    Row rowSkosfile = sheet.getRow(5);
    Row rowConstant = sheet.getRow(6);
    Row rowConstantValue = sheet.getRow(7);
    Row rowRelationKos = sheet.getRow(8);
    Row rowKosName = sheet.getRow(9);
    boolean cont = true;
    int columnReaded = 0;
    while (cont) {
        Cell cellName = rowName.getCell(columnReaded);
        Cell cellNameNormalized = rowNameNormalized.getCell(columnReaded);
        Cell cellNormalization = rowNormalization.getCell(columnReaded);
        Cell cellDimMesure = rowDimMesure.getCell(columnReaded);
        Cell cellType = rowType.getCell(columnReaded);
        Cell cellSkosfile = null;
        if (rowSkosfile != null)
            cellSkosfile = rowSkosfile.getCell(columnReaded);
        Cell cellConstant = null;
        if (rowConstant != null)
            cellConstant = rowConstant.getCell(columnReaded);
        Cell cellConstantValue = null;
        if (rowConstantValue != null)
            cellConstantValue = rowConstantValue.getCell(columnReaded);
        Cell cellRelationKos = null;
        if (rowRelationKos != null)
            cellRelationKos = rowRelationKos.getCell(columnReaded);
        Cell cellKosName = null;
        if (rowKosName != null)
            cellKosName = rowKosName.getCell(columnReaded);
        DataBean dataBean = new DataBean();
        if (cellName == null) {
            if (rowName.getCell((columnReaded + 1)) == null)
                cont = false;
            else
                columnReaded++;
        } else {
            dataBean.setName(cellName.getStringCellValue());
            dataBean.setNameNormalized(cellNameNormalized.getStringCellValue());
            dataBean.setNormalizacion(cellNormalization.getStringCellValue());
            dataBean.setDimensionMesure(cellDimMesure.getStringCellValue());
            dataBean.setIdConfig(configBean.getId());
            String type = "";
            if (cellType != null) {
                type = cellType.getStringCellValue();
            } else {
                type = "xsd:string";
            }
            dataBean.setType(type);
            if (cellSkosfile != null && !cellSkosfile.getStringCellValue().equals("")) {
                HashMap<String, SkosBean> mapSkos = readMappingFile(cellSkosfile.getStringCellValue());
                dataBean.setMapSkos(mapSkos);
                configBean.getMapData().put(dataBean.getNameNormalized(), dataBean);
                dataWithSkos.add(dataBean);
            } else {
                configBean.getMapData().put(dataBean.getNameNormalized(), dataBean);
            }
            if (Prop.addDataConstant && cellConstant != null && cellConstant.getStringCellValue().equals(Constants.constante)) {
                if (cellConstantValue != null) {
                    dataBean.setConstant(cellConstantValue.getStringCellValue() + "");
                    configBean.getListDataConstant().add(dataBean);
                }
            }
            if (cellRelationKos != null) {
                dataBean.setRelationKos(cellRelationKos.getStringCellValue());
                dataWithSkosHierarchical.add(dataBean);
            }
            if (cellKosName != null) {
                dataBean.setKosName(cellKosName.getStringCellValue());
            } else {
                dataBean.setKosName(dataBean.getNameNormalized());
            }
            columnReaded++;
        }
    }
    log.debug("End readXlsxFile");
}

Example 7 with DataBean

use of com.localidata.process.bean.DataBean in project local-data-aragopedia by aragonopendata.

the class TransformToRDF method addObservation.

private StringBuffer addObservation(String line, String fileName) {
    log.debug("Init addObservation " + line + " " + fileName);
    StringBuffer result = new StringBuffer();
    String endResult = "";
    boolean year = false;
    String cleanLine = Utils.weakClean(line);
    if (cleanLine.equals("")) {
        log.debug("End addObservation");
        return result;
    }
    String id = Utils.genUUIDHash(cleanLine);
    result.append("<" + Prop.host + "/" + Prop.eldaName + "/" + Prop.datasetName + "/observacion/" + fileName + "/" + id + "> a qb:Observation ;" + "\n");
    result.append("\tqb:DataSet <" + cubo + ">; \n");
    String[] cells = cleanLine.split("\t");
    int col = 1;
    for (String cell : cells) {
        String normalizedCell = Utils.urlify(cell);
        if (normalizedHeader.size() <= col - 1) {
            TransformToRDF.insertError(fileName + ". ERROR. " + "COLUMN NAME MISSING  ");
            log.error(fileName + ". ERROR. " + "COLUMN NAME MISSING  ");
            continue;
        }
        String header = normalizedHeader.get(col - 1);
        cleanHeader.get(col - 1);
        DataBean dataBean = configBean.getMapData().get(header);
        try {
            if (dataBean != null) {
                if (dataBean.getNormalizacion() != null) {
                    if (normalizedCell.equals("")) {
                        TransformToRDF.insertError(fileName + ". ERROR. Column " + header + ". NO VALUE ");
                    } else {
                        if (!dataBean.getType().contains(Constants.URIType)) {
                            if (!dataBean.getNormalizacion().equals("sdmx-dimension:refPeriod")) {
                                if (dataBean.getType().equals(Constants.skosType)) {
                                    if (dataBean.getMapSkos().get(normalizedCell) == null) {
                                        TransformToRDF.insertError(fileName + ". ERROR. Column " + header + ". NO SKOS VALID BY " + normalizedCell);
                                        log.error(fileName + ". ERROR. Column " + header + ". NO SKOS VALID BY " + normalizedCell);
                                    } else {
                                        result.append("\t" + dataBean.getNormalizacion() + " <" + dataBean.getMapSkos().get(normalizedCell).getURI() + "> ;" + "\n");
                                        dataBean.getMapSkos().get(normalizedCell).setLabel(Utils.weakClean(cell));
                                    }
                                } else {
                                    result.append("\t" + dataBean.getNormalizacion() + " \"" + normalizedCell + "\"^^" + dataBean.getType() + ";" + "\n");
                                }
                            } else {
                                if (dataBean.getNormalizacion().equals("sdmx-dimension:refPeriod")) {
                                    year = true;
                                    result.append("\t" + dataBean.getNormalizacion() + " <http://reference.data.gov.uk/id/year/" + normalizedCell + "> ;" + "\n");
                                }
                            }
                        } else {
                            String pattern = "([0-9]+)(-)(.*)";
                            Pattern r = Pattern.compile(pattern);
                            Matcher m = r.matcher(cell);
                            if (m.find()) {
                                TransformToRDF.insertError(fileName + ". WARNING. Column " + header + ". MIXED CODE AND VALUE");
                                cell = cell.substring(cell.indexOf("-") + 1, cell.length());
                            }
                            String urlRefArea = Utils.getUrlRefArea(header, cell, fileName);
                            result.append("\t" + dataBean.getDimensionMesureSDMX() + ":refArea " + urlRefArea + " ;" + "\n");
                        }
                    }
                }
            } else {
                if (header.equals("")) {
                    TransformToRDF.insertError(fileName + ". ERROR. HEADER COLUMN EMPTY " + ". CONFIGURATION FOR THIS COLUMN NOT FOUND ");
                    log.error(fileName + ". ERROR. HEADER COLUMN EMPTY " + ". CONFIGURATION FOR THIS COLUMN NOT FOUND ");
                }
                TransformToRDF.insertError(fileName + ". ERROR. Column " + header + ". CONFIGURATION FOR THIS COLUMN NOT FOUND ");
                log.error(fileName + ". ERROR. Column " + header + ". CONFIGURATION FOR THIS COLUMN NOT FOUND ");
            }
            col++;
        } catch (Exception e) {
            log.error("Error al añadir una observacion en " + configBean.getNameFile(), e);
        }
    }
    if (configBean.getListDataConstant().size() > 0) {
        for (DataBean data : configBean.getListDataConstant()) {
            result.append("\t" + data.getNormalizacion() + " " + "\"" + data.getConstant() + "\"");
            if (Utils.v(data.getType()))
                result.append("^^" + data.getType() + " ." + "\n");
            else
                result.append(" ." + "\n");
            if (data.getNormalizacion().equals("sdmx-dimension:refPeriod")) {
                year = true;
            }
        }
    }
    if (!year) {
        result.append("\tsdmx-dimension:refPeriod <http://reference.data.gov.uk/id/year/2011> ." + "\n");
    }
    endResult = (result.toString()).substring(0, result.length() - 2);
    endResult = endResult + "." + "\n";
    result = new StringBuffer(endResult);
    log.debug("End addObservation");
    return result;
}

Example 8 with DataBean

use of com.localidata.process.bean.DataBean in project local-data-aragopedia by aragonopendata.

the class GenerateConfig method extractSkosConcept.

private void extractSkosConcept(List<String> csvLines, ArrayList<DataBean> skosData) {
    log.debug("Init extractSkosConcept");
    String headerLine = Utils.weakClean(csvLines.get(0));
    String[] cells = headerLine.split("\t");
    int[] posColumn = new int[skosData.size()];
    for (int h = 0; h < skosData.size(); h++) {
        String name = skosData.get(h).getName();
        for (int i = 0; i < cells.length; i++) {
            if (cells[i].equalsIgnoreCase(name)) {
                boolean incluido = false;
                for (int j = 0; j < posColumn.length; j++) {
                    if (posColumn[j] == i)
                        incluido = true;
                }
                if (!incluido) {
                    posColumn[h] = i;
                    break;
                }
            }
        }
    }
    for (int h = 1; h < csvLines.size(); h++) {
        String line = Utils.weakClean(csvLines.get(h));
        if (Utils.v(line)) {
            cells = line.split("\t");
            for (int i = 0; i < skosData.size(); i++) {
                try {
                    String cell = cells[posColumn[i]];
                    SkosBean skosBean = new SkosBean();
                    String skosUrified = Utils.urlify(cell);
                    skosBean.setId(skosUrified);
                    skosBean.setLabel(Utils.weakClean(cell));
                    skosBean.setURI(Prop.host + "/kos/" + Prop.datasetName + "/" + Utils.urlify(skosData.get(i).getName()) + "/" + skosUrified);
                    DataBean dataBean = null;
                    if (skosExtrated.get(skosData.get(i).getName()) != null) {
                        dataBean = skosExtrated.get(skosData.get(i).getName());
                    } else {
                        dataBean = skosData.get(i);
                    }
                    if (dataBean.getMapSkos().get(skosBean.getId()) == null) {
                        dataBean.getMapSkos().put(skosBean.getId(), skosBean);
                        skosExtrated.put(dataBean.getName(), dataBean);
                    }
                } catch (ArrayIndexOutOfBoundsException e) {
                    log.error("ERROR al extraer los skos debido a incoherencia de columnas", e);
                    DataBean dataBean = null;
                    if (skosExtrated.get(skosData.get(i).getName()) != null) {
                        dataBean = skosExtrated.get(skosData.get(i).getName());
                    } else {
                        dataBean = skosData.get(i);
                    }
                    skosExtrated.put(dataBean.getName(), dataBean);
                }
            }
        }
    }
    log.debug("End extractSkosConcept");
}

Example 9 with DataBean

use of com.localidata.process.bean.DataBean in project local-data-aragopedia by aragonopendata.

the class GenerateConfig method generateSkosMapping.

public void generateSkosMapping() {
    log.debug("Init generateSkosMapping");
    String filedSeparator = "\"";
    String csvSeparator = ",";
    for (String key : GenerateConfig.skosExtrated.keySet()) {
        StringBuffer content = new StringBuffer();
        DataBean data = GenerateConfig.skosExtrated.get(key);
        for (String skosName : data.getMapSkos().keySet()) {
            SkosBean skosBean = data.getMapSkos().get(skosName);
            if (Utils.v(skosBean.getId()))
                content.append(filedSeparator + skosBean.getLabel() + filedSeparator + csvSeparator + filedSeparator + skosBean.getURI() + filedSeparator + System.getProperty("line.separator"));
        }
        String nameFile = "mapping-" + Utils.urlify(data.getName());
        String pathFile = configDirectoryString + File.separator + nameFile + ".csv";
        log.info("comienza a escribirse el archivo " + nameFile + ".csv");
        File file = new File(pathFile);
        try {
            Utils.stringToFile(content.toString(), file);
            if (Prop.publishDrive) {
                GoogleDriveAPI api = new GoogleDriveAPI();
                api.init();
                api.createSpreadsheetFromFile(Prop.idParentFolder, Prop.emailUserFile, "csv", nameFile, file, "text/csv");
            }
            log.info("finaliza de escribirse el archivo " + nameFile + ".csv");
        } catch (Exception e) {
            log.error("Error to generate skos mapping " + pathFile, e);
        }
    }
    log.debug("End generateSkosMapping");
}