Examples with SkosBean com.localidata.bean.SkosBean used on opensource projects

Example 1 with SkosBean

use of com.localidata.bean.SkosBean in project local-data-aragopedia by aragonopendata.

the class GenerateRDF method readConfig.

public void readConfig(HashMap<String, String> idDescription) {
    log.debug("Init readConfig");
    log.info("Se descarga la configuración");
    GoogleDriveAPI api = new GoogleDriveAPI();
    api.init();
    try {
        if (Prop.downloadDrive)
            api.downloadFolderFiles(configDirectoryString, Prop.idParentFolder);
    } catch (IOException e1) {
        e1.printStackTrace();
        String error = "Por problemas con la conexión de google drive no se puede descargar la configuración, intentelo más tarde.";
        log.error(error);
    }
    log.info("Comienza a extraerse la configuración");
    File configDirectoryFile = new File(configDirectoryString);
    File areasReportFile = new File(outputDirectoryString + File.separator + "areas.txt");
    Collection<File> listCSV = FileUtils.listFiles(configDirectoryFile, extensionsConfig, true);
    int cont = 0;
    int size = listCSV.size();
    for (File file : listCSV) {
        log.info("Se extrae el fichero " + file.getName() + " " + (++cont) + " " + size);
        if (!file.getName().startsWith("mapping") && !file.getName().startsWith(Prop.fileHashCSV)) {
            ConfigBean configBean = new ConfigBean();
            configBean.setNameFile(file.getName());
            String id = file.getName().substring(8);
            String areas = "";
            id = id.replace(".csv", "");
            id = id.replace(".xlsx", "");
            if (id.contains("TC")) {
                id = id.replace("TC", "");
                configBean.getLetters().add("TC");
                areas += "TC ";
            }
            if (id.contains("TM")) {
                id = id.replace("TM", "");
                configBean.getLetters().add("TM");
                areas += "TM ";
            }
            if (id.contains("TP")) {
                id = id.replace("TP", "");
                configBean.getLetters().add("TP");
                areas += "TP ";
            }
            if (id.contains("A")) {
                id = id.substring(0, id.lastIndexOf("A")) + id.substring(id.lastIndexOf("A") + 1, id.length());
                configBean.getLetters().add("A");
                areas += "A ";
            }
            while (id.charAt(id.length() - 1) == '-') {
                id = id.substring(0, id.length() - 1);
            }
            configBean.setId(id);
            if (Prop.formatConfig.equals("csv")) {
                readCsv(file, configBean);
            } else {
                readXlsxFile(file, configBean);
            }
            mapconfig.put(id, configBean);
            Utils.stringToFileAppend(id + " " + areas + "\n", areasReportFile);
        }
    }
    for (Iterator<DataBean> it1 = dataWithSkosHierarchical.iterator(); it1.hasNext(); ) {
        DataBean data1 = (DataBean) it1.next();
        if (data1.getRelationKos() != null) {
            DataBean data2 = mapconfig.get(data1.getIdConfig()).getMapData().get(data1.getRelationKos());
            HashMap<String, SkosBean> mapSkos = data1.mergeSkos(data2);
            if (mapSkos != null) {
                try {
                    log.info("Kos " + data1.getName() + " is parent of " + data2.getName());
                    data2.setWriteSkos(false);
                    data1.setMapSkos(mapSkos);
                    data2.setMapSkos(mapSkos);
                    mapconfig.get(data1.getIdConfig()).getMapData().get(data1.getNameNormalized()).setMapSkos(mapSkos);
                    mapconfig.get(data2.getIdConfig()).getMapData().get(data2.getNameNormalized()).setMapSkos(mapSkos);
                    data1.setNormalizacion(data1.getNormalizacion().replace(data1.getNameNormalized(), data1.getKosNameNormalized()));
                    data2.setNormalizacion(data2.getNormalizacion().replace(data2.getNameNormalized(), data2.getKosNameNormalized()));
                } catch (Exception e) {
                    log.error("[M]Error : ", e);
                }
            }
        }
    }
    if (idDescription == null) {
        this.idDescription = new HashMap<String, String>();
        File urlsFile = new File(urlsFileString);
        List<String> csvLines;
        try {
            csvLines = FileUtils.readLines(urlsFile, "UTF-8");
            for (int h = 1; h < csvLines.size(); h++) {
                String line = csvLines.get(h);
                String[] valores = line.split(",");
                valores[0] = valores[0].replaceAll("\"", "");
                valores[1] = valores[1].replaceAll("\"", "");
                valores[2] = valores[2].replaceAll("\"", "");
                this.idDescription.put(valores[1], valores[2]);
            }
        } catch (IOException e) {
            log.error("Error leyendo la configuración", e);
        }
    } else {
        this.idDescription = idDescription;
    }
    log.info("Finaliza de extraerse la configuración");
    log.debug("End readConfig");
}

Example 2 with SkosBean

use of com.localidata.bean.SkosBean in project local-data-aragopedia by aragonopendata.

the class GenerateRDF method readCsv.

private void readCsv(File file, ConfigBean configBean) {
    log.debug("Init readCsv");
    List<String> csvLines;
    try {
        csvLines = FileUtils.readLines(file, "UTF-8");
        String[] cellsName = csvLines.get(0).split(",");
        String[] cellsNameNormalized = csvLines.get(1).split(",");
        String[] cellsNormalization = csvLines.get(2).split(",");
        String[] cellsDimMesure = csvLines.get(3).split(",");
        String[] cellsType = csvLines.get(4).split(",");
        String[] cellsSkosfile = csvLines.get(5).split(",");
        String[] cellsConstant = null;
        String[] cellsConstantValue = null;
        String[] cellsRelationKos = null;
        String[] cellsKosName = null;
        String[] cellsKosNameNormalized = null;
        if (csvLines.size() == 7)
            cellsConstant = csvLines.get(6).split(",");
        if (csvLines.size() == 8)
            cellsConstantValue = csvLines.get(7).split(",");
        if (csvLines.size() == 9)
            cellsRelationKos = csvLines.get(9).split(",");
        if (csvLines.size() == 10)
            cellsKosName = csvLines.get(10).split(",");
        if (csvLines.size() == 11)
            cellsKosNameNormalized = csvLines.get(11).split(",");
        int columnReaded = 0;
        while (columnReaded < cellsName.length) {
            DataBean dataBean = new DataBean();
            if (cellsName[columnReaded] == null) {
                columnReaded++;
            } else {
                dataBean.setName(removeStartEndCaracter(cellsName[columnReaded]));
                dataBean.setNameNormalized(removeStartEndCaracter(cellsNameNormalized[columnReaded]));
                dataBean.setNormalizacion(removeStartEndCaracter(cellsNormalization[columnReaded]));
                dataBean.setDimensionMesure(removeStartEndCaracter(cellsDimMesure[columnReaded]));
                dataBean.setIdConfig(configBean.getId());
                String type = "";
                if (Utils.v(removeStartEndCaracter(cellsType[columnReaded]))) {
                    type = removeStartEndCaracter(cellsType[columnReaded]);
                } else {
                    type = "xsd:string";
                }
                dataBean.setType(type);
                if (Utils.v(removeStartEndCaracter(cellsSkosfile[columnReaded]))) {
                    HashMap<String, SkosBean> mapSkos = readMappingFileCSV(removeStartEndCaracter(cellsSkosfile[columnReaded]));
                    dataBean.setMapSkos(mapSkos);
                    configBean.getMapData().put(dataBean.getNameNormalized(), dataBean);
                    dataWithSkos.add(dataBean);
                } else {
                    configBean.getMapData().put(dataBean.getNameNormalized(), dataBean);
                }
                if (Prop.addDataConstant && cellsConstant != null && Utils.v(removeStartEndCaracter(cellsConstant[columnReaded])) && removeStartEndCaracter(cellsConstant[columnReaded]).equals(Constants.constante)) {
                    if (Utils.v(removeStartEndCaracter(cellsConstantValue[columnReaded]))) {
                        dataBean.setConstant(removeStartEndCaracter(cellsConstantValue[columnReaded]) + "");
                        configBean.getListDataConstant().add(dataBean);
                    }
                }
                if (cellsRelationKos != null && Utils.v(removeStartEndCaracter(cellsRelationKos[columnReaded]))) {
                    dataBean.setRelationKos(removeStartEndCaracter(cellsRelationKos[columnReaded]));
                    dataWithSkosHierarchical.add(dataBean);
                }
                if (cellsKosNameNormalized != null && Utils.v(removeStartEndCaracter(cellsKosNameNormalized[columnReaded]))) {
                    dataBean.setKosNameNormalized(removeStartEndCaracter(cellsKosNameNormalized[columnReaded]));
                } else {
                    dataBean.setKosNameNormalized(dataBean.getNameNormalized());
                }
                if (cellsKosName != null && Utils.v(removeStartEndCaracter(cellsKosName[columnReaded]))) {
                    dataBean.setKosName(removeStartEndCaracter(cellsKosName[columnReaded]));
                } else {
                    dataBean.setKosName(dataBean.getName());
                }
                columnReaded++;
            }
        }
    } catch (IOException e) {
        log.error("Error read csv ", e);
    }
    log.debug("End readCsv");
}

Example 3 with SkosBean

use of com.localidata.bean.SkosBean in project local-data-aragopedia by aragonopendata.

the class GenerateRDF method readMappingFile.

private HashMap<String, SkosBean> readMappingFile(String skosPath) {
    log.debug("Init readSkosFile");
    HashMap<String, SkosBean> mapSkos = new HashMap<String, SkosBean>();
    File skosMappingg = new File(configDirectoryString + File.separator + skosPath);
    InputStream inp = null;
    Workbook wb = null;
    try {
        inp = new FileInputStream(skosMappingg);
        wb = WorkbookFactory.create(inp);
    } catch (FileNotFoundException e) {
        log.error(e.getMessage());
    } catch (InvalidFormatException e) {
        log.error(e.getMessage());
    } catch (IOException e) {
        log.error(e.getMessage());
    }
    if (wb != null) {
        Sheet sheet = wb.getSheetAt(0);
        for (int i = 0; i < sheet.getPhysicalNumberOfRows(); i++) {
            Row row = sheet.getRow(i);
            Cell cellId = row.getCell(0);
            Cell cellUri = row.getCell(1);
            if (cellId == null || cellUri == null) {
                continue;
            }
            SkosBean skosBean = new SkosBean();
            SkosBean skosBeanExtra = new SkosBean();
            String idCell = "";
            if (cellId.getCellType() == 0) {
                Double d = new Double(cellId.getNumericCellValue());
                idCell = d.intValue() + "";
            } else {
                idCell = cellId.getStringCellValue();
            }
            skosBean.setLabel(idCell);
            idCell = Utils.urlify(idCell);
            skosBean.setId(idCell);
            String uriCell = "";
            if (cellUri.getCellType() == 0) {
                Double d = new Double(cellUri.getNumericCellValue());
                uriCell = d.intValue() + "";
            } else {
                uriCell = cellUri.getStringCellValue();
                String id = uriCell.substring(uriCell.lastIndexOf("/") + 1, uriCell.length());
                if (!idCell.equals(id)) {
                    skosBeanExtra.setId(id);
                    skosBeanExtra.setLabel(id);
                    skosBeanExtra.setURI(uriCell);
                    mapSkos.put(id, skosBeanExtra);
                }
            }
            skosBean.setURI(uriCell);
            mapSkos.put(idCell, skosBean);
        }
    }
    log.debug("End readSkosFile");
    return mapSkos;
}

Example 4 with SkosBean

use of com.localidata.bean.SkosBean in project local-data-aragopedia by aragonopendata.

the class GenerateConfig method updateConfig.

public void updateConfig(List<String> changes, List<String> news, HashMap<String, ConfigBean> configMap) {
    log.info("init updateConfig");
    HashMap<String, ConfigBean> configExtrated = generateUpdateConfig();
    this.configMap = configMap;
    GoogleDriveAPI drive = new GoogleDriveAPI();
    drive.init();
    String mensaje = "";
    String mensajeAux = "";
    String mensajesNuevos = "";
    String mensajesCambiosNuevaConf = "";
    String mensajeCambiosConf = "";
    String mensajeRegenerar = "";
    generateNewConfig(news);
    for (String nuevo : news) {
        String id = "";
        String letters = "";
        if (nuevo.endsWith("A")) {
            id = nuevo.substring(0, nuevo.length() - 1);
            letters = nuevo.substring(nuevo.length() - 1, nuevo.length());
        } else {
            id = nuevo.substring(0, nuevo.length() - 2);
            letters = nuevo.substring(nuevo.length() - 2, nuevo.length());
        }
        filesNotRDF.add(id + letters);
        com.google.api.services.drive.model.File f = drive.searchFile(id);
        mensajesNuevos = mensajesNuevos + "Se ha detectado un nuevo cubo de datos " + nuevo + ", revise la nueva configuración propuesta y sus mapping " + f.getDefaultOpenWithLink() + "\n\n";
    }
    for (String change : changes) {
        log.info("Detectando si hay cambios en " + change);
        String id = "";
        String letters = "";
        boolean detectadoCambio = false;
        mensajeAux = "";
        if (change.endsWith("A")) {
            id = change.substring(0, change.length() - 1);
            letters = change.substring(change.length() - 1, change.length());
        } else {
            id = change.substring(0, change.length() - 2);
            letters = change.substring(change.length() - 2, change.length());
        }
        try {
            com.google.api.services.drive.model.File f = drive.searchFile(id);
            File fileDrive = drive.downloadFile(configDirectoryString, f, Constants.CSV);
            if (fileDrive == null) {
                log.info("Error al descargar " + id + " f " + f + " configDirectoryString " + configDirectoryString + " fileDrive " + fileDrive);
                continue;
            }
            List<String> csvLinesDrive = FileUtils.readLines(fileDrive, "UTF-8");
            List cellsDriveList = new ArrayList();
            String lineClean = csvLinesDrive.get(0);
            log.debug("csvLinesDrive " + lineClean);
            lineClean = lineClean.replace("'", "");
            int charac = 0;
            int first = -1;
            int last = -1;
            while (lineClean.contains("\"")) {
                if (lineClean.charAt(charac) == '"' && first == -1) {
                    first = charac + 1;
                } else if (lineClean.charAt(charac) == '"' && last == -1) {
                    last = charac;
                    cellsDriveList.add(lineClean.substring(first, last));
                    String aux = lineClean.substring(0, first - 1);
                    String aux2 = lineClean.substring(last + 1, lineClean.length());
                    lineClean = aux + aux2;
                    first = -1;
                    last = -1;
                    charac = 0;
                }
                charac++;
            }
            Object[] cellsDriveArray = Utils.split(lineClean, ",");
            cellsDriveList.addAll(Arrays.asList(cellsDriveArray));
            ConfigBean config = configMap.get(id);
            File fileLocal = new File(inputDirectoryString + File.separator + change + ".csv");
            List<String> csvLinesLocal = FileUtils.readLines(fileLocal, "UTF-8");
            lineClean = csvLinesLocal.get(0);
            log.debug("csvLinesLocal " + lineClean);
            lineClean = lineClean.replace("'", "");
            lineClean = lineClean.replace("", "");
            Object[] cellsLocalArray = Utils.split(lineClean, "\t");
            List cellsLocalList = Arrays.asList(cellsLocalArray);
            ConfigBean configLocal = configExtrated.get(id + letters);
            if (ListUtils.subtract(cellsLocalList, cellsDriveList).size() > 0) {
                List list = ListUtils.subtract(cellsLocalList, cellsDriveList);
                mensajeAux = mensajeAux + "Se han añadido las columnas ";
                for (Object object : list) {
                    if (cellsLocalList.contains(object) && cellsDriveList.contains(object))
                        continue;
                    mensajeAux = mensajeAux + "'" + object + "', ";
                    detectadoCambio = true;
                }
                if (detectadoCambio) {
                    mensajeAux = mensajeAux + " al cubo " + change + ", por favor actualice la configuración " + f.getDefaultOpenWithLink() + "\n\n";
                    if (!mensajesCambiosNuevaConf.contains(mensajeAux))
                        mensajesCambiosNuevaConf = mensajesCambiosNuevaConf + mensajeAux;
                    List<String> lettersList = config.getLetters();
                    for (String letter : lettersList) {
                        filesNotRDF.add(config.getId() + letter);
                    }
                } else {
                    log.info("detectadoCambio " + detectadoCambio);
                }
            } else {
                log.info(change + " Sin nuevas columnas");
            }
            if (config != null) {
                for (String key : config.getMapData().keySet()) {
                    String provisionalMensaje = "Se han añadido los valores ";
                    boolean sendEmail = false;
                    DataBean data = config.getMapData().get(key);
                    if (data.getType() != null && data.getType().equals(Constants.skosType)) {
                        DataBean dataLocal = configLocal.getMapData().get(key);
                        if (dataLocal != null) {
                            for (String key2 : dataLocal.getMapSkos().keySet()) {
                                if (data.getMapSkos().get(key2) == null) {
                                    SkosBean skos = dataLocal.getMapSkos().get(key2);
                                    if (Utils.v(skos.getLabel())) {
                                        provisionalMensaje = provisionalMensaje + "'" + skos.getLabel() + "' ";
                                        sendEmail = true;
                                        detectadoCambio = true;
                                    } else {
                                        log.info("skos.getLabel() " + skos.getLabel());
                                    }
                                } else {
                                    log.info("data.getMapSkos().get(key2) " + data.getMapSkos().get(key2));
                                }
                            }
                        } else {
                            log.info("dataLocal " + dataLocal);
                        }
                    } else {
                        log.info("data.getType() " + data.getType());
                    }
                    if (sendEmail) {
                        String fileName = fileLocal.getName().substring(0, fileLocal.getName().length() - 4);
                        f = drive.searchFile(data.getNameNormalized());
                        provisionalMensaje = provisionalMensaje + "al codelist " + fileName + ", por favor actualice la configuración " + f.getDefaultOpenWithLink() + "\n\n";
                        if (!mensajeCambiosConf.contains(provisionalMensaje))
                            mensajeCambiosConf = mensajeCambiosConf + provisionalMensaje;
                        List<String> lettersList = config.getLetters();
                        for (String letter : lettersList) {
                            filesNotRDF.add(config.getId() + letter);
                        }
                    }
                }
            }
        } catch (IOException e) {
            log.error("Error chequeando las cambios ", e);
        }
        if (!detectadoCambio) {
            mensajeRegenerar = mensajeRegenerar + "En el cubo " + change + ", se han detectado nuevos registros y se va a regenerar el cubo de datos.\n\n";
        }
    }
    String content = "";
    ArrayList<String> idDataCubeVirtuoso = new ArrayList<>();
    try {
        content = Utils.processURLGet(Prop.urlQueryTodosGrafos);
        String[] splitConent = content.split("\"\n");
        for (int h = 1; h < splitConent.length; h++) {
            String line = splitConent[h];
            line = line.replaceAll("\"", "");
            if (line.contains("http://opendata.aragon.es/graph/datacube") && !line.equals("http://opendata.aragon.es/graph/datacube/commonData")) {
                idDataCubeVirtuoso.add(line.replace("http://opendata.aragon.es/graph/datacube/", ""));
            }
        }
    } catch (IOException e1) {
        log.error("Error al procesar la url de los grafos de virtuoso", e1);
    }
    String[] valores = null;
    ArrayList<String> idDataCubeBI = new ArrayList<>();
    File urlsFile = new File(urlsFileString);
    List<String> csvLines;
    try {
        csvLines = FileUtils.readLines(urlsFile, "UTF-8");
        for (int h = 1; h < csvLines.size(); h++) {
            String line = csvLines.get(h);
            valores = line.split(",");
            idDataCubeBI.add(valores[1].replaceAll("\"", ""));
        }
    } catch (IOException e) {
        log.error("Error al leer el archivo de id's de los cubos de datos.", e);
    }
    File dropFile = new File("dropDatacube.txt");
    String contentDropFile = "";
    String mensajesBorrados = "";
    if (ListUtils.subtract(idDataCubeVirtuoso, idDataCubeBI).size() > 0) {
        List<String> cubeDelete = ListUtils.subtract(idDataCubeVirtuoso, idDataCubeBI);
        for (String cube : cubeDelete) {
            mensajesBorrados += "El cubo de datos " + cube + " ya no se encuentra en la base de datos.\n\n";
            contentDropFile += cube + System.lineSeparator();
        }
    }
    try {
        Utils.stringToFile(contentDropFile, dropFile);
    } catch (Exception e) {
        log.error("Error escribiendo el fichero de cubos de datos a borrar", e);
    }
    if (Utils.v(mensajesNuevos) || Utils.v(mensajesCambiosNuevaConf) || Utils.v(mensajeCambiosConf) || Utils.v(mensajeRegenerar) || Utils.v(mensajesBorrados)) {
        if (true) {
            mensaje = mensajesNuevos + mensajesCambiosNuevaConf + mensajeCambiosConf + mensajeRegenerar + mensajesBorrados;
            String titulo = "Cambios en los datos del IAEsT " + Utils.getDate();
            if (Prop.createIssue)
                GithubApi.createIssue(titulo, mensaje);
        }
    }
    log.info("end updateConfig");
}

Example 5 with SkosBean

use of com.localidata.bean.SkosBean in project local-data-aragopedia by aragonopendata.

the class GenerateConfig method generateSkosMapping.

public void generateSkosMapping(boolean update) {
    log.debug("Init generateSkosMapping");
    String filedSeparator = "\"";
    String csvSeparator = ",";
    for (String key : GenerateConfig.skosExtrated.keySet()) {
        StringBuffer content = new StringBuffer();
        DataBean data = GenerateConfig.skosExtrated.get(key);
        for (String skosName : data.getMapSkos().keySet()) {
            SkosBean skosBean = data.getMapSkos().get(skosName);
            if (Utils.v(skosBean.getId()))
                content.append(filedSeparator + skosBean.getLabel() + filedSeparator + csvSeparator + filedSeparator + skosBean.getURI() + filedSeparator + System.getProperty("line.separator"));
        }
        String nameFile = "mapping-" + Utils.urlify(data.getName());
        String pathFile = configDirectoryString + File.separator + nameFile + ".csv";
        log.info("comienza a escribirse el archivo " + nameFile + ".csv");
        File file = new File(pathFile);
        try {
            Utils.stringToFile(content.toString(), file);
            if (Prop.publishDrive) {
                if (!update) {
                    GoogleDriveAPI api = new GoogleDriveAPI();
                    api.init();
                    com.google.api.services.drive.model.File f = api.searchFile(nameFile);
                    if (f == null)
                        api.createSpreadsheetFromFile(Prop.idParentFolder, Prop.emailUserFile, "csv", nameFile, file, "text/csv");
                }
            }
            log.info("finaliza de escribirse el archivo " + nameFile + ".csv");
        } catch (Exception e) {
            log.error("Error to generate skos mapping " + pathFile, e);
        }
    }
    log.debug("End generateSkosMapping");
}