Examples with GoogleDriveAPI com.localidata.generic.GoogleDriveAPI used on opensource projects

Example 1 with GoogleDriveAPI

use of com.localidata.generic.GoogleDriveAPI in project local-data-aragopedia by aragonopendata.

the class GenerateRDF method readConfig.

public void readConfig(HashMap<String, String> idDescription) {
    log.debug("Init readConfig");
    log.info("Se descarga la configuración");
    GoogleDriveAPI api = new GoogleDriveAPI();
    api.init();
    try {
        if (Prop.downloadDrive)
            api.downloadFolderFiles(configDirectoryString, Prop.idParentFolder);
    } catch (IOException e1) {
        e1.printStackTrace();
        String error = "Por problemas con la conexión de google drive no se puede descargar la configuración, intentelo más tarde.";
        log.error(error);
    }
    log.info("Comienza a extraerse la configuración");
    File configDirectoryFile = new File(configDirectoryString);
    File areasReportFile = new File(outputDirectoryString + File.separator + "areas.txt");
    Collection<File> listCSV = FileUtils.listFiles(configDirectoryFile, extensionsConfig, true);
    int cont = 0;
    int size = listCSV.size();
    for (File file : listCSV) {
        log.info("Se extrae el fichero " + file.getName() + " " + (++cont) + " " + size);
        if (!file.getName().startsWith("mapping") && !file.getName().startsWith(Prop.fileHashCSV)) {
            ConfigBean configBean = new ConfigBean();
            configBean.setNameFile(file.getName());
            String id = file.getName().substring(8);
            String areas = "";
            id = id.replace(".csv", "");
            id = id.replace(".xlsx", "");
            if (id.contains("TC")) {
                id = id.replace("TC", "");
                configBean.getLetters().add("TC");
                areas += "TC ";
            }
            if (id.contains("TM")) {
                id = id.replace("TM", "");
                configBean.getLetters().add("TM");
                areas += "TM ";
            }
            if (id.contains("TP")) {
                id = id.replace("TP", "");
                configBean.getLetters().add("TP");
                areas += "TP ";
            }
            if (id.contains("A")) {
                id = id.substring(0, id.lastIndexOf("A")) + id.substring(id.lastIndexOf("A") + 1, id.length());
                configBean.getLetters().add("A");
                areas += "A ";
            }
            while (id.charAt(id.length() - 1) == '-') {
                id = id.substring(0, id.length() - 1);
            }
            configBean.setId(id);
            if (Prop.formatConfig.equals("csv")) {
                readCsv(file, configBean);
            } else {
                readXlsxFile(file, configBean);
            }
            mapconfig.put(id, configBean);
            Utils.stringToFileAppend(id + " " + areas + "\n", areasReportFile);
        }
    }
    for (Iterator<DataBean> it1 = dataWithSkosHierarchical.iterator(); it1.hasNext(); ) {
        DataBean data1 = (DataBean) it1.next();
        if (data1.getRelationKos() != null) {
            DataBean data2 = mapconfig.get(data1.getIdConfig()).getMapData().get(data1.getRelationKos());
            HashMap<String, SkosBean> mapSkos = data1.mergeSkos(data2);
            if (mapSkos != null) {
                try {
                    log.info("Kos " + data1.getName() + " is parent of " + data2.getName());
                    data2.setWriteSkos(false);
                    data1.setMapSkos(mapSkos);
                    data2.setMapSkos(mapSkos);
                    mapconfig.get(data1.getIdConfig()).getMapData().get(data1.getNameNormalized()).setMapSkos(mapSkos);
                    mapconfig.get(data2.getIdConfig()).getMapData().get(data2.getNameNormalized()).setMapSkos(mapSkos);
                    data1.setNormalizacion(data1.getNormalizacion().replace(data1.getNameNormalized(), data1.getKosNameNormalized()));
                    data2.setNormalizacion(data2.getNormalizacion().replace(data2.getNameNormalized(), data2.getKosNameNormalized()));
                } catch (Exception e) {
                    log.error("[M]Error : ", e);
                }
            }
        }
    }
    if (idDescription == null) {
        this.idDescription = new HashMap<String, String>();
        File urlsFile = new File(urlsFileString);
        List<String> csvLines;
        try {
            csvLines = FileUtils.readLines(urlsFile, "UTF-8");
            for (int h = 1; h < csvLines.size(); h++) {
                String line = csvLines.get(h);
                String[] valores = line.split(",");
                valores[0] = valores[0].replaceAll("\"", "");
                valores[1] = valores[1].replaceAll("\"", "");
                valores[2] = valores[2].replaceAll("\"", "");
                this.idDescription.put(valores[1], valores[2]);
            }
        } catch (IOException e) {
            log.error("Error leyendo la configuración", e);
        }
    } else {
        this.idDescription = idDescription;
    }
    log.info("Finaliza de extraerse la configuración");
    log.debug("End readConfig");
}

Example 2 with GoogleDriveAPI

use of com.localidata.generic.GoogleDriveAPI in project local-data-aragopedia by aragonopendata.

the class GenerateConfig method updateConfig.

public void updateConfig(List<String> changes, List<String> news, HashMap<String, ConfigBean> configMap) {
    log.info("init updateConfig");
    HashMap<String, ConfigBean> configExtrated = generateUpdateConfig();
    this.configMap = configMap;
    GoogleDriveAPI drive = new GoogleDriveAPI();
    drive.init();
    String mensaje = "";
    String mensajeAux = "";
    String mensajesNuevos = "";
    String mensajesCambiosNuevaConf = "";
    String mensajeCambiosConf = "";
    String mensajeRegenerar = "";
    generateNewConfig(news);
    for (String nuevo : news) {
        String id = "";
        String letters = "";
        if (nuevo.endsWith("A")) {
            id = nuevo.substring(0, nuevo.length() - 1);
            letters = nuevo.substring(nuevo.length() - 1, nuevo.length());
        } else {
            id = nuevo.substring(0, nuevo.length() - 2);
            letters = nuevo.substring(nuevo.length() - 2, nuevo.length());
        }
        filesNotRDF.add(id + letters);
        com.google.api.services.drive.model.File f = drive.searchFile(id);
        mensajesNuevos = mensajesNuevos + "Se ha detectado un nuevo cubo de datos " + nuevo + ", revise la nueva configuración propuesta y sus mapping " + f.getDefaultOpenWithLink() + "\n\n";
    }
    for (String change : changes) {
        log.info("Detectando si hay cambios en " + change);
        String id = "";
        String letters = "";
        boolean detectadoCambio = false;
        mensajeAux = "";
        if (change.endsWith("A")) {
            id = change.substring(0, change.length() - 1);
            letters = change.substring(change.length() - 1, change.length());
        } else {
            id = change.substring(0, change.length() - 2);
            letters = change.substring(change.length() - 2, change.length());
        }
        try {
            com.google.api.services.drive.model.File f = drive.searchFile(id);
            File fileDrive = drive.downloadFile(configDirectoryString, f, Constants.CSV);
            if (fileDrive == null) {
                log.info("Error al descargar " + id + " f " + f + " configDirectoryString " + configDirectoryString + " fileDrive " + fileDrive);
                continue;
            }
            List<String> csvLinesDrive = FileUtils.readLines(fileDrive, "UTF-8");
            List cellsDriveList = new ArrayList();
            String lineClean = csvLinesDrive.get(0);
            log.debug("csvLinesDrive " + lineClean);
            lineClean = lineClean.replace("'", "");
            int charac = 0;
            int first = -1;
            int last = -1;
            while (lineClean.contains("\"")) {
                if (lineClean.charAt(charac) == '"' && first == -1) {
                    first = charac + 1;
                } else if (lineClean.charAt(charac) == '"' && last == -1) {
                    last = charac;
                    cellsDriveList.add(lineClean.substring(first, last));
                    String aux = lineClean.substring(0, first - 1);
                    String aux2 = lineClean.substring(last + 1, lineClean.length());
                    lineClean = aux + aux2;
                    first = -1;
                    last = -1;
                    charac = 0;
                }
                charac++;
            }
            Object[] cellsDriveArray = Utils.split(lineClean, ",");
            cellsDriveList.addAll(Arrays.asList(cellsDriveArray));
            ConfigBean config = configMap.get(id);
            File fileLocal = new File(inputDirectoryString + File.separator + change + ".csv");
            List<String> csvLinesLocal = FileUtils.readLines(fileLocal, "UTF-8");
            lineClean = csvLinesLocal.get(0);
            log.debug("csvLinesLocal " + lineClean);
            lineClean = lineClean.replace("'", "");
            lineClean = lineClean.replace("", "");
            Object[] cellsLocalArray = Utils.split(lineClean, "\t");
            List cellsLocalList = Arrays.asList(cellsLocalArray);
            ConfigBean configLocal = configExtrated.get(id + letters);
            if (ListUtils.subtract(cellsLocalList, cellsDriveList).size() > 0) {
                List list = ListUtils.subtract(cellsLocalList, cellsDriveList);
                mensajeAux = mensajeAux + "Se han añadido las columnas ";
                for (Object object : list) {
                    if (cellsLocalList.contains(object) && cellsDriveList.contains(object))
                        continue;
                    mensajeAux = mensajeAux + "'" + object + "', ";
                    detectadoCambio = true;
                }
                if (detectadoCambio) {
                    mensajeAux = mensajeAux + " al cubo " + change + ", por favor actualice la configuración " + f.getDefaultOpenWithLink() + "\n\n";
                    if (!mensajesCambiosNuevaConf.contains(mensajeAux))
                        mensajesCambiosNuevaConf = mensajesCambiosNuevaConf + mensajeAux;
                    List<String> lettersList = config.getLetters();
                    for (String letter : lettersList) {
                        filesNotRDF.add(config.getId() + letter);
                    }
                } else {
                    log.info("detectadoCambio " + detectadoCambio);
                }
            } else {
                log.info(change + " Sin nuevas columnas");
            }
            if (config != null) {
                for (String key : config.getMapData().keySet()) {
                    String provisionalMensaje = "Se han añadido los valores ";
                    boolean sendEmail = false;
                    DataBean data = config.getMapData().get(key);
                    if (data.getType() != null && data.getType().equals(Constants.skosType)) {
                        DataBean dataLocal = configLocal.getMapData().get(key);
                        if (dataLocal != null) {
                            for (String key2 : dataLocal.getMapSkos().keySet()) {
                                if (data.getMapSkos().get(key2) == null) {
                                    SkosBean skos = dataLocal.getMapSkos().get(key2);
                                    if (Utils.v(skos.getLabel())) {
                                        provisionalMensaje = provisionalMensaje + "'" + skos.getLabel() + "' ";
                                        sendEmail = true;
                                        detectadoCambio = true;
                                    } else {
                                        log.info("skos.getLabel() " + skos.getLabel());
                                    }
                                } else {
                                    log.info("data.getMapSkos().get(key2) " + data.getMapSkos().get(key2));
                                }
                            }
                        } else {
                            log.info("dataLocal " + dataLocal);
                        }
                    } else {
                        log.info("data.getType() " + data.getType());
                    }
                    if (sendEmail) {
                        String fileName = fileLocal.getName().substring(0, fileLocal.getName().length() - 4);
                        f = drive.searchFile(data.getNameNormalized());
                        provisionalMensaje = provisionalMensaje + "al codelist " + fileName + ", por favor actualice la configuración " + f.getDefaultOpenWithLink() + "\n\n";
                        if (!mensajeCambiosConf.contains(provisionalMensaje))
                            mensajeCambiosConf = mensajeCambiosConf + provisionalMensaje;
                        List<String> lettersList = config.getLetters();
                        for (String letter : lettersList) {
                            filesNotRDF.add(config.getId() + letter);
                        }
                    }
                }
            }
        } catch (IOException e) {
            log.error("Error chequeando las cambios ", e);
        }
        if (!detectadoCambio) {
            mensajeRegenerar = mensajeRegenerar + "En el cubo " + change + ", se han detectado nuevos registros y se va a regenerar el cubo de datos.\n\n";
        }
    }
    String content = "";
    ArrayList<String> idDataCubeVirtuoso = new ArrayList<>();
    try {
        content = Utils.processURLGet(Prop.urlQueryTodosGrafos);
        String[] splitConent = content.split("\"\n");
        for (int h = 1; h < splitConent.length; h++) {
            String line = splitConent[h];
            line = line.replaceAll("\"", "");
            if (line.contains("http://opendata.aragon.es/graph/datacube") && !line.equals("http://opendata.aragon.es/graph/datacube/commonData")) {
                idDataCubeVirtuoso.add(line.replace("http://opendata.aragon.es/graph/datacube/", ""));
            }
        }
    } catch (IOException e1) {
        log.error("Error al procesar la url de los grafos de virtuoso", e1);
    }
    String[] valores = null;
    ArrayList<String> idDataCubeBI = new ArrayList<>();
    File urlsFile = new File(urlsFileString);
    List<String> csvLines;
    try {
        csvLines = FileUtils.readLines(urlsFile, "UTF-8");
        for (int h = 1; h < csvLines.size(); h++) {
            String line = csvLines.get(h);
            valores = line.split(",");
            idDataCubeBI.add(valores[1].replaceAll("\"", ""));
        }
    } catch (IOException e) {
        log.error("Error al leer el archivo de id's de los cubos de datos.", e);
    }
    File dropFile = new File("dropDatacube.txt");
    String contentDropFile = "";
    String mensajesBorrados = "";
    if (ListUtils.subtract(idDataCubeVirtuoso, idDataCubeBI).size() > 0) {
        List<String> cubeDelete = ListUtils.subtract(idDataCubeVirtuoso, idDataCubeBI);
        for (String cube : cubeDelete) {
            mensajesBorrados += "El cubo de datos " + cube + " ya no se encuentra en la base de datos.\n\n";
            contentDropFile += cube + System.lineSeparator();
        }
    }
    try {
        Utils.stringToFile(contentDropFile, dropFile);
    } catch (Exception e) {
        log.error("Error escribiendo el fichero de cubos de datos a borrar", e);
    }
    if (Utils.v(mensajesNuevos) || Utils.v(mensajesCambiosNuevaConf) || Utils.v(mensajeCambiosConf) || Utils.v(mensajeRegenerar) || Utils.v(mensajesBorrados)) {
        if (true) {
            mensaje = mensajesNuevos + mensajesCambiosNuevaConf + mensajeCambiosConf + mensajeRegenerar + mensajesBorrados;
            String titulo = "Cambios en los datos del IAEsT " + Utils.getDate();
            if (Prop.createIssue)
                GithubApi.createIssue(titulo, mensaje);
        }
    }
    log.info("end updateConfig");
}

Example 3 with GoogleDriveAPI

use of com.localidata.generic.GoogleDriveAPI in project local-data-aragopedia by aragonopendata.

the class GenerateConfig method generateSkosMapping.

public void generateSkosMapping(boolean update) {
    log.debug("Init generateSkosMapping");
    String filedSeparator = "\"";
    String csvSeparator = ",";
    for (String key : GenerateConfig.skosExtrated.keySet()) {
        StringBuffer content = new StringBuffer();
        DataBean data = GenerateConfig.skosExtrated.get(key);
        for (String skosName : data.getMapSkos().keySet()) {
            SkosBean skosBean = data.getMapSkos().get(skosName);
            if (Utils.v(skosBean.getId()))
                content.append(filedSeparator + skosBean.getLabel() + filedSeparator + csvSeparator + filedSeparator + skosBean.getURI() + filedSeparator + System.getProperty("line.separator"));
        }
        String nameFile = "mapping-" + Utils.urlify(data.getName());
        String pathFile = configDirectoryString + File.separator + nameFile + ".csv";
        log.info("comienza a escribirse el archivo " + nameFile + ".csv");
        File file = new File(pathFile);
        try {
            Utils.stringToFile(content.toString(), file);
            if (Prop.publishDrive) {
                if (!update) {
                    GoogleDriveAPI api = new GoogleDriveAPI();
                    api.init();
                    com.google.api.services.drive.model.File f = api.searchFile(nameFile);
                    if (f == null)
                        api.createSpreadsheetFromFile(Prop.idParentFolder, Prop.emailUserFile, "csv", nameFile, file, "text/csv");
                }
            }
            log.info("finaliza de escribirse el archivo " + nameFile + ".csv");
        } catch (Exception e) {
            log.error("Error to generate skos mapping " + pathFile, e);
        }
    }
    log.debug("End generateSkosMapping");
}

Example 4 with GoogleDriveAPI

use of com.localidata.generic.GoogleDriveAPI in project local-data-aragopedia by aragonopendata.

the class ConfigBean method toCSV.

public String toCSV() {
    String csvSepartor = ",";
    String fieldSeparator = "\"";
    String content = "";
    DataBean[] dataArray = new DataBean[getMapData().keySet().size()];
    int cont = 0;
    for (String key : getMapData().keySet()) {
        DataBean data = getMapData().get(key);
        dataArray[cont++] = data;
    }
    for (DataBean dataBean : dataArray) {
        content = content + fieldSeparator + dataBean.getName() + fieldSeparator + csvSepartor;
    }
    for (DataBean dataBean : listDataConstant) {
        content = content + fieldSeparator + dataBean.getName() + fieldSeparator + csvSepartor;
    }
    content = content.substring(0, content.length() - 1) + System.getProperty("line.separator");
    for (DataBean dataBean : dataArray) {
        content = content + fieldSeparator + dataBean.getNameNormalized() + fieldSeparator + csvSepartor;
    }
    for (DataBean dataBean : listDataConstant) {
        content = content + fieldSeparator + dataBean.getNameNormalized() + fieldSeparator + csvSepartor;
    }
    content = content.substring(0, content.length() - 1) + System.getProperty("line.separator");
    for (DataBean dataBean : dataArray) {
        content = content + fieldSeparator + dataBean.getNormalizacion() + fieldSeparator + csvSepartor;
    }
    for (DataBean dataBean : listDataConstant) {
        content = content + fieldSeparator + dataBean.getNormalizacion() + fieldSeparator + csvSepartor;
    }
    content = content.substring(0, content.length() - 1) + System.getProperty("line.separator");
    for (DataBean dataBean : dataArray) {
        content = content + fieldSeparator + dataBean.getDimensionMesureEntry() + fieldSeparator + csvSepartor;
    }
    for (DataBean dataBean : listDataConstant) {
        content = content + fieldSeparator + dataBean.getDimensionMesureEntry() + fieldSeparator + csvSepartor;
    }
    content = content.substring(0, content.length() - 1) + System.getProperty("line.separator");
    for (DataBean dataBean : dataArray) {
        content = content + fieldSeparator + dataBean.getType() + fieldSeparator + csvSepartor;
    }
    for (DataBean dataBean : listDataConstant) {
        content = content + fieldSeparator + dataBean.getType() + fieldSeparator + csvSepartor;
    }
    content = content.substring(0, content.length() - 1) + System.getProperty("line.separator");
    for (DataBean dataBean : dataArray) {
        DataBean dataBeanAux = GenerateConfig.skosExtrated.get(dataBean.getName());
        if (dataBeanAux != null && dataBeanAux.getMapSkos() != null && dataBeanAux.getMapSkos().size() > 0) {
            String nameFile = dataBeanAux.generateSkosMapping();
            content = content + fieldSeparator + nameFile + fieldSeparator + csvSepartor;
        } else {
            content = content + fieldSeparator + fieldSeparator + csvSepartor;
        }
    }
    for (DataBean dataBean : listDataConstant) {
        content = content + fieldSeparator + dataBean.getConstant() + fieldSeparator + csvSepartor;
    }
    content = content.substring(0, content.length() - 1) + System.getProperty("line.separator");
    String nameFile = GenerateConfig.configDirectoryString + File.separator + getNameFile();
    File file = new File(nameFile);
    try {
        Utils.stringToFile(content, file);
    } catch (Exception e) {
        log.error("Error to generate config file " + getNameFile(), e);
    }
    if (Prop.publishDrive) {
        if (!isUpdated()) {
            GoogleDriveAPI api = new GoogleDriveAPI();
            api.init();
            api.createSpreadsheetFromFile(Prop.idParentFolder, Prop.emailUserFile, "csv", getNameFile().substring(0, getNameFile().length() - 4), file, "text/csv");
        }
    }
    return content;
}