Examples with MCRContentStore org.mycore.datamodel.ifs.MCRContentStore used on opensource projects

Example 1 with MCRContentStore

use of org.mycore.datamodel.ifs.MCRContentStore in project mycore by MyCoRe-Org.

the class MCRIFSCommands method moveFile.

@MCRCommand(syntax = "move ifs node {0} to store {1}", help = "Moves the MCRFile with IFSID {0} to a new MCRContentStore with ID {1}")
public static void moveFile(String ifsId, String storeID) throws IOException {
    MCRContentStore store = MCRContentStoreFactory.getStore(storeID);
    String storageID = moveFile(ifsId, store);
    LOGGER.debug("File id={} has storage ID {} in store {}.", ifsId, storageID, store);
}

Example 2 with MCRContentStore

use of org.mycore.datamodel.ifs.MCRContentStore in project mycore by MyCoRe-Org.

the class MCRIFSCommands method writeMissingNodesReport.

@MCRCommand(syntax = "generate missing nodes report in directory {0}", help = "generates XML report over all content stores about missing ifs nodes and write it in directory {0}")
public static void writeMissingNodesReport(String targetDirectory) throws SAXException, TransformerConfigurationException, IOException {
    File targetDir = getDirectory(targetDirectory);
    Map<String, MCRContentStore> availableStores = MCRContentStoreFactory.getAvailableStores();
    final String nsURI = NS_URI;
    final String ATT_TYPE = CDATA;
    SAXTransformerFactory tf = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
    Attributes2Impl atts = new Attributes2Impl();
    final String rootName = "missingnodes";
    EntityManager em = MCREntityManagerProvider.getCurrentEntityManager();
    for (MCRContentStore currentStore : availableStores.values()) {
        File baseDir;
        try {
            baseDir = currentStore.getBaseDir();
            if (baseDir == null) {
                LOGGER.warn("Could not get baseDir of store: {}", currentStore.getID());
                continue;
            }
        } catch (Exception e) {
            LOGGER.warn("Could not get baseDir of store: {}", currentStore.getID(), e);
            continue;
        }
        MCRStreamQuery<String> streamQuery = MCRStreamQuery.getInstance(em, "select storeageid from MCRFSNODES where type='F' and storeid=:storeid order by storageid", String.class).setParameter("storeid", currentStore.getID());
        boolean endOfList = false;
        String nameOfProject = MCRConfiguration.instance().getString("MCR.NameOfProject", "MyCoRe");
        String storeID = currentStore.getID();
        File outputFile = new File(targetDir, MessageFormat.format("{0}-{1}-{2}.xml", nameOfProject, storeID, rootName));
        StreamResult streamResult;
        try {
            streamResult = new StreamResult(new FileOutputStream(outputFile));
        } catch (FileNotFoundException e) {
            // should not happen as we checked it before
            LOGGER.error(e);
            return;
        }
        try {
            TransformerHandler th = tf.newTransformerHandler();
            Transformer serializer = th.getTransformer();
            serializer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
            serializer.setOutputProperty(OutputKeys.INDENT, "yes");
            th.setResult(streamResult);
            LOGGER.info("Writing to file: {}", outputFile.getAbsolutePath());
            th.startDocument();
            atts.clear();
            atts.addAttribute(nsURI, "project", "project", ATT_TYPE, nameOfProject);
            atts.addAttribute(nsURI, "store", "store", ATT_TYPE, storeID);
            atts.addAttribute(nsURI, "baseDir", "baseDir", ATT_TYPE, baseDir.getAbsolutePath());
            th.startElement(nsURI, rootName, rootName, atts);
            URI baseURI = baseDir.toURI();
            try (Stream<String> resultStream = streamQuery.getResultStream()) {
                Iterator<String> storageIds = resultStream.iterator();
                int rows = -1;
                for (File currentFile : new FileStoreIterator(baseDir)) {
                    if (currentFile.isDirectory()) {
                        String relative = baseURI.relativize(currentFile.toURI()).getPath();
                        LOGGER.info("Checking segment: {}", relative);
                    } else {
                        int checkFile = endOfList ? -1 : checkFile(baseURI, currentFile, storageIds, rows);
                        rows++;
                        endOfList = checkFile == -1;
                        if (endOfList || checkFile == 1) {
                            LOGGER.warn("Found orphaned file: {}", currentFile);
                            atts.clear();
                            atts.addAttribute(NS_URI, ATT_FILE_NAME, ATT_FILE_NAME, CDATA, baseURI.relativize(currentFile.toURI()).getPath());
                            th.startElement(NS_URI, ELEMENT_FILE, ELEMENT_FILE, atts);
                            th.endElement(NS_URI, ELEMENT_FILE, ELEMENT_FILE);
                        }
                    }
                }
            }
            th.endElement(nsURI, rootName, rootName);
            th.endDocument();
        } finally {
            OutputStream stream = streamResult.getOutputStream();
            if (stream != null) {
                stream.close();
            }
        }
    }
}

Example 3 with MCRContentStore

use of org.mycore.datamodel.ifs.MCRContentStore in project mycore by MyCoRe-Org.

the class MCRIFSCommands method checkMCRFSNODESForDerivatesWithProjectID.

@MCRCommand(syntax = "check derivates of mcrfsnodes with project id {0}", help = "check the entries of MCRFSNODES for all derivates with project ID {0}")
public static void checkMCRFSNODESForDerivatesWithProjectID(String project_id) {
    LOGGER.info("Start check of MCRFSNODES for derivates with project ID {}", project_id);
    if (project_id == null || project_id.length() == 0) {
        LOGGER.error("Project ID missed for check MCRFSNODES entries of derivates with project ID {0}");
        return;
    }
    Map<String, MCRContentStore> availableStores = MCRContentStoreFactory.getAvailableStores();
    Session session = MCRHIBConnection.instance().getSession();
    MCRXMLMetadataManager mgr = MCRXMLMetadataManager.instance();
    List<String> id_list = mgr.listIDsForBase(project_id + "_derivate");
    int counter = 0;
    int maxresults = id_list.size();
    EntityManager em = MCREntityManagerProvider.getCurrentEntityManager();
    CriteriaBuilder cb = em.getCriteriaBuilder();
    CriteriaQuery<MCRFSNODES> query = cb.createQuery(MCRFSNODES.class);
    Root<MCRFSNODES> nodes = query.from(MCRFSNODES.class);
    ParameterExpression<String> ownerID = cb.parameter(String.class);
    TypedQuery<MCRFSNODES> typedQuery = em.createQuery(query.where(cb.equal(nodes.get(MCRFSNODES_.owner), ownerID), cb.equal(nodes.get(MCRFSNODES_.type), "F")).orderBy(cb.asc(nodes.get(MCRFSNODES_.storageid))));
    for (String derid : id_list) {
        counter++;
        LOGGER.info("Processing dataset {} from {} with ID: {}", counter, maxresults, derid);
        // check mcrfsnodes entries
        try {
            AtomicInteger nodeCount = new AtomicInteger();
            typedQuery.setParameter(ownerID, derid);
            typedQuery.getResultList().stream().forEach(fsNode -> {
                nodeCount.incrementAndGet();
                String store_name = fsNode.getStoreid();
                String storageid = fsNode.getStorageid();
                String name = fsNode.getName();
                long size = fsNode.getSize();
                Date date = fsNode.getDate();
                GregorianCalendar datecal = new GregorianCalendar(TimeZone.getDefault(), Locale.getDefault());
                datecal.setTime(date);
                String fctid = fsNode.getFctid();
                String md5 = fsNode.getMd5();
                session.evict(fsNode);
                LOGGER.debug("File for [owner] {} [name] {} [storeid] {} [storageid] {} [fctid] {} [size] {} [md5] {}", derid, name, store_name, storageid, fctid, size, md5);
                // get path of file
                MCRContentStore fs_store = availableStores.get(store_name);
                if (fs_store == null) {
                    LOGGER.error("Can't find content store {}", store_name);
                    return;
                }
                try {
                    File content_file = fs_store.getLocalFile(storageid);
                    if (content_file == null || !content_file.canRead()) {
                        LOGGER.error("   !!!! Can't access to file {} of store {}", storageid, store_name);
                    }
                } catch (Exception e) {
                    LOGGER.error("   !!!! Can't access to file {} of store {}", storageid, store_name);
                }
            });
            if (nodeCount.get() == 0) {
                LOGGER.error("   !!!! Can't find file entries in MCRFSNODES for {}", derid);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            session.clear();
        }
    }
    LOGGER.info("Check done for {} entries", Integer.toString(counter));
}

Example 4 with MCRContentStore

use of org.mycore.datamodel.ifs.MCRContentStore in project mycore by MyCoRe-Org.

the class MCRIFSCommands method writeMD5SumFile.

@MCRCommand(syntax = "generate md5sum files in directory {0}", help = "writes md5sum files for every content store in directory {0}")
public static void writeMD5SumFile(String targetDirectory) throws IOException {
    File targetDir = getDirectory(targetDirectory);
    EntityManager em = MCREntityManagerProvider.getCurrentEntityManager();
    MCRStreamQuery<MCRFSNODES> streamQuery = MCRStreamQuery.getInstance(em, "from MCRFSNODES where type='F' order by storeid, storageid", MCRFSNODES.class);
    Map<String, MCRContentStore> availableStores = MCRContentStoreFactory.getAvailableStores();
    String currentStoreId = null;
    MCRContentStore currentStore = null;
    File currentStoreBaseDir = null;
    BufferedWriter bw = null;
    String nameOfProject = MCRConfiguration.instance().getString("MCR.NameOfProject", "MyCoRe");
    try {
        Iterator<MCRFSNODES> fsnodes = streamQuery.getResultStream().iterator();
        while (fsnodes.hasNext()) {
            MCRFSNODES fsNode = fsnodes.next();
            String storeID = fsNode.getStoreid();
            String storageID = fsNode.getStorageid();
            String md5 = fsNode.getMd5();
            em.detach(fsNode);
            if (!storeID.equals(currentStoreId)) {
                // initialize current store
                currentStoreId = storeID;
                currentStore = availableStores.get(storeID);
                if (bw != null) {
                    bw.close();
                }
                File outputFile = new File(targetDir, MessageFormat.format("{0}-{1}.md5", nameOfProject, storeID));
                LOGGER.info("Writing to file: {}", outputFile.getAbsolutePath());
                bw = Files.newBufferedWriter(outputFile.toPath(), Charset.defaultCharset(), StandardOpenOption.CREATE);
                try {
                    currentStoreBaseDir = currentStore.getBaseDir();
                } catch (Exception e) {
                    LOGGER.warn("Could not get baseDir of store: {}", storeID, e);
                    currentStoreBaseDir = null;
                }
            }
            String path = currentStoreBaseDir != null ? currentStore.getLocalFile(storageID).getAbsolutePath() : storageID;
            // current store initialized
            String line = MessageFormat.format("{0}  {1}\n", md5, path);
            bw.write(line);
        }
    } finally {
        if (bw != null) {
            try {
                bw.close();
            } catch (IOException e1) {
                LOGGER.warn("Error while closing file.", e1);
            }
        }
        em.clear();
    }
}

Example 5 with MCRContentStore

use of org.mycore.datamodel.ifs.MCRContentStore in project mycore by MyCoRe-Org.

the class MCRIFSCommands method writeReport.

private static void writeReport(File targetDir, FSNodeChecker checker) throws TransformerFactoryConfigurationError, SAXException, IOException, TransformerConfigurationException {
    EntityManager em = MCREntityManagerProvider.getCurrentEntityManager();
    MCRStreamQuery<MCRFSNODES> query = MCRStreamQuery.getInstance(em, "from MCRFSNODES where type='F' order by storeid, owner, name", MCRFSNODES.class);
    Map<String, MCRContentStore> availableStores = MCRContentStoreFactory.getAvailableStores();
    String currentStoreId = null;
    MCRContentStore currentStore = null;
    File currentStoreBaseDir = null;
    StreamResult streamResult = null;
    String nameOfProject = MCRConfiguration.instance().getString("MCR.NameOfProject", "MyCoRe");
    SAXTransformerFactory tf = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
    TransformerHandler th = null;
    Attributes2Impl atts = new Attributes2Impl();
    final String rootName = checker.getName();
    final String elementName = ELEMENT_FILE;
    final String ATT_BASEDIR = "basedir";
    final String nsURI = NS_URI;
    final String ATT_TYPE = CDATA;
    String owner = null;
    try (Stream<MCRFSNODES> resultStream = query.getResultStream()) {
        Iterator<MCRFSNODES> fsnodes = resultStream.iterator();
        while (fsnodes.hasNext()) {
            MCRFSNODES fsNode = fsnodes.next();
            String storeID = fsNode.getStoreid();
            String storageID = fsNode.getStorageid();
            em.detach(fsNode);
            if (!storeID.equals(currentStoreId)) {
                // initialize current store
                currentStoreId = storeID;
                currentStore = availableStores.get(storeID);
                if (th != null) {
                    th.endElement(nsURI, rootName, rootName);
                    th.endDocument();
                    OutputStream outputStream = streamResult.getOutputStream();
                    if (outputStream != null) {
                        outputStream.close();
                    }
                }
                File outputFile = new File(targetDir, MessageFormat.format("{0}-{1}-{2}.xml", nameOfProject, storeID, rootName));
                streamResult = new StreamResult(new FileOutputStream(outputFile));
                th = tf.newTransformerHandler();
                Transformer serializer = th.getTransformer();
                serializer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
                serializer.setOutputProperty(OutputKeys.INDENT, "yes");
                th.setResult(streamResult);
                LOGGER.info("Writing to file: {}", outputFile.getAbsolutePath());
                th.startDocument();
                atts.clear();
                atts.addAttribute(nsURI, "project", "project", ATT_TYPE, nameOfProject);
                try {
                    currentStoreBaseDir = currentStore.getBaseDir();
                    atts.addAttribute(nsURI, ATT_BASEDIR, ATT_BASEDIR, ATT_TYPE, currentStoreBaseDir.getAbsolutePath());
                } catch (Exception e) {
                    LOGGER.warn("Could not get baseDir of store: {}", storeID, e);
                    currentStoreBaseDir = null;
                }
                th.startElement(nsURI, rootName, rootName, atts);
            }
            if (!fsNode.getOwner().equals(owner)) {
                owner = fsNode.getOwner();
                LOGGER.info("Checking owner/derivate: {}", owner);
            }
            File f = null;
            try {
                f = currentStore.getLocalFile(storageID);
            } catch (IOException e) {
                LOGGER.warn("Missing file with storageID: {}", storageID);
            }
            if (!checker.checkNode(fsNode, f, atts)) {
                th.startElement(nsURI, elementName, elementName, atts);
                th.endElement(nsURI, elementName, elementName);
            }
        }
    } finally {
        em.clear();
        if (th != null) {
            try {
                th.endElement(nsURI, rootName, rootName);
                th.endDocument();
                OutputStream outputStream = streamResult.getOutputStream();
                if (outputStream != null) {
                    outputStream.close();
                }
            } catch (IOException e1) {
                LOGGER.warn("Error while closing file.", e1);
            }
        }
    }
}