Examples with TrpDocMetadata eu.transkribus.core.model.beans.TrpDocMetadata used on opensource projects

Example 1 with TrpDocMetadata

use of eu.transkribus.core.model.beans.TrpDocMetadata in project TranskribusCore by Transkribus.

the class GoobiMetsImporter method loadDocFromGoobiMets.

/**
 * Reads the Mets metadata and fetches all files with help of the contained URLs
 * into an temporarily folder and creates a TrpDoc out of it
 *
 * @param metsPath: path to the Goobi mets file
 * @return
 * @throws IOException
 * @throws SAXException
 * @throws JAXBException
 */
public TrpDoc loadDocFromGoobiMets(File metsFile, String localDirPath) throws IOException, JAXBException, SAXException {
    TrpDocMetadata md;
    Mets mets = JaxbUtils.unmarshal(metsFile, Mets.class, TrpDocMetadata.class);
    String metsPath = metsFile.getAbsolutePath();
    updateStatus("Reading metadata...");
    // unmarshal TrpDocMetadata
    md = readModsMetadata(XmlUtils.getDocumentFromFileWOE(metsPath));
    // String localDir = System.getProperty("user.home") + File.separator + "GoobiTest" + File.separator + md.getTitle() + File.separator;
    logger.debug("the local user home dir = " + localDirPath);
    // System.in.read();
    // collect files into "user.home" + "/GoobiTest/" + mods title
    // fetchFiles(localDirPath, mets);
    md.setLocalFolder(new File(localDirPath));
    /*
		 * next line can disorder the ORDER of the pages of the Mets when filename length is not equal and we store 
		 * the files temporary in a local folder instead of importing directly as we did now
		 */
    // final TrpDoc doc = LocalDocReader.load(localDirPath, true);
    // overwrite metadata with the metadata read from the MODS section in the METS file
    final TrpDoc doc = new TrpDoc();
    doc.setMd(md);
    doc.setPages(fetchFiles(localDirPath, mets));
    return doc;
}

Example 2 with TrpDocMetadata

use of eu.transkribus.core.model.beans.TrpDocMetadata in project TranskribusCore by Transkribus.

the class LocalDocReader method load.

/**
 * Read a local doc based on its mets.xml file that has to contain file
 * paths relative to parentDir
 *
 * @param mets the mets object
 * @param parentDir the directory of the local document
 * @return the constructed Document
 * @throws IOException if the path can't be read
 */
public static TrpDoc load(Mets mets, File parentDir) throws IOException {
    final TrpDoc doc = new TrpDoc();
    TrpDocMetadata md;
    List<TrpPage> pages;
    // FIXME set TRP_METS_VERSION to PROFILE, not TYPE
    if (mets.getPROFILE().equals(TrpMetsBuilder.TRP_METS_PROFILE)) {
        // unmarshal TrpDocMetadata
        md = MetsUtil.getTrpDocMd(mets);
        // collect files
        pages = MetsUtil.getTrpPages(mets, parentDir);
    // } else if (mets.getPROFILE().equals(EnmapMetsBuilder.ENMAP_METS_PROFILE)){
    // md = EnmapMetsBuilder.getTrpDocMd(mets);
    // pages = EnmapMetsBuilder.getTrpPages(mets, parentDir);
    } else {
        throw new IOException("Unsupported METS PROFILE: " + mets.getPROFILE());
    }
    md.setLocalFolder(parentDir);
    doc.setMd(md);
    doc.setPages(pages);
    return doc;
}

Example 3 with TrpDocMetadata

use of eu.transkribus.core.model.beans.TrpDocMetadata in project TranskribusCore by Transkribus.

the class LocalDocReader method load.

public static TrpDoc load(TrpUpload upload) throws IOException {
    // validate most necessary things
    if (upload == null) {
        throw new IllegalArgumentException("Upload is null.");
    }
    if (upload.getUploadId() < 1) {
        throw new IllegalArgumentException("Invalid upload ID: " + upload.getUploadId());
    }
    if (!upload.canReadDirectories()) {
        throw new IllegalArgumentException("Directories are not readable: " + upload.getUploadTmpDir().getAbsolutePath());
    }
    // transform the upload object into a TRP document
    TrpDoc doc = new TrpDoc();
    TrpDocMetadata md = upload.getMd();
    md.setLocalFolder(upload.getUploadTmpDir());
    doc.setMd(md);
    File baseDir = upload.getUploadTmpDir();
    File xmlDir = upload.getUploadPageTmpDir();
    File thumbDir = new File(baseDir.getAbsolutePath() + File.separatorChar + LocalDocConst.THUMBS_FILE_SUB_FOLDER);
    for (PageUploadDescriptor p : upload.getPages()) {
        final int pageNr = p.getPageNr();
        File img = new File(baseDir.getAbsolutePath() + File.separator + p.getFileName());
        if (!img.isFile()) {
            throw new FileNotFoundException("Image for page " + pageNr + " does not exist: " + img.getAbsolutePath());
        }
        // try to read image dimension in any case to detect corrupt files immediately!
        Dimension dim = null;
        String imageRemark = null;
        try {
            dim = ImgUtils.readImageDimensions(img);
        } catch (CorruptImageException cie) {
            logger.error("Image is corrupt: " + img.getAbsolutePath(), cie);
            imageRemark = getCorruptImgMsg(img.getName());
        }
        final String imgBaseName = FilenameUtils.getBaseName(img.getName());
        File thumb = getThumbFile(thumbDir, imgBaseName);
        File pageXml = null;
        if (!StringUtils.isEmpty(p.getPageXmlName())) {
            pageXml = new File(xmlDir.getAbsolutePath() + File.separator + p.getPageXmlName());
            if (!pageXml.isFile()) {
                throw new FileNotFoundException("PAGE XML for page " + pageNr + " does not exist: " + img.getAbsolutePath());
            }
        } else if (StringUtils.isEmpty(imageRemark)) {
            // if a problem occured when reading the image
            File pageOutFile = new File(xmlDir.getAbsolutePath() + File.separatorChar + imgBaseName + ".xml");
            PcGtsType pc = PageXmlUtils.createEmptyPcGtsType(img, dim);
            try {
                pageXml = JaxbUtils.marshalToFile(pc, pageOutFile);
            } catch (JAXBException je) {
                logger.error(je.getMessage(), je);
                throw new IOException("Could not create empty PageXml on disk!", je);
            }
        }
        TrpPage page = buildPage(baseDir, pageNr, img, pageXml, thumb, dim, imageRemark);
        doc.getPages().add(page);
    }
    return doc;
}

Example 4 with TrpDocMetadata

use of eu.transkribus.core.model.beans.TrpDocMetadata in project TranskribusCore by Transkribus.

the class LocalDocReader method load.

/**
 * Loads a document from path.<br>
 *
 * Document metadata has to be in an XML called "metadata.xml".<br>
 *
 * Image files and corresponding XML/txt files have to have the same name. <br>
 * Lexicographic order of image names will imply order of pages.<br>
 * Types of transcript source files are searched in this order:
 * <ol>
 * <li>./page: PAGE XMLs according to schema 2010/2013</li>
 * <li>./ocr: Abbyy Finereader XMLs schema version 10</li>
 * <li>./alto: ALTO v2 XMls
 * <li>./txt: txt files with transcription fulltext only
 * </ol>
 * Testdoc is in $dea_scratch/TRP/TrpTestDoc <br>
 * No versioning of files for local use!<br>
 *
 * @param path the path where the document is stored
 * @param config {@link DocLoadConfig}
 * @return the constructed document
 * @throws IOException if the path can't be read or is malformed
 *
 * @todo implement monitor feedback!
 * @todo Respect Storage.uploadDocument where the monitor will be used by the upload itself later.
 */
public static TrpDoc load(final String path, DocLoadConfig config, IProgressMonitor monitor) throws IOException {
    // create the document
    TrpDoc doc = new TrpDoc();
    // check OS and adjust URL protocol
    final String os = System.getProperty("os.name");
    /*
		 * FIXME use SysUtils.isWin() here?
		 */
    if (os.toLowerCase().contains("win")) {
        LocalDocConst.URL_PROT_CONST = "file:///";
    }
    // else: keep default
    final File inputDir = new File(path);
    final File docXml = new File(inputDir.getAbsolutePath() + File.separator + LocalDocConst.DOC_XML_FILENAME);
    // validate input path ======================================================
    checkInputDir(inputDir);
    // search for IMG files
    TreeMap<String, File> pageMap = findImgFiles(inputDir);
    logger.info("Found " + pageMap.entrySet().size() + " page images.");
    if (pageMap.isEmpty()) {
        throw new FileNotFoundException("The directory does not contain any images: " + inputDir.getAbsolutePath());
    }
    TrpDocMetadata docMd = null;
    boolean doRefresh = true;
    // try to read doc structure from disk
    if (docXml.isFile()) {
        doc = loadDocXml(docXml);
        if (isValid(doc, pageMap.size(), config.isForceCreatePageXml())) {
            logger.info("Loaded document structure from disk.");
            docMd = doc.getMd();
            // no refresh is necessary as doc structure matches the input dir content
            doRefresh = false;
        } else {
            if (doc != null && doc.getMd() != null) {
                // keep any existing metadata if invalid doc structure was found
                docMd = doc.getMd();
            }
            logger.info("Removing faulty doc XML from disk and doing reload.");
            docXml.delete();
            doc = new TrpDoc();
        }
    }
    logger.info("Reading document at " + inputDir.getAbsolutePath());
    // find metadata file if not extracted from doc.xml =============================================
    if (docMd == null) {
        try {
            docMd = loadDocMd(inputDir);
        } catch (IOException ioe) {
            docMd = new TrpDocMetadata();
        }
    }
    initDocMd(docMd, inputDir, config.isStripServerRelatedMetadata());
    // Set the docMd
    doc.setMd(docMd);
    if (!doRefresh) {
        // Stop now and reuse doc structure from file
        return doc;
    }
    // Construct the input dir with pageXml Files.
    File pageInputDir = getPageXmlInputDir(inputDir);
    if (config.isForceCreatePageXml() && !pageInputDir.isDirectory()) {
        pageInputDir.mkdir();
    }
    // abbyy XML search path
    File ocrInputDir = getOcrXmlInputDir(inputDir);
    // alto XML search path
    File altoInputDir = getAltoXmlInputDir(inputDir);
    // alto XML search path
    File txtInputDir = getTxtInputDir(inputDir);
    // backupfolder for outdated page format files, if any
    final String backupFolderName = XmlFormat.PAGE_2010.toString().toLowerCase() + "_backup";
    final String backupPath = pageInputDir.getAbsolutePath() + File.separator + backupFolderName;
    // iterate imgList, search for corresponding XML files and build TrpPages
    int pageNr = 1;
    List<TrpPage> pages = new ArrayList<TrpPage>(pageMap.entrySet().size());
    // need a special variable to test whether we are in sync mode (only then do the following!!!!)
    if (pageMap.entrySet().size() == 0 && config.isEnableSyncWithoutImages()) {
        pageMap = createDummyImgFilesForXmls(inputDir, pageInputDir);
    }
    for (Entry<String, File> e : pageMap.entrySet()) {
        File imgFile = e.getValue();
        // the img file name without extension
        final String imgFileName = e.getKey();
        // check for a page XML of this name
        File pageXml = findXml(imgFileName, pageInputDir);
        // TODO thumbURL dir + imgFile.getName())+".jpg"
        File thumbFile = getThumbFile(inputDir, imgFileName);
        if (pageXml != null) {
            XmlFormat xmlFormat = XmlUtils.getXmlFormat(pageXml);
            switch(xmlFormat) {
                case PAGE_2010:
                    Page2010Converter.updatePageFormatSingleFile(pageXml, backupPath);
                    break;
                case PAGE_2013:
                    break;
                default:
                    throw new IOException("Incompatible XML file in PAGE XML path! " + pageXml.getAbsolutePath());
            }
        }
        // try to read image dimension in any case to detect corrupt files immediately!
        // FIXME this is taking too long and is only necessary on initial loading
        Dimension dim = null;
        String imageRemark = null;
        try {
            dim = ImgUtils.readImageDimensions(imgFile);
        } catch (CorruptImageException cie) {
            logger.error("Image is corrupt: " + imgFile.getAbsolutePath(), cie);
            imageRemark = getCorruptImgMsg(imgFile.getName());
        }
        if (pageXml == null && config.isForceCreatePageXml()) {
            // if no page XML, then create one at this path
            File pageOutFile = new File(pageInputDir.getAbsolutePath() + File.separatorChar + imgFileName + ".xml");
            File abbyyXml = findXml(imgFileName, ocrInputDir);
            File altoXml = findXml(imgFileName, altoInputDir);
            File txtFile = findFile(imgFileName, txtInputDir, "txt");
            pageXml = createPageXml(pageOutFile, false, abbyyXml, altoXml, txtFile, config.isPreserveOcrFontFamily(), config.isPreserveOcrTxtStyles(), config.isReplaceBadChars(), imgFile.getName(), dim);
        }
        TrpPage page = buildPage(inputDir, pageNr++, imgFile, pageXml, thumbFile, dim, imageRemark);
        pages.add(page);
    }
    doc.setPages(pages);
    doc.getMd().setNrOfPages(doc.getPages().size());
    // set editorial declaration:
    List<EdFeature> features = readEditDeclFeatures(doc.getMd().getLocalFolder());
    doc.setEdDeclList(features);
    logger.debug(doc.toString());
    // store doc on disk to save time on next load
    LocalDocWriter.writeDocXml(doc, docXml);
    return doc;
}

Example 5 with TrpDocMetadata

use of eu.transkribus.core.model.beans.TrpDocMetadata in project TranskribusCore by Transkribus.

the class LocalDocReader method findDocMd.

/**
 * Quick access to docMd without loading and checking all the stuff which load() would do.
 *
 * @param dir
 * @return existing metadata if found, a freshly initiated instance otherwise
 */
public static TrpDocMetadata findDocMd(File dir) {
    final File docXml = new File(dir.getAbsolutePath() + File.separator + LocalDocConst.DOC_XML_FILENAME);
    TrpDocMetadata docMd = null;
    TrpDoc doc;
    if (docXml.isFile() && (doc = loadDocXml(docXml)) != null) {
        docMd = doc.getMd();
    }
    // try find legacy metadata.xml
    if (docMd == null) {
        try {
            docMd = loadDocMd(dir);
        } catch (IOException ioe) {
        }
    }
    // init object as load() would do
    initDocMd(docMd, dir, false);
    return docMd;
}