Examples with AnalysedDirectory org.alfresco.repo.bulkimport.AnalysedDirectory used on opensource projects

Example 1 with AnalysedDirectory

use of org.alfresco.repo.bulkimport.AnalysedDirectory in project alfresco-repository by Alfresco.

the class DirectoryAnalyserImpl method analyseDirectory.

/**
 * @see org.alfresco.repo.bulkimport.DirectoryAnalyser#analyseDirectory(org.alfresco.repo.bulkimport.ImportableItem, java.nio.file.DirectoryStream.Filter)
 */
public AnalysedDirectory analyseDirectory(ImportableItem directory, DirectoryStream.Filter<Path> filter) {
    Path directoryFile = directory.getHeadRevision().getContentFile();
    AnalysedDirectory result = new AnalysedDirectory(listFiles(directoryFile, filter));
    if (log.isDebugEnabled()) {
        log.debug("Analysing directory " + FileUtils.getFileName(directoryFile) + "...");
    }
    // Build up the list of ImportableItems from the directory listing
    for (Path file : result.getOriginalPaths()) {
        // MNT-9763 bulkimport fails when there is a very large LastModified timestamp.
        String isoDate = null;
        try {
            isoDate = ISO8601DateFormat.format(new Date(Files.getLastModifiedTime(file, LinkOption.NOFOLLOW_LINKS).toMillis()));
            ISO8601DateFormat.parse(isoDate);
        } catch (PlatformRuntimeException | IOException e) {
            log.warn("Failed to convert date " + isoDate + " to string for " + file.getFileName(), e);
            importStatus.incrementNumberOfUnreadableEntries();
            continue;
        }
        if (log.isTraceEnabled()) {
            log.trace("Scanning file " + FileUtils.getFileName(file) + "...");
        }
        if (Files.isReadable(file)) {
            try {
                nameChecker.evaluate(file.getFileName().toString());
            } catch (ConstraintException e) {
                if (log.isWarnEnabled()) {
                    log.warn("Skipping file with invalid name: '" + FileUtils.getFileName(file) + "'.");
                }
                // mark file with invalid name as unreadable
                importStatus.incrementNumberOfUnreadableEntries();
                continue;
            }
            if (isVersionFile(file)) {
                addVersionFile(directory, result, file);
                importStatus.incrementNumberOfFilesScanned();
            } else if (isMetadataFile(file)) {
                addMetadataFile(directory, result, file);
                importStatus.incrementNumberOfFilesScanned();
            } else {
                boolean isDirectory = addParentFile(directory, result, file);
                if (isDirectory) {
                    importStatus.incrementNumberOfFoldersScanned();
                } else {
                    importStatus.incrementNumberOfFilesScanned();
                }
            }
        } else {
            if (log.isWarnEnabled()) {
                log.warn("Skipping unreadable file '" + FileUtils.getFileName(file) + "'.");
            }
            importStatus.incrementNumberOfUnreadableEntries();
        }
    }
    // Finally, remove any items from the list that aren't valid (don't have either a
    // contentFile or a metadataFile)
    Iterator<ImportableItem> iter = result.getImportableItems().iterator();
    while (iter.hasNext()) {
        ImportableItem importableItem = iter.next();
        if (!importableItem.isValid() || !isMetadataValid(importableItem)) {
            iter.remove();
        }
    }
    iter = result.getImportableDirectories().iterator();
    while (iter.hasNext()) {
        ImportableItem importableItem = iter.next();
        if (!importableItem.isValid()) {
            iter.remove();
        }
    }
    if (log.isDebugEnabled()) {
        log.debug("Finished analysing directory " + FileUtils.getFileName(directoryFile) + ".");
    }
    return result;
}

Example 2 with AnalysedDirectory

use of org.alfresco.repo.bulkimport.AnalysedDirectory in project alfresco-repository by Alfresco.

the class StripingFilesystemTracker method getImportableItems.

protected List<ImportableItem> getImportableItems(int count) {
    while (toProcess.size() < count) {
        ImportableItem directory = getDirectoryToProcess();
        if (directory != null) {
            AnalysedDirectory analysedDirectory = getImportableItemsInDirectory(directory);
            addItemsToProcess(analysedDirectory.getImportableDirectories());
            addItemsToProcess(analysedDirectory.getImportableItems());
            // add new directories to process in next level
            getDirectoriesToProcess(currentLevel + 1).addAll(analysedDirectory.getImportableDirectories());
        } else {
            break;
        }
    }
    int size = (toProcess.size() >= count ? count : toProcess.size());
    List<ImportableItem> result = new ArrayList<ImportableItem>(size);
    int i = size;
    while (i > 0) {
        // we can assume that there are items in toProcess to remove because the size has been pre-calculated above
        ImportableItem importableItem = toProcess.remove(0);
        if (importableItem != null) {
            result.add(importableItem);
            i--;
        } else {
            logger.warn("Unexpected empty toProcess queue");
        }
    }
    if (result.size() == 0) {
        // this level has been exhausted, increment level
        incrementLevel();
    }
    return result;
}

Example 3 with AnalysedDirectory

use of org.alfresco.repo.bulkimport.AnalysedDirectory in project alfresco-repository by Alfresco.

the class AbstractFilesystemTracker method getImportableDirectoriesInDirectory.

protected final AnalysedDirectory getImportableDirectoriesInDirectory(ImportableItem directory, final int count) {
    DirectoryStream.Filter<Path> filter = null;
    if (count != -1) {
        filter = new DirectoryStream.Filter<Path>() {

            private int i = count;

            @Override
            public boolean accept(Path entry) throws IOException {
                return Files.isDirectory(entry) && i-- > 0;
            }
        };
    } else {
        filter = new DirectoryStream.Filter<Path>() {

            @Override
            public boolean accept(Path entry) throws IOException {
                return Files.isDirectory(entry);
            }
        };
    }
    AnalysedDirectory analysedDirectory = directoryAnalyser.analyseDirectory(directory, filter);
    return analysedDirectory;
}