Examples with ImportableItem org.alfresco.repo.bulkimport.ImportableItem used on opensource projects

Example 1 with ImportableItem

use of org.alfresco.repo.bulkimport.ImportableItem in project alfresco-repository by Alfresco.

the class StripingFilesystemTrackerTest method test1.

@Test
public void test1() throws FileNotFoundException {
    final File sourceFolder = ResourceUtils.getFile("classpath:bulkimport");
    final StripingFilesystemTracker tracker = new StripingFilesystemTracker(directoryAnalyser, new NodeRef("workspace", "SpacesStore", "123"), sourceFolder, Integer.MAX_VALUE);
    List<ImportableItem> items = tracker.getImportableItems(Integer.MAX_VALUE);
    assertEquals("", 11, items.size());
    tracker.incrementLevel();
    items = tracker.getImportableItems(Integer.MAX_VALUE);
    assertEquals("", 2, items.size());
    tracker.incrementLevel();
    items = tracker.getImportableItems(Integer.MAX_VALUE);
    assertEquals("", 31, items.size());
}

Example 2 with ImportableItem

use of org.alfresco.repo.bulkimport.ImportableItem in project alfresco-repository by Alfresco.

the class DirectoryAnalyserImpl method analyseDirectory.

/**
 * @see org.alfresco.repo.bulkimport.DirectoryAnalyser#analyseDirectory(org.alfresco.repo.bulkimport.ImportableItem, java.nio.file.DirectoryStream.Filter)
 */
public AnalysedDirectory analyseDirectory(ImportableItem directory, DirectoryStream.Filter<Path> filter) {
    Path directoryFile = directory.getHeadRevision().getContentFile();
    AnalysedDirectory result = new AnalysedDirectory(listFiles(directoryFile, filter));
    if (log.isDebugEnabled()) {
        log.debug("Analysing directory " + FileUtils.getFileName(directoryFile) + "...");
    }
    // Build up the list of ImportableItems from the directory listing
    for (Path file : result.getOriginalPaths()) {
        // MNT-9763 bulkimport fails when there is a very large LastModified timestamp.
        String isoDate = null;
        try {
            isoDate = ISO8601DateFormat.format(new Date(Files.getLastModifiedTime(file, LinkOption.NOFOLLOW_LINKS).toMillis()));
            ISO8601DateFormat.parse(isoDate);
        } catch (PlatformRuntimeException | IOException e) {
            log.warn("Failed to convert date " + isoDate + " to string for " + file.getFileName(), e);
            importStatus.incrementNumberOfUnreadableEntries();
            continue;
        }
        if (log.isTraceEnabled()) {
            log.trace("Scanning file " + FileUtils.getFileName(file) + "...");
        }
        if (Files.isReadable(file)) {
            try {
                nameChecker.evaluate(file.getFileName().toString());
            } catch (ConstraintException e) {
                if (log.isWarnEnabled()) {
                    log.warn("Skipping file with invalid name: '" + FileUtils.getFileName(file) + "'.");
                }
                // mark file with invalid name as unreadable
                importStatus.incrementNumberOfUnreadableEntries();
                continue;
            }
            if (isVersionFile(file)) {
                addVersionFile(directory, result, file);
                importStatus.incrementNumberOfFilesScanned();
            } else if (isMetadataFile(file)) {
                addMetadataFile(directory, result, file);
                importStatus.incrementNumberOfFilesScanned();
            } else {
                boolean isDirectory = addParentFile(directory, result, file);
                if (isDirectory) {
                    importStatus.incrementNumberOfFoldersScanned();
                } else {
                    importStatus.incrementNumberOfFilesScanned();
                }
            }
        } else {
            if (log.isWarnEnabled()) {
                log.warn("Skipping unreadable file '" + FileUtils.getFileName(file) + "'.");
            }
            importStatus.incrementNumberOfUnreadableEntries();
        }
    }
    // Finally, remove any items from the list that aren't valid (don't have either a
    // contentFile or a metadataFile)
    Iterator<ImportableItem> iter = result.getImportableItems().iterator();
    while (iter.hasNext()) {
        ImportableItem importableItem = iter.next();
        if (!importableItem.isValid() || !isMetadataValid(importableItem)) {
            iter.remove();
        }
    }
    iter = result.getImportableDirectories().iterator();
    while (iter.hasNext()) {
        ImportableItem importableItem = iter.next();
        if (!importableItem.isValid()) {
            iter.remove();
        }
    }
    if (log.isDebugEnabled()) {
        log.debug("Finished analysing directory " + FileUtils.getFileName(directoryFile) + ".");
    }
    return result;
}

Example 3 with ImportableItem

use of org.alfresco.repo.bulkimport.ImportableItem in project alfresco-repository by Alfresco.

the class DirectoryAnalyserImpl method findOrCreateImportableItem.

private ImportableItem findOrCreateImportableItem(ImportableItem parent, AnalysedDirectory analysedDirectory, Path contentFile) {
    ImportableItem result = findImportableItem(analysedDirectory, contentFile);
    // We didn't find it, so create it
    if (result == null) {
        result = new ImportableItem();
        result.setParent(parent);
        result.getHeadRevision().setContentFile(contentFile);
        if (!shouldFilter(result)) {
            analysedDirectory.addImportableItem(result);
        }
    }
    return (result);
}

Example 4 with ImportableItem

use of org.alfresco.repo.bulkimport.ImportableItem in project alfresco-repository by Alfresco.

the class DirectoryAnalyserImpl method addMetadataFile.

private void addMetadataFile(ImportableItem parent, AnalysedDirectory analysedDirectory, Path metadataFile) {
    Path parentContentfile = getParentOfMetadatafile(metadataFile);
    ImportableItem importableItem = findOrCreateImportableItem(parent, analysedDirectory, parentContentfile);
    importableItem.getHeadRevision().setMetadataFile(metadataFile);
}

Example 5 with ImportableItem

use of org.alfresco.repo.bulkimport.ImportableItem in project alfresco-repository by Alfresco.

the class StripingFilesystemTracker method getImportableItems.

protected List<ImportableItem> getImportableItems(int count) {
    while (toProcess.size() < count) {
        ImportableItem directory = getDirectoryToProcess();
        if (directory != null) {
            AnalysedDirectory analysedDirectory = getImportableItemsInDirectory(directory);
            addItemsToProcess(analysedDirectory.getImportableDirectories());
            addItemsToProcess(analysedDirectory.getImportableItems());
            // add new directories to process in next level
            getDirectoriesToProcess(currentLevel + 1).addAll(analysedDirectory.getImportableDirectories());
        } else {
            break;
        }
    }
    int size = (toProcess.size() >= count ? count : toProcess.size());
    List<ImportableItem> result = new ArrayList<ImportableItem>(size);
    int i = size;
    while (i > 0) {
        // we can assume that there are items in toProcess to remove because the size has been pre-calculated above
        ImportableItem importableItem = toProcess.remove(0);
        if (importableItem != null) {
            result.add(importableItem);
            i--;
        } else {
            logger.warn("Unexpected empty toProcess queue");
        }
    }
    if (result.size() == 0) {
        // this level has been exhausted, increment level
        incrementLevel();
    }
    return result;
}