Search in sources :

Example 36 with Files

use of java.nio.file.Files in project sirix by sirixdb.

the class FMSETest method test.

/**
 * Test a folder of XML files.
 *
 * @param FOLDER path string
 * @throws Exception if any exception occurs
 */
private void test(final String FOLDER) throws Exception {
    Database database = TestHelper.getDatabase(PATHS.PATH1.getFile());
    ResourceManager resource = database.getResourceManager(new ResourceManagerConfiguration.Builder(TestHelper.RESOURCE).build());
    final Path folder = Paths.get(FOLDER);
    final List<Path> list = Files.list(folder).filter(path -> path.getFileName().endsWith(".xml")).collect(toList());
    // Sort files list according to file names.
    list.sort((first, second) -> {
        final String firstName = first.getFileName().toString().substring(0, first.getFileName().toString().indexOf('.'));
        final String secondName = second.getFileName().toString().substring(0, second.getFileName().toString().indexOf('.'));
        if (Integer.parseInt(firstName) < Integer.parseInt(secondName)) {
            return -1;
        } else if (Integer.parseInt(firstName) > Integer.parseInt(secondName)) {
            return +1;
        } else {
            return 0;
        }
    });
    boolean first = true;
    // Shredder files.
    for (final Path file : list) {
        if (file.getFileName().endsWith(".xml")) {
            if (first) {
                first = false;
                try (final XdmNodeWriteTrx wtx = resource.beginNodeWriteTrx()) {
                    final XMLShredder shredder = new XMLShredder.Builder(wtx, XMLShredder.createFileReader(file), Insert.ASFIRSTCHILD).commitAfterwards().build();
                    shredder.call();
                }
            } else {
                FMSEImport.main(new String[] { PATHS.PATH1.getFile().toAbsolutePath().toString(), file.toAbsolutePath().toString() });
            }
            resource.close();
            resource = database.getResourceManager(new ResourceManagerConfiguration.Builder(TestHelper.RESOURCE).build());
            final OutputStream out = new ByteArrayOutputStream();
            final XMLSerializer serializer = new XMLSerializerBuilder(resource, out).build();
            serializer.call();
            final StringBuilder sBuilder = TestHelper.readFile(file, false);
            final Diff diff = new Diff(sBuilder.toString(), out.toString());
            final DetailedDiff detDiff = new DetailedDiff(diff);
            @SuppressWarnings("unchecked") final List<Difference> differences = detDiff.getAllDifferences();
            for (final Difference difference : differences) {
                System.err.println("***********************");
                System.err.println(difference);
                System.err.println("***********************");
            }
            assertTrue("pieces of XML are similar " + diff, diff.similar());
            assertTrue("but are they identical? " + diff, diff.identical());
        }
    }
    database.close();
}
Also used : Path(java.nio.file.Path) XMLUnit(org.custommonkey.xmlunit.XMLUnit) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Difference(org.custommonkey.xmlunit.Difference) XMLSerializer(org.sirix.service.xml.serialize.XMLSerializer) XMLTestCase(org.custommonkey.xmlunit.XMLTestCase) PATHS(org.sirix.TestHelper.PATHS) ResourceManagerConfiguration(org.sirix.access.conf.ResourceManagerConfiguration) TestHelper(org.sirix.TestHelper) After(org.junit.After) XMLSerializerBuilder(org.sirix.service.xml.serialize.XMLSerializer.XMLSerializerBuilder) DetailedDiff(org.custommonkey.xmlunit.DetailedDiff) Path(java.nio.file.Path) Before(org.junit.Before) OutputStream(java.io.OutputStream) SirixException(org.sirix.exception.SirixException) Files(java.nio.file.Files) IOException(java.io.IOException) Test(org.junit.Test) FMSEImport(org.sirix.diff.service.FMSEImport) File(java.io.File) XdmNodeWriteTrx(org.sirix.api.XdmNodeWriteTrx) Insert(org.sirix.service.xml.shredder.Insert) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List) XMLShredder(org.sirix.service.xml.shredder.XMLShredder) ResourceManager(org.sirix.api.ResourceManager) Paths(java.nio.file.Paths) Database(org.sirix.api.Database) Diff(org.custommonkey.xmlunit.Diff) XdmNodeWriteTrx(org.sirix.api.XdmNodeWriteTrx) XMLSerializer(org.sirix.service.xml.serialize.XMLSerializer) DetailedDiff(org.custommonkey.xmlunit.DetailedDiff) DetailedDiff(org.custommonkey.xmlunit.DetailedDiff) Diff(org.custommonkey.xmlunit.Diff) XMLSerializerBuilder(org.sirix.service.xml.serialize.XMLSerializer.XMLSerializerBuilder) ByteArrayOutputStream(java.io.ByteArrayOutputStream) OutputStream(java.io.OutputStream) ResourceManagerConfiguration(org.sirix.access.conf.ResourceManagerConfiguration) ResourceManager(org.sirix.api.ResourceManager) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Difference(org.custommonkey.xmlunit.Difference) XMLSerializerBuilder(org.sirix.service.xml.serialize.XMLSerializer.XMLSerializerBuilder) Database(org.sirix.api.Database) XMLShredder(org.sirix.service.xml.shredder.XMLShredder)

Example 37 with Files

use of java.nio.file.Files in project rom-manager by Jakz.

the class Scanner method scanForRoms.

public void scanForRoms(boolean total) throws IOException {
    ignoredPaths.clear();
    foundFiles.clear();
    clones.clear();
    if (total) {
        logger.i(LogTarget.romset(set), "Scanning for roms");
        set.resetStatus();
    } else {
        logger.i(LogTarget.romset(set), "Scanning for new roms");
        set.romStream().filter(r -> r.isPresent()).map(r -> r.handle().path()).forEach(ignoredPaths::add);
    }
    ignoredPaths.addAll(settings.getIgnoredPaths());
    Path folder = settings.romsPath;
    if (!canProceedWithScan())
        return;
    FolderScanner folderScanner = new FolderScanner(ignoredPaths, true);
    final Set<Path> pathsToScan = folderScanner.scan(folder);
    List<VerifierEntry> foundEntries = Collections.synchronizedList(new ArrayList<>());
    /* scanning task */
    {
        Operation<Path, List<VerifierEntry>> operation = path -> {
            logger.d(LogTarget.file(path), "> Scanning %s", path.getFileName().toString());
            try {
                return scanner.scanFile(path);
            } catch (IOException e) {
                e.printStackTrace();
                return null;
            } catch (RuntimeException e) {
                logger.e(LogTarget.file(path), "Exception while scanning %s", path.toString());
                e.printStackTrace();
                if (!SevenZip.isInitializedSuccessfully()) {
                    Throwable ee = SevenZip.getLastInitializationException();
                    ee.printStackTrace();
                }
                return null;
            }
        };
        BiConsumer<Long, Float> guiProgress = (i, f) -> {
            Main.progress.update(f, "Scanning " + i + " of " + pathsToScan.size() + "...");
        };
        Consumer<List<VerifierEntry>> callback = results -> {
            foundEntries.addAll(results);
        };
        Runnable onComplete = () -> {
            SwingUtilities.invokeLater(() -> {
                Main.progress.finished();
                List<VerifierEntry> transformedEntries = foundEntries.stream().collect(Collectors.toList());
                HandleSet handleSet = new HandleSet(transformedEntries);
                HandleSet.Stats stats = handleSet.stats();
                logger.i(LogTarget.romset(set), "Found %d potential matches (%d binary, %d inside archives, %d nested inside %d archives).", stats.totalHandles, stats.binaryCount, stats.archivedCount, stats.nestedArchiveInnerCount, stats.nestedArchiveCount);
                transformedEntries.forEach(h -> logger.d(LogTarget.romset(set), "> %s", h.toString()));
                Runnable verifyTask = verifyTask(transformedEntries, total);
                verifyTask.run();
            });
        };
        AsyncGuiPoolWorker<Path, List<VerifierEntry>> worker = new AsyncGuiPoolWorker<>(operation, guiProgress, 1);
        SwingUtilities.invokeLater(() -> {
            Main.progress.show(Main.mainFrame, "Scanning " + pathsToScan.size() + " files", () -> worker.cancel());
        });
        worker.compute(pathsToScan, callback, onComplete);
    }
}
Also used : IntStream(java.util.stream.IntStream) Game(com.github.jakz.romlib.data.game.Game) Dialogs(jack.rm.gui.Dialogs) MyGameSetFeatures(jack.rm.data.romset.MyGameSetFeatures) ScannerPlugin(jack.rm.plugins.types.ScannerPlugin) Function(java.util.function.Function) TreeSet(java.util.TreeSet) Handle(com.pixbits.lib.io.archive.handles.Handle) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) GameSet(com.github.jakz.romlib.data.set.GameSet) SwingUtilities(javax.swing.SwingUtilities) HandleSet(com.pixbits.lib.io.archive.HandleSet) StreamException(com.pixbits.lib.functional.StreamException) Log(com.pixbits.lib.log.Log) BiConsumer(java.util.function.BiConsumer) Rom(com.github.jakz.romlib.data.game.Rom) Path(java.nio.file.Path) Files(java.nio.file.Files) LogSource(jack.rm.log.LogSource) LogTarget(jack.rm.log.LogTarget) Operation(com.pixbits.lib.concurrent.Operation) PluginRealType(jack.rm.plugins.PluginRealType) Set(java.util.Set) FormatSupportPlugin(jack.rm.plugins.types.FormatSupportPlugin) IOException(java.io.IOException) GameStatus(com.github.jakz.romlib.data.game.GameStatus) Collectors(java.util.stream.Collectors) ExecutionException(java.util.concurrent.ExecutionException) Consumer(java.util.function.Consumer) FolderScanner(com.pixbits.lib.io.FolderScanner) List(java.util.List) Logger(com.pixbits.lib.log.Logger) VerifierPlugin(jack.rm.plugins.types.VerifierPlugin) SevenZip(net.sf.sevenzipjbinding.SevenZip) Collections(java.util.Collections) VerifierEntry(com.pixbits.lib.io.archive.VerifierEntry) Main(jack.rm.Main) AsyncGuiPoolWorker(com.pixbits.lib.concurrent.AsyncGuiPoolWorker) Settings(jack.rm.data.romset.Settings) Path(java.nio.file.Path) Operation(com.pixbits.lib.concurrent.Operation) IOException(java.io.IOException) AsyncGuiPoolWorker(com.pixbits.lib.concurrent.AsyncGuiPoolWorker) VerifierEntry(com.pixbits.lib.io.archive.VerifierEntry) BiConsumer(java.util.function.BiConsumer) Consumer(java.util.function.Consumer) FolderScanner(com.pixbits.lib.io.FolderScanner) BiConsumer(java.util.function.BiConsumer) HandleSet(com.pixbits.lib.io.archive.HandleSet)

Example 38 with Files

use of java.nio.file.Files in project rom-manager by Jakz.

the class MoveUnknownFilesPlugin method execute.

@Override
public void execute(GameSet set) {
    try {
        counter = 0;
        if (!Files.exists(path) || !Files.isDirectory(path))
            Files.createDirectory(path);
        Set<Path> existing = set.romStream().filter(r -> r.isPresent()).map(r -> r.handle().path()).collect(Collectors.toSet());
        Settings settings = getGameSetSettings();
        Set<Path> total = new FolderScanner(FileSystems.getDefault().getPathMatcher("glob:*.*"), settings.getIgnoredPaths(), true).scan(settings.romsPath);
        total.removeAll(existing);
        total.stream().filter(f -> !f.getParent().equals(path)).forEach(f -> {
            Path dest = path.resolve(f.getFileName());
            int i = 1;
            while (Files.exists(dest)) dest = path.resolve(f.getFileName().toString() + (i++));
            try {
                Files.move(f, dest);
                ++counter;
            } catch (IOException e) {
                e.printStackTrace();
            /* TODO: log */
            }
        });
        message("Moved " + counter + " unknown files");
    } catch (IOException e) {
        e.printStackTrace();
    // TODO: log
    }
}
Also used : Path(java.nio.file.Path) Arrays(java.util.Arrays) CleanupPlugin(jack.rm.plugins.types.CleanupPlugin) Files(java.nio.file.Files) ExposedParameter(com.pixbits.lib.plugin.ExposedParameter) Set(java.util.Set) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) HashSet(java.util.HashSet) GameSet(com.github.jakz.romlib.data.set.GameSet) FolderScanner(com.pixbits.lib.io.FolderScanner) Path(java.nio.file.Path) PluginWithIgnorePaths(jack.rm.plugins.PluginWithIgnorePaths) FileSystems(java.nio.file.FileSystems) Settings(jack.rm.data.romset.Settings) FolderScanner(com.pixbits.lib.io.FolderScanner) IOException(java.io.IOException) Settings(jack.rm.data.romset.Settings)

Example 39 with Files

use of java.nio.file.Files in project ant by apache.

the class FileUtils method hasErrorInCase.

/**
 * test whether a file or directory exists, with an error in the
 * upper/lower case spelling of the name.
 * Using this method is only interesting on case insensitive file systems
 * (Windows).
 * <p>
 * It will return true only if 3 conditions are met:
 * </p>
 * <ul>
 *   <li>operating system is case insensitive</li>
 *   <li>file exists</li>
 *   <li>actual name from directory reading is different from the
 *       supplied argument</li>
 * </ul>
 * <p>
 * The purpose is to identify files or directories on case-insensitive
 * filesystems whose case is not what is expected.
 * </p>
 * Possibly to rename them afterwards to the desired upper/lowercase
 * combination.
 *
 * @param localFile file to test
 * @return true if the file exists and the case of the actual file
 *              is not the case of the parameter
 * @since Ant 1.7.1
 */
public boolean hasErrorInCase(File localFile) {
    localFile = normalize(localFile.getAbsolutePath());
    if (!localFile.exists()) {
        return false;
    }
    final String localFileName = localFile.getName();
    FilenameFilter ff = (dir, name) -> name.equalsIgnoreCase(localFileName) && (!name.equals(localFileName));
    String[] names = localFile.getParentFile().list(ff);
    return names != null && names.length == 1;
}
Also used : HttpURLConnection(java.net.HttpURLConnection) FilenameFilter(java.io.FilenameFilter) Arrays(java.util.Arrays) PathTokenizer(org.apache.tools.ant.PathTokenizer) URL(java.net.URL) FilterChain(org.apache.tools.ant.types.FilterChain) FilterSetCollection(org.apache.tools.ant.types.FilterSetCollection) Random(java.util.Random) JarFile(java.util.jar.JarFile) FileResource(org.apache.tools.ant.types.resources.FileResource) Stack(java.util.Stack) ArrayList(java.util.ArrayList) Vector(java.util.Vector) URLConnection(java.net.URLConnection) StringTokenizer(java.util.StringTokenizer) Project(org.apache.tools.ant.Project) JarURLConnection(java.net.JarURLConnection) Path(java.nio.file.Path) Channel(java.nio.channels.Channel) OutputStream(java.io.OutputStream) MalformedURLException(java.net.MalformedURLException) Files(java.nio.file.Files) StandardOpenOption(java.nio.file.StandardOpenOption) DecimalFormat(java.text.DecimalFormat) IOException(java.io.IOException) BuildException(org.apache.tools.ant.BuildException) Reader(java.io.Reader) InputStreamReader(java.io.InputStreamReader) Collectors(java.util.stream.Collectors) File(java.io.File) Locator(org.apache.tools.ant.launch.Locator) List(java.util.List) Os(org.apache.tools.ant.taskdefs.condition.Os) Paths(java.nio.file.Paths) Writer(java.io.Writer) InputStream(java.io.InputStream) FilenameFilter(java.io.FilenameFilter)

Example 40 with Files

use of java.nio.file.Files in project irida by phac-nml.

the class AnalysisController method getAjaxDownloadAnalysisSubmissionIndividualFile.

/**
 * Download single output files from an {@link AnalysisSubmission}
 *
 * @param analysisSubmissionId Id for a {@link AnalysisSubmission}
 * @param fileId               the id of the file to download
 * @param response             {@link HttpServletResponse}
 */
@RequestMapping(value = "/ajax/download/{analysisSubmissionId}/file/{fileId}")
public void getAjaxDownloadAnalysisSubmissionIndividualFile(@PathVariable Long analysisSubmissionId, @PathVariable Long fileId, HttpServletResponse response) {
    AnalysisSubmission analysisSubmission = analysisSubmissionService.read(analysisSubmissionId);
    Analysis analysis = analysisSubmission.getAnalysis();
    Set<AnalysisOutputFile> files = analysis.getAnalysisOutputFiles();
    Optional<AnalysisOutputFile> optFile = files.stream().filter(f -> f.getId().equals(fileId)).findAny();
    if (!optFile.isPresent()) {
        throw new EntityNotFoundException("Could not find file with id " + fileId);
    }
    FileUtilities.createSingleFileResponse(response, optFile.get());
}
Also used : PreAuthorize(org.springframework.security.access.prepost.PreAuthorize) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) AnalysisType(ca.corefacility.bioinformatics.irida.model.enums.AnalysisType) ProjectService(ca.corefacility.bioinformatics.irida.service.ProjectService) FileUtilities(ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities) Model(org.springframework.ui.Model) MetadataTemplateField(ca.corefacility.bioinformatics.irida.model.sample.MetadataTemplateField) DataTablesResponse(ca.corefacility.bioinformatics.irida.ria.web.components.datatables.DataTablesResponse) AnalysisSubmissionService(ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionService) MetadataTemplate(ca.corefacility.bioinformatics.irida.model.sample.MetadataTemplate) SecurityContextHolder(org.springframework.security.core.context.SecurityContextHolder) TypeReference(com.fasterxml.jackson.core.type.TypeReference) Path(java.nio.file.Path) JsonParseException(com.fasterxml.jackson.core.JsonParseException) ProjectMetadataTemplateJoin(ca.corefacility.bioinformatics.irida.model.joins.impl.ProjectMetadataTemplateJoin) IridaWorkflowNotFoundException(ca.corefacility.bioinformatics.irida.exceptions.IridaWorkflowNotFoundException) AnalysisSubmission(ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission) SequencingObjectService(ca.corefacility.bioinformatics.irida.service.SequencingObjectService) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) MediaType(org.springframework.http.MediaType) Collectors(java.util.stream.Collectors) AnalysesListingService(ca.corefacility.bioinformatics.irida.ria.web.services.AnalysesListingService) Principal(java.security.Principal) org.springframework.web.bind.annotation(org.springframework.web.bind.annotation) User(ca.corefacility.bioinformatics.irida.model.user.User) SampleService(ca.corefacility.bioinformatics.irida.service.sample.SampleService) JsonMappingException(com.fasterxml.jackson.databind.JsonMappingException) Authentication(org.springframework.security.core.Authentication) java.util(java.util) ExecutionManagerException(ca.corefacility.bioinformatics.irida.exceptions.ExecutionManagerException) UpdateAnalysisSubmissionPermission(ca.corefacility.bioinformatics.irida.security.permissions.analysis.UpdateAnalysisSubmissionPermission) Analysis(ca.corefacility.bioinformatics.irida.model.workflow.analysis.Analysis) EntityNotFoundException(ca.corefacility.bioinformatics.irida.exceptions.EntityNotFoundException) Controller(org.springframework.stereotype.Controller) MetadataEntry(ca.corefacility.bioinformatics.irida.model.sample.metadata.MetadataEntry) MetadataTemplateService(ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService) ToolExecution(ca.corefacility.bioinformatics.irida.model.workflow.analysis.ToolExecution) MessageSource(org.springframework.context.MessageSource) Logger(org.slf4j.Logger) Files(java.nio.file.Files) AnalysisState(ca.corefacility.bioinformatics.irida.model.enums.AnalysisState) DataTablesRequest(ca.corefacility.bioinformatics.irida.ria.web.components.datatables.config.DataTablesRequest) IridaWorkflowsService(ca.corefacility.bioinformatics.irida.service.workflow.IridaWorkflowsService) HttpServletResponse(javax.servlet.http.HttpServletResponse) SequenceFilePair(ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Sample(ca.corefacility.bioinformatics.irida.model.sample.Sample) Project(ca.corefacility.bioinformatics.irida.model.project.Project) ProjectAnalysisSubmissionJoin(ca.corefacility.bioinformatics.irida.model.workflow.submission.ProjectAnalysisSubmissionJoin) AnalysisOutputFileInfo(ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.AnalysisOutputFileInfo) AnalysisOutputFile(ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile) java.io(java.io) UserService(ca.corefacility.bioinformatics.irida.service.user.UserService) JobError(ca.corefacility.bioinformatics.irida.model.workflow.analysis.JobError) IridaWorkflow(ca.corefacility.bioinformatics.irida.model.workflow.IridaWorkflow) DataTablesParams(ca.corefacility.bioinformatics.irida.ria.web.components.datatables.DataTablesParams) Analysis(ca.corefacility.bioinformatics.irida.model.workflow.analysis.Analysis) AnalysisSubmission(ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission) EntityNotFoundException(ca.corefacility.bioinformatics.irida.exceptions.EntityNotFoundException) AnalysisOutputFile(ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile)

Aggregations

Files (java.nio.file.Files)247 IOException (java.io.IOException)213 Path (java.nio.file.Path)199 List (java.util.List)177 Collectors (java.util.stream.Collectors)157 Paths (java.nio.file.Paths)135 File (java.io.File)130 ArrayList (java.util.ArrayList)117 Map (java.util.Map)111 Set (java.util.Set)97 Collections (java.util.Collections)89 Arrays (java.util.Arrays)81 Stream (java.util.stream.Stream)78 HashMap (java.util.HashMap)75 HashSet (java.util.HashSet)58 InputStream (java.io.InputStream)56 Collection (java.util.Collection)55 Logger (org.slf4j.Logger)54 Pattern (java.util.regex.Pattern)53 Optional (java.util.Optional)51