Examples with SourceFile io.dockstore.webservice.core.SourceFile used on opensource projects

Example 1 with SourceFile

use of io.dockstore.webservice.core.SourceFile in project dockstore by dockstore.

the class NextflowHandlerIT method testProcessImportsSamtools.

/**
 * Tests:
 * Nextflow DSL2 with main descriptor deep inside GitHub repo with import that's up one level that imports something else
 * Import also has the same file name as another file
 */
@Test
public void testProcessImportsSamtools() {
    final String githubRepository = "dockstore-testing/modules";
    WorkflowVersion workflowVersion = new WorkflowVersion();
    workflowVersion.setName("0.0.1");
    workflowVersion.setReference("0.0.1");
    String mainDescriptorContents = sourceCodeRepoInterface.readFile(githubRepository, "software/samtools/flagstat/test/nextflow.config", "master");
    Map<String, SourceFile> stringSourceFileMap = sourceCodeRepoInterface.resolveImports(githubRepository, mainDescriptorContents, DescriptorLanguage.FileType.NEXTFLOW_CONFIG, workflowVersion, "/software/samtools/flagstat/test/nextflow.config");
    List<String> knownFileNames = Arrays.asList("main.nf", "/software/samtools/flagstat/main.nf", "lib/checksum.groovy", "/software/samtools/flagstat/functions.nf");
    int size = knownFileNames.size();
    checkAllSourceFiles(stringSourceFileMap, size);
    Assert.assertEquals(size, stringSourceFileMap.size());
    knownFileNames.forEach(knownFile -> {
        SourceFile sourceFile = stringSourceFileMap.get(knownFile);
        checkSourceFile(sourceFile);
    });
}

Example 2 with SourceFile

use of io.dockstore.webservice.core.SourceFile in project dockstore by dockstore.

the class NextflowHandlerIT method testProcessImportsCalliNGS.

/**
 * Tests:
 * Nextflow DSL2 with an import statement spans multiple lines
 */
@Test
public void testProcessImportsCalliNGS() {
    final String githubRepository = "dockstore-testing/CalliNGS-NF";
    WorkflowVersion workflowVersion = new WorkflowVersion();
    workflowVersion.setName("1.0.1");
    workflowVersion.setReference("1.0.1");
    String mainDescriptorContents = sourceCodeRepoInterface.readFile(githubRepository, "nextflow.config", "master");
    Map<String, SourceFile> stringSourceFileMap = sourceCodeRepoInterface.resolveImports(githubRepository, mainDescriptorContents, DescriptorLanguage.FileType.NEXTFLOW_CONFIG, workflowVersion, "/nextflow.config");
    List<String> knownFileNames = Arrays.asList("main.nf", "bin/gghist.R", "/modules.nf");
    int size = knownFileNames.size();
    checkAllSourceFiles(stringSourceFileMap, size);
    Assert.assertEquals(size, stringSourceFileMap.size());
    knownFileNames.forEach(knownFile -> {
        SourceFile sourceFile = stringSourceFileMap.get(knownFile);
        checkSourceFile(sourceFile);
    });
}

Example 3 with SourceFile

use of io.dockstore.webservice.core.SourceFile in project dockstore by dockstore.

the class LanguagePluginHandler method processImports.

@Override
public Map<String, SourceFile> processImports(String repositoryId, String content, Version version, SourceCodeRepoInterface sourceCodeRepoInterface, String filepath) {
    MinimalLanguageInterface.FileReader reader = new MinimalLanguageInterface.FileReader() {

        @Override
        public String readFile(String path) {
            return sourceCodeRepoInterface.readFile(repositoryId, path, version.getReference());
        }

        @Override
        public List<String> listFiles(String pathToDirectory) {
            return sourceCodeRepoInterface.listFiles(repositoryId, pathToDirectory, version.getReference());
        }
    };
    final Map<String, Pair<String, MinimalLanguageInterface.GenericFileType>> stringPairMap = minimalLanguageInterface.indexWorkflowFiles(filepath, content, reader);
    Map<String, SourceFile> results = new HashMap<>();
    for (Map.Entry<String, Pair<String, MinimalLanguageInterface.GenericFileType>> entry : stringPairMap.entrySet()) {
        final SourceFile sourceFile = new SourceFile();
        sourceFile.setPath(entry.getKey());
        sourceFile.setContent(entry.getValue().getLeft());
        if (minimalLanguageInterface.getDescriptorLanguage().isServiceLanguage()) {
            // TODO: this needs to be more sophisticated
            sourceFile.setType(DescriptorLanguage.FileType.DOCKSTORE_SERVICE_YML);
        }
        sourceFile.setAbsolutePath(entry.getKey());
        results.put(entry.getKey(), sourceFile);
    }
    return results;
}

Example 4 with SourceFile

use of io.dockstore.webservice.core.SourceFile in project dockstore by dockstore.

the class LanguagePluginHandler method sourcefilesToIndexedFiles.

/**
 * Converts a set of sourcesfiles into the generic indexed files
 * @param sourceFiles set of sourcefiles
 * @return Generic indexed files mapping
 */
private Map<String, Pair<String, MinimalLanguageInterface.GenericFileType>> sourcefilesToIndexedFiles(Set<SourceFile> sourceFiles) {
    Map<String, Pair<String, MinimalLanguageInterface.GenericFileType>> indexedFiles = new HashMap<>();
    for (SourceFile file : sourceFiles) {
        String content = file.getContent();
        String absolutePath = file.getAbsolutePath();
        MinimalLanguageInterface.GenericFileType fileType;
        switch(file.getType()) {
            case DOCKSTORE_CWL:
            case DOCKSTORE_WDL:
            case NEXTFLOW_CONFIG:
            case NEXTFLOW:
            case DOCKSTORE_SERVICE_YML:
            case DOCKSTORE_SERVICE_OTHER:
            case DOCKSTORE_YML:
                fileType = MinimalLanguageInterface.GenericFileType.IMPORTED_DESCRIPTOR;
                break;
            case CWL_TEST_JSON:
            case WDL_TEST_JSON:
            case NEXTFLOW_TEST_PARAMS:
            case DOCKSTORE_SERVICE_TEST_JSON:
                fileType = MinimalLanguageInterface.GenericFileType.TEST_PARAMETER_FILE;
                break;
            default:
                fileType = MinimalLanguageInterface.GenericFileType.CONTAINERFILE;
                break;
        }
        Pair<String, MinimalLanguageInterface.GenericFileType> indexedFile = new ImmutablePair<>(content, fileType);
        indexedFiles.put(absolutePath, indexedFile);
    }
    return indexedFiles;
}

Example 5 with SourceFile

use of io.dockstore.webservice.core.SourceFile in project dockstore by dockstore.

the class NextflowHandler method processImports.

@Override
public Map<String, SourceFile> processImports(String repositoryId, String content, Version version, SourceCodeRepoInterface sourceCodeRepoInterface, String filepath) {
    LOG.info("Processing import for repository {}, version {}, filepath {}", repositoryId, version.getName(), filepath);
    // FIXME: see{@link NextflowUtilities#grabConfig(String) grabConfig} method for comments on why
    // we have to look at imports in this crummy way
    final Matcher matcher = INCLUDE_CONFIG_PATTERN.matcher(content);
    Set<String> suspectedConfigImports = new HashSet<>();
    while (matcher.find()) {
        suspectedConfigImports.add(CharMatcher.is('\'').trimFrom(matcher.group(1).trim()));
    }
    Map<String, SourceFile> imports = new HashMap<>();
    Configuration configuration;
    try {
        configuration = NextflowUtilities.grabConfig(content);
    } catch (Exception e) {
        createValidationMessageForGeneralFailure(version, filepath);
        return imports;
    }
    // add the Nextflow scripts
    String mainScriptPath = "main.nf";
    if (configuration.containsKey("manifest.mainScript")) {
        mainScriptPath = configuration.getString("manifest.mainScript");
    }
    suspectedConfigImports.add(mainScriptPath);
    for (String filename : suspectedConfigImports) {
        String filenameAbsolutePath = convertRelativePathToAbsolutePath(filepath, filename);
        Optional<SourceFile> sourceFile = sourceCodeRepoInterface.readFile(repositoryId, version, DescriptorLanguage.FileType.NEXTFLOW, filenameAbsolutePath);
        if (sourceFile.isPresent()) {
            sourceFile.get().setPath(filename);
            imports.put(filename, sourceFile.get());
            imports.putAll(processOtherImports(repositoryId, sourceFile.get().getContent(), version, sourceCodeRepoInterface, sourceFile.get().getAbsolutePath()));
        }
    }
    // source files in /lib seem to be automatically added to the script classpath
    // binaries are also there and will need to be ignored
    List<String> strings = sourceCodeRepoInterface.listFiles(repositoryId, "/", version.getReference());
    handleNextflowImports(repositoryId, version, sourceCodeRepoInterface, imports, strings, "lib");
    handleNextflowImports(repositoryId, version, sourceCodeRepoInterface, imports, strings, "bin");
    return imports;
}