Examples with FileFormatDAO io.dockstore.webservice.jdbi.FileFormatDAO used on opensource projects

Example 1 with FileFormatDAO

use of io.dockstore.webservice.jdbi.FileFormatDAO in project dockstore by dockstore.

the class AbstractImageRegistry method refreshTools.

/**
 * Updates/Adds/Deletes tools and their associated tags
 *
 * @param userId            The ID of the user
 * @param userDAO           ...
 * @param toolDAO           ...
 * @param tagDAO            ...
 * @param fileDAO           ...
 * @param githubToken       The user's GitHub token
 * @param bitbucketToken    The user's Bitbucket token
 * @param gitlabToken       The user's GitLab token
 * @param organization      If not null, only refresh tools belonging to the specific organization. Otherwise, refresh all.
 * @param dashboardPrefix   A string that prefixes logging statements to indicate that it will be used for Cloudwatch & Grafana.
 * @return The list of tools that have been updated
 */
@SuppressWarnings("checkstyle:parameternumber")
public List<Tool> refreshTools(final long userId, final UserDAO userDAO, final ToolDAO toolDAO, final TagDAO tagDAO, final FileDAO fileDAO, final FileFormatDAO fileFormatDAO, final Token githubToken, final Token bitbucketToken, final Token gitlabToken, String organization, final EventDAO eventDAO, final String dashboardPrefix) {
    // Get all the namespaces for the given registry
    List<String> namespaces;
    if (organization != null) {
        namespaces = Collections.singletonList(organization);
    } else {
        namespaces = getNamespaces();
    }
    // Get all the tools based on the found namespaces
    List<Tool> apiTools = getToolsFromNamespace(namespaces);
    String registryString = getRegistry().getDockerPath();
    // Add manual tools to list of api tools
    User user = userDAO.findById(userId);
    List<Tool> userTools = toolDAO.findByUserRegistryNamespace(userId, registryString, organization);
    // manualTools:
    // - isManualMode
    // - isTool
    // - belongs to user
    // - correct registry
    // - correct organization/namespace
    List<Tool> manualTools = userTools.stream().filter(tool -> ToolMode.MANUAL_IMAGE_PATH.equals(tool.getMode())).collect(Collectors.toList());
    // notManualTools is similar except it's not manualMode
    List<Tool> notManualTools = userTools.stream().filter(tool -> !ToolMode.MANUAL_IMAGE_PATH.equals(tool.getMode())).collect(Collectors.toList());
    apiTools.addAll(manualTools);
    // Update api tools with build information
    updateAPIToolsWithBuildInformation(apiTools);
    // Update db tools by copying over from api tools
    List<Tool> newDBTools = updateTools(apiTools, notManualTools, user, toolDAO);
    // Get tags and update for each tool
    for (Tool tool : newDBTools) {
        logToolRefresh(dashboardPrefix, tool);
        List<Tag> toolTags = getTags(tool);
        final SourceCodeRepoInterface sourceCodeRepo = SourceCodeRepoFactory.createSourceCodeRepo(tool.getGitUrl(), bitbucketToken == null ? null : bitbucketToken.getContent(), gitlabToken == null ? null : gitlabToken.getContent(), githubToken);
        updateTags(toolTags, tool, sourceCodeRepo, tagDAO, fileDAO, toolDAO, fileFormatDAO, eventDAO, user);
    }
    return newDBTools;
}

Example 2 with FileFormatDAO

use of io.dockstore.webservice.jdbi.FileFormatDAO in project dockstore by dockstore.

the class AbstractImageRegistry method refreshTool.

@SuppressWarnings("checkstyle:ParameterNumber")
public void refreshTool(final long userId, final UserDAO userDAO, final ToolDAO toolDAO, final TagDAO tagDAO, final FileDAO fileDAO, final FileFormatDAO fileFormatDAO, final Token githubToken, final Token bitbucketToken, final Token gitlabToken, String organization, final EventDAO eventDAO, final String dashboardPrefix, String repository) {
    // Get all the tools based on the found namespaces
    List<Tool> apiTools;
    if (repository != null) {
        apiTools = new ArrayList<>(Collections.singletonList(getToolFromNamespaceAndRepo(organization, repository)));
    } else {
        throw new CustomWebApplicationException("Trying to refresh/register a tool without a repository name", HttpStatus.SC_BAD_REQUEST);
    }
    // Add manual tools to list of api tools
    String registryString = getRegistry().getDockerPath();
    User user = userDAO.findById(userId);
    List<Tool> userRegistryNamespaceRepositoryTools = toolDAO.findByUserRegistryNamespaceRepository(userId, registryString, organization, repository);
    List<Tool> manualTools = userRegistryNamespaceRepositoryTools.stream().filter(tool -> ToolMode.MANUAL_IMAGE_PATH.equals(tool.getMode())).collect(Collectors.toList());
    List<Tool> notManualTools = userRegistryNamespaceRepositoryTools.stream().filter(tool -> !ToolMode.MANUAL_IMAGE_PATH.equals(tool.getMode())).collect(Collectors.toList());
    apiTools.addAll(manualTools);
    // Update api tools with build information
    updateAPIToolsWithBuildInformation(apiTools);
    // Update db tools by copying over from api tools
    List<Tool> newDBTools = updateTools(apiTools, notManualTools, user, toolDAO);
    // Get tags and update for each tool
    for (Tool tool : newDBTools) {
        logToolRefresh(dashboardPrefix, tool);
        List<Tag> toolTags = getTags(tool);
        final SourceCodeRepoInterface sourceCodeRepo = SourceCodeRepoFactory.createSourceCodeRepo(tool.getGitUrl(), bitbucketToken == null ? null : bitbucketToken.getContent(), gitlabToken == null ? null : gitlabToken.getContent(), githubToken);
        updateTags(toolTags, tool, sourceCodeRepo, tagDAO, fileDAO, toolDAO, fileFormatDAO, eventDAO, user);
    }
}

Example 3 with FileFormatDAO

use of io.dockstore.webservice.jdbi.FileFormatDAO in project dockstore by dockstore.

the class AbstractImageRegistry method updateTags.

/**
 * Updates/Adds/Deletes tags for a specific tool
 *
 * @param newTags
 * @param tool
 * @param tagDAO
 * @param fileDAO
 * @param toolDAO
 */
@SuppressWarnings("checkstyle:ParameterNumber")
private void updateTags(List<Tag> newTags, @NotNull Tool tool, SourceCodeRepoInterface sourceCodeRepoInterface, final TagDAO tagDAO, final FileDAO fileDAO, final ToolDAO toolDAO, final FileFormatDAO fileFormatDAO, final EventDAO eventDAO, final User user) {
    // Get all existing tags
    List<Tag> existingTags = new ArrayList<>(tool.getWorkflowVersions());
    if (tool.getMode() != ToolMode.MANUAL_IMAGE_PATH || (tool.getRegistry().equals(Registry.QUAY_IO.getDockerPath()) && existingTags.isEmpty())) {
        if (newTags == null) {
            LOG.info(tool.getToolPath() + " : Tags for tool {} did not get updated because new tags were not found", tool.getPath());
            return;
        }
        List<Tag> toDelete = new ArrayList<>(0);
        for (Iterator<Tag> iterator = existingTags.iterator(); iterator.hasNext(); ) {
            Tag oldTag = iterator.next();
            boolean exists = false;
            for (Tag newTag : newTags) {
                if (newTag.getName().equals(oldTag.getName())) {
                    exists = true;
                    break;
                }
            }
            if (!exists) {
                toDelete.add(oldTag);
                iterator.remove();
            }
        }
        for (Tag newTag : newTags) {
            boolean exists = false;
            // Find if user already has the tag
            for (Tag oldTag : existingTags) {
                if (newTag.getName().equals(oldTag.getName())) {
                    exists = true;
                    updateImageInformation(tool, newTag, oldTag);
                    oldTag.update(newTag);
                    // Update tag with default paths if dirty bit not set
                    if (!oldTag.isDirtyBit()) {
                        // Has not been modified => set paths
                        oldTag.setCwlPath(tool.getDefaultCwlPath());
                        oldTag.setWdlPath(tool.getDefaultWdlPath());
                        oldTag.setDockerfilePath(tool.getDefaultDockerfilePath());
                        // TODO: keep an eye on this, this used to always create new test params no matter what
                        if (tool.getDefaultTestCwlParameterFile() != null && oldTag.getSourceFiles().stream().noneMatch(file -> file.getPath().equals(tool.getDefaultTestCwlParameterFile()))) {
                            oldTag.getSourceFiles().add(createSourceFile(tool.getDefaultTestCwlParameterFile(), DescriptorLanguage.FileType.CWL_TEST_JSON));
                        }
                        if (tool.getDefaultTestWdlParameterFile() != null && oldTag.getSourceFiles().stream().noneMatch(file -> file.getPath().equals(tool.getDefaultTestWdlParameterFile()))) {
                            oldTag.getSourceFiles().add(createSourceFile(tool.getDefaultTestWdlParameterFile(), DescriptorLanguage.FileType.WDL_TEST_JSON));
                        }
                    }
                    break;
                }
            }
            // Tag does not already exist
            if (!exists) {
                // this could result in the same tag being added to multiple containers with the same path, need to clone
                Tag clonedTag = new Tag();
                clonedTag.clone(newTag);
                clonedTag.getImages().addAll(newTag.getImages());
                if (tool.getDefaultTestCwlParameterFile() != null) {
                    clonedTag.getSourceFiles().add(createSourceFile(tool.getDefaultTestCwlParameterFile(), DescriptorLanguage.FileType.CWL_TEST_JSON));
                }
                if (tool.getDefaultTestWdlParameterFile() != null) {
                    clonedTag.getSourceFiles().add(createSourceFile(tool.getDefaultTestWdlParameterFile(), DescriptorLanguage.FileType.WDL_TEST_JSON));
                }
                existingTags.add(clonedTag);
            }
        }
        boolean allAutomated = true;
        for (Tag tag : existingTags) {
            // create and add a tag if it does not already exist
            if (!tool.getWorkflowVersions().contains(tag)) {
                LOG.info(tool.getToolPath() + " : Updating tag {}", tag.getName());
                tag.setParent(tool);
                long id = tagDAO.create(tag);
                tag = tagDAO.findById(id);
                eventDAO.createAddTagToEntryEvent(user, tool, tag);
                tool.addWorkflowVersion(tag);
                if (!tag.isAutomated()) {
                    allAutomated = false;
                }
            }
        }
        // delete tool if it has no users
        deleteToolWithNoUsers(tool, toDelete);
        if (tool.getMode() != ToolMode.MANUAL_IMAGE_PATH) {
            if (allAutomated) {
                tool.setMode(ToolMode.AUTO_DETECT_QUAY_TAGS_AUTOMATED_BUILDS);
            } else {
                tool.setMode(ToolMode.AUTO_DETECT_QUAY_TAGS_WITH_MIXED);
            }
        }
    }
    // For tools from dockerhub, grab/update the image and checksum information
    if (tool.getRegistry().equals(Registry.DOCKER_HUB.getDockerPath()) || tool.getRegistry().equals(Registry.GITLAB.getDockerPath())) {
        updateNonQuayImageInformation(newTags, tool, existingTags);
    }
    // Now grab default/main tag to grab general information (defaults to github/bitbucket "main branch")
    if (sourceCodeRepoInterface != null) {
        // Grab files for each version/tag and check if valid
        Set<Tag> tags = tool.getWorkflowVersions();
        for (Tag tag : tags) {
            // check to see whether the commit id has changed
            // TODO: calls validation eventually, may simplify if we take into account metadata parsing below
            updateFiles(tool, tag, fileDAO, sourceCodeRepoInterface, sourceCodeRepoInterface.gitUsername);
        // Grab and parse files to get tool information
        // Add for new descriptor types
        }
        if (tool.getDefaultCwlPath() != null) {
            LOG.info(tool.getToolPath() + " " + sourceCodeRepoInterface.gitUsername + " : Parsing CWL...");
            sourceCodeRepoInterface.updateEntryMetadata(tool, DescriptorLanguage.CWL);
        }
        if (tool.getDefaultWdlPath() != null) {
            LOG.info(tool.getToolPath() + " " + sourceCodeRepoInterface.gitUsername + " : Parsing WDL...");
            sourceCodeRepoInterface.updateEntryMetadata(tool, DescriptorLanguage.WDL);
        }
    }
    FileFormatHelper.updateFileFormats(tool, tool.getWorkflowVersions(), fileFormatDAO, true);
    // ensure updated tags are saved to the database, not sure why this is necessary. See GeneralIT#testImageIDUpdateDuringRefresh
    tool.getWorkflowVersions().forEach(tagDAO::create);
    toolDAO.create(tool);
}

Example 4 with FileFormatDAO

use of io.dockstore.webservice.jdbi.FileFormatDAO in project dockstore by dockstore.

the class FileFormatHelper method updateFileFormats.

/**
 * Updates the given tool/workflow to show which file formats are associated with its sourcefiles
 * @param entry The tool or workflow to update
 * @param versions  A tool/workflow's versions (tags/workflowVersions)
 * @param fileFormatDAO  The FileFormatDAO to check the FileFormat table
 * @param updateAllVersions If updating all versions, then this will clear the entry's input/output formats and reset them. Otherwise it will only add the file formats
 */
public static void updateFileFormats(Entry entry, Set<? extends Version> versions, final FileFormatDAO fileFormatDAO, boolean updateAllVersions) {
    SortedSet<FileFormat> entrysInputFileFormats = new TreeSet<>();
    SortedSet<FileFormat> entrysOutputFileFormats = new TreeSet<>();
    CWLHandler cwlHandler = new CWLHandler();
    versions.stream().filter(tag -> !((Version) tag).isFrozen()).forEach(tag -> {
        SortedSet<FileFormat> inputFileFormats = new TreeSet<>();
        SortedSet<FileFormat> outputFileFormats = new TreeSet<>();
        SortedSet<SourceFile> sourceFiles = tag.getSourceFiles();
        List<SourceFile> cwlFiles = sourceFiles.stream().filter(sourceFile -> sourceFile.getType() == DescriptorLanguage.FileType.DOCKSTORE_CWL).collect(Collectors.toList());
        cwlFiles.stream().filter(cwlFile -> cwlFile.getContent() != null).forEach(cwlFile -> {
            inputFileFormats.addAll(cwlHandler.getFileFormats(cwlFile.getContent(), "inputs"));
            outputFileFormats.addAll(cwlHandler.getFileFormats(cwlFile.getContent(), "outputs"));
        });
        SortedSet<FileFormat> realInputFileFormats = getFileFormatsFromDatabase(fileFormatDAO, inputFileFormats);
        SortedSet<FileFormat> realOutputFileFormats = getFileFormatsFromDatabase(fileFormatDAO, outputFileFormats);
        tag.setInputFileFormats(realInputFileFormats);
        tag.setOutputFileFormats(realOutputFileFormats);
        entrysInputFileFormats.addAll(realInputFileFormats);
        entrysOutputFileFormats.addAll(realOutputFileFormats);
    });
    if (updateAllVersions) {
        entry.getInputFileFormats().clear();
        entry.getOutputFileFormats().clear();
    }
    entry.getInputFileFormats().addAll(entrysInputFileFormats);
    entry.getOutputFileFormats().addAll(entrysOutputFileFormats);
}