Examples with Registry io.dockstore.common.Registry used on opensource projects

Example 1 with Registry

use of io.dockstore.common.Registry in project dockstore by dockstore.

the class LanguageHandlerInterface method getURLFromEntry.

/**
 * Given a docker entry (quay or dockerhub), return a URL to the given entry
 *
 * @param dockerEntry has the docker name
 * @return URL
 */
// TODO: Potentially add support for other registries and add message that the registry is unsupported
default String getURLFromEntry(String dockerEntry, ToolDAO toolDAO) {
    // For now ignore tag, later on it may be more useful
    String quayIOPath = "https://quay.io/repository/";
    // For type repo/subrepo:tag
    String dockerHubPathR = "https://hub.docker.com/r/";
    // For type repo:tag
    String dockerHubPathUnderscore = "https://hub.docker.com/_/";
    // Update to tools once UI is updated to use /tools instead of /containers
    String dockstorePath = "https://www.dockstore.org/containers/";
    String url;
    // Remove tag if exists
    Pattern p = Pattern.compile("([^:]+):?(\\S+)?");
    Matcher m = p.matcher(dockerEntry);
    if (m.matches()) {
        dockerEntry = m.group(1);
    }
    if (dockerEntry.isEmpty()) {
        return null;
    }
    // Regex for determining registry requires a tag; add a fake "0" tag
    Optional<Registry> registry = determineImageRegistry(dockerEntry + ":0");
    // TODO: How do we check that the URL is valid? If not then the entry is likely a local docker build
    if (registry.isPresent() && registry.get().equals(Registry.QUAY_IO)) {
        List<Tool> byPath = toolDAO.findAllByPath(dockerEntry, true);
        if (byPath == null || byPath.isEmpty()) {
            // when we cannot find a published tool on Dockstore, link to quay.io
            url = dockerEntry.replaceFirst("quay\\.io/", quayIOPath);
        } else {
            // when we found a published tool, link to the tool on Dockstore
            url = dockstorePath + dockerEntry;
        }
    } else if (registry.isEmpty() || !registry.get().equals(Registry.DOCKER_HUB)) {
        // if the registry is neither Quay nor Docker Hub, return the entry as the url
        url = "https://" + dockerEntry;
    } else {
        // DOCKER_HUB
        String[] parts = dockerEntry.split("/");
        if (parts.length == 2) {
            // if the path looks like pancancer/pcawg-oxog-tools
            List<Tool> publishedByPath = toolDAO.findAllByPath("registry.hub.docker.com/" + dockerEntry, true);
            if (publishedByPath == null || publishedByPath.isEmpty()) {
                // when we cannot find a published tool on Dockstore, link to docker hub
                url = dockerHubPathR + dockerEntry;
            } else {
                // when we found a published tool, link to the tool on Dockstore
                url = dockstorePath + "registry.hub.docker.com/" + dockerEntry;
            }
        } else {
            // if the path looks like debian:8 or debian
            url = dockerHubPathUnderscore + dockerEntry;
        }
    }
    return url;
}

Example 2 with Registry

use of io.dockstore.common.Registry in project dockstore by dockstore.

the class DockerRepoResource method registerManual.

@POST
@Timed
@UnitOfWork
@Path("/registerManual")
@Operation(operationId = "registerManual", description = "Register a tool manually, along with tags.", security = @SecurityRequirement(name = OPENAPI_JWT_SECURITY_DEFINITION_NAME))
@ApiOperation(value = "Register a tool manually, along with tags.", authorizations = { @Authorization(value = JWT_SECURITY_DEFINITION_NAME) }, response = Tool.class)
public Tool registerManual(@ApiParam(hidden = true) @Parameter(hidden = true, name = "user") @Auth User user, @ApiParam(value = "Tool to be registered", required = true) Tool toolParam) {
    // Check for custom docker registries
    Registry registry = toolParam.getRegistryProvider();
    if (registry == null) {
        throw new CustomWebApplicationException("The provided registry is not valid. If you are using a custom registry please ensure that it matches the allowed paths.", HttpStatus.SC_BAD_REQUEST);
    }
    Tool duplicate = toolDAO.findByPath(toolParam.getToolPath(), false);
    if (duplicate != null) {
        LOG.info(user.getUsername() + ": duplicate tool found: {}" + toolParam.getToolPath());
        throw new CustomWebApplicationException("Tool " + toolParam.getToolPath() + " already exists.", HttpStatus.SC_BAD_REQUEST);
    }
    // Check if tool has tags
    if (toolParam.getRegistry().equals(Registry.QUAY_IO.getDockerPath()) && !checkContainerForTags(toolParam, user.getId())) {
        LOG.info(user.getUsername() + ": tool has no tags.");
        throw new CustomWebApplicationException("Tool " + toolParam.getToolPath() + " has no tags. Quay containers must have at least one tag.", HttpStatus.SC_BAD_REQUEST);
    }
    // Check if user owns repo, or if user is in the organization which owns the tool
    if (toolParam.getRegistry().equals(Registry.QUAY_IO.getDockerPath()) && !checkIfUserOwns(toolParam, user.getId())) {
        LOG.info(user.getUsername() + ": User does not own the given Quay Repo.");
        throw new CustomWebApplicationException("User does not own the tool " + toolParam.getPath() + ". You can only add Quay repositories that you own or are part of the organization", HttpStatus.SC_BAD_REQUEST);
    }
    final Set<Tag> workflowVersionsFromParam = Sets.newHashSet(toolParam.getWorkflowVersions());
    toolParam.setWorkflowVersions(Sets.newHashSet());
    // cannot create tool with a transient version hanging on it
    long id = toolDAO.create(toolParam);
    Tool tool = toolDAO.findById(id);
    // put the hanging versions back
    toolParam.setWorkflowVersions(workflowVersionsFromParam);
    if (registry.isPrivateOnly() && !tool.isPrivateAccess()) {
        throw new CustomWebApplicationException("The registry " + registry.getFriendlyName() + " is a private only registry.", HttpStatus.SC_BAD_REQUEST);
    }
    if (tool.isPrivateAccess() && Strings.isNullOrEmpty(tool.getToolMaintainerEmail())) {
        throw new CustomWebApplicationException("Tool maintainer email is required for private tools.", HttpStatus.SC_BAD_REQUEST);
    }
    // populate user in tool
    tool.addUser(user);
    // create dependent Tags before creating tool
    Set<Tag> createdTags = new HashSet<>();
    for (Tag tag : toolParam.getWorkflowVersions()) {
        tag.setParent(tool);
        final long l = tagDAO.create(tag);
        Tag byId = tagDAO.findById(l);
        createdTags.add(byId);
        this.eventDAO.createAddTagToEntryEvent(user, tool, byId);
    }
    tool.getWorkflowVersions().clear();
    tool.getWorkflowVersions().addAll(createdTags);
    // create dependent Labels before creating tool
    Set<Label> createdLabels = new HashSet<>();
    for (Label label : tool.getLabels()) {
        final long l = labelDAO.create(label);
        createdLabels.add(labelDAO.findById(l));
    }
    tool.getLabels().clear();
    tool.getLabels().addAll(createdLabels);
    if (!isGit(tool.getGitUrl())) {
        tool.setGitUrl(convertHttpsToSsh(tool.getGitUrl()));
    }
    // Can't set tool license information here, far too many tests register a tool without a GitHub token
    setToolLicenseInformation(user, tool);
    return toolDAO.findById(id);
}

Example 3 with Registry

use of io.dockstore.common.Registry in project dockstore by dockstore.

the class AbstractImageRegistry method refreshTools.

/**
 * Updates/Adds/Deletes tools and their associated tags
 *
 * @param userId            The ID of the user
 * @param userDAO           ...
 * @param toolDAO           ...
 * @param tagDAO            ...
 * @param fileDAO           ...
 * @param githubToken       The user's GitHub token
 * @param bitbucketToken    The user's Bitbucket token
 * @param gitlabToken       The user's GitLab token
 * @param organization      If not null, only refresh tools belonging to the specific organization. Otherwise, refresh all.
 * @param dashboardPrefix   A string that prefixes logging statements to indicate that it will be used for Cloudwatch & Grafana.
 * @return The list of tools that have been updated
 */
@SuppressWarnings("checkstyle:parameternumber")
public List<Tool> refreshTools(final long userId, final UserDAO userDAO, final ToolDAO toolDAO, final TagDAO tagDAO, final FileDAO fileDAO, final FileFormatDAO fileFormatDAO, final Token githubToken, final Token bitbucketToken, final Token gitlabToken, String organization, final EventDAO eventDAO, final String dashboardPrefix) {
    // Get all the namespaces for the given registry
    List<String> namespaces;
    if (organization != null) {
        namespaces = Collections.singletonList(organization);
    } else {
        namespaces = getNamespaces();
    }
    // Get all the tools based on the found namespaces
    List<Tool> apiTools = getToolsFromNamespace(namespaces);
    String registryString = getRegistry().getDockerPath();
    // Add manual tools to list of api tools
    User user = userDAO.findById(userId);
    List<Tool> userTools = toolDAO.findByUserRegistryNamespace(userId, registryString, organization);
    // manualTools:
    // - isManualMode
    // - isTool
    // - belongs to user
    // - correct registry
    // - correct organization/namespace
    List<Tool> manualTools = userTools.stream().filter(tool -> ToolMode.MANUAL_IMAGE_PATH.equals(tool.getMode())).collect(Collectors.toList());
    // notManualTools is similar except it's not manualMode
    List<Tool> notManualTools = userTools.stream().filter(tool -> !ToolMode.MANUAL_IMAGE_PATH.equals(tool.getMode())).collect(Collectors.toList());
    apiTools.addAll(manualTools);
    // Update api tools with build information
    updateAPIToolsWithBuildInformation(apiTools);
    // Update db tools by copying over from api tools
    List<Tool> newDBTools = updateTools(apiTools, notManualTools, user, toolDAO);
    // Get tags and update for each tool
    for (Tool tool : newDBTools) {
        logToolRefresh(dashboardPrefix, tool);
        List<Tag> toolTags = getTags(tool);
        final SourceCodeRepoInterface sourceCodeRepo = SourceCodeRepoFactory.createSourceCodeRepo(tool.getGitUrl(), bitbucketToken == null ? null : bitbucketToken.getContent(), gitlabToken == null ? null : gitlabToken.getContent(), githubToken);
        updateTags(toolTags, tool, sourceCodeRepo, tagDAO, fileDAO, toolDAO, fileFormatDAO, eventDAO, user);
    }
    return newDBTools;
}

Example 4 with Registry

use of io.dockstore.common.Registry in project dockstore by dockstore.

the class UserResource method checkToolTokens.

private void checkToolTokens(User authUser, Long userId, String organization) {
    List<Token> tokens = tokenDAO.findByUserId(userId);
    List<Tool> tools = userContainers(authUser, userId);
    if (organization != null && !organization.isEmpty()) {
        tools.removeIf(tool -> !tool.getNamespace().equals(organization));
    }
    Token gitLabToken = Token.extractToken(tokens, TokenType.GITLAB_COM);
    Token quayioToken = Token.extractToken(tokens, TokenType.QUAY_IO);
    Set<Registry> uniqueRegistry = new HashSet<>();
    tools.forEach(tool -> uniqueRegistry.add(tool.getRegistryProvider()));
    if (uniqueRegistry.size() == 0 && quayioToken == null) {
        throw new CustomWebApplicationException("You have no tools and no Quay.io token to automatically add tools. Please add a Quay.io token.", HttpStatus.SC_BAD_REQUEST);
    }
    if (uniqueRegistry.contains(Registry.QUAY_IO) && quayioToken == null) {
        throw new CustomWebApplicationException("You have Quay.io tools but no Quay.io token to refresh the tools with. Please add a Quay.io token.", HttpStatus.SC_BAD_REQUEST);
    }
    if (uniqueRegistry.contains(Registry.GITLAB) && gitLabToken == null) {
        throw new CustomWebApplicationException("You have GitLab tools but no GitLab token to refresh the tools with. Please add a GitLab token", HttpStatus.SC_BAD_REQUEST);
    }
}

Example 5 with Registry

use of io.dockstore.common.Registry in project dockstore by dockstore.

the class UserResource method getGitRepositoryMap.

/**
 * For a given user and git registry, retrieve a map of git url to repository path
 * @param user
 * @param gitRegistry
 * @return mapping of git url to repository path
 */
private Map<String, String> getGitRepositoryMap(User user, SourceControl gitRegistry) {
    List<Token> scTokens = getAndRefreshTokens(user, tokenDAO, client, bitbucketClientID, bitbucketClientSecret).stream().filter(token -> Objects.equals(token.getTokenSource().getSourceControl(), gitRegistry)).collect(Collectors.toList());
    if (scTokens.size() > 0) {
        Token scToken = scTokens.get(0);
        SourceCodeRepoInterface sourceCodeRepo = SourceCodeRepoFactory.createSourceCodeRepo(scToken);
        return sourceCodeRepo.getWorkflowGitUrl2RepositoryId();
    } else {
        return new HashMap<>();
    }
}