Examples with TagDAO io.dockstore.webservice.jdbi.TagDAO used on opensource projects

Example 1 with TagDAO

use of io.dockstore.webservice.jdbi.TagDAO in project dockstore by dockstore.

the class AbstractImageRegistry method refreshTools.

/**
 * Updates/Adds/Deletes tools and their associated tags
 *
 * @param userId            The ID of the user
 * @param userDAO           ...
 * @param toolDAO           ...
 * @param tagDAO            ...
 * @param fileDAO           ...
 * @param githubToken       The user's GitHub token
 * @param bitbucketToken    The user's Bitbucket token
 * @param gitlabToken       The user's GitLab token
 * @param organization      If not null, only refresh tools belonging to the specific organization. Otherwise, refresh all.
 * @param dashboardPrefix   A string that prefixes logging statements to indicate that it will be used for Cloudwatch & Grafana.
 * @return The list of tools that have been updated
 */
@SuppressWarnings("checkstyle:parameternumber")
public List<Tool> refreshTools(final long userId, final UserDAO userDAO, final ToolDAO toolDAO, final TagDAO tagDAO, final FileDAO fileDAO, final FileFormatDAO fileFormatDAO, final Token githubToken, final Token bitbucketToken, final Token gitlabToken, String organization, final EventDAO eventDAO, final String dashboardPrefix) {
    // Get all the namespaces for the given registry
    List<String> namespaces;
    if (organization != null) {
        namespaces = Collections.singletonList(organization);
    } else {
        namespaces = getNamespaces();
    }
    // Get all the tools based on the found namespaces
    List<Tool> apiTools = getToolsFromNamespace(namespaces);
    String registryString = getRegistry().getDockerPath();
    // Add manual tools to list of api tools
    User user = userDAO.findById(userId);
    List<Tool> userTools = toolDAO.findByUserRegistryNamespace(userId, registryString, organization);
    // manualTools:
    // - isManualMode
    // - isTool
    // - belongs to user
    // - correct registry
    // - correct organization/namespace
    List<Tool> manualTools = userTools.stream().filter(tool -> ToolMode.MANUAL_IMAGE_PATH.equals(tool.getMode())).collect(Collectors.toList());
    // notManualTools is similar except it's not manualMode
    List<Tool> notManualTools = userTools.stream().filter(tool -> !ToolMode.MANUAL_IMAGE_PATH.equals(tool.getMode())).collect(Collectors.toList());
    apiTools.addAll(manualTools);
    // Update api tools with build information
    updateAPIToolsWithBuildInformation(apiTools);
    // Update db tools by copying over from api tools
    List<Tool> newDBTools = updateTools(apiTools, notManualTools, user, toolDAO);
    // Get tags and update for each tool
    for (Tool tool : newDBTools) {
        logToolRefresh(dashboardPrefix, tool);
        List<Tag> toolTags = getTags(tool);
        final SourceCodeRepoInterface sourceCodeRepo = SourceCodeRepoFactory.createSourceCodeRepo(tool.getGitUrl(), bitbucketToken == null ? null : bitbucketToken.getContent(), gitlabToken == null ? null : gitlabToken.getContent(), githubToken);
        updateTags(toolTags, tool, sourceCodeRepo, tagDAO, fileDAO, toolDAO, fileFormatDAO, eventDAO, user);
    }
    return newDBTools;
}

Example 2 with TagDAO

use of io.dockstore.webservice.jdbi.TagDAO in project dockstore by dockstore.

the class AbstractImageRegistry method refreshTool.

@SuppressWarnings("checkstyle:ParameterNumber")
public void refreshTool(final long userId, final UserDAO userDAO, final ToolDAO toolDAO, final TagDAO tagDAO, final FileDAO fileDAO, final FileFormatDAO fileFormatDAO, final Token githubToken, final Token bitbucketToken, final Token gitlabToken, String organization, final EventDAO eventDAO, final String dashboardPrefix, String repository) {
    // Get all the tools based on the found namespaces
    List<Tool> apiTools;
    if (repository != null) {
        apiTools = new ArrayList<>(Collections.singletonList(getToolFromNamespaceAndRepo(organization, repository)));
    } else {
        throw new CustomWebApplicationException("Trying to refresh/register a tool without a repository name", HttpStatus.SC_BAD_REQUEST);
    }
    // Add manual tools to list of api tools
    String registryString = getRegistry().getDockerPath();
    User user = userDAO.findById(userId);
    List<Tool> userRegistryNamespaceRepositoryTools = toolDAO.findByUserRegistryNamespaceRepository(userId, registryString, organization, repository);
    List<Tool> manualTools = userRegistryNamespaceRepositoryTools.stream().filter(tool -> ToolMode.MANUAL_IMAGE_PATH.equals(tool.getMode())).collect(Collectors.toList());
    List<Tool> notManualTools = userRegistryNamespaceRepositoryTools.stream().filter(tool -> !ToolMode.MANUAL_IMAGE_PATH.equals(tool.getMode())).collect(Collectors.toList());
    apiTools.addAll(manualTools);
    // Update api tools with build information
    updateAPIToolsWithBuildInformation(apiTools);
    // Update db tools by copying over from api tools
    List<Tool> newDBTools = updateTools(apiTools, notManualTools, user, toolDAO);
    // Get tags and update for each tool
    for (Tool tool : newDBTools) {
        logToolRefresh(dashboardPrefix, tool);
        List<Tag> toolTags = getTags(tool);
        final SourceCodeRepoInterface sourceCodeRepo = SourceCodeRepoFactory.createSourceCodeRepo(tool.getGitUrl(), bitbucketToken == null ? null : bitbucketToken.getContent(), gitlabToken == null ? null : gitlabToken.getContent(), githubToken);
        updateTags(toolTags, tool, sourceCodeRepo, tagDAO, fileDAO, toolDAO, fileFormatDAO, eventDAO, user);
    }
}

Example 3 with TagDAO

use of io.dockstore.webservice.jdbi.TagDAO in project dockstore by dockstore.

the class AbstractImageRegistry method updateTags.

/**
 * Updates/Adds/Deletes tags for a specific tool
 *
 * @param newTags
 * @param tool
 * @param tagDAO
 * @param fileDAO
 * @param toolDAO
 */
@SuppressWarnings("checkstyle:ParameterNumber")
private void updateTags(List<Tag> newTags, @NotNull Tool tool, SourceCodeRepoInterface sourceCodeRepoInterface, final TagDAO tagDAO, final FileDAO fileDAO, final ToolDAO toolDAO, final FileFormatDAO fileFormatDAO, final EventDAO eventDAO, final User user) {
    // Get all existing tags
    List<Tag> existingTags = new ArrayList<>(tool.getWorkflowVersions());
    if (tool.getMode() != ToolMode.MANUAL_IMAGE_PATH || (tool.getRegistry().equals(Registry.QUAY_IO.getDockerPath()) && existingTags.isEmpty())) {
        if (newTags == null) {
            LOG.info(tool.getToolPath() + " : Tags for tool {} did not get updated because new tags were not found", tool.getPath());
            return;
        }
        List<Tag> toDelete = new ArrayList<>(0);
        for (Iterator<Tag> iterator = existingTags.iterator(); iterator.hasNext(); ) {
            Tag oldTag = iterator.next();
            boolean exists = false;
            for (Tag newTag : newTags) {
                if (newTag.getName().equals(oldTag.getName())) {
                    exists = true;
                    break;
                }
            }
            if (!exists) {
                toDelete.add(oldTag);
                iterator.remove();
            }
        }
        for (Tag newTag : newTags) {
            boolean exists = false;
            // Find if user already has the tag
            for (Tag oldTag : existingTags) {
                if (newTag.getName().equals(oldTag.getName())) {
                    exists = true;
                    updateImageInformation(tool, newTag, oldTag);
                    oldTag.update(newTag);
                    // Update tag with default paths if dirty bit not set
                    if (!oldTag.isDirtyBit()) {
                        // Has not been modified => set paths
                        oldTag.setCwlPath(tool.getDefaultCwlPath());
                        oldTag.setWdlPath(tool.getDefaultWdlPath());
                        oldTag.setDockerfilePath(tool.getDefaultDockerfilePath());
                        // TODO: keep an eye on this, this used to always create new test params no matter what
                        if (tool.getDefaultTestCwlParameterFile() != null && oldTag.getSourceFiles().stream().noneMatch(file -> file.getPath().equals(tool.getDefaultTestCwlParameterFile()))) {
                            oldTag.getSourceFiles().add(createSourceFile(tool.getDefaultTestCwlParameterFile(), DescriptorLanguage.FileType.CWL_TEST_JSON));
                        }
                        if (tool.getDefaultTestWdlParameterFile() != null && oldTag.getSourceFiles().stream().noneMatch(file -> file.getPath().equals(tool.getDefaultTestWdlParameterFile()))) {
                            oldTag.getSourceFiles().add(createSourceFile(tool.getDefaultTestWdlParameterFile(), DescriptorLanguage.FileType.WDL_TEST_JSON));
                        }
                    }
                    break;
                }
            }
            // Tag does not already exist
            if (!exists) {
                // this could result in the same tag being added to multiple containers with the same path, need to clone
                Tag clonedTag = new Tag();
                clonedTag.clone(newTag);
                clonedTag.getImages().addAll(newTag.getImages());
                if (tool.getDefaultTestCwlParameterFile() != null) {
                    clonedTag.getSourceFiles().add(createSourceFile(tool.getDefaultTestCwlParameterFile(), DescriptorLanguage.FileType.CWL_TEST_JSON));
                }
                if (tool.getDefaultTestWdlParameterFile() != null) {
                    clonedTag.getSourceFiles().add(createSourceFile(tool.getDefaultTestWdlParameterFile(), DescriptorLanguage.FileType.WDL_TEST_JSON));
                }
                existingTags.add(clonedTag);
            }
        }
        boolean allAutomated = true;
        for (Tag tag : existingTags) {
            // create and add a tag if it does not already exist
            if (!tool.getWorkflowVersions().contains(tag)) {
                LOG.info(tool.getToolPath() + " : Updating tag {}", tag.getName());
                tag.setParent(tool);
                long id = tagDAO.create(tag);
                tag = tagDAO.findById(id);
                eventDAO.createAddTagToEntryEvent(user, tool, tag);
                tool.addWorkflowVersion(tag);
                if (!tag.isAutomated()) {
                    allAutomated = false;
                }
            }
        }
        // delete tool if it has no users
        deleteToolWithNoUsers(tool, toDelete);
        if (tool.getMode() != ToolMode.MANUAL_IMAGE_PATH) {
            if (allAutomated) {
                tool.setMode(ToolMode.AUTO_DETECT_QUAY_TAGS_AUTOMATED_BUILDS);
            } else {
                tool.setMode(ToolMode.AUTO_DETECT_QUAY_TAGS_WITH_MIXED);
            }
        }
    }
    // For tools from dockerhub, grab/update the image and checksum information
    if (tool.getRegistry().equals(Registry.DOCKER_HUB.getDockerPath()) || tool.getRegistry().equals(Registry.GITLAB.getDockerPath())) {
        updateNonQuayImageInformation(newTags, tool, existingTags);
    }
    // Now grab default/main tag to grab general information (defaults to github/bitbucket "main branch")
    if (sourceCodeRepoInterface != null) {
        // Grab files for each version/tag and check if valid
        Set<Tag> tags = tool.getWorkflowVersions();
        for (Tag tag : tags) {
            // check to see whether the commit id has changed
            // TODO: calls validation eventually, may simplify if we take into account metadata parsing below
            updateFiles(tool, tag, fileDAO, sourceCodeRepoInterface, sourceCodeRepoInterface.gitUsername);
        // Grab and parse files to get tool information
        // Add for new descriptor types
        }
        if (tool.getDefaultCwlPath() != null) {
            LOG.info(tool.getToolPath() + " " + sourceCodeRepoInterface.gitUsername + " : Parsing CWL...");
            sourceCodeRepoInterface.updateEntryMetadata(tool, DescriptorLanguage.CWL);
        }
        if (tool.getDefaultWdlPath() != null) {
            LOG.info(tool.getToolPath() + " " + sourceCodeRepoInterface.gitUsername + " : Parsing WDL...");
            sourceCodeRepoInterface.updateEntryMetadata(tool, DescriptorLanguage.WDL);
        }
    }
    FileFormatHelper.updateFileFormats(tool, tool.getWorkflowVersions(), fileFormatDAO, true);
    // ensure updated tags are saved to the database, not sure why this is necessary. See GeneralIT#testImageIDUpdateDuringRefresh
    tool.getWorkflowVersions().forEach(tagDAO::create);
    toolDAO.create(tool);
}

Example 4 with TagDAO

use of io.dockstore.webservice.jdbi.TagDAO in project dockstore by dockstore.

the class DockstoreWebserviceApplication method run.

@Override
public void run(DockstoreWebserviceConfiguration configuration, Environment environment) {
    BeanConfig beanConfig = new BeanConfig();
    beanConfig.setSchemes(new String[] { configuration.getExternalConfig().getScheme() });
    String portFragment = configuration.getExternalConfig().getPort() == null ? "" : ":" + configuration.getExternalConfig().getPort();
    beanConfig.setHost(configuration.getExternalConfig().getHostname() + portFragment);
    beanConfig.setBasePath(MoreObjects.firstNonNull(configuration.getExternalConfig().getBasePath(), "/"));
    beanConfig.setResourcePackage("io.dockstore.webservice.resources,io.swagger.api,io.openapi.api");
    beanConfig.setScan(true);
    final DefaultPluginManager languagePluginManager = LanguagePluginManager.getInstance(getFilePluginLocation(configuration));
    describeAvailableLanguagePlugins(languagePluginManager);
    LanguageHandlerFactory.setLanguagePluginManager(languagePluginManager);
    final PublicStateManager publicStateManager = PublicStateManager.getInstance();
    publicStateManager.setConfig(configuration);
    final TRSListener trsListener = new TRSListener();
    publicStateManager.addListener(trsListener);
    environment.jersey().property(CommonProperties.FEATURE_AUTO_DISCOVERY_DISABLE, true);
    environment.jersey().register(new JsonProcessingExceptionMapper(true));
    final TemplateHealthCheck healthCheck = new TemplateHealthCheck(configuration.getTemplate());
    environment.healthChecks().register("template", healthCheck);
    final ElasticSearchHealthCheck elasticSearchHealthCheck = new ElasticSearchHealthCheck(new ToolsExtendedApi());
    environment.healthChecks().register("elasticSearch", elasticSearchHealthCheck);
    environment.lifecycle().manage(new ElasticSearchHelper(configuration.getEsConfiguration()));
    final UserDAO userDAO = new UserDAO(hibernate.getSessionFactory());
    final TokenDAO tokenDAO = new TokenDAO(hibernate.getSessionFactory());
    final DeletedUsernameDAO deletedUsernameDAO = new DeletedUsernameDAO(hibernate.getSessionFactory());
    final ToolDAO toolDAO = new ToolDAO(hibernate.getSessionFactory());
    final FileDAO fileDAO = new FileDAO(hibernate.getSessionFactory());
    final WorkflowDAO workflowDAO = new WorkflowDAO(hibernate.getSessionFactory());
    final TagDAO tagDAO = new TagDAO(hibernate.getSessionFactory());
    final EventDAO eventDAO = new EventDAO(hibernate.getSessionFactory());
    final VersionDAO versionDAO = new VersionDAO(hibernate.getSessionFactory());
    final BioWorkflowDAO bioWorkflowDAO = new BioWorkflowDAO(hibernate.getSessionFactory());
    LOG.info("Cache directory for OkHttp is: " + cache.directory().getAbsolutePath());
    LOG.info("This is our custom logger saying that we're about to load authenticators");
    // setup authentication to allow session access in authenticators, see https://github.com/dropwizard/dropwizard/pull/1361
    SimpleAuthenticator authenticator = new UnitOfWorkAwareProxyFactory(getHibernate()).create(SimpleAuthenticator.class, new Class[] { TokenDAO.class, UserDAO.class }, new Object[] { tokenDAO, userDAO });
    CachingAuthenticator<String, User> cachingAuthenticator = new CachingAuthenticator<>(environment.metrics(), authenticator, configuration.getAuthenticationCachePolicy());
    environment.jersey().register(new AuthDynamicFeature(new OAuthCredentialAuthFilter.Builder<User>().setAuthenticator(cachingAuthenticator).setAuthorizer(new SimpleAuthorizer()).setPrefix("Bearer").setRealm("Dockstore User Authentication").buildAuthFilter()));
    environment.jersey().register(new AuthValueFactoryProvider.Binder<>(User.class));
    environment.jersey().register(RolesAllowedDynamicFeature.class);
    final HttpClient httpClient = new HttpClientBuilder(environment).using(configuration.getHttpClientConfiguration()).build(getName());
    final PermissionsInterface authorizer = PermissionsFactory.createAuthorizer(tokenDAO, configuration);
    final EntryResource entryResource = new EntryResource(tokenDAO, toolDAO, versionDAO, userDAO, configuration);
    environment.jersey().register(entryResource);
    final WorkflowResource workflowResource = new WorkflowResource(httpClient, hibernate.getSessionFactory(), authorizer, entryResource, configuration);
    environment.jersey().register(workflowResource);
    final ServiceResource serviceResource = new ServiceResource(httpClient, hibernate.getSessionFactory(), entryResource, configuration);
    environment.jersey().register(serviceResource);
    // Note workflow resource must be passed to the docker repo resource, as the workflow resource refresh must be called for checker workflows
    final DockerRepoResource dockerRepoResource = new DockerRepoResource(httpClient, hibernate.getSessionFactory(), configuration, workflowResource, entryResource);
    environment.jersey().register(dockerRepoResource);
    environment.jersey().register(new DockerRepoTagResource(toolDAO, tagDAO, eventDAO, versionDAO));
    environment.jersey().register(new TokenResource(tokenDAO, userDAO, deletedUsernameDAO, httpClient, cachingAuthenticator, configuration));
    environment.jersey().register(new UserResource(httpClient, getHibernate().getSessionFactory(), workflowResource, serviceResource, dockerRepoResource, cachingAuthenticator, authorizer, configuration));
    MetadataResourceHelper.init(configuration);
    environment.jersey().register(new UserResourceDockerRegistries(getHibernate().getSessionFactory()));
    environment.jersey().register(new MetadataResource(getHibernate().getSessionFactory(), configuration));
    environment.jersey().register(new HostedToolResource(getHibernate().getSessionFactory(), authorizer, configuration.getLimitConfig()));
    environment.jersey().register(new HostedWorkflowResource(getHibernate().getSessionFactory(), authorizer, configuration.getLimitConfig()));
    environment.jersey().register(new OrganizationResource(getHibernate().getSessionFactory()));
    environment.jersey().register(new LambdaEventResource(getHibernate().getSessionFactory()));
    environment.jersey().register(new NotificationResource(getHibernate().getSessionFactory()));
    environment.jersey().register(new CollectionResource(getHibernate().getSessionFactory()));
    environment.jersey().register(new EventResource(eventDAO, userDAO));
    environment.jersey().register(new ToolTesterResource(configuration));
    environment.jersey().register(new CloudInstanceResource(getHibernate().getSessionFactory()));
    // disable odd extra endpoints showing up
    final SwaggerConfiguration swaggerConfiguration = new SwaggerConfiguration().prettyPrint(true);
    swaggerConfiguration.setIgnoredRoutes(Lists.newArrayList("/application.wadl", "/pprof"));
    BaseOpenApiResource openApiResource = new OpenApiResource().openApiConfiguration(swaggerConfiguration);
    environment.jersey().register(openApiResource);
    final AliasResource aliasResource = new AliasResource(hibernate.getSessionFactory(), workflowResource);
    environment.jersey().register(aliasResource);
    // attach the container dao statically to avoid too much modification of generated code
    ToolsApiServiceImpl.setToolDAO(toolDAO);
    ToolsApiServiceImpl.setWorkflowDAO(workflowDAO);
    ToolsApiServiceImpl.setBioWorkflowDAO(bioWorkflowDAO);
    ToolsApiServiceImpl.setFileDAO(fileDAO);
    ToolsApiServiceImpl.setConfig(configuration);
    ToolsApiServiceImpl.setTrsListener(trsListener);
    ToolsApiExtendedServiceImpl.setStateManager(publicStateManager);
    ToolsApiExtendedServiceImpl.setToolDAO(toolDAO);
    ToolsApiExtendedServiceImpl.setWorkflowDAO(workflowDAO);
    ToolsApiExtendedServiceImpl.setConfig(configuration);
    DOIGeneratorFactory.setConfig(configuration);
    GoogleHelper.setConfig(configuration);
    registerAPIsAndMisc(environment);
    // optional CORS support
    // Enable CORS headers
    // final FilterRegistration.Dynamic cors = environment.servlets().addFilter("CORS", CrossOriginFilter.class);
    final FilterHolder filterHolder = environment.getApplicationContext().addFilter(CrossOriginFilter.class, "/*", EnumSet.of(REQUEST));
    filterHolder.setInitParameter(ACCESS_CONTROL_ALLOW_METHODS_HEADER, "GET,POST,DELETE,PUT,OPTIONS,PATCH");
    filterHolder.setInitParameter(ALLOWED_ORIGINS_PARAM, "*");
    filterHolder.setInitParameter(ALLOWED_METHODS_PARAM, "GET,POST,DELETE,PUT,OPTIONS,PATCH");
    filterHolder.setInitParameter(ALLOWED_HEADERS_PARAM, "Authorization, X-Auth-Username, X-Auth-Password, X-Requested-With,Content-Type,Accept,Origin,Access-Control-Request-Headers,cache-control");
    // Initialize GitHub App Installation Access Token cache
    CacheConfigManager cacheConfigManager = CacheConfigManager.getInstance();
    cacheConfigManager.initCache();
}