Examples with FeedMetadata com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata used on opensource projects

Example 1 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedModelTransform method domainToFeedMetadata.

/**
 * Transforms the specified Metadata feed to a Feed Manager feed.
 *
 * @param domain       the Metadata feed
 * @param userFieldMap cache map from category to user-defined fields, or {@code null}
 * @return the Feed Manager feed
 */
@Nonnull
private FeedMetadata domainToFeedMetadata(@Nonnull final Feed domain, @Nullable final Map<Category, Set<UserFieldDescriptor>> userFieldMap) {
    FeedMetadata feed = deserializeFeedMetadata(domain, false);
    feed.setId(domain.getId().toString());
    feed.setFeedId(domain.getId().toString());
    feed.setFeedName(domain.getDisplayName());
    feed.setSystemFeedName(domain.getName());
    feed.setDescription(domain.getDescription());
    feed.setAllowIndexing(domain.isAllowIndexing());
    feed.setHistoryReindexingStatus(domain.getCurrentHistoryReindexingStatus().getHistoryReindexingState().toString());
    feed.setOwner(domain.getOwner() != null ? new User(domain.getOwner().getName()) : null);
    if (domain.getCreatedTime() != null) {
        feed.setCreateDate(domain.getCreatedTime().toDate());
    }
    if (domain.getModifiedTime() != null) {
        feed.setUpdateDate(domain.getModifiedTime().toDate());
    }
    FeedManagerTemplate template = domain.getTemplate();
    if (template != null) {
        RegisteredTemplate registeredTemplate = templateModelTransform.DOMAIN_TO_REGISTERED_TEMPLATE.apply(template);
        feed.setRegisteredTemplate(registeredTemplate);
        feed.setTemplateId(registeredTemplate.getId());
        feed.setTemplateName(registeredTemplate.getTemplateName());
    }
    Category category = domain.getCategory();
    if (category != null) {
        feed.setCategory(categoryModelTransform.domainToFeedCategorySimple(category));
    }
    feed.setState(domain.getState() != null ? domain.getState().name() : null);
    feed.setVersionName(domain.getVersionName() != null ? domain.getVersionName() : null);
    // Set user-defined properties
    final Set<UserFieldDescriptor> userFields;
    if (userFieldMap == null) {
        userFields = getUserFields(category);
    } else if (userFieldMap.containsKey(category)) {
        userFields = userFieldMap.get(category);
    } else {
        userFields = getUserFields(category);
        userFieldMap.put(category, userFields);
    }
    @SuppressWarnings("unchecked") final Set<UserProperty> userProperties = UserPropertyTransform.toUserProperties(domain.getUserProperties(), userFields);
    feed.setUserProperties(userProperties);
    // Convert JCR securitygroup to DTO
    List<com.thinkbiganalytics.feedmgr.rest.model.HadoopSecurityGroup> restSecurityGroups = new ArrayList<>();
    if (domain.getSecurityGroups() != null && domain.getSecurityGroups().size() > 0) {
        for (Object group : domain.getSecurityGroups()) {
            HadoopSecurityGroup hadoopSecurityGroup = (HadoopSecurityGroup) group;
            com.thinkbiganalytics.feedmgr.rest.model.HadoopSecurityGroup restSecurityGroup = new com.thinkbiganalytics.feedmgr.rest.model.HadoopSecurityGroup();
            restSecurityGroup.setDescription(hadoopSecurityGroup.getDescription());
            restSecurityGroup.setId(hadoopSecurityGroup.getGroupId());
            restSecurityGroup.setName(hadoopSecurityGroup.getName());
            restSecurityGroups.add(restSecurityGroup);
        }
    }
    feed.setSecurityGroups(restSecurityGroups);
    feed.setTags(domain.getTags().stream().map(name -> new DefaultTag(name)).collect(Collectors.toList()));
    if (domain.getUsedByFeeds() != null) {
        final List<FeedSummary> usedByFeeds = domain.getUsedByFeeds().stream().map(this::domainToFeedSummary).collect(Collectors.toList());
        feed.setUsedByFeeds(usedByFeeds);
    }
    // add in access control items
    securityTransform.applyAccessControl(domain, feed);
    return feed;
}

Example 2 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedRestController method profileSummary.

@GET
@Path("/{feedId}/profile-summary")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Gets a summary of the feed profiles.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the profile summaries.", response = Map.class, responseContainer = "List"), @ApiResponse(code = 500, message = "The profiles are unavailable.", response = RestResponseStatus.class) })
public Response profileSummary(@PathParam("feedId") String feedId) {
    FeedMetadata feedMetadata = getMetadataService().getFeedById(feedId);
    final String profileTable = HiveUtils.quoteIdentifier(feedMetadata.getProfileTableName());
    String query = "SELECT * from " + profileTable + " where columnname = '(ALL)'";
    List<Map<String, Object>> rows = new ArrayList<>();
    try {
        QueryResult results = hiveService.query(query);
        rows.addAll(results.getRows());
        // add in the archive date time fields if applicipable
        String ARCHIVE_PROCESSOR_TYPE = "com.thinkbiganalytics.nifi.GetTableData";
        if (feedMetadata.getInputProcessorType().equalsIgnoreCase(ARCHIVE_PROCESSOR_TYPE)) {
            NifiProperty property = NifiPropertyUtil.findPropertyByProcessorType(feedMetadata.getProperties(), ARCHIVE_PROCESSOR_TYPE, "Date Field");
            if (property != null && property.getValue() != null) {
                String field = property.getValue();
                if (field.contains(".")) {
                    field = StringUtils.substringAfterLast(field, ".");
                }
                query = "SELECT * from " + profileTable + " where metrictype IN('MIN_TIMESTAMP','MAX_TIMESTAMP') AND columnname = " + HiveUtils.quoteString(field);
                QueryResult dateRows = hiveService.query(query);
                if (dateRows != null && !dateRows.isEmpty()) {
                    rows.addAll(dateRows.getRows());
                }
            }
        }
    } catch (DataAccessException e) {
        if (e.getCause() instanceof org.apache.hive.service.cli.HiveSQLException && e.getCause().getMessage().contains("Table not found")) {
        // this exception is ok to swallow since it just means no profile data exists yet
        } else if (e.getCause().getMessage().contains("HiveAccessControlException Permission denied")) {
            throw new AccessControlException("You do not have permission to execute this hive query");
        } else {
            throw e;
        }
    }
    return Response.ok(rows).build();
}

Example 3 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class EntityLevelAccessIT method assertAnalystCantEditFeed.

private void assertAnalystCantEditFeed(String errorMessage) {
    LOG.debug("EntityLevelAccessIT.assertAnalystCantEditFeed");
    runAs(ANALYST);
    FeedMetadata editFeedRequest = getEditFeedRequest();
    NifiFeed feed = createFeed(editFeedRequest);
    Assert.assertEquals(1, feed.getErrorMessages().size());
    Assert.assertTrue(feed.getErrorMessages().get(0).startsWith(errorMessage));
}

Example 4 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedIT method testDataIngestFeed.

@Test
public void testDataIngestFeed() throws Exception {
    prepare();
    importSystemFeeds();
    copyDataToDropzone();
    // create new category
    FeedCategory category = createCategory(CATEGORY_NAME);
    ImportTemplate ingest = importDataIngestTemplate();
    // create standard ingest feed
    FeedMetadata feed = getCreateFeedRequest(category, ingest, createNewFeedName());
    FeedMetadata response = createFeed(feed).getFeedMetadata();
    Assert.assertEquals(feed.getFeedName(), response.getFeedName());
    waitForFeedToComplete();
    assertExecutedJobs(response.getFeedName(), response.getFeedId());
    failJobs(response.getCategoryAndFeedName());
    abandonAllJobs(response.getCategoryAndFeedName());
}

Example 5 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedRestController method convertFeedIdToDisplayName.

@POST
@Path("/feed-system-name-to-display-name")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Maps feed ids to feed display names")
@ApiResponses(@ApiResponse(code = 200, message = "Returns a list of feed ids mapped to feed display names", response = Pair.class, responseContainer = "List"))
public Response convertFeedIdToDisplayName(@Nonnull final List<String> feedSystemNames) {
    Pair nullPair = new Pair(UUID.randomUUID().toString(), UUID.randomUUID().toString());
    List<Pair> names = feedSystemNames.stream().map(systemName -> {
        int dotIdx = systemName.indexOf(".");
        String categorySystemName = systemName.substring(0, dotIdx);
        String feedSystemName = systemName.substring(dotIdx + 1);
        FeedMetadata feed = getMetadataService().getFeedByName(categorySystemName, feedSystemName);
        if (feed != null) {
            return new Pair(systemName, feed.getCategoryAndFeedDisplayName());
        } else {
            return nullPair;
        }
    }).filter(pair -> !pair.equals(nullPair)).collect(Collectors.toList());
    return Response.ok(names).build();
}