Examples with FeedMetadata com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata used on opensource projects

Example 26 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class CreateFeedBuilder method fetchInputProcessorForProcessGroup.

private ProcessorDTO fetchInputProcessorForProcessGroup(ProcessGroupDTO entity) {
    // Find first processor by type
    final List<ProcessorDTO> inputProcessors = NifiProcessUtil.getInputProcessors(entity);
    String inputProcessorName = feedMetadata != null ? feedMetadata.getInputProcessorName() : null;
    final ProcessorDTO input = Optional.ofNullable(NifiProcessUtil.findFirstProcessorsByTypeAndName(inputProcessors, inputProcessorType, inputProcessorName)).orElseGet(() -> inputProcessors.stream().filter(processor -> !processor.getType().equals(NifiProcessUtil.CLEANUP_TYPE)).findFirst().orElse(null));
    // Update cached type
    if (input != null) {
        inputProcessorType = input.getType();
    }
    return input;
}

Example 27 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedRestController method profileSummary.

@GET
@Path("/{feedId}/profile-summary")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Gets a summary of the feed profiles.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the profile summaries.", response = Map.class, responseContainer = "List"), @ApiResponse(code = 500, message = "The profiles are unavailable.", response = RestResponseStatus.class) })
public Response profileSummary(@PathParam("feedId") String feedId) {
    FeedMetadata feedMetadata = getMetadataService().getFeedById(feedId);
    final String profileTable = HiveUtils.quoteIdentifier(feedMetadata.getProfileTableName());
    String query = "SELECT * from " + profileTable + " where columnname = '(ALL)'";
    List<Map<String, Object>> rows = new ArrayList<>();
    try {
        QueryResult results = hiveService.query(query);
        rows.addAll(results.getRows());
        // add in the archive date time fields if applicipable
        String ARCHIVE_PROCESSOR_TYPE = "com.thinkbiganalytics.nifi.GetTableData";
        if (feedMetadata.getInputProcessorType().equalsIgnoreCase(ARCHIVE_PROCESSOR_TYPE)) {
            NifiProperty property = NifiPropertyUtil.findPropertyByProcessorType(feedMetadata.getProperties(), ARCHIVE_PROCESSOR_TYPE, "Date Field");
            if (property != null && property.getValue() != null) {
                String field = property.getValue();
                if (field.contains(".")) {
                    field = StringUtils.substringAfterLast(field, ".");
                }
                query = "SELECT * from " + profileTable + " where metrictype IN('MIN_TIMESTAMP','MAX_TIMESTAMP') AND columnname = " + HiveUtils.quoteString(field);
                QueryResult dateRows = hiveService.query(query);
                if (dateRows != null && !dateRows.isEmpty()) {
                    rows.addAll(dateRows.getRows());
                }
            }
        }
    } catch (DataAccessException e) {
        if (e.getCause() instanceof org.apache.hive.service.cli.HiveSQLException && e.getCause().getMessage().contains("Table not found")) {
        // this exception is ok to swallow since it just means no profile data exists yet
        } else if (e.getCause().getMessage().contains("HiveAccessControlException Permission denied")) {
            throw new AccessControlException("You do not have permission to execute this hive query");
        } else {
            throw e;
        }
    }
    return Response.ok(rows).build();
}

Example 28 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedRestController method updateFeedMetadata.

private void updateFeedMetadata(FeedMetadata targetFeedMetadata, FeedMetadata modifiedFeedMetadata, FeedPropertySection feedPropertySection) {
    AnnotationFieldNameResolver annotationFieldNameResolver = new AnnotationFieldNameResolver(FeedPropertyType.class);
    List<AnnotatedFieldProperty> list = annotationFieldNameResolver.getProperties(FeedMetadata.class);
    List<AnnotatedFieldProperty> sectionList = list.stream().filter(annotatedFieldProperty -> feedPropertySection.equals(((FeedPropertyType) annotatedFieldProperty.getAnnotation()).section())).collect(Collectors.toList());
    sectionList.forEach(annotatedFieldProperty -> {
        try {
            Object value = FieldUtils.readField(annotatedFieldProperty.getField(), modifiedFeedMetadata);
            FieldUtils.writeField(annotatedFieldProperty.getField(), targetFeedMetadata, value);
        } catch (IllegalAccessException e) {
            log.warn("Unable to update FeedMetadata field: {}.  Exception: {} ", annotatedFieldProperty.getField(), e.getMessage(), e);
        }
    });
}

Example 29 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedRestController method convertFeedIdToDisplayName.

@POST
@Path("/feed-system-name-to-display-name")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Maps feed ids to feed display names")
@ApiResponses(@ApiResponse(code = 200, message = "Returns a list of feed ids mapped to feed display names", response = Pair.class, responseContainer = "List"))
public Response convertFeedIdToDisplayName(@Nonnull final List<String> feedSystemNames) {
    Pair nullPair = new Pair(UUID.randomUUID().toString(), UUID.randomUUID().toString());
    List<Pair> names = feedSystemNames.stream().map(systemName -> {
        int dotIdx = systemName.indexOf(".");
        String categorySystemName = systemName.substring(0, dotIdx);
        String feedSystemName = systemName.substring(dotIdx + 1);
        FeedMetadata feed = getMetadataService().getFeedByName(categorySystemName, feedSystemName);
        if (feed != null) {
            return new Pair(systemName, feed.getFeedName());
        } else {
            return nullPair;
        }
    }).filter(pair -> !pair.equals(nullPair)).collect(Collectors.toList());
    return Response.ok(names).build();
}

Example 30 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedsController method updateDataHistoryReindexStatus.

@POST
@Path("{id}/update-data-history-reindex-status")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@ApiOperation("Sets the data history reindexing status for the specified feed")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the feed's updated data history reindex status and columns to index", response = FeedDataHistoryReindexParams.class), @ApiResponse(code = 500, message = "The data history reindex status could not be updated for the feed", response = RestResponseStatus.class) })
public FeedDataHistoryReindexParams updateDataHistoryReindexStatus(@PathParam("id") String feedIdStr, HistoryReindexingStatus status) {
    LOG.debug("Update data history reindexing status for feed with id: {}", feedIdStr);
    FeedDataHistoryReindexParams feedDataHistoryReindexParams = this.metadata.commit(() -> {
        this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_FEEDS);
        com.thinkbiganalytics.metadata.api.feed.Feed.ID feedId = feedProvider.resolveFeed(feedIdStr);
        com.thinkbiganalytics.metadata.api.feed.Feed feed = feedProvider.getFeed(feedId);
        if (feed != null) {
            FeedDataHistoryReindexParams reindexParams = new FeedDataHistoryReindexParams();
            com.thinkbiganalytics.metadata.api.feed.reindex.HistoryReindexingState newHistoryReindexingState = com.thinkbiganalytics.metadata.api.feed.reindex.HistoryReindexingState.valueOf(status.getHistoryReindexingState().name());
            com.thinkbiganalytics.metadata.api.feed.Feed updatedDomainFeed = feed.updateHistoryReindexingStatus(new com.thinkbiganalytics.metadata.api.feed.reindex.HistoryReindexingStatus(newHistoryReindexingState));
            Feed updatedRestFeed = metadataTransform.domainToFeed().apply(updatedDomainFeed);
            // updated reindexing status
            reindexParams.setHistoryReindexingStatus(updatedRestFeed.getCurrentHistoryReindexingStatus());
            reindexParams.setFeedId(updatedRestFeed.getId());
            reindexParams.setFeedSystemName(updatedRestFeed.getSystemName());
            reindexParams.setCategorySystemName(updatedRestFeed.getCategory().getSystemName());
            if (updatedDomainFeed != null) {
                FeedMetadata feedMetadata = getMetadataService().getFeedById(updatedDomainFeed.getId().toString());
                if (feedMetadata != null && feedMetadata.getTable() != null) {
                    List<String> commaSepColumns = new ArrayList<>();
                    List<FieldPolicy> fieldPolicies = feedMetadata.getTable().getFieldPolicies();
                    for (FieldPolicy fieldPolicy : fieldPolicies) {
                        if (fieldPolicy.isIndex()) {
                            commaSepColumns.add(fieldPolicy.getFieldName().toLowerCase().trim());
                        }
                    }
                    // columns to index as comma separated string
                    reindexParams.setCommaSeparatedColumnsForIndexing(StringUtils.join(commaSepColumns.toArray(), ","));
                }
            }
            return reindexParams;
        } else {
            throw new WebApplicationException("A feed with the given ID does not exist: " + feedId, Status.NOT_FOUND);
        }
    });
    return feedDataHistoryReindexParams;
}