Examples with FeedMetadata com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata used on opensource projects

Example 61 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedRestController method getFeedByName.

@GET
@Path("/by-name/{feedName}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Gets the specified feed.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the feed.", response = FeedMetadata.class), @ApiResponse(code = 500, message = "The feed is unavailable.", response = RestResponseStatus.class) })
public Response getFeedByName(@PathParam("feedName") String feedName) {
    String categorySystemName = FeedNameUtil.category(feedName);
    String feedSystemName = FeedNameUtil.feed(feedName);
    if (StringUtils.isNotBlank(categorySystemName) && StringUtils.isNotBlank(feedSystemName)) {
        FeedMetadata feed = getMetadataService().getFeedByName(categorySystemName, feedSystemName);
        return Response.ok(feed).build();
    } else {
        throw new NotFoundException("Unable to find the feed for name: " + feedName);
    }
}

Example 62 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedRestController method profileStats.

@GET
@Path("/{feedId}/profile-stats")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Gets the profile statistics for the specified job.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the profile statistics.", response = Map.class, responseContainer = "List"), @ApiResponse(code = 500, message = "The profile is unavailable.", response = RestResponseStatus.class) })
public Response profileStats(@PathParam("feedId") String feedId, @QueryParam("processingdttm") String processingdttm) {
    FeedMetadata feedMetadata = getMetadataService().getFeedById(feedId);
    String profileTable = feedMetadata.getProfileTableName();
    String query = "SELECT * from " + HiveUtils.quoteIdentifier(profileTable) + " where processing_dttm = " + HiveUtils.quoteString(processingdttm);
    QueryResult rows = hiveService.query(query);
    return Response.ok(rows.getRows()).build();
}

Example 63 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class RegisteredTemplateService method mergeTemplatePropertiesWithFeed.

public FeedMetadata mergeTemplatePropertiesWithFeed(FeedMetadata feedMetadata) {
    // gets the feed data and then gets the latest template associated with that feed and merges the properties into the feed
    RegisteredTemplate registeredTemplate = findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(feedMetadata.getTemplateId()).isFeedEdit(true).nifiTemplateId(feedMetadata.getTemplateId()).includeAllProperties(true).build());
    if (registeredTemplate != null) {
        feedMetadata.setTemplateId(registeredTemplate.getId());
        List<NifiProperty> templateProperties = registeredTemplate.getProperties().stream().map(nifiProperty -> new NifiProperty(nifiProperty)).collect(Collectors.toList());
        NifiPropertyUtil.matchAndSetPropertyByProcessorName(templateProperties, feedMetadata.getProperties(), NifiPropertyUtil.PROPERTY_MATCH_AND_UPDATE_MODE.FEED_DETAILS_MATCH_TEMPLATE);
        // detect template properties that dont match the feed.properties from the registeredtemplate
        ensureFeedPropertiesExistInTemplate(feedMetadata, templateProperties);
        feedMetadata.setProperties(templateProperties);
        registeredTemplate.setProperties(templateProperties);
        registeredTemplate.initializeProcessors();
        feedMetadata.setRegisteredTemplate(registeredTemplate);
    }
    return feedMetadata;
}

Example 64 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class ImportUtil method applyImportPropertiesToFeed.

public static boolean applyImportPropertiesToFeed(FeedMetadata metadata, ImportFeed importFeed, ImportComponent component) {
    ImportComponentOption option = importFeed.getImportOptions().findImportComponentOption(component);
    if (!option.getProperties().isEmpty() && option.getProperties().stream().anyMatch(importProperty -> StringUtils.isBlank(importProperty.getPropertyValue()))) {
        importFeed.setSuccess(false);
        if (importFeed.getTemplate() == null) {
            ImportTemplate importTemplate = new ImportTemplate(importFeed.getFileName());
            importFeed.setTemplate(importTemplate);
        }
        String feedCategory = importFeed.getImportOptions().getCategorySystemName() != null ? importFeed.getImportOptions().getCategorySystemName() : metadata.getSystemCategoryName();
        String msg = "The feed " + FeedNameUtil.fullName(feedCategory, metadata.getSystemFeedName()) + " needs additional properties to be supplied before importing.";
        importFeed.addErrorMessage(metadata, msg);
        option.getErrorMessages().add(msg);
        return false;
    } else {
        metadata.getSensitiveProperties().forEach(nifiProperty -> {
            ImportProperty userSuppliedValue = importFeed.getImportOptions().getProperties(ImportComponent.FEED_DATA).stream().filter(importFeedProperty -> {
                return nifiProperty.getProcessorId().equalsIgnoreCase(importFeedProperty.getProcessorId()) && nifiProperty.getKey().equalsIgnoreCase(importFeedProperty.getPropertyKey());
            }).findFirst().orElse(null);
            // deal with nulls?
            if (userSuppliedValue != null) {
                nifiProperty.setValue(userSuppliedValue.getPropertyValue());
            }
        });
        return true;
    }
}

Example 65 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class ImportUtil method replaceDatasource.

/**
 * Replaces the specified data source id with a new data source id.
 *
 * @param metadata the feed metadata
 * @param oldDatasourceId the id of the data source to be replaced
 * @param newDatasourceId the id of the new data source
 */
@SuppressWarnings("unchecked")
public static void replaceDatasource(@Nonnull final FeedMetadata metadata, @Nonnull final String oldDatasourceId, @Nonnull final String newDatasourceId) {
    // Update data transformation
    final FeedDataTransformation transformation = metadata.getDataTransformation();
    if (transformation != null) {
        // Update chart view model
        Optional.of(transformation.getChartViewModel()).map(model -> (List<Map<String, Object>>) model.get("nodes")).ifPresent(nodes -> nodes.forEach(node -> {
            final String nodeDatasourceId = (String) node.get("datasourceId");
            if (nodeDatasourceId != null && oldDatasourceId.equals(nodeDatasourceId)) {
                node.put("datasourceId", newDatasourceId);
            }
        }));
        // Update data source id list
        transformation.getDatasourceIds().replaceAll(id -> oldDatasourceId.equals(id) ? newDatasourceId : id);
        // Update transform script
        final String updatedDataTransformScript = transformation.getDataTransformScript().replace(oldDatasourceId, newDatasourceId);
        transformation.setDataTransformScript(updatedDataTransformScript);
    }
    // Update processor properties
    metadata.getProperties().forEach(property -> {
        final String value = property.getValue();
        if (value != null && !value.isEmpty()) {
            property.setValue(value.replace(oldDatasourceId, newDatasourceId));
        }
    });
}