Examples with FeedMetadata com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata used on opensource projects

Example 31 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class InMemoryFeedManagerTemplateService method saveRegisteredTemplate.

protected RegisteredTemplate saveRegisteredTemplate(RegisteredTemplate registeredTemplate) {
    // ensure that the incoming template name doesnt already exist.
    // if so remove and replace with this one
    RegisteredTemplate template = getRegisteredTemplateByName(registeredTemplate.getTemplateName());
    if (template != null && !template.getId().equalsIgnoreCase(registeredTemplate.getId())) {
        // remove the old one with the same name
        registeredTemplates.remove(template.getId());
        // update those feeds that were pointing to this old one, to this new one
        List<FeedMetadata> feedsToUpdate = feedProvider.getFeedsWithTemplate(registeredTemplate.getId());
        if (feedsToUpdate != null && !feedsToUpdate.isEmpty()) {
            for (FeedMetadata feedMetadata : feedsToUpdate) {
                feedMetadata.setTemplateId(template.getId());
            }
            // save the feeds
            FileObjectPersistence.getInstance().writeFeedsToFile(feedProvider.getFeeds());
        }
    }
    registeredTemplates.put(registeredTemplate.getId(), registeredTemplate);
    if (registeredTemplates.containsKey(registeredTemplate.getNifiTemplateId())) {
        registeredTemplates.remove(registeredTemplate.getNifiTemplateId());
    }
    FileObjectPersistence.getInstance().writeTemplatesToFile(registeredTemplates.values());
    return registeredTemplate;
}

Example 32 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class RegisteredTemplateService method ensureFeedPropertiesExistInTemplate.

/**
 * If a Template changes the Processor Names the Feed Properties will no longer be associated to the correct processors
 * This will match any feed properties to a whose processor name has changed in
 * the template to the template processor/property based upon the template processor type.
 */
private void ensureFeedPropertiesExistInTemplate(FeedMetadata feed, List<NifiProperty> templateProperties) {
    Set<String> templateProcessors = templateProperties.stream().map(property -> property.getProcessorName()).collect(Collectors.toSet());
    // Store the template Properties
    Map<String, String> templateProcessorIdProcessorNameMap = new HashMap<>();
    Map<String, String> templateProcessorTypeProcessorIdMap = new HashMap<>();
    templateProperties.stream().filter(property -> !templateProcessorIdProcessorNameMap.containsKey(property.getProcessorId())).forEach(property1 -> {
        templateProcessorIdProcessorNameMap.put(property1.getProcessorId(), property1.getProcessorName());
        templateProcessorTypeProcessorIdMap.put(property1.getProcessorType(), property1.getProcessorId());
    });
    Map<String, Map<String, NifiProperty>> templatePropertiesByProcessorIdMap = new HashMap<>();
    templateProperties.stream().forEach(property -> {
        templatePropertiesByProcessorIdMap.computeIfAbsent(property.getProcessorId(), key -> new HashMap<String, NifiProperty>()).put(property.getKey(), property);
    });
    // store the Feed Properties
    Map<String, String> processorIdProcessorTypeMap = new HashMap<>();
    feed.getProperties().stream().filter(property -> !processorIdProcessorTypeMap.containsKey(property.getProcessorId())).forEach(property1 -> {
        processorIdProcessorTypeMap.put(property1.getProcessorId(), property1.getProcessorType());
    });
    feed.getProperties().stream().filter(property -> !templateProcessors.contains(property.getProcessorName())).forEach(property -> {
        // if the property doesn't match the template but the type matches try to merge in the feed properties overwriting the template ones
        String processorType = processorIdProcessorTypeMap.get(property.getProcessorId());
        if (processorType != null) {
            String templateProcessorId = templateProcessorTypeProcessorIdMap.get(processorType);
            if (templateProcessorId != null && templateProcessorIdProcessorNameMap.containsKey(templateProcessorId)) {
                NifiProperty templateProperty = templatePropertiesByProcessorIdMap.get(templateProcessorId).get(property.getKey());
                if (templateProperty != null) {
                    // replace it from the collection with a copy
                    NifiProperty copy = new NifiProperty(templateProperty);
                    copy.setValue(property.getValue());
                    copy.setRenderType(property.getRenderType());
                    copy.setRenderOptions(property.getRenderOptions());
                    templateProperties.remove(templateProperty);
                    templateProperties.add(copy);
                }
            }
        }
    });
}

Example 33 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class DefaultServiceLevelAgreementService method saveAndScheduleSla.

/**
 * In order to Save an SLA if it is related to a Feed(s) the user needs to have EDIT_DETAILS permission on the Feed(s)
 *
 * @param serviceLevelAgreement the sla to save
 * @param feed                  an option Feed to relate to this SLA.  If this is not present the related feeds are also embedded in the SLA policies.  The Feed is a pointer access to the current
 *                              feed the user is editing if they are creating an SLA from the Feed Details page. If creating an SLA from the main SLA page the feed property will not be populated.
 */
private ServiceLevelAgreement saveAndScheduleSla(ServiceLevelAgreementGroup serviceLevelAgreement, FeedMetadata feed) {
    // ensure user has permissions to edit the SLA
    if (serviceLevelAgreement != null) {
        ServiceLevelAgreementMetricTransformerHelper transformer = new ServiceLevelAgreementMetricTransformerHelper();
        // Read the feeds on the SLA as a Service. Then verify the current user has access to edit these feeds
        List<String> feedsOnSla = metadataAccess.read(() -> {
            List<String> feedIds = new ArrayList<>();
            // all referencing Feeds
            List<String> systemCategoryAndFeedNames = transformer.getCategoryFeedNames(serviceLevelAgreement);
            for (String categoryAndFeed : systemCategoryAndFeedNames) {
                // fetch and update the reference to the sla
                String categoryName = StringUtils.trim(StringUtils.substringBefore(categoryAndFeed, "."));
                String feedName = StringUtils.trim(StringUtils.substringAfterLast(categoryAndFeed, "."));
                Feed feedEntity = feedProvider.findBySystemName(categoryName, feedName);
                if (feedEntity != null) {
                    feedIds.add(feedEntity.getId().toString());
                }
            }
            return feedIds;
        }, MetadataAccess.SERVICE);
        boolean allowedToEdit = feedsOnSla.isEmpty() ? true : feedsOnSla.stream().allMatch(feedId -> feedManagerFeedService.checkFeedPermission(feedId, FeedAccessControl.EDIT_DETAILS));
        if (allowedToEdit) {
            return metadataAccess.commit(() -> {
                // Re read back in the Feeds for this session
                Set<Feed> slaFeeds = new HashSet<Feed>();
                Set<Feed.ID> slaFeedIds = new HashSet<Feed.ID>();
                feedsOnSla.stream().forEach(feedId -> {
                    Feed feedEntity = feedProvider.findById(feedProvider.resolveId(feedId));
                    if (feedEntity != null) {
                        slaFeeds.add(feedEntity);
                        slaFeedIds.add(feedEntity.getId());
                    }
                });
                if (feed != null) {
                    feedManagerFeedService.checkFeedPermission(feed.getId(), FeedAccessControl.EDIT_DETAILS);
                }
                if (feed != null) {
                    transformer.applyFeedNameToCurrentFeedProperties(serviceLevelAgreement, feed.getCategory().getSystemName(), feed.getSystemFeedName());
                }
                ServiceLevelAgreement sla = transformer.getServiceLevelAgreement(serviceLevelAgreement);
                ServiceLevelAgreementBuilder slaBuilder = null;
                com.thinkbiganalytics.metadata.sla.api.ServiceLevelAgreement.ID existingId = null;
                if (StringUtils.isNotBlank(sla.getId())) {
                    existingId = slaProvider.resolve(sla.getId());
                }
                if (existingId != null) {
                    slaBuilder = slaProvider.builder(existingId);
                } else {
                    slaBuilder = slaProvider.builder();
                }
                slaBuilder.name(sla.getName()).description(sla.getDescription());
                for (com.thinkbiganalytics.metadata.rest.model.sla.ObligationGroup group : sla.getGroups()) {
                    ObligationGroupBuilder groupBuilder = slaBuilder.obligationGroupBuilder(ObligationGroup.Condition.valueOf(group.getCondition()));
                    for (Obligation o : group.getObligations()) {
                        groupBuilder.obligationBuilder().metric(o.getMetrics()).description(o.getDescription()).build();
                    }
                    groupBuilder.build();
                }
                com.thinkbiganalytics.metadata.sla.api.ServiceLevelAgreement savedSla = slaBuilder.build();
                List<ServiceLevelAgreementActionConfiguration> actions = transformer.getActionConfigurations(serviceLevelAgreement);
                // now assign the sla checks
                slaProvider.slaCheckBuilder(savedSla.getId()).removeSlaChecks().actionConfigurations(actions).build();
                // relate them
                Set<Feed.ID> feedIds = new HashSet<>();
                FeedServiceLevelAgreementRelationship feedServiceLevelAgreementRelationship = feedSlaProvider.relateFeeds(savedSla, slaFeeds);
                if (feedServiceLevelAgreementRelationship != null && feedServiceLevelAgreementRelationship.getFeeds() != null) {
                    feedIds = feedServiceLevelAgreementRelationship.getFeeds().stream().map(f -> f.getId()).collect(Collectors.toSet());
                }
                Set<VelocityTemplate.ID> velocityTemplates = findVelocityTemplates(serviceLevelAgreement);
                // Update the JPA mapping in Ops Manager for this SLA and its related Feeds
                serviceLevelAgreementDescriptionProvider.updateServiceLevelAgreement(savedSla.getId(), savedSla.getName(), savedSla.getDescription(), feedIds, velocityTemplates);
                com.thinkbiganalytics.metadata.rest.model.sla.FeedServiceLevelAgreement restModel = serviceLevelAgreementTransform.toModel(savedSla, slaFeeds, true);
                // schedule it
                serviceLevelAgreementScheduler.scheduleServiceLevelAgreement(savedSla);
                return restModel;
            });
        }
    }
    return null;
}

Example 34 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class PropertyExpressionResolverTest method testFeedMetadataProperties.

@Test
public void testFeedMetadataProperties() {
    FeedMetadata metadata = new FeedMetadata();
    metadata.setSystemFeedName("feedSystemName");
    metadata.setCategory(new FeedCategory());
    metadata.setTable(new TableSetup());
    metadata.getTable().setSourceTableSchema(new DefaultTableSchema());
    metadata.getTable().setTableSchema(new DefaultTableSchema());
    metadata.getTable().getSourceTableSchema().setName("sourceTableName");
    metadata.getTable().getTableSchema().setName("tableSchemaName");
    final NifiProperty prop1 = createProperty("${metadata.table.sourceTableSchema.name}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop1));
    Assert.assertEquals("sourceTableName", prop1.getValue());
}

Example 35 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class PropertyExpressionResolverTest method resolveExpression.

/**
 * Verifies resolving expressions in property values.
 */
@Test
public void resolveExpression() {
    final FeedMetadata metadata = new FeedMetadata();
    metadata.setSystemFeedName("myfeed");
    // Verify config variable
    final NifiProperty prop1 = createProperty("${config.test.value}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop1));
    Assert.assertEquals("hello-world", prop1.getValue());
    // Verify metadata variable
    final NifiProperty prop2 = createProperty("${metadata.systemFeedName}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop2));
    Assert.assertEquals("myfeed", prop2.getValue());
    // Verify static config
    final NifiProperty prop3 = createProperty(STATIC_KEY, "${metadata.systemFeedName}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop3));
    Assert.assertEquals("myapp", prop3.getValue());
    final NifiProperty prop4 = createProperty(STATIC_KEY, "${config.test.value}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop4));
    Assert.assertEquals("hello-world", prop4.getValue());
    final NifiProperty prop5 = createProperty(STATIC_KEY, "");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop5));
    Assert.assertEquals("myapp", prop5.getValue());
    // Verify multiple variables
    final NifiProperty prop6 = createProperty("${metadata.systemFeedName}.${config.test.value}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop6));
    Assert.assertEquals("myfeed.hello-world", prop6.getValue());
    // Verify multiple variables
    final NifiProperty prop7 = createProperty("$${${metadata.systemFeedName}.${config.test.value}}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop7));
    Assert.assertEquals("${myfeed.hello-world}", prop7.getValue());
    // Verify multiple variables
    final NifiProperty prop8 = createProperty("${config.${metadata.systemFeedName}.${config.test.value}}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop8));
    Assert.assertEquals("runtime value", prop8.getValue());
    // Verify static text
    final NifiProperty prop9 = createProperty("config.test.value");
    Assert.assertFalse(resolver.resolveExpression(metadata, prop9));
    Assert.assertEquals("config.test.value", prop9.getValue());
    // verify replacement with NiFi el
    final NifiProperty prop10 = createProperty("property1", "a value");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop10));
    Assert.assertEquals("/path/to/property1,${nifi.expression.property}", prop10.getValue());
    // verify replacement without NiFi el
    final NifiProperty prop11 = createProperty("Another Processor", "property1", "a value");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop11));
    Assert.assertEquals("/path/to/another_processor/property1/location", prop11.getValue());
    // verify replacement without NiFi el using default processor type replacement
    final NifiProperty prop12 = createProperty("My New Processor", "property1", "a value");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop12));
    Assert.assertEquals("/path/to/property1/location", prop12.getValue());
    // verify replacement without NiFi el using default processor type replacement
    final NifiProperty extraFiles = createProperty("extra_files", "a value");
    Assert.assertTrue(resolver.resolveExpression(metadata, extraFiles));
    Assert.assertEquals("${table_field_policy_json_file},/usr/hdp/current/spark-client/conf/hive-site.xml", extraFiles.getValue());
    Assert.assertTrue(resolver.resolveExpression(metadata, extraFiles));
    Assert.assertEquals("${table_field_policy_json_file},/usr/hdp/current/spark-client/conf/hive-site.xml", extraFiles.getValue());
    final NifiProperty hiveSchema = createProperty(STATIC_KEY, "${config.hive.schema}");
    Assert.assertTrue(resolver.resolveExpression(metadata, hiveSchema));
    Assert.assertEquals("hive", hiveSchema.getValue());
}