Examples with Tag com.thinkbiganalytics.discovery.schema.Tag used on opensource projects

Example 1 with Tag

use of com.thinkbiganalytics.discovery.schema.Tag in project kylo by Teradata.

the class FeedModelTransform method feedToDomain.

/**
 * Transforms the specified Feed Manager feed to a Metadata feed.
 *
 * @param feedMetadata the Feed Manager feed
 * @return the Metadata feed
 */
@Nonnull
public Feed feedToDomain(@Nonnull final FeedMetadata feedMetadata) {
    // resolve the id
    Feed.ID domainId = feedMetadata.getId() != null ? feedProvider.resolveId(feedMetadata.getId()) : null;
    Feed domain = domainId != null ? feedProvider.findById(domainId) : null;
    FeedCategory restCategoryModel = feedMetadata.getCategory();
    Category category = null;
    if (restCategoryModel != null && (domain == null || domain.getCategory() == null)) {
        category = categoryProvider.findById(categoryProvider.resolveId(restCategoryModel.getId()));
    }
    if (domain == null) {
        // ensure the Category exists
        if (category == null) {
            final String categoryId = (restCategoryModel != null) ? restCategoryModel.getId() : "(null)";
            throw new RuntimeException("Category cannot be found while creating feed " + feedMetadata.getSystemFeedName() + ".  Category Id is " + categoryId);
        }
        domain = feedProvider.ensureFeed(category.getId(), feedMetadata.getSystemFeedName());
        domainId = domain.getId();
        Feed.State state = Feed.State.valueOf(feedMetadata.getState());
        domain.setState(state);
        // reassign the domain data back to the ui model....
        feedMetadata.setFeedId(domainId.toString());
        feedMetadata.setState(state.name());
    }
    domain.setDisplayName(feedMetadata.getFeedName());
    if (feedMetadata.getDescription() == null) {
        feedMetadata.setDescription("");
    }
    domain.setDescription(feedMetadata.getDescription());
    domain.setAllowIndexing(feedMetadata.isAllowIndexing());
    if (StringUtils.isNotBlank(feedMetadata.getHistoryReindexingStatus())) {
        domain.updateHistoryReindexingStatus(new HistoryReindexingStatus(com.thinkbiganalytics.metadata.api.feed.reindex.HistoryReindexingState.valueOf(feedMetadata.getHistoryReindexingStatus())));
    }
    feedMetadata.setId(domain.getId().toString());
    if (StringUtils.isNotBlank(feedMetadata.getState())) {
        Feed.State state = Feed.State.valueOf(feedMetadata.getState().toUpperCase());
        domain.setState(state);
    }
    domain.setNifiProcessGroupId(feedMetadata.getNifiProcessGroupId());
    // clear out the state as that
    RegisteredTemplate template = feedMetadata.getRegisteredTemplate();
    prepareForSave(feedMetadata);
    feedMetadata.setRegisteredTemplate(template);
    if (domain.getTemplate() == null) {
        FeedManagerTemplate.ID templateId = templateProvider.resolveId(feedMetadata.getTemplateId());
        FeedManagerTemplate domainTemplate = templateProvider.findById(templateId);
        domain.setTemplate(domainTemplate);
    }
    // Set user-defined properties
    if (feedMetadata.getUserProperties() != null) {
        final Set<UserFieldDescriptor> userFields = getUserFields(category);
        domain.setUserProperties(UserPropertyTransform.toMetadataProperties(feedMetadata.getUserProperties()), userFields);
    }
    // Set the hadoop security groups
    final List<HadoopSecurityGroup> securityGroups = new ArrayList<>();
    if (feedMetadata.getSecurityGroups() != null) {
        for (com.thinkbiganalytics.feedmgr.rest.model.HadoopSecurityGroup securityGroup : feedMetadata.getSecurityGroups()) {
            JcrHadoopSecurityGroup hadoopSecurityGroup = (JcrHadoopSecurityGroup) hadoopSecurityGroupProvider.ensureSecurityGroup(securityGroup.getName());
            hadoopSecurityGroup.setGroupId(securityGroup.getId());
            hadoopSecurityGroup.setDescription(securityGroup.getDescription());
            securityGroups.add(hadoopSecurityGroup);
        }
    }
    domain.setSecurityGroups(securityGroups);
    domain.setVersionName(domain.getVersionName());
    if (feedMetadata.getTags() != null) {
        domain.setTags(feedMetadata.getTags().stream().map(Tag::getName).collect(Collectors.toSet()));
    }
    // Create a new feed metadata stripped of any excess data that does
    // not need to be serialized and stored in the feed domain entity.
    FeedMetadata stripped = stripMetadata(feedMetadata);
    domain.setJson(ObjectMapperSerializer.serialize(stripped));
    return domain;
}

Example 2 with Tag

use of com.thinkbiganalytics.discovery.schema.Tag in project kylo by Teradata.

the class IntegrationTestBase method makeCreateFeedRequest.

protected FeedMetadata makeCreateFeedRequest(FeedCategory category, ImportTemplate template, String feedName, String testFile) {
    FeedMetadata feed = new FeedMetadata();
    feed.setFeedName(feedName);
    feed.setSystemFeedName(feedName.toLowerCase());
    feed.setCategory(category);
    feed.setTemplateId(template.getTemplateId());
    feed.setTemplateName(template.getTemplateName());
    feed.setDescription("Created by functional test");
    feed.setInputProcessorType("org.apache.nifi.processors.standard.GetFile");
    List<NifiProperty> properties = new ArrayList<>();
    NifiProperty fileFilter = new NifiProperty("764d053d-015e-1000-b8a2-763cd17080e1", "cffa8f24-d097-3c7a-7d04-26b7feff81ab", "File Filter", testFile);
    fileFilter.setProcessGroupName("NiFi Flow");
    fileFilter.setProcessorName("GetFile");
    fileFilter.setProcessorType("org.apache.nifi.processors.standard.GetFile");
    fileFilter.setTemplateValue("mydata\\d{1,3}.csv");
    fileFilter.setInputProperty(true);
    fileFilter.setUserEditable(true);
    properties.add(fileFilter);
    feed.setProperties(properties);
    FeedSchedule schedule = new FeedSchedule();
    schedule.setConcurrentTasks(1);
    schedule.setSchedulingPeriod("15 sec");
    schedule.setSchedulingStrategy("TIMER_DRIVEN");
    feed.setSchedule(schedule);
    feed.setDataOwner("Marketing");
    List<Tag> tags = new ArrayList<>();
    tags.add(new DefaultTag("functional tests"));
    tags.add(new DefaultTag("for category " + category.getName()));
    feed.setTags(tags);
    User owner = new User();
    owner.setSystemName("dladmin");
    owner.setDisplayName("Data Lake Admin");
    Set<String> groups = new HashSet<>();
    groups.add("admin");
    groups.add("user");
    owner.setGroups(groups);
    feed.setOwner(owner);
    return feed;
}

Example 3 with Tag

use of com.thinkbiganalytics.discovery.schema.Tag in project kylo by Teradata.

the class FeedIT method getCreateFeedRequest.

protected FeedMetadata getCreateFeedRequest(FeedCategory category, ImportTemplate template, String name) throws Exception {
    FeedMetadata feed = new FeedMetadata();
    feed.setFeedName(name);
    feed.setSystemFeedName(name.toLowerCase());
    feed.setCategory(category);
    feed.setTemplateId(template.getTemplateId());
    feed.setTemplateName(template.getTemplateName());
    feed.setDescription("Created by functional test");
    feed.setInputProcessorType("org.apache.nifi.processors.standard.GetFile");
    List<NifiProperty> properties = new ArrayList<>();
    NifiProperty fileFilter = new NifiProperty("305363d8-015a-1000-0000-000000000000", "1f67e296-2ff8-4b5d-0000-000000000000", "File Filter", USERDATA1_CSV);
    fileFilter.setProcessGroupName("NiFi Flow");
    fileFilter.setProcessorName("Filesystem");
    fileFilter.setProcessorType("org.apache.nifi.processors.standard.GetFile");
    fileFilter.setTemplateValue("mydata\\d{1,3}.csv");
    fileFilter.setInputProperty(true);
    fileFilter.setUserEditable(true);
    properties.add(fileFilter);
    NifiProperty inputDir = new NifiProperty("305363d8-015a-1000-0000-000000000000", "1f67e296-2ff8-4b5d-0000-000000000000", "Input Directory", VAR_DROPZONE);
    inputDir.setProcessGroupName("NiFi Flow");
    inputDir.setProcessorName("Filesystem");
    inputDir.setProcessorType("org.apache.nifi.processors.standard.GetFile");
    inputDir.setInputProperty(true);
    inputDir.setUserEditable(true);
    properties.add(inputDir);
    NifiProperty loadStrategy = new NifiProperty("305363d8-015a-1000-0000-000000000000", "6aeabec7-ec36-4ed5-0000-000000000000", "Load Strategy", "FULL_LOAD");
    loadStrategy.setProcessGroupName("NiFi Flow");
    loadStrategy.setProcessorName("GetTableData");
    loadStrategy.setProcessorType("com.thinkbiganalytics.nifi.v2.ingest.GetTableData");
    properties.add(loadStrategy);
    feed.setProperties(properties);
    FeedSchedule schedule = new FeedSchedule();
    schedule.setConcurrentTasks(1);
    schedule.setSchedulingPeriod("15 sec");
    schedule.setSchedulingStrategy("TIMER_DRIVEN");
    feed.setSchedule(schedule);
    TableSetup table = new TableSetup();
    DefaultTableSchema schema = new DefaultTableSchema();
    schema.setName("test1");
    List<Field> fields = new ArrayList<>();
    fields.add(newTimestampField("registration_dttm"));
    fields.add(newBigIntField("id"));
    fields.add(newStringField("first_name"));
    fields.add(newStringField("second_name"));
    fields.add(newStringField("email"));
    fields.add(newStringField("gender"));
    fields.add(newStringField("ip_address"));
    fields.add(newBinaryField("cc"));
    fields.add(newStringField("country"));
    fields.add(newStringField("birthdate"));
    fields.add(newStringField("salary"));
    schema.setFields(fields);
    table.setTableSchema(schema);
    table.setSourceTableSchema(schema);
    table.setFeedTableSchema(schema);
    table.setTargetMergeStrategy("DEDUPE_AND_MERGE");
    table.setFeedFormat("ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',' ,'escapeChar' = '\\\\' ,'quoteChar' = '\\'') STORED AS TEXTFILE");
    table.setTargetFormat("STORED AS ORC");
    List<FieldPolicy> policies = new ArrayList<>();
    policies.add(newPolicyBuilder("registration_dttm").toPolicy());
    policies.add(newPolicyBuilder("id").toPolicy());
    policies.add(newPolicyBuilder("first_name").withStandardisation(toUpperCase).withProfile().withIndex().toPolicy());
    policies.add(newPolicyBuilder("second_name").withProfile().withIndex().toPolicy());
    policies.add(newPolicyBuilder("email").withValidation(email).toPolicy());
    policies.add(newPolicyBuilder("gender").withValidation(lookup, notNull).toPolicy());
    policies.add(newPolicyBuilder("ip_address").withValidation(ipAddress).toPolicy());
    policies.add(newPolicyBuilder("cc").withStandardisation(base64EncodeBinary).withProfile().toPolicy());
    policies.add(newPolicyBuilder("country").withStandardisation(base64EncodeBinary, base64DecodeBinary, base64EncodeString, base64DecodeString).withValidation(notNull, length).withProfile().toPolicy());
    policies.add(newPolicyBuilder("birthdate").toPolicy());
    policies.add(newPolicyBuilder("salary").toPolicy());
    table.setFieldPolicies(policies);
    List<PartitionField> partitions = new ArrayList<>();
    partitions.add(byYear("registration_dttm"));
    table.setPartitions(partitions);
    TableOptions options = new TableOptions();
    options.setCompressionFormat("SNAPPY");
    options.setAuditLogging(true);
    table.setOptions(options);
    table.setTableType("SNAPSHOT");
    feed.setTable(table);
    feed.setOptions(new FeedProcessingOptions());
    feed.getOptions().setSkipHeader(true);
    feed.setDataOwner("Marketing");
    List<Tag> tags = new ArrayList<>();
    tags.add(new DefaultTag("users"));
    tags.add(new DefaultTag("registrations"));
    feed.setTags(tags);
    User owner = new User();
    owner.setSystemName("dladmin");
    owner.setDisplayName("Data Lake Admin");
    Set<String> groups = new HashSet<>();
    groups.add("admin");
    groups.add("user");
    owner.setGroups(groups);
    feed.setOwner(owner);
    return feed;
}

Example 4 with Tag

use of com.thinkbiganalytics.discovery.schema.Tag in project kylo by Teradata.

the class FeedTestUtil method getCreateFeedRequest.

public static FeedMetadata getCreateFeedRequest(FeedCategory category, ImportTemplate template, String name, String inputProcessorType, List<NifiProperty> properties) throws Exception {
    FeedMetadata feed = new FeedMetadata();
    feed.setFeedName(name);
    feed.setSystemFeedName(name.toLowerCase());
    feed.setCategory(category);
    feed.setTemplateId(template.getTemplateId());
    feed.setTemplateName(template.getTemplateName());
    feed.setDescription("Created by functional test");
    feed.setInputProcessorType(inputProcessorType);
    feed.setProperties(properties);
    FeedSchedule schedule = new FeedSchedule();
    schedule.setConcurrentTasks(1);
    schedule.setSchedulingPeriod("15 sec");
    schedule.setSchedulingStrategy("TIMER_DRIVEN");
    feed.setSchedule(schedule);
    feed.setDataOwner("Marketing");
    List<Tag> tags = new ArrayList<>();
    tags.add(new DefaultTag("users"));
    tags.add(new DefaultTag("registrations"));
    feed.setTags(tags);
    User owner = new User();
    owner.setSystemName("dladmin");
    owner.setDisplayName("Data Lake Admin");
    Set<String> groups = new HashSet<>();
    groups.add("admin");
    groups.add("user");
    owner.setGroups(groups);
    feed.setOwner(owner);
    return feed;
}