Examples with FieldPolicy com.thinkbiganalytics.policy.rest.model.FieldPolicy used on opensource projects

Example 1 with FieldPolicy

use of com.thinkbiganalytics.policy.rest.model.FieldPolicy in project kylo by Teradata.

the class FeedsController method updateDataHistoryReindexStatus.

@POST
@Path("{id}/update-data-history-reindex-status")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@ApiOperation("Sets the data history reindexing status for the specified feed")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the feed's updated data history reindex status and columns to index", response = FeedDataHistoryReindexParams.class), @ApiResponse(code = 500, message = "The data history reindex status could not be updated for the feed", response = RestResponseStatus.class) })
public FeedDataHistoryReindexParams updateDataHistoryReindexStatus(@PathParam("id") String feedIdStr, HistoryReindexingStatus status) {
    LOG.debug("Update data history reindexing status for feed with id: {}", feedIdStr);
    FeedDataHistoryReindexParams feedDataHistoryReindexParams = this.metadata.commit(() -> {
        this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_FEEDS);
        com.thinkbiganalytics.metadata.api.feed.Feed.ID feedId = feedProvider.resolveFeed(feedIdStr);
        com.thinkbiganalytics.metadata.api.feed.Feed feed = feedProvider.getFeed(feedId);
        if (feed != null) {
            FeedDataHistoryReindexParams reindexParams = new FeedDataHistoryReindexParams();
            com.thinkbiganalytics.metadata.api.feed.reindex.HistoryReindexingState newHistoryReindexingState = com.thinkbiganalytics.metadata.api.feed.reindex.HistoryReindexingState.valueOf(status.getHistoryReindexingState().name());
            com.thinkbiganalytics.metadata.api.feed.Feed updatedDomainFeed = feed.updateHistoryReindexingStatus(new com.thinkbiganalytics.metadata.api.feed.reindex.HistoryReindexingStatus(newHistoryReindexingState));
            Feed updatedRestFeed = metadataTransform.domainToFeed().apply(updatedDomainFeed);
            // updated reindexing status
            reindexParams.setHistoryReindexingStatus(updatedRestFeed.getCurrentHistoryReindexingStatus());
            reindexParams.setFeedId(updatedRestFeed.getId());
            reindexParams.setFeedSystemName(updatedRestFeed.getSystemName());
            reindexParams.setCategorySystemName(updatedRestFeed.getCategory().getSystemName());
            if (updatedDomainFeed != null) {
                FeedMetadata feedMetadata = getMetadataService().getFeedById(updatedDomainFeed.getId().toString());
                if (feedMetadata != null && feedMetadata.getTable() != null) {
                    List<String> commaSepColumns = new ArrayList<>();
                    List<FieldPolicy> fieldPolicies = feedMetadata.getTable().getFieldPolicies();
                    for (FieldPolicy fieldPolicy : fieldPolicies) {
                        if (fieldPolicy.isIndex()) {
                            commaSepColumns.add(fieldPolicy.getFieldName().toLowerCase().trim());
                        }
                    }
                    // columns to index as comma separated string
                    reindexParams.setCommaSeparatedColumnsForIndexing(StringUtils.join(commaSepColumns.toArray(), ","));
                }
            }
            return reindexParams;
        } else {
            throw new WebApplicationException("A feed with the given ID does not exist: " + feedId, Status.NOT_FOUND);
        }
    });
    return feedDataHistoryReindexParams;
}

Example 2 with FieldPolicy

use of com.thinkbiganalytics.policy.rest.model.FieldPolicy in project kylo by Teradata.

the class FieldPolicyRuleBuilder method toPolicy.

public FieldPolicy toPolicy() {
    FieldPolicy policy = new FieldPolicy();
    policy.setFieldName(fieldName);
    policy.setFeedFieldName(fieldName);
    policy.setProfile(profile);
    policy.setIndex(index);
    policy.setStandardization(standardisations);
    policy.setValidation(validations);
    return policy;
}

Example 3 with FieldPolicy

use of com.thinkbiganalytics.policy.rest.model.FieldPolicy in project kylo by Teradata.

the class TableSetup method updateFieldIndexString.

@JsonIgnore
public void updateFieldIndexString() {
    StringBuffer sb = new StringBuffer();
    if (tableSchema != null && tableSchema.getFields() != null && fieldPolicies != null) {
        int idx = 0;
        for (FieldPolicy field : fieldPolicies) {
            if (field.isIndex() && StringUtils.isNotBlank(sb.toString())) {
                sb.append(",");
            }
            if (field.isIndex()) {
                sb.append(tableSchema.getFields().get(idx).getName());
            }
            idx++;
        }
    }
    fieldIndexString = sb.toString();
}

Example 4 with FieldPolicy

use of com.thinkbiganalytics.policy.rest.model.FieldPolicy in project kylo by Teradata.

the class TableSetup method updateFieldPolicyNames.

@JsonIgnore
public void updateFieldPolicyNames() {
    if (tableSchema != null && tableSchema.getFields() != null && fieldPolicies != null && fieldPolicies.size() == tableSchema.getFields().size()) {
        if (sourceTargetFieldMap == null) {
            sourceTargetFieldMap = new HashMap<>();
        }
        if (targetSourceFieldMap == null) {
            targetSourceFieldMap = new HashMap<>();
        }
        int idx = 0;
        for (FieldPolicy field : fieldPolicies) {
            field.setFieldName(tableSchema.getFields().get(idx).getName());
            sourceTargetFieldMap.put(field.getFeedFieldName(), field.getFieldName());
            targetSourceFieldMap.put(field.getFieldName(), field.getFeedFieldName());
            idx++;
        }
    }
}

Example 5 with FieldPolicy

use of com.thinkbiganalytics.policy.rest.model.FieldPolicy in project kylo by Teradata.

the class FeedIT method getCreateFeedRequest.

protected FeedMetadata getCreateFeedRequest(FeedCategory category, ImportTemplate template, String name) throws Exception {
    FeedMetadata feed = new FeedMetadata();
    feed.setFeedName(name);
    feed.setSystemFeedName(name.toLowerCase());
    feed.setCategory(category);
    feed.setTemplateId(template.getTemplateId());
    feed.setTemplateName(template.getTemplateName());
    feed.setDescription("Created by functional test");
    feed.setInputProcessorType("org.apache.nifi.processors.standard.GetFile");
    List<NifiProperty> properties = new ArrayList<>();
    NifiProperty fileFilter = new NifiProperty("305363d8-015a-1000-0000-000000000000", "1f67e296-2ff8-4b5d-0000-000000000000", "File Filter", USERDATA1_CSV);
    fileFilter.setProcessGroupName("NiFi Flow");
    fileFilter.setProcessorName("Filesystem");
    fileFilter.setProcessorType("org.apache.nifi.processors.standard.GetFile");
    fileFilter.setTemplateValue("mydata\\d{1,3}.csv");
    fileFilter.setInputProperty(true);
    fileFilter.setUserEditable(true);
    properties.add(fileFilter);
    NifiProperty inputDir = new NifiProperty("305363d8-015a-1000-0000-000000000000", "1f67e296-2ff8-4b5d-0000-000000000000", "Input Directory", VAR_DROPZONE);
    inputDir.setProcessGroupName("NiFi Flow");
    inputDir.setProcessorName("Filesystem");
    inputDir.setProcessorType("org.apache.nifi.processors.standard.GetFile");
    inputDir.setInputProperty(true);
    inputDir.setUserEditable(true);
    properties.add(inputDir);
    NifiProperty loadStrategy = new NifiProperty("305363d8-015a-1000-0000-000000000000", "6aeabec7-ec36-4ed5-0000-000000000000", "Load Strategy", "FULL_LOAD");
    loadStrategy.setProcessGroupName("NiFi Flow");
    loadStrategy.setProcessorName("GetTableData");
    loadStrategy.setProcessorType("com.thinkbiganalytics.nifi.v2.ingest.GetTableData");
    properties.add(loadStrategy);
    feed.setProperties(properties);
    FeedSchedule schedule = new FeedSchedule();
    schedule.setConcurrentTasks(1);
    schedule.setSchedulingPeriod("15 sec");
    schedule.setSchedulingStrategy("TIMER_DRIVEN");
    feed.setSchedule(schedule);
    TableSetup table = new TableSetup();
    DefaultTableSchema schema = new DefaultTableSchema();
    schema.setName("test1");
    List<Field> fields = new ArrayList<>();
    fields.add(newTimestampField("registration_dttm"));
    fields.add(newBigIntField("id"));
    fields.add(newStringField("first_name"));
    fields.add(newStringField("second_name"));
    fields.add(newStringField("email"));
    fields.add(newStringField("gender"));
    fields.add(newStringField("ip_address"));
    fields.add(newBinaryField("cc"));
    fields.add(newStringField("country"));
    fields.add(newStringField("birthdate"));
    fields.add(newStringField("salary"));
    schema.setFields(fields);
    table.setTableSchema(schema);
    table.setSourceTableSchema(schema);
    table.setFeedTableSchema(schema);
    table.setTargetMergeStrategy("DEDUPE_AND_MERGE");
    table.setFeedFormat("ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',' ,'escapeChar' = '\\\\' ,'quoteChar' = '\\'') STORED AS TEXTFILE");
    table.setTargetFormat("STORED AS ORC");
    List<FieldPolicy> policies = new ArrayList<>();
    policies.add(newPolicyBuilder("registration_dttm").toPolicy());
    policies.add(newPolicyBuilder("id").toPolicy());
    policies.add(newPolicyBuilder("first_name").withStandardisation(toUpperCase).withProfile().withIndex().toPolicy());
    policies.add(newPolicyBuilder("second_name").withProfile().withIndex().toPolicy());
    policies.add(newPolicyBuilder("email").withValidation(email).toPolicy());
    policies.add(newPolicyBuilder("gender").withValidation(lookup, notNull).toPolicy());
    policies.add(newPolicyBuilder("ip_address").withValidation(ipAddress).toPolicy());
    policies.add(newPolicyBuilder("cc").withStandardisation(base64EncodeBinary).withProfile().toPolicy());
    policies.add(newPolicyBuilder("country").withStandardisation(base64EncodeBinary, base64DecodeBinary, base64EncodeString, base64DecodeString).withValidation(notNull, length).withProfile().toPolicy());
    policies.add(newPolicyBuilder("birthdate").toPolicy());
    policies.add(newPolicyBuilder("salary").toPolicy());
    table.setFieldPolicies(policies);
    List<PartitionField> partitions = new ArrayList<>();
    partitions.add(byYear("registration_dttm"));
    table.setPartitions(partitions);
    TableOptions options = new TableOptions();
    options.setCompressionFormat("SNAPPY");
    options.setAuditLogging(true);
    table.setOptions(options);
    table.setTableType("SNAPSHOT");
    feed.setTable(table);
    feed.setOptions(new FeedProcessingOptions());
    feed.getOptions().setSkipHeader(true);
    feed.setDataOwner("Marketing");
    List<Tag> tags = new ArrayList<>();
    tags.add(new DefaultTag("users"));
    tags.add(new DefaultTag("registrations"));
    feed.setTags(tags);
    User owner = new User();
    owner.setSystemName("dladmin");
    owner.setDisplayName("Data Lake Admin");
    Set<String> groups = new HashSet<>();
    groups.add("admin");
    groups.add("user");
    owner.setGroups(groups);
    feed.setOwner(owner);
    return feed;
}