Search in sources :

Example 11 with FieldStandardizationRule

use of com.thinkbiganalytics.policy.rest.model.FieldStandardizationRule in project kylo by Teradata.

the class TestStandardizationTransform method testUppercaseStandardizer.

@Test
public void testUppercaseStandardizer() throws IOException {
    UppercaseStandardizer standardizer = UppercaseStandardizer.instance();
    FieldStandardizationRule uiModel = StandardizationAnnotationTransformer.instance().toUIModel(standardizer);
    UppercaseStandardizer convertedPolicy = fromUI(uiModel, UppercaseStandardizer.class);
    Assert.assertEquals(standardizer, convertedPolicy);
}
Also used : UppercaseStandardizer(com.thinkbiganalytics.policy.standardization.UppercaseStandardizer) FieldStandardizationRule(com.thinkbiganalytics.policy.rest.model.FieldStandardizationRule) Test(org.junit.Test)

Example 12 with FieldStandardizationRule

use of com.thinkbiganalytics.policy.rest.model.FieldStandardizationRule in project kylo by Teradata.

the class TestStandardizationTransform method testDefaultValue.

@Test
public void testDefaultValue() throws IOException {
    String INPUT = "My Default";
    DefaultValueStandardizer standardizer = new DefaultValueStandardizer(INPUT);
    FieldStandardizationRule uiModel = StandardizationAnnotationTransformer.instance().toUIModel(standardizer);
    DefaultValueStandardizer convertedPolicy = fromUI(uiModel, DefaultValueStandardizer.class);
    Assert.assertEquals(INPUT, convertedPolicy.getDefaultStr());
}
Also used : DefaultValueStandardizer(com.thinkbiganalytics.policy.standardization.DefaultValueStandardizer) FieldStandardizationRule(com.thinkbiganalytics.policy.rest.model.FieldStandardizationRule) Test(org.junit.Test)

Example 13 with FieldStandardizationRule

use of com.thinkbiganalytics.policy.rest.model.FieldStandardizationRule in project kylo by Teradata.

the class FieldPolicyRuleBuilder method withStandardisation.

public FieldPolicyRuleBuilder withStandardisation(FieldStandardizationRule... rules) {
    for (FieldStandardizationRule rule : rules) {
        rule.setSequence(seq++);
        standardisations.add(rule);
    }
    return this;
}
Also used : FieldStandardizationRule(com.thinkbiganalytics.policy.rest.model.FieldStandardizationRule)

Example 14 with FieldStandardizationRule

use of com.thinkbiganalytics.policy.rest.model.FieldStandardizationRule in project kylo by Teradata.

the class FeedIT method testEditFeed.

@Test
public void testEditFeed() throws Exception {
    // Prepare environment
    prepare();
    final FeedCategory category = createCategory(CATEGORY_NAME);
    final ImportTemplate template = importDataIngestTemplate();
    // Create feed
    FeedMetadata feed = getCreateFeedRequest(category, template, createNewFeedName());
    feed.setDescription("Test feed");
    feed.setDataOwner("Some Guy");
    FeedMetadata response = createFeed(feed).getFeedMetadata();
    Assert.assertEquals(feed.getFeedName(), response.getFeedName());
    Assert.assertEquals(feed.getDataOwner(), response.getDataOwner());
    // Edit feed
    feed.setId(response.getId());
    feed.setFeedId(response.getFeedId());
    feed.setIsNew(false);
    feed.setDescription(null);
    feed.setDataOwner("Some Other Guy");
    NifiProperty fileFilter = feed.getProperties().get(0);
    fileFilter.setValue("some-file.csv");
    List<FieldPolicy> policies = feed.getTable().getFieldPolicies();
    FieldPolicy id = policies.get(1);
    // add new validator
    id.getValidation().add(notNull);
    feed.getTable().setPrimaryKeyFields("id");
    FieldPolicy firstName = policies.get(2);
    // flip profiling
    firstName.setProfile(false);
    FieldPolicy secondName = policies.get(3);
    // flip indexing
    secondName.setIndex(false);
    // add new standardiser
    secondName.getStandardization().add(toUpperCase);
    FieldPolicy email = policies.get(4);
    // remove validators
    email.setValidation(Collections.emptyList());
    FieldPolicy gender = policies.get(5);
    FieldValidationRule lookup = gender.getValidation().get(0);
    // change existing validator property
    lookup.getProperties().get(0).setValue("new value");
    // add profiling
    gender.setProfile(true);
    // add indexing
    gender.setIndex(true);
    FieldPolicy creditCard = policies.get(7);
    FieldStandardizationRule base64EncodeBinary = creditCard.getStandardization().get(0);
    // change existing standardiser property
    base64EncodeBinary.getProperties().get(0).setValue("STRING");
    feed.getOptions().setSkipHeader(false);
    feed.getTable().setTargetMergeStrategy("ROLLING_SYNC");
    feed.getTags().add(new DefaultTag("updated"));
    feed.getSchedule().setSchedulingPeriod("20 sec");
    response = createFeed(feed).getFeedMetadata();
    Assert.assertEquals(feed.getFeedName(), response.getFeedName());
    Assert.assertEquals(feed.getDescription(), response.getDescription());
    FeedVersions feedVersions = getVersions(feed.getFeedId());
    List<EntityVersion> versions = feedVersions.getVersions();
    Assert.assertEquals(2, versions.size());
    EntityVersionDifference entityDiff = getVersionDiff(feed.getFeedId(), versions.get(1).getId(), versions.get(0).getId());
    EntityDifference diff = entityDiff.getDifference();
    JsonNode patch = diff.getPatch();
    ArrayNode diffs = (ArrayNode) patch;
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/properties/0/value", "some-file.csv")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/schedule/schedulingPeriod", "20 sec")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("remove", "/description")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("add", "/tags/1")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/dataOwner", "Some Other Guy")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("add", "/table/fieldPolicies/1/validation/0")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/2/profile", "false")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/3/index", "false")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("add", "/table/fieldPolicies/3/standardization/0")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("remove", "/table/fieldPolicies/4/validation/0")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/5/profile", "true")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/5/index", "true")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/5/validation/0/properties/0/value", "new value")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/7/standardization/0/properties/0/value", "STRING")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/8/standardization/0/properties/0/value", "STRING")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/targetMergeStrategy", "ROLLING_SYNC")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldIndexString", "first_name,gender")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/options/skipHeader", "false")));
}
Also used : FeedCategory(com.thinkbiganalytics.feedmgr.rest.model.FeedCategory) EntityDifference(com.thinkbiganalytics.feedmgr.rest.model.EntityDifference) FieldPolicy(com.thinkbiganalytics.policy.rest.model.FieldPolicy) FieldValidationRule(com.thinkbiganalytics.policy.rest.model.FieldValidationRule) Diff(com.thinkbiganalytics.integration.Diff) FeedMetadata(com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata) ImportTemplate(com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate) JsonNode(com.fasterxml.jackson.databind.JsonNode) EntityVersion(com.thinkbiganalytics.feedmgr.rest.model.EntityVersion) FeedVersions(com.thinkbiganalytics.feedmgr.rest.model.FeedVersions) EntityVersionDifference(com.thinkbiganalytics.feedmgr.rest.model.EntityVersionDifference) NifiProperty(com.thinkbiganalytics.nifi.rest.model.NifiProperty) ArrayNode(com.fasterxml.jackson.databind.node.ArrayNode) DefaultTag(com.thinkbiganalytics.discovery.model.DefaultTag) FieldStandardizationRule(com.thinkbiganalytics.policy.rest.model.FieldStandardizationRule) Test(org.junit.Test)

Aggregations

FieldStandardizationRule (com.thinkbiganalytics.policy.rest.model.FieldStandardizationRule)14 Test (org.junit.Test)11 FieldValidationRule (com.thinkbiganalytics.policy.rest.model.FieldValidationRule)4 DefaultValueStandardizer (com.thinkbiganalytics.policy.standardization.DefaultValueStandardizer)3 ArrayList (java.util.ArrayList)3 DateTimeStandardizer (com.thinkbiganalytics.policy.standardization.DateTimeStandardizer)2 JsonNode (com.fasterxml.jackson.databind.JsonNode)1 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 ArrayNode (com.fasterxml.jackson.databind.node.ArrayNode)1 DefaultTag (com.thinkbiganalytics.discovery.model.DefaultTag)1 EntityDifference (com.thinkbiganalytics.feedmgr.rest.model.EntityDifference)1 EntityVersion (com.thinkbiganalytics.feedmgr.rest.model.EntityVersion)1 EntityVersionDifference (com.thinkbiganalytics.feedmgr.rest.model.EntityVersionDifference)1 FeedCategory (com.thinkbiganalytics.feedmgr.rest.model.FeedCategory)1 FeedMetadata (com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata)1 FeedVersions (com.thinkbiganalytics.feedmgr.rest.model.FeedVersions)1 ImportTemplate (com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate)1 PredicateImpl (com.thinkbiganalytics.guava.PredicateImpl)1 Diff (com.thinkbiganalytics.integration.Diff)1 NifiProperty (com.thinkbiganalytics.nifi.rest.model.NifiProperty)1