Examples with FeedMetadata com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata used on opensource projects

Example 71 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedIT method testEditFeed.

@Test
public void testEditFeed() throws Exception {
    // Prepare environment
    prepare();
    final FeedCategory category = createCategory(CATEGORY_NAME);
    final ImportTemplate template = importDataIngestTemplate();
    // Create feed
    FeedMetadata feed = getCreateFeedRequest(category, template, createNewFeedName());
    feed.setDescription("Test feed");
    feed.setDataOwner("Some Guy");
    FeedMetadata response = createFeed(feed).getFeedMetadata();
    Assert.assertEquals(feed.getFeedName(), response.getFeedName());
    Assert.assertEquals(feed.getDataOwner(), response.getDataOwner());
    // Edit feed
    feed.setId(response.getId());
    feed.setFeedId(response.getFeedId());
    feed.setIsNew(false);
    feed.setDescription(null);
    feed.setDataOwner("Some Other Guy");
    NifiProperty fileFilter = feed.getProperties().get(0);
    fileFilter.setValue("some-file.csv");
    List<FieldPolicy> policies = feed.getTable().getFieldPolicies();
    FieldPolicy id = policies.get(1);
    // add new validator
    id.getValidation().add(notNull);
    feed.getTable().setPrimaryKeyFields("id");
    FieldPolicy firstName = policies.get(2);
    // flip profiling
    firstName.setProfile(false);
    FieldPolicy secondName = policies.get(3);
    // flip indexing
    secondName.setIndex(false);
    // add new standardiser
    secondName.getStandardization().add(toUpperCase);
    FieldPolicy email = policies.get(4);
    // remove validators
    email.setValidation(Collections.emptyList());
    FieldPolicy gender = policies.get(5);
    FieldValidationRule lookup = gender.getValidation().get(0);
    // change existing validator property
    lookup.getProperties().get(0).setValue("new value");
    // add profiling
    gender.setProfile(true);
    // add indexing
    gender.setIndex(true);
    FieldPolicy creditCard = policies.get(7);
    FieldStandardizationRule base64EncodeBinary = creditCard.getStandardization().get(0);
    // change existing standardiser property
    base64EncodeBinary.getProperties().get(0).setValue("STRING");
    feed.getOptions().setSkipHeader(false);
    feed.getTable().setTargetMergeStrategy("ROLLING_SYNC");
    feed.getTags().add(new DefaultTag("updated"));
    feed.getSchedule().setSchedulingPeriod("20 sec");
    response = createFeed(feed).getFeedMetadata();
    Assert.assertEquals(feed.getFeedName(), response.getFeedName());
    Assert.assertEquals(feed.getDescription(), response.getDescription());
    FeedVersions feedVersions = getVersions(feed.getFeedId());
    List<EntityVersion> versions = feedVersions.getVersions();
    Assert.assertEquals(2, versions.size());
    EntityVersionDifference entityDiff = getVersionDiff(feed.getFeedId(), versions.get(1).getId(), versions.get(0).getId());
    EntityDifference diff = entityDiff.getDifference();
    JsonNode patch = diff.getPatch();
    ArrayNode diffs = (ArrayNode) patch;
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/properties/0/value", "some-file.csv")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/schedule/schedulingPeriod", "20 sec")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("remove", "/description")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("add", "/tags/1")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/dataOwner", "Some Other Guy")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("add", "/table/fieldPolicies/1/validation/0")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/2/profile", "false")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/3/index", "false")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("add", "/table/fieldPolicies/3/standardization/0")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("remove", "/table/fieldPolicies/4/validation/0")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/5/profile", "true")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/5/index", "true")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/5/validation/0/properties/0/value", "new value")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/7/standardization/0/properties/0/value", "STRING")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldPolicies/8/standardization/0/properties/0/value", "STRING")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/targetMergeStrategy", "ROLLING_SYNC")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/table/fieldIndexString", "first_name,gender")));
    Assert.assertTrue(versionPatchContains(diffs, new Diff("replace", "/options/skipHeader", "false")));
}

Example 72 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedLoadTest method loadTest.

/**
 * Bulk load a template into NiFi and Kylo
 * categories are named  'category_#
 * feeds are named  cat_#_feedName_#
 *
 * @param templatePath the template to use
 * @param feedName the name of the feed
 * @param categories the number of categories to use/create
 * @param feedsInCategory the number of feeds to create in each category
 * @param startingCategoryId the category number to use for the first category
 * @throws Exception
 */
private void loadTest(String templatePath, String feedName, int categories, int feedsInCategory, int startingCategoryId) throws Exception {
    ImportTemplate ingestTemplate = importTemplate(templatePath);
    for (int i = startingCategoryId; i < (startingCategoryId + categories); i++) {
        // create new category
        String categoryName = "category_" + i;
        FeedCategory category = getorCreateCategoryByName(categoryName);
        Integer maxFeedId = category.getRelatedFeeds();
        maxFeedId += 1;
        for (int j = maxFeedId; j < (feedsInCategory + maxFeedId); j++) {
            try {
                String updateFeedName = "cat_" + i + "_" + feedName + "_" + j;
                FeedMetadata feed = getCreateFeedRequest(category, ingestTemplate, updateFeedName);
                long start = System.currentTimeMillis();
                FeedMetadata response = createFeed(feed).getFeedMetadata();
                LOG.info("Time to save: {} was: {} ms ", updateFeedName, (System.currentTimeMillis() - start));
            } catch (Exception e) {
            }
        }
    }
}

Example 73 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedTestUtil method getCreateFeedRequest.

public static FeedMetadata getCreateFeedRequest(FeedCategory category, ImportTemplate template, String name, String inputProcessorType, List<NifiProperty> properties) throws Exception {
    FeedMetadata feed = new FeedMetadata();
    feed.setFeedName(name);
    feed.setSystemFeedName(name.toLowerCase());
    feed.setCategory(category);
    feed.setTemplateId(template.getTemplateId());
    feed.setTemplateName(template.getTemplateName());
    feed.setDescription("Created by functional test");
    feed.setInputProcessorType(inputProcessorType);
    feed.setProperties(properties);
    FeedSchedule schedule = new FeedSchedule();
    schedule.setConcurrentTasks(1);
    schedule.setSchedulingPeriod("15 sec");
    schedule.setSchedulingStrategy("TIMER_DRIVEN");
    feed.setSchedule(schedule);
    feed.setDataOwner("Marketing");
    List<Tag> tags = new ArrayList<>();
    tags.add(new DefaultTag("users"));
    tags.add(new DefaultTag("registrations"));
    feed.setTags(tags);
    User owner = new User();
    owner.setSystemName("dladmin");
    owner.setDisplayName("Data Lake Admin");
    Set<String> groups = new HashSet<>();
    groups.add("admin");
    groups.add("user");
    owner.setGroups(groups);
    feed.setOwner(owner);
    return feed;
}

Example 74 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedWithConnectedReusableTemplateIT method testConnectedFeed.

@Test
public void testConnectedFeed() throws Exception {
    ConnectedTemplate connectedTemplate = registerConnectedReusableTemplate();
    URL resource = IntegrationTestBase.class.getResource("feedconnectedflow.template.zip");
    // Create a feed using this template
    // Register simple feed template
    ImportTemplate template = importTemplate(resource.getPath());
    FeedCategory feedCategory = createCategory("connected flows");
    // create a feed using this template
    FeedMetadata feed = FeedTestUtil.getCreateGenerateFlowFileFeedRequest(feedCategory, template, "connected_feed_" + System.currentTimeMillis());
    NifiFeed createdFeed = createFeed(feed);
    Assert.assertNotNull(createdFeed);
    waitForFeedToComplete();
    assertExecutedJobs(feed);
}

Example 75 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class NifiRestTest method testLoad.

// @Test
public void testLoad() {
    // setup constants for the test
    String templateName = "Data Ingest";
    int num = 10;
    String processGroupName = "LoadTest";
    String feedPrefix = "LT_";
    String inputType = "org.apache.nifi.processors.standard.GetFile";
    List<NifiProperty> templateProperties = new ArrayList<>();
    String schedulePeriod = "10 sec";
    String GET_FILE_PROCESSOR_NAME = "Poll filesystem";
    String UPDATE_PARAMETERS_PROCESSOR_NAME = "Update flow parameters";
    String INPUT_DIRECTORY_PROPERTY = "Input Directory";
    String SOURCE_PROPERTY = "source";
    String ENTITY_PROPERTY = "entity";
    try {
        TemplateDTO template = restClient.getTemplateByName(templateName);
        List<NifiProperty> propertyList = restClient.getPropertiesForTemplate(template.getId(), true);
        NifiProperty inputDirectory = NifiPropertyUtil.getProperty(GET_FILE_PROCESSOR_NAME, INPUT_DIRECTORY_PROPERTY, propertyList);
        NifiProperty entity = NifiPropertyUtil.getProperty(UPDATE_PARAMETERS_PROCESSOR_NAME, SOURCE_PROPERTY, propertyList);
        NifiProperty source = NifiPropertyUtil.getProperty(UPDATE_PARAMETERS_PROCESSOR_NAME, ENTITY_PROPERTY, propertyList);
        templateProperties.add(inputDirectory);
        templateProperties.add(entity);
        templateProperties.add(source);
        NifiProcessorSchedule schedule = new NifiProcessorSchedule();
        schedule.setSchedulingStrategy("TIMER_DRIVEN");
        schedule.setSchedulingPeriod(schedulePeriod);
        for (int i = 0; i < num; i++) {
            String feedName = feedPrefix + i;
            List<NifiProperty> instanceProperties = NifiPropertyUtil.copyProperties(templateProperties);
            // update the properties
            NifiPropertyUtil.getProperty(GET_FILE_PROCESSOR_NAME, INPUT_DIRECTORY_PROPERTY, instanceProperties).setValue("/tmp/" + feedName);
            NifiPropertyUtil.getProperty(UPDATE_PARAMETERS_PROCESSOR_NAME, SOURCE_PROPERTY, instanceProperties).setValue(processGroupName);
            NifiPropertyUtil.getProperty(UPDATE_PARAMETERS_PROCESSOR_NAME, ENTITY_PROPERTY, instanceProperties).setValue(feedName);
            FeedMetadata feedMetadata = new FeedMetadata();
            feedMetadata.setCategory(new FeedCategory());
            feedMetadata.getCategory().setSystemName(processGroupName);
            feedMetadata.setSystemFeedName("feedPrefix + i");
            CreateFeedBuilder.newFeed(restClient, nifiFlowCache, feedMetadata, template.getId(), new PropertyExpressionResolver(), propertyDescriptorTransform, createFeedBuilderCache, templateConnectionUtil).inputProcessorType(inputType).feedSchedule(schedule).properties(instanceProperties).build();
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
}