Examples with Referenceable org.apache.atlas.typesystem.Referenceable used on opensource projects

Example 26 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class FalconBridge method createHiveDatabaseInstance.

private static Referenceable createHiveDatabaseInstance(String clusterName, String dbName) {
    Referenceable dbRef = new Referenceable(HiveDataTypes.HIVE_DB.getName());
    dbRef.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName);
    dbRef.set(AtlasClient.NAME, dbName);
    dbRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, HiveMetaStoreBridge.getDBQualifiedName(clusterName, dbName));
    return dbRef;
}

Example 27 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class FalconBridge method getFeedDataSetReference.

private static Referenceable getFeedDataSetReference(Feed feed, Referenceable clusterReference) {
    LOG.info("Getting reference for entity {}", feed.getName());
    Referenceable feedDatasetRef = new Referenceable(FalconDataTypes.FALCON_FEED.getName());
    feedDatasetRef.set(AtlasClient.NAME, feed.getName());
    feedDatasetRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getFeedQualifiedName(feed.getName(), (String) clusterReference.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME)));
    feedDatasetRef.set(FalconBridge.STOREDIN, clusterReference);
    feedDatasetRef.set(FalconBridge.FREQUENCY, feed.getFrequency());
    return feedDatasetRef;
}

Example 28 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class FalconBridge method createFeedCreationEntity.

public static List<Referenceable> createFeedCreationEntity(Feed feed, ConfigurationStore falconStore) throws Exception {
    LOG.info("Creating feed : {}", feed.getName());
    List<Referenceable> entities = new ArrayList<>();
    if (feed.getClusters() != null) {
        List<Referenceable> replicationInputs = new ArrayList<>();
        List<Referenceable> replicationOutputs = new ArrayList<>();
        for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) {
            org.apache.falcon.entity.v0.cluster.Cluster cluster = falconStore.get(EntityType.CLUSTER, feedCluster.getName());
            // set cluster
            Referenceable clusterReferenceable = getClusterEntityReference(cluster.getName(), cluster.getColo());
            entities.add(clusterReferenceable);
            // input as hive_table or hdfs_path, output as falcon_feed dataset
            List<Referenceable> inputs = new ArrayList<>();
            List<Referenceable> inputReferenceables = getInputEntities(cluster, feed);
            if (inputReferenceables != null) {
                entities.addAll(inputReferenceables);
                inputs.add(inputReferenceables.get(inputReferenceables.size() - 1));
            }
            List<Referenceable> outputs = new ArrayList<>();
            Referenceable feedEntity = createFeedEntity(feed, clusterReferenceable);
            if (feedEntity != null) {
                entities.add(feedEntity);
                outputs.add(feedEntity);
            }
            if (!inputs.isEmpty() || !outputs.isEmpty()) {
                Referenceable feedCreateEntity = new Referenceable(FalconDataTypes.FALCON_FEED_CREATION.getName());
                String feedQualifiedName = getFeedQualifiedName(feed.getName(), cluster.getName());
                feedCreateEntity.set(AtlasClient.NAME, feed.getName());
                feedCreateEntity.set(AtlasClient.DESCRIPTION, "Feed creation - " + feed.getName());
                feedCreateEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, feedQualifiedName);
                if (!inputs.isEmpty()) {
                    feedCreateEntity.set(AtlasClient.PROCESS_ATTRIBUTE_INPUTS, inputs);
                }
                if (!outputs.isEmpty()) {
                    feedCreateEntity.set(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS, outputs);
                }
                feedCreateEntity.set(FalconBridge.STOREDIN, clusterReferenceable);
                entities.add(feedCreateEntity);
            }
            if (ClusterType.SOURCE == feedCluster.getType()) {
                replicationInputs.add(feedEntity);
            } else if (ClusterType.TARGET == feedCluster.getType()) {
                replicationOutputs.add(feedEntity);
            }
        }
        if (!replicationInputs.isEmpty() && !replicationInputs.isEmpty()) {
            Referenceable feedReplicationEntity = new Referenceable(FalconDataTypes.FALCON_FEED_REPLICATION.getName());
            feedReplicationEntity.set(AtlasClient.NAME, feed.getName());
            feedReplicationEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, feed.getName());
            feedReplicationEntity.set(AtlasClient.PROCESS_ATTRIBUTE_INPUTS, replicationInputs);
            feedReplicationEntity.set(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS, replicationOutputs);
            entities.add(feedReplicationEntity);
        }
    }
    return entities;
}

Example 29 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class FalconBridge method createClusterEntity.

/**
     * Creates cluster entity
     *
     * @param cluster ClusterEntity
     * @return cluster instance reference
     */
public static Referenceable createClusterEntity(final org.apache.falcon.entity.v0.cluster.Cluster cluster) {
    LOG.info("Creating cluster Entity : {}", cluster.getName());
    Referenceable clusterRef = new Referenceable(FalconDataTypes.FALCON_CLUSTER.getName());
    clusterRef.set(AtlasClient.NAME, cluster.getName());
    clusterRef.set(AtlasClient.DESCRIPTION, cluster.getDescription());
    clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, cluster.getName());
    clusterRef.set(FalconBridge.COLO, cluster.getColo());
    if (cluster.getACL() != null) {
        clusterRef.set(AtlasClient.OWNER, cluster.getACL().getGroup());
    }
    if (StringUtils.isNotEmpty(cluster.getTags())) {
        clusterRef.set(FalconBridge.TAGS, EventUtil.convertKeyValueStringToMap(cluster.getTags()));
    }
    return clusterRef;
}

Example 30 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class FalconBridge method fillHDFSDataSet.

private static List<Referenceable> fillHDFSDataSet(final String pathUri, final String clusterName) {
    List<Referenceable> entities = new ArrayList<>();
    Referenceable ref = new Referenceable(HiveMetaStoreBridge.HDFS_PATH);
    ref.set("path", pathUri);
    //        Path path = new Path(pathUri);
    //        ref.set("name", path.getName());
    //TODO - Fix after ATLAS-542 to shorter Name
    Path path = new Path(pathUri);
    ref.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
    ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathUri);
    ref.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName);
    entities.add(ref);
    return entities;
}