Examples with Referenceable org.apache.atlas.v1.model.instance.Referenceable used on opensource projects

Example 16 with Referenceable

use of org.apache.atlas.v1.model.instance.Referenceable in project atlas by apache.

the class FalconBridge method getFeedDataSetReference.

private static Referenceable getFeedDataSetReference(Feed feed, Referenceable clusterReference) {
    LOG.info("Getting reference for entity {}", feed.getName());
    Referenceable feedDatasetRef = new Referenceable(FalconDataTypes.FALCON_FEED.getName());
    feedDatasetRef.set(AtlasClient.NAME, feed.getName());
    feedDatasetRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getFeedQualifiedName(feed.getName(), (String) clusterReference.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME)));
    feedDatasetRef.set(FalconBridge.STOREDIN, clusterReference);
    feedDatasetRef.set(FalconBridge.FREQUENCY, feed.getFrequency());
    return feedDatasetRef;
}

Example 17 with Referenceable

use of org.apache.atlas.v1.model.instance.Referenceable in project atlas by apache.

the class FalconBridge method createFeedCreationEntity.

public static List<Referenceable> createFeedCreationEntity(Feed feed, ConfigurationStore falconStore) throws FalconException, URISyntaxException {
    LOG.info("Creating feed : {}", feed.getName());
    List<Referenceable> entities = new ArrayList<>();
    if (feed.getClusters() != null) {
        List<Referenceable> replicationInputs = new ArrayList<>();
        List<Referenceable> replicationOutputs = new ArrayList<>();
        for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) {
            org.apache.falcon.entity.v0.cluster.Cluster cluster = falconStore.get(EntityType.CLUSTER, feedCluster.getName());
            // set cluster
            Referenceable clusterReferenceable = getClusterEntityReference(cluster.getName(), cluster.getColo());
            entities.add(clusterReferenceable);
            // input as hive_table or hdfs_path, output as falcon_feed dataset
            List<Referenceable> inputs = new ArrayList<>();
            List<Referenceable> inputReferenceables = getInputEntities(cluster, feed);
            if (inputReferenceables != null) {
                entities.addAll(inputReferenceables);
                inputs.add(inputReferenceables.get(inputReferenceables.size() - 1));
            }
            List<Referenceable> outputs = new ArrayList<>();
            Referenceable feedEntity = createFeedEntity(feed, clusterReferenceable);
            if (feedEntity != null) {
                entities.add(feedEntity);
                outputs.add(feedEntity);
            }
            if (!inputs.isEmpty() || !outputs.isEmpty()) {
                Referenceable feedCreateEntity = new Referenceable(FalconDataTypes.FALCON_FEED_CREATION.getName());
                String feedQualifiedName = getFeedQualifiedName(feed.getName(), cluster.getName());
                feedCreateEntity.set(AtlasClient.NAME, feed.getName());
                feedCreateEntity.set(AtlasClient.DESCRIPTION, "Feed creation - " + feed.getName());
                feedCreateEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, feedQualifiedName);
                if (!inputs.isEmpty()) {
                    feedCreateEntity.set(AtlasClient.PROCESS_ATTRIBUTE_INPUTS, inputs);
                }
                if (!outputs.isEmpty()) {
                    feedCreateEntity.set(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS, outputs);
                }
                feedCreateEntity.set(FalconBridge.STOREDIN, clusterReferenceable);
                entities.add(feedCreateEntity);
            }
            if (ClusterType.SOURCE == feedCluster.getType()) {
                replicationInputs.add(feedEntity);
            } else if (ClusterType.TARGET == feedCluster.getType()) {
                replicationOutputs.add(feedEntity);
            }
        }
        if (!replicationInputs.isEmpty() && !replicationInputs.isEmpty()) {
            Referenceable feedReplicationEntity = new Referenceable(FalconDataTypes.FALCON_FEED_REPLICATION.getName());
            feedReplicationEntity.set(AtlasClient.NAME, feed.getName());
            feedReplicationEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, feed.getName());
            feedReplicationEntity.set(AtlasClient.PROCESS_ATTRIBUTE_INPUTS, replicationInputs);
            feedReplicationEntity.set(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS, replicationOutputs);
            entities.add(feedReplicationEntity);
        }
    }
    return entities;
}

Example 18 with Referenceable

use of org.apache.atlas.v1.model.instance.Referenceable in project atlas by apache.

the class FalconBridge method createProcessEntity.

/**
 * Creates process entity
 *
 * @param process process entity
 * @param falconStore config store
 * @return process instance reference
 *
 * @throws FalconException if retrieving from the configuration store fail
 */
public static List<Referenceable> createProcessEntity(org.apache.falcon.entity.v0.process.Process process, ConfigurationStore falconStore) throws FalconException {
    LOG.info("Creating process Entity : {}", process.getName());
    // The requirement is for each cluster, create a process entity with name
    // clustername.processname
    List<Referenceable> entities = new ArrayList<>();
    if (process.getClusters() != null) {
        for (Cluster processCluster : process.getClusters().getClusters()) {
            org.apache.falcon.entity.v0.cluster.Cluster cluster = falconStore.get(EntityType.CLUSTER, processCluster.getName());
            Referenceable clusterReferenceable = getClusterEntityReference(cluster.getName(), cluster.getColo());
            entities.add(clusterReferenceable);
            List<Referenceable> inputs = new ArrayList<>();
            if (process.getInputs() != null) {
                for (Input input : process.getInputs().getInputs()) {
                    Feed feed = falconStore.get(EntityType.FEED, input.getFeed());
                    Referenceable inputReferenceable = getFeedDataSetReference(feed, clusterReferenceable);
                    entities.add(inputReferenceable);
                    inputs.add(inputReferenceable);
                }
            }
            List<Referenceable> outputs = new ArrayList<>();
            if (process.getOutputs() != null) {
                for (Output output : process.getOutputs().getOutputs()) {
                    Feed feed = falconStore.get(EntityType.FEED, output.getFeed());
                    Referenceable outputReferenceable = getFeedDataSetReference(feed, clusterReferenceable);
                    entities.add(outputReferenceable);
                    outputs.add(outputReferenceable);
                }
            }
            if (!inputs.isEmpty() || !outputs.isEmpty()) {
                Referenceable processEntity = new Referenceable(FalconDataTypes.FALCON_PROCESS.getName());
                processEntity.set(AtlasClient.NAME, process.getName());
                processEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getProcessQualifiedName(process.getName(), cluster.getName()));
                processEntity.set(FalconBridge.FREQUENCY, process.getFrequency().toString());
                if (!inputs.isEmpty()) {
                    processEntity.set(AtlasClient.PROCESS_ATTRIBUTE_INPUTS, inputs);
                }
                if (!outputs.isEmpty()) {
                    processEntity.set(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS, outputs);
                }
                // set cluster
                processEntity.set(FalconBridge.RUNSON, clusterReferenceable);
                // Set user
                if (process.getACL() != null) {
                    processEntity.set(AtlasClient.OWNER, process.getACL().getOwner());
                }
                if (StringUtils.isNotEmpty(process.getTags())) {
                    processEntity.set(FalconBridge.TAGS, EventUtil.convertKeyValueStringToMap(process.getTags()));
                }
                if (process.getPipelines() != null) {
                    processEntity.set(FalconBridge.PIPELINES, process.getPipelines());
                }
                processEntity.set(FalconBridge.WFPROPERTIES, getProcessEntityWFProperties(process.getWorkflow(), process.getName()));
                entities.add(processEntity);
            }
        }
    }
    return entities;
}

Example 19 with Referenceable

use of org.apache.atlas.v1.model.instance.Referenceable in project atlas by apache.

the class FalconBridge method getClusterEntityReference.

private static Referenceable getClusterEntityReference(final String clusterName, final String colo) {
    LOG.info("Getting reference for entity {}", clusterName);
    Referenceable clusterRef = new Referenceable(FalconDataTypes.FALCON_CLUSTER.getName());
    clusterRef.set(AtlasClient.NAME, String.format("%s", clusterName));
    clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, clusterName);
    clusterRef.set(FalconBridge.COLO, colo);
    return clusterRef;
}

Example 20 with Referenceable

use of org.apache.atlas.v1.model.instance.Referenceable in project atlas by apache.

the class FalconBridge method createClusterEntity.

/**
 * Creates cluster entity
 *
 * @param cluster ClusterEntity
 * @return cluster instance reference
 */
public static Referenceable createClusterEntity(final org.apache.falcon.entity.v0.cluster.Cluster cluster) {
    LOG.info("Creating cluster Entity : {}", cluster.getName());
    Referenceable clusterRef = new Referenceable(FalconDataTypes.FALCON_CLUSTER.getName());
    clusterRef.set(AtlasClient.NAME, cluster.getName());
    clusterRef.set(AtlasClient.DESCRIPTION, cluster.getDescription());
    clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, cluster.getName());
    clusterRef.set(FalconBridge.COLO, cluster.getColo());
    if (cluster.getACL() != null) {
        clusterRef.set(AtlasClient.OWNER, cluster.getACL().getGroup());
    }
    if (StringUtils.isNotEmpty(cluster.getTags())) {
        clusterRef.set(FalconBridge.TAGS, EventUtil.convertKeyValueStringToMap(cluster.getTags()));
    }
    return clusterRef;
}