Examples with Id org.apache.atlas.v1.model.instance.Id used on opensource projects

Example 51 with Id

use of org.apache.atlas.v1.model.instance.Id in project atlas by apache.

the class QuickStart method createEntities.

void createEntities() throws Exception {
    Id salesDB = database(SALES_DB, SALES_DB_DESCRIPTION, "John ETL", "hdfs://host:8000/apps/warehouse/sales");
    Referenceable sd = rawStorageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true);
    List<Referenceable> salesFactColumns = Arrays.asList(rawColumn(TIME_ID_COLUMN, "int", "time id"), rawColumn("product_id", "int", "product id"), rawColumn("customer_id", "int", "customer id", "PII_v1"), rawColumn("sales", "double", "product id", "Metric_v1"));
    List<Referenceable> logFactColumns = Arrays.asList(rawColumn("time_id", "int", "time id"), rawColumn("app_id", "int", "app id"), rawColumn("machine_id", "int", "machine id"), rawColumn("log", "string", "log data", "Log Data_v1"));
    Id salesFact = table(SALES_FACT_TABLE, SALES_FACT_TABLE_DESCRIPTION, salesDB, sd, "Joe", "Managed", salesFactColumns, FACT_TRAIT);
    List<Referenceable> productDimColumns = Arrays.asList(rawColumn("product_id", "int", "product id"), rawColumn("product_name", "string", "product name"), rawColumn("brand_name", "int", "brand name"));
    Id productDim = table(PRODUCT_DIM_TABLE, "product dimension table", salesDB, sd, "John Doe", "Managed", productDimColumns, "Dimension_v1");
    List<Referenceable> timeDimColumns = Arrays.asList(rawColumn("time_id", "int", "time id"), rawColumn("dayOfYear", "int", "day Of Year"), rawColumn("weekDay", "int", "week Day"));
    Id timeDim = table(TIME_DIM_TABLE, "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns, "Dimension_v1");
    List<Referenceable> customerDimColumns = Arrays.asList(rawColumn("customer_id", "int", "customer id", "PII_v1"), rawColumn("name", "string", "customer name", "PII_v1"), rawColumn("address", "string", "customer address", "PII_v1"));
    Id customerDim = table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External", customerDimColumns, "Dimension_v1");
    Id reportingDB = database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting");
    Id logDB = database("Logging", "logging database", "Tim ETL", "hdfs://host:8000/apps/warehouse/logging");
    Id salesFactDaily = table(SALES_FACT_DAILY_MV_TABLE, "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed", salesFactColumns, "Metric_v1");
    Id loggingFactDaily = table("log_fact_daily_mv", "log fact daily materialized view", logDB, sd, "Tim ETL", "Managed", logFactColumns, "Log Data_v1");
    loadProcess(LOAD_SALES_DAILY_PROCESS, LOAD_SALES_DAILY_PROCESS_DESCRIPTION, "John ETL", Arrays.asList(salesFact, timeDim), Collections.singletonList(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL_v1");
    view(PRODUCT_DIM_VIEW, reportingDB, Collections.singletonList(productDim), "Dimension_v1", "JdbcAccess_v1");
    view("customer_dim_view", reportingDB, Collections.singletonList(customerDim), "Dimension_v1", "JdbcAccess_v1");
    Id salesFactMonthly = table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB, sd, "Jane BI", "Managed", salesFactColumns, "Metric_v1");
    loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL", Collections.singletonList(salesFactDaily), Collections.singletonList(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL_v1");
    Id loggingFactMonthly = table("logging_fact_monthly_mv", "logging fact monthly materialized view", logDB, sd, "Tim ETL", "Managed", logFactColumns, "Log Data_v1");
    loadProcess("loadLogsMonthly", "hive query for monthly summary", "Tim ETL", Collections.singletonList(loggingFactDaily), Collections.singletonList(loggingFactMonthly), "create table as select ", "plan", "id", "graph", "ETL_v1");
}

Example 52 with Id

use of org.apache.atlas.v1.model.instance.Id in project atlas by apache.

the class QuickStart method table.

Id table(String name, String description, Id dbId, Referenceable sd, String owner, String tableType, List<Referenceable> columns, String... traitNames) throws AtlasBaseException {
    try {
        Referenceable referenceable = new Referenceable(TABLE_TYPE, traitNames);
        referenceable.set("name", name);
        referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
        referenceable.set("description", description);
        referenceable.set("owner", owner);
        referenceable.set("tableType", tableType);
        referenceable.set("createTime", System.currentTimeMillis());
        referenceable.set("lastAccessTime", System.currentTimeMillis());
        referenceable.set("retention", System.currentTimeMillis());
        referenceable.set("db", dbId);
        referenceable.set("sd", sd);
        referenceable.set("columns", columns);
        return createInstance(referenceable);
    } catch (Exception e) {
        throw new AtlasBaseException(AtlasErrorCode.QUICK_START, e, String.format("%s table entity creation failed", name));
    }
}

Example 53 with Id

use of org.apache.atlas.v1.model.instance.Id in project atlas by apache.

the class QuickStart method loadProcess.

Id loadProcess(String name, String description, String user, List<Id> inputTables, List<Id> outputTables, String queryText, String queryPlan, String queryId, String queryGraph, String... traitNames) throws AtlasBaseException {
    try {
        Referenceable referenceable = new Referenceable(LOAD_PROCESS_TYPE, traitNames);
        // super type attributes
        referenceable.set(AtlasClient.NAME, name);
        referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
        referenceable.set("description", description);
        referenceable.set(INPUTS_ATTRIBUTE, inputTables);
        referenceable.set(OUTPUTS_ATTRIBUTE, outputTables);
        referenceable.set("user", user);
        referenceable.set("startTime", System.currentTimeMillis());
        referenceable.set("endTime", System.currentTimeMillis() + 10000);
        referenceable.set("queryText", queryText);
        referenceable.set("queryPlan", queryPlan);
        referenceable.set("queryId", queryId);
        referenceable.set("queryGraph", queryGraph);
        return createInstance(referenceable);
    } catch (Exception e) {
        throw new AtlasBaseException(AtlasErrorCode.QUICK_START, e, String.format("%s process entity creation failed", name));
    }
}

Example 54 with Id

use of org.apache.atlas.v1.model.instance.Id in project atlas by apache.

the class AtlasEntityFormatConverter method fromV2ToV1.

@Override
public Object fromV2ToV1(Object v2Obj, AtlasType type, ConverterContext context) throws AtlasBaseException {
    Object ret = null;
    if (v2Obj != null) {
        AtlasEntityType entityType = (AtlasEntityType) type;
        if (v2Obj instanceof Map) {
            Map v2Map = (Map) v2Obj;
            String idStr = (String) v2Map.get(AtlasObjectId.KEY_GUID);
            String typeName = type.getTypeName();
            if (StringUtils.isEmpty(idStr)) {
                throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND);
            }
            final Map v2Attribs = (Map) v2Map.get(ATTRIBUTES_PROPERTY_KEY);
            if (MapUtils.isEmpty(v2Attribs)) {
                ret = new Id(idStr, 0, typeName);
            } else {
                ret = new Referenceable(idStr, typeName, super.fromV2ToV1(entityType, v2Attribs, context));
            }
        } else if (v2Obj instanceof AtlasEntity) {
            AtlasEntity entity = (AtlasEntity) v2Obj;
            Status status = entity.getStatus();
            if (status == null) {
                status = Status.ACTIVE;
            }
            Referenceable referenceable = new Referenceable(entity.getGuid(), entity.getTypeName(), status.name(), fromV2ToV1(entityType, entity.getAttributes(), context), new AtlasSystemAttributes(entity.getCreatedBy(), entity.getUpdatedBy(), entity.getCreateTime(), entity.getUpdateTime()));
            if (CollectionUtils.isNotEmpty(entity.getClassifications())) {
                for (AtlasClassification classification : entity.getClassifications()) {
                    String traitName = classification.getTypeName();
                    AtlasClassificationType classificationType = typeRegistry.getClassificationTypeByName(traitName);
                    AtlasFormatConverter formatConverter = classificationType != null ? converterRegistry.getConverter(classificationType.getTypeCategory()) : null;
                    Struct trait = formatConverter != null ? (Struct) formatConverter.fromV2ToV1(classification, classificationType, context) : null;
                    if (trait != null) {
                        referenceable.getTraitNames().add(trait.getTypeName());
                        referenceable.getTraits().put(trait.getTypeName(), trait);
                    }
                }
            }
            ret = referenceable;
        } else if (v2Obj instanceof AtlasObjectId) {
            // transient-id
            AtlasEntity entity = context.getById(((AtlasObjectId) v2Obj).getGuid());
            if (entity == null) {
                throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Could not find entity ", v2Obj.toString());
            }
            ret = this.fromV2ToV1(entity, typeRegistry.getType(((AtlasObjectId) v2Obj).getTypeName()), context);
        } else {
            throw new AtlasBaseException(AtlasErrorCode.UNEXPECTED_TYPE, "Map or AtlasEntity or String", v2Obj.getClass().getCanonicalName());
        }
    }
    return ret;
}

Example 55 with Id

use of org.apache.atlas.v1.model.instance.Id in project atlas by apache.

the class FalconHookIT method testCreateProcessWithHDFSFeed.

@Test
public void testCreateProcessWithHDFSFeed() throws Exception {
    Cluster cluster = loadEntity(EntityType.CLUSTER, CLUSTER_RESOURCE, "cluster" + random());
    STORE.publish(EntityType.CLUSTER, cluster);
    TypesUtil.Pair<String, Feed> result = getHDFSFeed(FEED_HDFS_RESOURCE, cluster.getName());
    Feed infeed = result.right;
    String infeedId = result.left;
    Feed outfeed = getTableFeed(FEED_RESOURCE, cluster.getName());
    String outfeedId = atlasClient.getEntity(FalconDataTypes.FALCON_FEED.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, FalconBridge.getFeedQualifiedName(outfeed.getName(), cluster.getName())).getId()._getId();
    Process process = loadEntity(EntityType.PROCESS, PROCESS_RESOURCE, "process" + random());
    process.getClusters().getClusters().get(0).setName(cluster.getName());
    process.getInputs().getInputs().get(0).setFeed(infeed.getName());
    process.getOutputs().getOutputs().get(0).setFeed(outfeed.getName());
    STORE.publish(EntityType.PROCESS, process);
    String pid = assertProcessIsRegistered(process, cluster.getName());
    Referenceable processEntity = atlasClient.getEntity(pid);
    assertEquals(processEntity.get(AtlasClient.NAME), process.getName());
    assertEquals(processEntity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), FalconBridge.getProcessQualifiedName(process.getName(), cluster.getName()));
    assertEquals(((List<Id>) processEntity.get("inputs")).get(0)._getId(), infeedId);
    assertEquals(((List<Id>) processEntity.get("outputs")).get(0)._getId(), outfeedId);
}