Examples with Referenceable org.apache.atlas.typesystem.Referenceable used on opensource projects

Example 56 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class BaseRepositoryTest method column.

protected Referenceable column(String name, String dataType, String comment, String... traitNames) throws Exception {
    Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames);
    referenceable.set("name", name);
    referenceable.set("dataType", dataType);
    referenceable.set("comment", comment);
    return referenceable;
}

Example 57 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class BaseRepositoryTest method table.

protected Id table(String name, String description, Id dbId, Referenceable sd, String owner, String tableType, List<Referenceable> columns, String... traitNames) throws Exception {
    Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames);
    referenceable.set("name", name);
    referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, "qualified:" + name);
    referenceable.set("description", description);
    referenceable.set("owner", owner);
    referenceable.set("tableType", tableType);
    referenceable.set("temporary", false);
    referenceable.set("createTime", new Date(System.currentTimeMillis()));
    referenceable.set("lastAccessTime", System.currentTimeMillis());
    referenceable.set("retention", System.currentTimeMillis());
    referenceable.set("db", dbId);
    // todo - uncomment this, something is broken
    referenceable.set("sd", sd);
    referenceable.set("columns", columns);
    ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, HIVE_TABLE_TYPE);
    return createInstance(referenceable, clsType);
}

Example 58 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class BaseRepositoryTest method datasetSubType.

Id datasetSubType(final String name, String owner) throws Exception {
    Referenceable referenceable = new Referenceable(DATASET_SUBTYPE);
    referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
    referenceable.set(AtlasClient.NAME, name);
    referenceable.set("owner", owner);
    ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, DATASET_SUBTYPE);
    return createInstance(referenceable, clsType);
}

Example 59 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class BaseRepositoryTest method setupInstances.

private void setupInstances() throws Exception {
    Id salesDB = database("Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
    Referenceable sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
    List<Referenceable> salesFactColumns = ImmutableList.of(column("time_id", "int", "time id"), column("product_id", "int", "product id"), column("customer_id", "int", "customer id", "PII"), column("sales", "double", "product id", "Metric"));
    Id salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
    List<Referenceable> logFactColumns = ImmutableList.of(column("time_id", "int", "time id"), column("app_id", "int", "app id"), column("machine_id", "int", "machine id"), column("log", "string", "log data", "Log Data"));
    List<Referenceable> timeDimColumns = ImmutableList.of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"), column("weekDay", "int", "week Day"));
    Id timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns, "Dimension");
    Id reportingDB = database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting");
    Id salesFactDaily = table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed", salesFactColumns, "Metric");
    Id circularLineageTable1 = table("table1", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric");
    Id circularLineageTable2 = table("table2", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric");
    loadProcess("circularLineage1", "hive query for daily summary", "John ETL", ImmutableList.of(circularLineageTable1), ImmutableList.of(circularLineageTable2), "create table as select ", "plan", "id", "graph", "ETL");
    loadProcess("circularLineage2", "hive query for daily summary", "John ETL", ImmutableList.of(circularLineageTable2), ImmutableList.of(circularLineageTable1), "create table as select ", "plan", "id", "graph", "ETL");
    loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL", ImmutableList.of(salesFact, timeDim), ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL");
    Id logDB = database("Logging", "logging database", "Tim ETL", "hdfs://host:8000/apps/warehouse/logging");
    Id loggingFactDaily = table("log_fact_daily_mv", "log fact daily materialized view", logDB, sd, "Tim ETL", "Managed", logFactColumns, "Log Data");
    List<Referenceable> productDimColumns = ImmutableList.of(column("product_id", "int", "product id"), column("product_name", "string", "product name"), column("brand_name", "int", "brand name"));
    Id productDim = table("product_dim", "product dimension table", salesDB, sd, "John Doe", "Managed", productDimColumns, "Dimension");
    view("product_dim_view", reportingDB, ImmutableList.of(productDim), "Dimension", "JdbcAccess");
    List<Referenceable> customerDimColumns = ImmutableList.of(column("customer_id", "int", "customer id", "PII"), column("name", "string", "customer name", "PII"), column("address", "string", "customer address", "PII"));
    Id customerDim = table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External", customerDimColumns, "Dimension");
    view("customer_dim_view", reportingDB, ImmutableList.of(customerDim), "Dimension", "JdbcAccess");
    Id salesFactMonthly = table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB, sd, "Jane BI", "Managed", salesFactColumns, "Metric");
    loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL", ImmutableList.of(salesFactDaily), ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL");
    Id loggingFactMonthly = table("logging_fact_monthly_mv", "logging fact monthly materialized view", logDB, sd, "Tim ETL", "Managed", logFactColumns, "Log Data");
    loadProcess("loadLogsMonthly", "hive query for monthly summary", "Tim ETL", ImmutableList.of(loggingFactDaily), ImmutableList.of(loggingFactMonthly), "create table as select ", "plan", "id", "graph", "ETL");
    partition(new ArrayList() {

        {
            add("2015-01-01");
        }
    }, salesFactDaily);
    datasetSubType("dataSetSubTypeInst1", "testOwner");
}

Example 60 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class BaseRepositoryTest method storageDescriptor.

protected Referenceable storageDescriptor(String location, String inputFormat, String outputFormat, boolean compressed, List<Referenceable> columns) throws Exception {
    Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE);
    referenceable.set("location", location);
    referenceable.set("inputFormat", inputFormat);
    referenceable.set("outputFormat", outputFormat);
    referenceable.set("compressed", compressed);
    referenceable.set("cols", columns);
    return referenceable;
}