use of org.apache.atlas.model.typedef.AtlasClassificationDef in project incubator-atlas by apache.
the class TestUtilsV2 method defineHiveTypes.
public static AtlasTypesDef defineHiveTypes() {
String _description = "_description";
AtlasEntityDef superTypeDefinition = AtlasTypeUtil.createClassTypeDef(SUPER_TYPE_NAME, "SuperType_description", ImmutableSet.<String>of(), AtlasTypeUtil.createOptionalAttrDef("namespace", "string"), AtlasTypeUtil.createOptionalAttrDef("cluster", "string"), AtlasTypeUtil.createOptionalAttrDef("colo", "string"));
AtlasEntityDef databaseTypeDefinition = AtlasTypeUtil.createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME), AtlasTypeUtil.createUniqueRequiredAttrDef(NAME, "string"), AtlasTypeUtil.createOptionalAttrDef("isReplicated", "boolean"), AtlasTypeUtil.createOptionalAttrDef("created", "string"), AtlasTypeUtil.createOptionalAttrDef("parameters", "map<string,string>"), AtlasTypeUtil.createRequiredAttrDef("description", "string"));
AtlasStructDef structTypeDefinition = new AtlasStructDef("serdeType", "serdeType" + _description, "1.0", Arrays.asList(AtlasTypeUtil.createRequiredAttrDef("name", "string"), AtlasTypeUtil.createRequiredAttrDef("serde", "string"), AtlasTypeUtil.createOptionalAttrDef("description", "string")));
AtlasEnumElementDef[] values = { new AtlasEnumElementDef("MANAGED", "Element Description", 1), new AtlasEnumElementDef("EXTERNAL", "Element Description", 2) };
AtlasEnumDef enumTypeDefinition = new AtlasEnumDef("tableType", "tableType" + _description, "1.0", Arrays.asList(values));
AtlasEntityDef columnsDefinition = AtlasTypeUtil.createClassTypeDef(COLUMN_TYPE, COLUMN_TYPE + "_description", ImmutableSet.<String>of(), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), AtlasTypeUtil.createRequiredAttrDef("type", "string"), AtlasTypeUtil.createOptionalAttrDef("description", "string"), new AtlasAttributeDef("table", TABLE_TYPE, true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, new ArrayList<AtlasStructDef.AtlasConstraintDef>() {
{
add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {
{
put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "columns");
}
}));
}
}));
AtlasStructDef partitionDefinition = new AtlasStructDef("partition_struct_type", "partition_struct_type" + _description, "1.0", Arrays.asList(AtlasTypeUtil.createRequiredAttrDef("name", "string")));
AtlasAttributeDef[] attributeDefinitions = new AtlasAttributeDef[] { new AtlasAttributeDef("location", "string", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), new AtlasAttributeDef("inputFormat", "string", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), new AtlasAttributeDef("outputFormat", "string", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), new AtlasAttributeDef("compressed", "boolean", false, AtlasAttributeDef.Cardinality.SINGLE, 1, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), new AtlasAttributeDef("numBuckets", "int", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()) };
AtlasEntityDef storageDescClsDef = new AtlasEntityDef(STORAGE_DESC_TYPE, STORAGE_DESC_TYPE + _description, "1.0", Arrays.asList(attributeDefinitions), ImmutableSet.of(SUPER_TYPE_NAME));
AtlasAttributeDef[] partClsAttributes = new AtlasAttributeDef[] { new AtlasAttributeDef("values", "array<string>", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), new AtlasAttributeDef("table", TABLE_TYPE, true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), new AtlasAttributeDef("createTime", "long", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), new AtlasAttributeDef("lastAccessTime", "long", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), new AtlasAttributeDef("sd", STORAGE_DESC_TYPE, false, AtlasAttributeDef.Cardinality.SINGLE, 1, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), new AtlasAttributeDef("columns", String.format("array<%s>", COLUMN_TYPE), true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), new AtlasAttributeDef("parameters", String.format("map<%s,%s>", "string", "string"), true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()) };
AtlasEntityDef partClsDef = new AtlasEntityDef("partition_class_type", "partition_class_type" + _description, "1.0", Arrays.asList(partClsAttributes), ImmutableSet.of(SUPER_TYPE_NAME));
AtlasEntityDef processClsType = new AtlasEntityDef(PROCESS_TYPE, PROCESS_TYPE + _description, "1.0", Arrays.asList(new AtlasAttributeDef("outputs", "array<" + TABLE_TYPE + ">", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList())), ImmutableSet.<String>of());
AtlasEntityDef tableTypeDefinition = AtlasTypeUtil.createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), AtlasTypeUtil.createOptionalAttrDef("description", "string"), AtlasTypeUtil.createRequiredAttrDef("type", "string"), AtlasTypeUtil.createOptionalAttrDef("created", "date"), // enum
new AtlasAttributeDef("tableType", "tableType", false, AtlasAttributeDef.Cardinality.SINGLE, 1, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), // array of strings
new AtlasAttributeDef("columnNames", String.format("array<%s>", "string"), true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), // array of classes
new AtlasAttributeDef("columns", String.format("array<%s>", COLUMN_TYPE), true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, new ArrayList<AtlasStructDef.AtlasConstraintDef>() {
{
add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF));
}
}), // array of structs
new AtlasAttributeDef("partitions", String.format("array<%s>", "partition_struct_type"), true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), // map of primitives
new AtlasAttributeDef("parametersMap", String.format("map<%s,%s>", "string", "string"), true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), //map of classes -
new AtlasAttributeDef(COLUMNS_MAP, String.format("map<%s,%s>", "string", COLUMN_TYPE), true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, new ArrayList<AtlasStructDef.AtlasConstraintDef>() {
{
add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF));
}
}), //map of structs
new AtlasAttributeDef("partitionsMap", String.format("map<%s,%s>", "string", "partition_struct_type"), true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), // struct reference
new AtlasAttributeDef("serde1", "serdeType", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), new AtlasAttributeDef("serde2", "serdeType", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), // class reference
new AtlasAttributeDef("database", DATABASE_TYPE, false, AtlasAttributeDef.Cardinality.SINGLE, 1, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), //class reference as composite
new AtlasAttributeDef("databaseComposite", DATABASE_TYPE, true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, new ArrayList<AtlasStructDef.AtlasConstraintDef>() {
{
add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF));
}
}));
AtlasClassificationDef piiTypeDefinition = AtlasTypeUtil.createTraitTypeDef(PII, PII + _description, ImmutableSet.<String>of());
AtlasClassificationDef classificationTypeDefinition = AtlasTypeUtil.createTraitTypeDef(CLASSIFICATION, CLASSIFICATION + _description, ImmutableSet.<String>of(), AtlasTypeUtil.createRequiredAttrDef("tag", "string"));
AtlasClassificationDef fetlClassificationTypeDefinition = AtlasTypeUtil.createTraitTypeDef("fetl" + CLASSIFICATION, "fetl" + CLASSIFICATION + _description, ImmutableSet.of(CLASSIFICATION), AtlasTypeUtil.createRequiredAttrDef("tag", "string"));
AtlasClassificationDef phiTypeDefinition = AtlasTypeUtil.createTraitTypeDef(PHI, PHI + _description, ImmutableSet.<String>of(), AtlasTypeUtil.createRequiredAttrDef("stringAttr", "string"), AtlasTypeUtil.createRequiredAttrDef("booleanAttr", "boolean"), AtlasTypeUtil.createRequiredAttrDef("integerAttr", "int"));
return AtlasTypeUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition, partitionDefinition), ImmutableList.of(classificationTypeDefinition, fetlClassificationTypeDefinition, piiTypeDefinition, phiTypeDefinition), ImmutableList.of(superTypeDefinition, databaseTypeDefinition, columnsDefinition, tableTypeDefinition, storageDescClsDef, partClsDef, processClsType));
}
use of org.apache.atlas.model.typedef.AtlasClassificationDef in project incubator-atlas by apache.
the class TestUtilsV2 method simpleType.
public static AtlasTypesDef simpleType() {
AtlasEntityDef superTypeDefinition = AtlasTypeUtil.createClassTypeDef("h_type", ImmutableSet.<String>of(), AtlasTypeUtil.createOptionalAttrDef("attr", "string"));
AtlasStructDef structTypeDefinition = new AtlasStructDef("s_type", "structType", "1.0", Arrays.asList(AtlasTypeUtil.createRequiredAttrDef("name", "string")));
AtlasClassificationDef traitTypeDefinition = AtlasTypeUtil.createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of());
AtlasEnumDef enumTypeDefinition = new AtlasEnumDef("e_type", "enumType", "1.0", Arrays.asList(new AtlasEnumElementDef("ONE", "Element Description", 1)));
return AtlasTypeUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition), ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition));
}
use of org.apache.atlas.model.typedef.AtlasClassificationDef in project incubator-atlas by apache.
the class TestUtilsV2 method defineDeptEmployeeTypes.
/**
* Class Hierarchy is:
* Department(name : String, employees : Array[Person])
* Person(name : String, department : Department, manager : Manager)
* Manager(subordinates : Array[Person]) extends Person
* <p/>
* Persons can have SecurityClearance(level : Int) clearance.
*/
public static AtlasTypesDef defineDeptEmployeeTypes() {
String _description = "_description";
AtlasEnumDef orgLevelEnum = new AtlasEnumDef("OrgLevel", "OrgLevel" + _description, "1.0", Arrays.asList(new AtlasEnumElementDef("L1", "Element" + _description, 1), new AtlasEnumElementDef("L2", "Element" + _description, 2)));
AtlasStructDef addressDetails = createStructTypeDef("Address", "Address" + _description, AtlasTypeUtil.createRequiredAttrDef("street", "string"), AtlasTypeUtil.createRequiredAttrDef("city", "string"));
AtlasEntityDef deptTypeDef = AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE, "Department" + _description, ImmutableSet.<String>of(), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), new AtlasAttributeDef("employees", String.format("array<%s>", "Employee"), true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, new ArrayList<AtlasStructDef.AtlasConstraintDef>() {
{
add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF));
}
}));
AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person" + _description, ImmutableSet.<String>of(), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), AtlasTypeUtil.createOptionalAttrDef("address", "Address"), AtlasTypeUtil.createOptionalAttrDef("birthday", "date"), AtlasTypeUtil.createOptionalAttrDef("hasPets", "boolean"), AtlasTypeUtil.createOptionalAttrDef("numberOfCars", "byte"), AtlasTypeUtil.createOptionalAttrDef("houseNumber", "short"), AtlasTypeUtil.createOptionalAttrDef("carMileage", "int"), AtlasTypeUtil.createOptionalAttrDef("age", "float"), AtlasTypeUtil.createOptionalAttrDef("numberOfStarsEstimate", "biginteger"), AtlasTypeUtil.createOptionalAttrDef("approximationOfPi", "bigdecimal"));
AtlasEntityDef employeeTypeDef = AtlasTypeUtil.createClassTypeDef("Employee", "Employee" + _description, ImmutableSet.of("Person"), AtlasTypeUtil.createOptionalAttrDef("orgLevel", "OrgLevel"), new AtlasAttributeDef("department", "Department", false, AtlasAttributeDef.Cardinality.SINGLE, 1, 1, false, false, new ArrayList<AtlasConstraintDef>()), new AtlasAttributeDef("manager", "Manager", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, new ArrayList<AtlasConstraintDef>() {
{
add(new AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {
{
put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "subordinates");
}
}));
}
}), new AtlasAttributeDef("mentor", EMPLOYEE_TYPE, true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), AtlasTypeUtil.createOptionalAttrDef("shares", "long"), AtlasTypeUtil.createOptionalAttrDef("salary", "double"));
employeeTypeDef.getAttribute("department").addConstraint(new AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {
{
put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "employees");
}
}));
AtlasEntityDef managerTypeDef = AtlasTypeUtil.createClassTypeDef("Manager", "Manager" + _description, ImmutableSet.of("Employee"), new AtlasAttributeDef("subordinates", String.format("array<%s>", "Employee"), false, AtlasAttributeDef.Cardinality.SET, 1, 10, false, false, Collections.<AtlasConstraintDef>emptyList()));
AtlasClassificationDef securityClearanceTypeDef = AtlasTypeUtil.createTraitTypeDef("SecurityClearance", "SecurityClearance" + _description, ImmutableSet.<String>of(), AtlasTypeUtil.createRequiredAttrDef("level", "int"));
return new AtlasTypesDef(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails), ImmutableList.of(securityClearanceTypeDef), ImmutableList.of(deptTypeDef, personTypeDef, employeeTypeDef, managerTypeDef));
}
use of org.apache.atlas.model.typedef.AtlasClassificationDef in project incubator-atlas by apache.
the class TypeConverterUtil method toAtlasTypesDef.
public static AtlasTypesDef toAtlasTypesDef(String typeDefinition, AtlasTypeRegistry registry) throws AtlasBaseException {
AtlasTypesDef ret = new AtlasTypesDef();
try {
if (StringUtils.isEmpty(typeDefinition)) {
throw new AtlasBaseException(INVALID_TYPE_DEFINITION, typeDefinition);
}
TypesDef typesDef = TypesSerialization.fromJson(typeDefinition);
if (CollectionUtils.isNotEmpty(typesDef.enumTypesAsJavaList())) {
List<AtlasEnumDef> enumDefs = toAtlasEnumDefs(typesDef.enumTypesAsJavaList());
ret.setEnumDefs(enumDefs);
}
if (CollectionUtils.isNotEmpty(typesDef.structTypesAsJavaList())) {
List<AtlasStructDef> structDefs = toAtlasStructDefs(typesDef.structTypesAsJavaList());
ret.setStructDefs(structDefs);
}
if (CollectionUtils.isNotEmpty(typesDef.classTypesAsJavaList())) {
List<AtlasEntityDef> entityDefs = toAtlasEntityDefs(typesDef.classTypesAsJavaList(), registry);
ret.setEntityDefs(entityDefs);
}
if (CollectionUtils.isNotEmpty(typesDef.traitTypesAsJavaList())) {
List<AtlasClassificationDef> classificationDefs = toAtlasClassificationDefs(typesDef.traitTypesAsJavaList());
ret.setClassificationDefs(classificationDefs);
}
} catch (Exception e) {
LOG.error("Invalid type definition = {}", typeDefinition, e);
throw new AtlasBaseException(INVALID_TYPE_DEFINITION, typeDefinition);
}
return ret;
}
use of org.apache.atlas.model.typedef.AtlasClassificationDef in project incubator-atlas by apache.
the class AtlasTypeDefGraphStore method deleteTypesDef.
@Override
@GraphTransaction
public void deleteTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeDefGraphStore.deleteTypesDef(enums={}, structs={}, classfications={}, entities={})", CollectionUtils.size(typesDef.getEnumDefs()), CollectionUtils.size(typesDef.getStructDefs()), CollectionUtils.size(typesDef.getClassificationDefs()), CollectionUtils.size(typesDef.getEntityDefs()));
}
AtlasTransientTypeRegistry ttr = lockTypeRegistryAndReleasePostCommit();
AtlasEnumDefStore enumDefStore = getEnumDefStore(ttr);
AtlasStructDefStore structDefStore = getStructDefStore(ttr);
AtlasClassificationDefStore classifiDefStore = getClassificationDefStore(ttr);
AtlasEntityDefStore entityDefStore = getEntityDefStore(ttr);
List<Object> preDeleteStructDefs = new ArrayList<>();
List<Object> preDeleteClassifiDefs = new ArrayList<>();
List<Object> preDeleteEntityDefs = new ArrayList<>();
if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) {
for (AtlasStructDef structDef : typesDef.getStructDefs()) {
if (StringUtils.isNotBlank(structDef.getGuid())) {
preDeleteStructDefs.add(structDefStore.preDeleteByGuid(structDef.getGuid()));
} else {
preDeleteStructDefs.add(structDefStore.preDeleteByName(structDef.getName()));
}
}
}
if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) {
for (AtlasClassificationDef classifiDef : typesDef.getClassificationDefs()) {
if (StringUtils.isNotBlank(classifiDef.getGuid())) {
preDeleteClassifiDefs.add(classifiDefStore.preDeleteByGuid(classifiDef.getGuid()));
} else {
preDeleteClassifiDefs.add(classifiDefStore.preDeleteByName(classifiDef.getName()));
}
}
}
if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) {
for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) {
if (StringUtils.isNotBlank(entityDef.getGuid())) {
preDeleteEntityDefs.add(entityDefStore.preDeleteByGuid(entityDef.getGuid()));
} else {
preDeleteEntityDefs.add(entityDefStore.preDeleteByName(entityDef.getName()));
}
}
}
if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) {
int i = 0;
for (AtlasStructDef structDef : typesDef.getStructDefs()) {
if (StringUtils.isNotBlank(structDef.getGuid())) {
structDefStore.deleteByGuid(structDef.getGuid(), preDeleteStructDefs.get(i));
} else {
structDefStore.deleteByName(structDef.getName(), preDeleteStructDefs.get(i));
}
i++;
}
}
if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) {
int i = 0;
for (AtlasClassificationDef classifiDef : typesDef.getClassificationDefs()) {
if (StringUtils.isNotBlank(classifiDef.getGuid())) {
classifiDefStore.deleteByGuid(classifiDef.getGuid(), preDeleteClassifiDefs.get(i));
} else {
classifiDefStore.deleteByName(classifiDef.getName(), preDeleteClassifiDefs.get(i));
}
i++;
}
}
if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) {
int i = 0;
for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) {
if (StringUtils.isNotBlank(entityDef.getGuid())) {
entityDefStore.deleteByGuid(entityDef.getGuid(), preDeleteEntityDefs.get(i));
} else {
entityDefStore.deleteByName(entityDef.getName(), preDeleteEntityDefs.get(i));
}
i++;
}
}
if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) {
for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) {
if (StringUtils.isNotBlank(enumDef.getGuid())) {
enumDefStore.deleteByGuid(enumDef.getGuid());
} else {
enumDefStore.deleteByName(enumDef.getName());
}
}
}
// Remove all from
ttr.removeTypesDef(typesDef);
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasTypeDefGraphStore.deleteTypesDef(enums={}, structs={}, classfications={}, entities={})", CollectionUtils.size(typesDef.getEnumDefs()), CollectionUtils.size(typesDef.getStructDefs()), CollectionUtils.size(typesDef.getClassificationDefs()), CollectionUtils.size(typesDef.getEntityDefs()));
}
}
Aggregations