Examples with Referenceable org.apache.atlas.typesystem.Referenceable used on opensource projects

Example 41 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class HiveHookIT method testCreateDatabase.

@Test
public void testCreateDatabase() throws Exception {
    String dbName = "db" + random();
    runCommand("create database " + dbName + " WITH DBPROPERTIES ('p1'='v1', 'p2'='v2')");
    String dbId = assertDatabaseIsRegistered(dbName);
    Referenceable definition = atlasClient.getEntity(dbId);
    Map params = (Map) definition.get(HiveMetaStoreBridge.PARAMETERS);
    Assert.assertNotNull(params);
    Assert.assertEquals(params.size(), 2);
    Assert.assertEquals(params.get("p1"), "v1");
    //There should be just one entity per dbname
    runCommand("drop database " + dbName);
    assertDBIsNotRegistered(dbName);
    runCommand("create database " + dbName);
    String dbid = assertDatabaseIsRegistered(dbName);
    //assert on qualified name
    Referenceable dbEntity = atlasClient.getEntity(dbid);
    Assert.assertEquals(dbEntity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), dbName.toLowerCase() + "@" + CLUSTER_NAME);
}

Example 42 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class SqoopHook method createSqoopProcessInstance.

private Referenceable createSqoopProcessInstance(Referenceable dbStoreRef, Referenceable hiveTableRef, SqoopJobDataPublisher.Data data, String clusterName) {
    Referenceable procRef = new Referenceable(SqoopDataTypes.SQOOP_PROCESS.getName());
    final String sqoopProcessName = getSqoopProcessName(data, clusterName);
    procRef.set(AtlasClient.NAME, sqoopProcessName);
    procRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, sqoopProcessName);
    procRef.set(SqoopHook.OPERATION, data.getOperation());
    if (isImportOperation(data)) {
        procRef.set(SqoopHook.INPUTS, dbStoreRef);
        procRef.set(SqoopHook.OUTPUTS, hiveTableRef);
    } else {
        procRef.set(SqoopHook.INPUTS, hiveTableRef);
        procRef.set(SqoopHook.OUTPUTS, dbStoreRef);
    }
    procRef.set(SqoopHook.USER, data.getUser());
    procRef.set(SqoopHook.START_TIME, new Date(data.getStartTime()));
    procRef.set(SqoopHook.END_TIME, new Date(data.getEndTime()));
    Map<String, String> sqoopOptionsMap = new HashMap<>();
    Properties options = data.getOptions();
    for (Object k : options.keySet()) {
        sqoopOptionsMap.put((String) k, (String) options.get(k));
    }
    procRef.set(SqoopHook.CMD_LINE_OPTS, sqoopOptionsMap);
    return procRef;
}

Example 43 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class SqoopHook method publish.

@Override
public void publish(SqoopJobDataPublisher.Data data) throws AtlasHookException {
    try {
        Configuration atlasProperties = ApplicationProperties.get();
        String clusterName = atlasProperties.getString(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
        Referenceable dbStoreRef = createDBStoreInstance(data);
        Referenceable dbRef = createHiveDatabaseInstance(clusterName, data.getHiveDB());
        Referenceable hiveTableRef = createHiveTableInstance(clusterName, dbRef, data.getHiveTable(), data.getHiveDB());
        Referenceable procRef = createSqoopProcessInstance(dbStoreRef, hiveTableRef, data, clusterName);
        int maxRetries = atlasProperties.getInt(HOOK_NUM_RETRIES, 3);
        HookNotification.HookNotificationMessage message = new HookNotification.EntityCreateRequest(AtlasHook.getUser(), dbStoreRef, dbRef, hiveTableRef, procRef);
        AtlasHook.notifyEntities(Arrays.asList(message), maxRetries);
    } catch (Exception e) {
        throw new AtlasHookException("SqoopHook.publish() failed.", e);
    }
}

Example 44 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class SqoopHook method createHiveTableInstance.

public Referenceable createHiveTableInstance(String clusterName, Referenceable dbRef, String tableName, String dbName) {
    Referenceable tableRef = new Referenceable(HiveDataTypes.HIVE_TABLE.getName());
    tableRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, HiveMetaStoreBridge.getTableQualifiedName(clusterName, dbName, tableName));
    tableRef.set(AtlasClient.NAME, tableName.toLowerCase());
    tableRef.set(HiveMetaStoreBridge.DB, dbRef);
    return tableRef;
}

Example 45 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class StormAtlasHook method createSpoutInstance.

private Referenceable createSpoutInstance(String spoutName, SpoutSpec stormSpout) throws IllegalAccessException {
    Referenceable spoutReferenceable = new Referenceable(StormDataTypes.STORM_SPOUT.getName());
    spoutReferenceable.set(AtlasClient.NAME, spoutName);
    Serializable instance = Utils.javaDeserialize(stormSpout.get_spout_object().get_serialized_java(), Serializable.class);
    spoutReferenceable.set("driverClass", instance.getClass().getName());
    Map<String, String> flatConfigMap = StormTopologyUtil.getFieldValues(instance, true, null);
    spoutReferenceable.set("conf", flatConfigMap);
    return spoutReferenceable;
}