Examples with AtlasObjectId org.apache.atlas.model.instance.AtlasObjectId used on opensource projects

Example 56 with AtlasObjectId

use of org.apache.atlas.model.instance.AtlasObjectId in project atlas by apache.

the class BaseImpalaEvent method toTableEntity.

protected AtlasEntity toTableEntity(AtlasObjectId dbId, ImpalaNode table, AtlasEntityExtInfo entityExtInfo) throws Exception {
    String tblQualifiedName = getQualifiedName(table);
    AtlasEntity ret = context.getEntity(tblQualifiedName);
    if (ret != null) {
        return ret;
    }
    // a table created in Impala still uses HIVE_TYPE_TABLE to allow both Impala and Hive operate
    // on the same table
    ret = new AtlasEntity(HIVE_TYPE_TABLE);
    // Impala hook should not send meta data entity to Atlas. set 'guid' to null - which will:
    // - result in this entity to be not included in 'referredEntities'
    // - cause Atlas server to resolve the entity by its qualifiedName
    // TODO: enable this once HMS hook is in. Disable this before that.
    ret.setGuid(null);
    long createTime = getTableCreateTime(table);
    long lastAccessTime = createTime;
    ret.setAttribute(ATTRIBUTE_DB, dbId);
    ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, tblQualifiedName);
    ret.setAttribute(ATTRIBUTE_NAME, table.getNodeName().toLowerCase());
    // just fake it. It should not be sent to Atlas once HMS hook is in
    ret.setAttribute(ATTRIBUTE_OWNER, getUserName());
    ret.setAttribute(ATTRIBUTE_CREATE_TIME, createTime);
    ret.setAttribute(ATTRIBUTE_LAST_ACCESS_TIME, lastAccessTime);
    AtlasObjectId tableId = getObjectId(ret);
    List<AtlasEntity> columns = getColumnEntities(tableId, table);
    if (entityExtInfo != null) {
        if (columns != null) {
            for (AtlasEntity column : columns) {
                entityExtInfo.addReferredEntity(column);
            }
        }
    }
    ret.setAttribute(ATTRIBUTE_COLUMNS, getObjectIds(columns));
    context.putEntity(tblQualifiedName, ret);
    return ret;
}

Example 57 with AtlasObjectId

use of org.apache.atlas.model.instance.AtlasObjectId in project atlas by apache.

the class ImpalaLineageToolIT method testCreateTableAsSelectFromFile.

/**
 * This tests
 * 1) ImpalaLineageTool can parse one lineage file that contains "create table as select" command lineage,
 *    there is table vertex with createTime.
 * 2) Lineage is sent to Atlas
 * 3) Atlas can get this lineage from Atlas
 */
@Test
public void testCreateTableAsSelectFromFile() throws Exception {
    String IMPALA = dir + "impalaCreateTableAsSelect.json";
    String IMPALA_WAL = dir + "WALimpala.wal";
    ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
    // create database and tables to simulate Impala behavior that Impala updates metadata
    // to HMS and HMSHook sends the metadata to Atlas, which has to happen before
    // Atlas can handle lineage notification
    String dbName = "db_3";
    createDatabase(dbName);
    String sourceTableName = "table_1";
    createTable(dbName, sourceTableName, "(id string, count int)", false);
    String targetTableName = "table_2";
    createTable(dbName, targetTableName, "(count int, id string)", false);
    // process lineage record, and send corresponding notification to Atlas
    String[] args = new String[] { "-d", "./", "-p", "impala" };
    ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
    toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA, IMPALA_WAL);
    // verify the process is saved in Atlas
    // the value is from info in IMPALA_4.
    String createTime = new Long(TABLE_CREATE_TIME * 1000).toString();
    String processQFName = dbName + "." + targetTableName + AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE + CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS + createTime;
    processQFName = processQFName.toLowerCase();
    String queryString = "create table " + dbName + "." + targetTableName + " as select count, id from " + dbName + "." + sourceTableName;
    AtlasEntity processEntity1 = validateProcess(processQFName, queryString);
    AtlasEntity processExecutionEntity1 = validateProcessExecution(processEntity1, queryString);
    AtlasObjectId process1 = toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(BaseImpalaEvent.ATTRIBUTE_PROCESS));
    Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
    Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
    String guid = assertTableIsRegistered(dbName, targetTableName);
    AtlasEntity entity = atlasClientV2.getEntityByGuid(guid).getEntity();
    List ddlQueries = (List) entity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
    assertNotNull(ddlQueries);
    assertEquals(ddlQueries.size(), 1);
}

Example 58 with AtlasObjectId

use of org.apache.atlas.model.instance.AtlasObjectId in project atlas by apache.

the class ImpalaLineageToolIT method testCreateTableAsSelectVertexIdNoTableNameFromFile.

/**
 * This tests
 * 1) ImpalaLineageTool can parse one lineage file that contains "create table as select" command lineage,
 *    there is table vertex with createTime. The target vertex's vertexId does not contain db name and table name
 * 2) Lineage is sent to Atlas
 * 3) Atlas can get this lineage from Atlas
 */
@Test
public void testCreateTableAsSelectVertexIdNoTableNameFromFile() throws Exception {
    String IMPALA = dir + "impalaCreateTableAsSelectVertexIdNoTableName.json";
    String IMPALA_WAL = dir + "WALimpala.wal";
    ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
    // create database and tables to simulate Impala behavior that Impala updates metadata
    // to HMS and HMSHook sends the metadata to Atlas, which has to happen before
    // Atlas can handle lineage notification
    String dbName = "sales_db";
    createDatabase(dbName);
    String sourceTableName = "sales_asia";
    createTable(dbName, sourceTableName, "(id string, name string)", false);
    String targetTableName = "sales_china";
    createTable(dbName, targetTableName, "(id string, name string)", false);
    // process lineage record, and send corresponding notification to Atlas
    String[] args = new String[] { "-d", "./", "-p", "impala" };
    ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
    toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA, IMPALA_WAL);
    // verify the process is saved in Atlas
    // the value is from info in IMPALA_4.
    String createTime = new Long((long) 1560885039 * 1000).toString();
    String processQFName = dbName + "." + targetTableName + AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE + CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS + createTime;
    processQFName = processQFName.toLowerCase();
    String queryString = "create table " + targetTableName + " as select * from " + sourceTableName;
    AtlasEntity processEntity1 = validateProcess(processQFName, queryString);
    AtlasEntity processExecutionEntity1 = validateProcessExecution(processEntity1, queryString);
    AtlasObjectId process1 = toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(BaseImpalaEvent.ATTRIBUTE_PROCESS));
    Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
    Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
    String guid = assertTableIsRegistered(dbName, targetTableName);
    AtlasEntity entity = atlasClientV2.getEntityByGuid(guid).getEntity();
    List ddlQueries = (List) entity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
    assertNotNull(ddlQueries);
    assertEquals(ddlQueries.size(), 1);
}

Example 59 with AtlasObjectId

use of org.apache.atlas.model.instance.AtlasObjectId in project atlas by apache.

the class ImpalaLineageToolIT method testCreateViewWithCommentSpacesFromFile.

/**
 * This tests is for create view query with extra comment and spaces added in between:
 * 1) ImpalaLineageTool can parse one lineage file that contains " create   view" command lineage
 * 2) Lineage is sent to Atlas
 * 3) Atlas can get this lineage from Atlas
 */
@Test
public void testCreateViewWithCommentSpacesFromFile() {
    // this file contains a single lineage record for "create view".
    // It has table vertex with createTime
    String IMPALA = dir + "impalaCreateViewWithCommentSpaces.json";
    String IMPALA_WAL = dir + "WALimpala.wal";
    List<ImpalaQuery> lineageList = new ArrayList<>();
    ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
    try {
        // create database and tables to simulate Impala behavior that Impala updates metadata
        // to HMS and HMSHook sends the metadata to Atlas, which has to happen before
        // Atlas can handle lineage notification
        String dbName = "db_8";
        createDatabase(dbName);
        String sourceTableName = "table_1";
        createTable(dbName, sourceTableName, "(id string, count int)", false);
        String targetTableName = "view_1";
        createTable(dbName, targetTableName, "(count int, id string)", false);
        // process lineage record, and send corresponding notification to Atlas
        String[] args = new String[] { "-d", "./", "-p", "impala" };
        ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
        toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA, IMPALA_WAL);
        // verify the process is saved in Atlas
        // the value is from info in IMPALA_3
        String createTime = new Long((long) (1554750072) * 1000).toString();
        String processQFName = "db_8.view_1" + AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE + CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS + createTime;
        processQFName = processQFName.toLowerCase();
        String queryString = " create   /* comment1 */ view db_8.view_1 as   select /* comment2 */ count, id from db_8.table_1";
        AtlasEntity processEntity1 = validateProcess(processQFName, queryString);
        AtlasEntity processExecutionEntity1 = validateProcessExecution(processEntity1, queryString);
        AtlasObjectId process1 = toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(BaseImpalaEvent.ATTRIBUTE_PROCESS));
        Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
        Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
        String guid = assertTableIsRegistered(dbName, targetTableName);
        AtlasEntity entity = atlasClientV2.getEntityByGuid(guid).getEntity();
        List ddlQueries = (List) entity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
        assertNotNull(ddlQueries);
        assertEquals(ddlQueries.size(), 1);
    } catch (Exception e) {
        System.out.print("Appending file error");
    }
}

Example 60 with AtlasObjectId

use of org.apache.atlas.model.instance.AtlasObjectId in project atlas by apache.

the class ImpalaLineageToolIT method testAlterViewAsSelectWithCommentSpacesFromFile.

/**
 * This tests is for extra comment and spaces present in alter view as select query
 * 1) ImpalaLineageTool can parse one lineage file that contains "alter view as select" command lineage,
 *    there is table vertex with createTime.
 * 2) Lineage is sent to Atlas
 * 3) Atlas can get this lineage from Atlas
 */
@Test
public void testAlterViewAsSelectWithCommentSpacesFromFile() throws Exception {
    String IMPALA = dir + "impalaAlterViewAsSelectWithCommentSpaces.json";
    String IMPALA_WAL = dir + "WALimpala.wal";
    ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
    // create database and tables to simulate Impala behavior that Impala updates metadata
    // to HMS and HMSHook sends the metadata to Atlas, which has to happen before
    // Atlas can handle lineage notification
    String dbName = "db_10";
    createDatabase(dbName);
    String sourceTableName = "table_1";
    createTable(dbName, sourceTableName, "(id string, count int)", false);
    String targetTableName = "view_1";
    createTable(dbName, targetTableName, "(count int, id string)", false);
    // process lineage record, and send corresponding notification to Atlas
    String[] args = new String[] { "-d", "./", "-p", "impala" };
    ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
    toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA, IMPALA_WAL);
    // verify the process is saved in Atlas
    // the value is from info in IMPALA_4.
    String createTime = new Long(TABLE_CREATE_TIME * 1000).toString();
    String processQFName = dbName + "." + targetTableName + AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE + CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS + createTime;
    processQFName = processQFName.toLowerCase();
    String queryString = "alter   /* comment1 */ view " + dbName + "." + targetTableName + " as   select /* comment1 */ count, id from " + dbName + "." + sourceTableName;
    AtlasEntity processEntity1 = validateProcess(processQFName, queryString);
    AtlasEntity processExecutionEntity1 = validateProcessExecution(processEntity1, queryString);
    AtlasObjectId process1 = toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(BaseImpalaEvent.ATTRIBUTE_PROCESS));
    Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
    Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
    String guid = assertTableIsRegistered(dbName, targetTableName);
    AtlasEntity entity = atlasClientV2.getEntityByGuid(guid).getEntity();
    List ddlQueries = (List) entity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
    assertNotNull(ddlQueries);
    assertEquals(ddlQueries.size(), 1);
}