Examples with AtlasLineageInfo org.apache.atlas.model.lineage.AtlasLineageInfo used on opensource projects

Example 6 with AtlasLineageInfo

use of org.apache.atlas.model.lineage.AtlasLineageInfo in project atlas by apache.

the class HiveHookIT method testAlterViewAsSelect.

@Test
public void testAlterViewAsSelect() throws Exception {
    // Create the view from table1
    String table1Name = createTable();
    String viewName = tableName();
    String query = "create view " + viewName + " as select * from " + table1Name;
    runCommand(query);
    String table1Id = assertTableIsRegistered(DEFAULT_DB, table1Name);
    assertProcessIsRegistered(constructEvent(query, HiveOperation.CREATEVIEW, getInputs(table1Name, Entity.Type.TABLE), getOutputs(viewName, Entity.Type.TABLE)));
    String viewId = assertTableIsRegistered(DEFAULT_DB, viewName);
    // Check lineage which includes table1
    String datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName);
    String tableId = assertTableIsRegistered(DEFAULT_DB, viewName);
    AtlasLineageInfo inputLineageInfo = atlasClientV2.getLineageInfo(tableId, AtlasLineageInfo.LineageDirection.INPUT, 0);
    Map<String, AtlasEntityHeader> entityMap = inputLineageInfo.getGuidEntityMap();
    assertTrue(entityMap.containsKey(viewId));
    assertTrue(entityMap.containsKey(table1Id));
    // Alter the view from table2
    String table2Name = createTable();
    query = "alter view " + viewName + " as select * from " + table2Name;
    runCommand(query);
    // Check if alter view process is reqistered
    assertProcessIsRegistered(constructEvent(query, HiveOperation.CREATEVIEW, getInputs(table2Name, Entity.Type.TABLE), getOutputs(viewName, Entity.Type.TABLE)));
    String table2Id = assertTableIsRegistered(DEFAULT_DB, table2Name);
    Assert.assertEquals(assertTableIsRegistered(DEFAULT_DB, viewName), viewId);
    datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName);
    String tableId1 = assertTableIsRegistered(DEFAULT_DB, viewName);
    AtlasLineageInfo inputLineageInfo1 = atlasClientV2.getLineageInfo(tableId1, AtlasLineageInfo.LineageDirection.INPUT, 0);
    Map<String, AtlasEntityHeader> entityMap1 = inputLineageInfo1.getGuidEntityMap();
    assertTrue(entityMap1.containsKey(viewId));
    // This is through the alter view process
    assertTrue(entityMap1.containsKey(table2Id));
    // This is through the Create view process
    assertTrue(entityMap1.containsKey(table1Id));
    // Outputs dont exist
    AtlasLineageInfo outputLineageInfo = atlasClientV2.getLineageInfo(tableId1, AtlasLineageInfo.LineageDirection.OUTPUT, 0);
    Map<String, AtlasEntityHeader> entityMap2 = outputLineageInfo.getGuidEntityMap();
    assertEquals(entityMap2.size(), 0);
}

Example 7 with AtlasLineageInfo

use of org.apache.atlas.model.lineage.AtlasLineageInfo in project atlas by apache.

the class HiveHookIT method testLineage.

@Test
public void testLineage() throws Exception {
    String table1 = createTable(false);
    String db2 = createDatabase();
    String table2 = tableName();
    String query = String.format("create table %s.%s as select * from %s", db2, table2, table1);
    runCommand(query);
    String table1Id = assertTableIsRegistered(DEFAULT_DB, table1);
    String table2Id = assertTableIsRegistered(db2, table2);
    AtlasLineageInfo inputLineage = atlasClientV2.getLineageInfo(table2Id, AtlasLineageInfo.LineageDirection.INPUT, 0);
    Map<String, AtlasEntityHeader> entityMap = inputLineage.getGuidEntityMap();
    assertTrue(entityMap.containsKey(table1Id));
    assertTrue(entityMap.containsKey(table2Id));
    AtlasLineageInfo inputLineage1 = atlasClientV2.getLineageInfo(table1Id, AtlasLineageInfo.LineageDirection.OUTPUT, 0);
    Map<String, AtlasEntityHeader> entityMap1 = inputLineage1.getGuidEntityMap();
    assertTrue(entityMap1.containsKey(table1Id));
    assertTrue(entityMap1.containsKey(table2Id));
}

Example 8 with AtlasLineageInfo

use of org.apache.atlas.model.lineage.AtlasLineageInfo in project atlas by apache.

the class EntityLineageJerseyResourceIT method testInputLineageInfo.

@Test
public void testInputLineageInfo() throws Exception {
    String tableId = atlasClientV1.getEntity(HIVE_TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, salesMonthlyTable).getId()._getId();
    MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
    queryParams.add(DIRECTION_PARAM, INPUT_DIRECTION);
    queryParams.add(DEPTH_PARAM, "5");
    ObjectNode response = atlasClientV1.callAPI(LINEAGE_V2_API, ObjectNode.class, queryParams, tableId);
    Assert.assertNotNull(response);
    System.out.println("input lineage info = " + response);
    AtlasLineageInfo inputLineageInfo = AtlasType.fromJson(response.toString(), AtlasLineageInfo.class);
    Map<String, AtlasEntityHeader> entities = inputLineageInfo.getGuidEntityMap();
    Assert.assertNotNull(entities);
    Set<AtlasLineageInfo.LineageRelation> relations = inputLineageInfo.getRelations();
    Assert.assertNotNull(relations);
    Assert.assertEquals(entities.size(), 6);
    Assert.assertEquals(relations.size(), 5);
    Assert.assertEquals(inputLineageInfo.getLineageDirection(), AtlasLineageInfo.LineageDirection.INPUT);
    Assert.assertEquals(inputLineageInfo.getLineageDepth(), 5);
    Assert.assertEquals(inputLineageInfo.getBaseEntityGuid(), tableId);
}

Example 9 with AtlasLineageInfo

use of org.apache.atlas.model.lineage.AtlasLineageInfo in project incubator-atlas by apache.

the class EntityLineageService method getBothLineageInfo.

private AtlasLineageInfo getBothLineageInfo(String guid, int depth) throws AtlasBaseException {
    AtlasLineageInfo inputLineage = getLineageInfo(guid, LineageDirection.INPUT, depth);
    AtlasLineageInfo outputLineage = getLineageInfo(guid, LineageDirection.OUTPUT, depth);
    AtlasLineageInfo ret = inputLineage;
    ret.getRelations().addAll(outputLineage.getRelations());
    ret.getGuidEntityMap().putAll(outputLineage.getGuidEntityMap());
    ret.setLineageDirection(LineageDirection.BOTH);
    return ret;
}

Example 10 with AtlasLineageInfo

use of org.apache.atlas.model.lineage.AtlasLineageInfo in project incubator-atlas by apache.

the class EntityLineageServiceTest method testNewLineageWithDelete.

@Test
public void testNewLineageWithDelete() throws Exception {
    TestUtils.skipForGremlin3EnabledGraphDb();
    String tableName = "table" + random();
    createTable(tableName, 3, true);
    String entityGuid = getEntityId(HIVE_TABLE_TYPE, "name", tableName);
    AtlasLineageInfo inputLineage = getInputLineageInfo(entityGuid, 5);
    assertNotNull(inputLineage);
    System.out.println("input lineage = " + inputLineage);
    Map<String, AtlasEntityHeader> entitiesInput = inputLineage.getGuidEntityMap();
    assertNotNull(entitiesInput);
    assertEquals(entitiesInput.size(), 3);
    Set<LineageRelation> relationsInput = inputLineage.getRelations();
    assertNotNull(relationsInput);
    assertEquals(relationsInput.size(), 2);
    AtlasEntityHeader tableEntityInput = entitiesInput.get(entityGuid);
    assertEquals(tableEntityInput.getStatus(), Status.ACTIVE);
    AtlasLineageInfo outputLineage = getOutputLineageInfo(entityGuid, 5);
    assertNotNull(outputLineage);
    System.out.println("output lineage = " + outputLineage);
    Map<String, AtlasEntityHeader> entitiesOutput = outputLineage.getGuidEntityMap();
    assertNotNull(entitiesOutput);
    assertEquals(entitiesOutput.size(), 3);
    Set<LineageRelation> relationsOutput = outputLineage.getRelations();
    assertNotNull(relationsOutput);
    assertEquals(relationsOutput.size(), 2);
    AtlasEntityHeader tableEntityOutput = entitiesOutput.get(entityGuid);
    assertEquals(tableEntityOutput.getStatus(), Status.ACTIVE);
    AtlasLineageInfo bothLineage = getBothLineageInfo(entityGuid, 5);
    assertNotNull(bothLineage);
    System.out.println("both lineage = " + bothLineage);
    Map<String, AtlasEntityHeader> entitiesBoth = bothLineage.getGuidEntityMap();
    assertNotNull(entitiesBoth);
    assertEquals(entitiesBoth.size(), 5);
    Set<LineageRelation> relationsBoth = bothLineage.getRelations();
    assertNotNull(relationsBoth);
    assertEquals(relationsBoth.size(), 4);
    AtlasEntityHeader tableEntityBoth = entitiesBoth.get(entityGuid);
    assertEquals(tableEntityBoth.getStatus(), Status.ACTIVE);
    // Delete the table entity. Lineage for entity returns the same results as before.
    // Lineage for table name throws EntityNotFoundException
    EntityResult deleteResult = repository.deleteEntities(Arrays.asList(entityGuid));
    assertTrue(deleteResult.getDeletedEntities().contains(entityGuid));
    inputLineage = getInputLineageInfo(entityGuid, 5);
    tableEntityInput = inputLineage.getGuidEntityMap().get(entityGuid);
    assertEquals(tableEntityInput.getStatus(), Status.DELETED);
    assertEquals(inputLineage.getGuidEntityMap().size(), 3);
    outputLineage = getOutputLineageInfo(entityGuid, 5);
    tableEntityOutput = outputLineage.getGuidEntityMap().get(entityGuid);
    assertEquals(tableEntityOutput.getStatus(), Status.DELETED);
    assertEquals(outputLineage.getGuidEntityMap().size(), 3);
    bothLineage = getBothLineageInfo(entityGuid, 5);
    tableEntityBoth = bothLineage.getGuidEntityMap().get(entityGuid);
    assertEquals(tableEntityBoth.getStatus(), Status.DELETED);
    assertEquals(bothLineage.getGuidEntityMap().size(), 5);
}