Examples with Referenceable org.apache.atlas.typesystem.Referenceable used on opensource projects

Example 71 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class HiveHook method renameTable.

private void renameTable(HiveMetaStoreBridge dgiBridge, HiveEventContext event) throws AtlasHookException {
    try {
        //crappy, no easy of getting new name
        assert event.getInputs() != null && event.getInputs().size() == 1;
        assert event.getOutputs() != null && event.getOutputs().size() > 0;
        //Update entity if not exists
        ReadEntity oldEntity = event.getInputs().iterator().next();
        Table oldTable = oldEntity.getTable();
        for (WriteEntity writeEntity : event.getOutputs()) {
            if (writeEntity.getType() == Entity.Type.TABLE) {
                Table newTable = writeEntity.getTable();
                //Hive sends with both old and new table names in the outputs which is weird. So skipping that with the below check
                if (!newTable.getDbName().equals(oldTable.getDbName()) || !newTable.getTableName().equals(oldTable.getTableName())) {
                    final String oldQualifiedName = HiveMetaStoreBridge.getTableQualifiedName(dgiBridge.getClusterName(), oldTable);
                    final String newQualifiedName = HiveMetaStoreBridge.getTableQualifiedName(dgiBridge.getClusterName(), newTable);
                    //Create/update old table entity - create entity with oldQFNme and old tableName if it doesnt exist. If exists, will update
                    //We always use the new entity while creating the table since some flags, attributes of the table are not set in inputEntity and Hive.getTable(oldTableName) also fails since the table doesnt exist in hive anymore
                    final LinkedHashMap<Type, Referenceable> tables = createOrUpdateEntities(dgiBridge, event, writeEntity, true);
                    Referenceable tableEntity = tables.get(Type.TABLE);
                    //Reset regular column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits
                    replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveMetaStoreBridge.COLUMNS), oldQualifiedName, newQualifiedName);
                    //Reset partition key column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits
                    replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveMetaStoreBridge.PART_COLS), oldQualifiedName, newQualifiedName);
                    //Reset SD QF Name to old Name and create a new partial notification request to replace old SD QFName to newName to retain any existing traits
                    replaceSDQFName(event, tableEntity, oldQualifiedName, newQualifiedName);
                    //Reset Table QF Name to old Name and create a new partial notification request to replace old Table QFName to newName
                    replaceTableQFName(event, oldTable, newTable, tableEntity, oldQualifiedName, newQualifiedName);
                }
            }
        }
    } catch (Exception e) {
        throw new AtlasHookException("HiveHook.renameTable() failed.", e);
    }
}

Example 72 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class HiveHook method processHiveEntity.

private <T extends Entity> void processHiveEntity(HiveMetaStoreBridge dgiBridge, HiveEventContext event, T entity, Set<String> dataSetsProcessed, SortedMap<T, Referenceable> dataSets, Set<Referenceable> entities) throws AtlasHookException {
    try {
        if (entity.getType() == Type.TABLE || entity.getType() == Type.PARTITION) {
            final String tblQFName = HiveMetaStoreBridge.getTableQualifiedName(dgiBridge.getClusterName(), entity.getTable());
            if (!dataSetsProcessed.contains(tblQFName)) {
                LinkedHashMap<Type, Referenceable> result = createOrUpdateEntities(dgiBridge, event, entity, false);
                dataSets.put(entity, result.get(Type.TABLE));
                dataSetsProcessed.add(tblQFName);
                entities.addAll(result.values());
            }
        } else if (entity.getType() == Type.DFS_DIR) {
            URI location = entity.getLocation();
            if (location != null) {
                final String pathUri = lower(new Path(location).toString());
                LOG.debug("Registering DFS Path {} ", pathUri);
                if (!dataSetsProcessed.contains(pathUri)) {
                    Referenceable hdfsPath = dgiBridge.fillHDFSDataSet(pathUri);
                    dataSets.put(entity, hdfsPath);
                    dataSetsProcessed.add(pathUri);
                    entities.add(hdfsPath);
                }
            }
        }
    } catch (Exception e) {
        throw new AtlasHookException("HiveHook.processHiveEntity() failed.", e);
    }
}

Example 73 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class HiveHook method renameColumn.

private void renameColumn(HiveMetaStoreBridge dgiBridge, HiveEventContext event) throws AtlasHookException {
    try {
        assert event.getInputs() != null && event.getInputs().size() == 1;
        assert event.getOutputs() != null && event.getOutputs().size() > 0;
        Table oldTable = event.getInputs().iterator().next().getTable();
        List<FieldSchema> oldColList = oldTable.getAllCols();
        Table outputTbl = event.getOutputs().iterator().next().getTable();
        outputTbl = dgiBridge.hiveClient.getTable(outputTbl.getDbName(), outputTbl.getTableName());
        List<FieldSchema> newColList = outputTbl.getAllCols();
        assert oldColList.size() == newColList.size();
        Pair<String, String> changedColNamePair = findChangedColNames(oldColList, newColList);
        String oldColName = changedColNamePair.getLeft();
        String newColName = changedColNamePair.getRight();
        for (WriteEntity writeEntity : event.getOutputs()) {
            if (writeEntity.getType() == Type.TABLE) {
                Table newTable = writeEntity.getTable();
                createOrUpdateEntities(dgiBridge, event, writeEntity, true, oldTable);
                final String newQualifiedTableName = HiveMetaStoreBridge.getTableQualifiedName(dgiBridge.getClusterName(), newTable);
                String oldColumnQFName = HiveMetaStoreBridge.getColumnQualifiedName(newQualifiedTableName, oldColName);
                String newColumnQFName = HiveMetaStoreBridge.getColumnQualifiedName(newQualifiedTableName, newColName);
                Referenceable newColEntity = new Referenceable(HiveDataTypes.HIVE_COLUMN.getName());
                newColEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, newColumnQFName);
                event.addMessage(new HookNotification.EntityPartialUpdateRequest(event.getUser(), HiveDataTypes.HIVE_COLUMN.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, oldColumnQFName, newColEntity));
            }
        }
        handleEventOutputs(dgiBridge, event, Type.TABLE);
    } catch (Exception e) {
        throw new AtlasHookException("HiveHook.renameColumn() failed.", e);
    }
}

Example 74 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class HiveHook method replaceColumnQFName.

private List<Referenceable> replaceColumnQFName(final HiveEventContext event, final List<Referenceable> cols, final String oldTableQFName, final String newTableQFName) {
    List<Referenceable> newColEntities = new ArrayList<>();
    for (Referenceable col : cols) {
        final String colName = (String) col.get(AtlasClient.NAME);
        String oldColumnQFName = HiveMetaStoreBridge.getColumnQualifiedName(oldTableQFName, colName);
        String newColumnQFName = HiveMetaStoreBridge.getColumnQualifiedName(newTableQFName, colName);
        col.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, oldColumnQFName);
        Referenceable newColEntity = new Referenceable(HiveDataTypes.HIVE_COLUMN.getName());
        ///Only QF Name changes
        newColEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, newColumnQFName);
        event.addMessage(new HookNotification.EntityPartialUpdateRequest(event.getUser(), HiveDataTypes.HIVE_COLUMN.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, oldColumnQFName, newColEntity));
        newColEntities.add(newColEntity);
    }
    return newColEntities;
}

Example 75 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class HiveMetaStoreBridge method getColumns.

public List<Referenceable> getColumns(List<FieldSchema> schemaList, Referenceable tableReference) throws AtlasHookException {
    List<Referenceable> colList = new ArrayList<>();
    int columnPosition = 0;
    for (FieldSchema fs : schemaList) {
        LOG.debug("Processing field {}", fs);
        Referenceable colReferenceable = new Referenceable(HiveDataTypes.HIVE_COLUMN.getName());
        colReferenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getColumnQualifiedName((String) tableReference.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), fs.getName()));
        colReferenceable.set(AtlasClient.NAME, fs.getName());
        colReferenceable.set(AtlasClient.OWNER, tableReference.get(AtlasClient.OWNER));
        colReferenceable.set("type", fs.getType());
        colReferenceable.set(POSITION, columnPosition++);
        colReferenceable.set(COMMENT, fs.getComment());
        colReferenceable.set(TABLE, tableReference.getId());
        colList.add(colReferenceable);
    }
    return colList;
}