Examples with Referenceable org.apache.atlas.typesystem.Referenceable used on opensource projects

Example 81 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class HiveMetaStoreBridge method registerDatabase.

/**
     * Checks if db is already registered, else creates and registers db entity
     * @param databaseName
     * @return
     * @throws Exception
     */
private Referenceable registerDatabase(String databaseName) throws Exception {
    Referenceable dbRef = getDatabaseReference(clusterName, databaseName);
    Database db = hiveClient.getDatabase(databaseName);
    if (db != null) {
        if (dbRef == null) {
            dbRef = createDBInstance(db);
            dbRef = registerInstance(dbRef);
        } else {
            LOG.info("Database {} is already registered with id {}. Updating it.", databaseName, dbRef.getId().id);
            dbRef = createOrUpdateDBInstance(db, dbRef);
            updateInstance(dbRef);
        }
    }
    return dbRef;
}

Example 82 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class HiveMetaStoreBridge method createOrUpdateTableInstance.

private Referenceable createOrUpdateTableInstance(Referenceable dbReference, Referenceable tableReference, final Table hiveTable) throws AtlasHookException {
    LOG.info("Importing objects from {}.{}", hiveTable.getDbName(), hiveTable.getTableName());
    if (tableReference == null) {
        tableReference = new Referenceable(HiveDataTypes.HIVE_TABLE.getName());
    }
    String tableQualifiedName = getTableQualifiedName(clusterName, hiveTable);
    tableReference.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableQualifiedName);
    tableReference.set(AtlasClient.NAME, hiveTable.getTableName().toLowerCase());
    tableReference.set(AtlasClient.OWNER, hiveTable.getOwner());
    Date createDate = new Date();
    if (hiveTable.getTTable() != null) {
        try {
            createDate = getTableCreatedTime(hiveTable);
            LOG.debug("Setting create time to {} ", createDate);
            tableReference.set(CREATE_TIME, createDate);
        } catch (Exception ne) {
            LOG.error("Error while setting createTime for the table {} ", hiveTable.getCompleteName(), ne);
        }
    }
    Date lastAccessTime = createDate;
    if (hiveTable.getLastAccessTime() > 0) {
        lastAccessTime = new Date(hiveTable.getLastAccessTime() * MILLIS_CONVERT_FACTOR);
    }
    tableReference.set(LAST_ACCESS_TIME, lastAccessTime);
    tableReference.set("retention", hiveTable.getRetention());
    tableReference.set(COMMENT, hiveTable.getParameters().get(COMMENT));
    // add reference to the database
    tableReference.set(DB, dbReference);
    // add reference to the StorageDescriptor
    Referenceable sdReferenceable = fillStorageDesc(hiveTable.getSd(), tableQualifiedName, getStorageDescQFName(tableQualifiedName), tableReference.getId());
    tableReference.set(STORAGE_DESC, sdReferenceable);
    tableReference.set(PARAMETERS, hiveTable.getParameters());
    if (hiveTable.getViewOriginalText() != null) {
        tableReference.set("viewOriginalText", hiveTable.getViewOriginalText());
    }
    if (hiveTable.getViewExpandedText() != null) {
        tableReference.set("viewExpandedText", hiveTable.getViewExpandedText());
    }
    tableReference.set(TABLE_TYPE_ATTR, hiveTable.getTableType().name());
    tableReference.set("temporary", hiveTable.isTemporary());
    // add reference to the Partition Keys
    List<Referenceable> partKeys = getColumns(hiveTable.getPartitionKeys(), tableReference);
    tableReference.set("partitionKeys", partKeys);
    tableReference.set(COLUMNS, getColumns(hiveTable.getCols(), tableReference));
    return tableReference;
}

Example 83 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class HiveITBase method validateHDFSPaths.

protected void validateHDFSPaths(Referenceable processReference, String attributeName, String... testPaths) throws Exception {
    List<Id> hdfsPathRefs = (List<Id>) processReference.get(attributeName);
    for (String testPath : testPaths) {
        final Path path = new Path(testPath);
        final String testPathNormed = lower(path.toString());
        String hdfsPathId = assertHDFSPathIsRegistered(testPathNormed);
        Assert.assertEquals(hdfsPathRefs.get(0)._getId(), hdfsPathId);
        Referenceable hdfsPathRef = atlasClient.getEntity(hdfsPathId);
        Assert.assertEquals(hdfsPathRef.get("path"), testPathNormed);
        Assert.assertEquals(hdfsPathRef.get(NAME), Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
        Assert.assertEquals(hdfsPathRef.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), testPathNormed);
    }
}

Example 84 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class HiveHookIT method getSortedProcessDataSets.

private <T extends Entity> SortedMap<T, Referenceable> getSortedProcessDataSets(Set<T> inputTbls) {
    SortedMap<T, Referenceable> inputs = new TreeMap<>(entityComparator);
    if (inputTbls != null) {
        for (final T tbl : inputTbls) {
            Referenceable inputTableRef = new Referenceable(getDSTypeName(tbl), new HashMap<String, Object>() {

                {
                    put(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tbl.getName());
                }
            });
            inputs.put(tbl, inputTableRef);
        }
    }
    return inputs;
}

Example 85 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class HiveHookIT method validateProcess.

private Referenceable validateProcess(HiveHook.HiveEventContext event, Set<ReadEntity> inputTables, Set<WriteEntity> outputTables) throws Exception {
    String processId = assertProcessIsRegistered(event, inputTables, outputTables);
    Referenceable process = atlasClient.getEntity(processId);
    if (inputTables == null) {
        Assert.assertNull(process.get(INPUTS));
    } else {
        Assert.assertEquals(((List<Referenceable>) process.get(INPUTS)).size(), inputTables.size());
        validateInputTables(process, inputTables);
    }
    if (outputTables == null) {
        Assert.assertNull(process.get(OUTPUTS));
    } else {
        Assert.assertEquals(((List<Id>) process.get(OUTPUTS)).size(), outputTables.size());
        validateOutputTables(process, outputTables);
    }
    return process;
}