Examples with Referenceable org.apache.atlas.typesystem.Referenceable used on opensource projects

Example 6 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class HiveMetaStoreBridge method getColumns.

public List<Referenceable> getColumns(List<FieldSchema> schemaList, Referenceable tableReference) throws AtlasHookException {
    List<Referenceable> colList = new ArrayList<>();
    int columnPosition = 0;
    for (FieldSchema fs : schemaList) {
        LOG.debug("Processing field {}", fs);
        Referenceable colReferenceable = new Referenceable(HiveDataTypes.HIVE_COLUMN.getName());
        colReferenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getColumnQualifiedName((String) tableReference.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), fs.getName()));
        colReferenceable.set(AtlasClient.NAME, fs.getName());
        colReferenceable.set(AtlasClient.OWNER, tableReference.get(AtlasClient.OWNER));
        colReferenceable.set("type", fs.getType());
        colReferenceable.set(POSITION, columnPosition++);
        colReferenceable.set(COMMENT, fs.getComment());
        colReferenceable.set(TABLE, tableReference.getId());
        colList.add(colReferenceable);
    }
    return colList;
}

Example 7 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class HiveMetaStoreBridge method fillStorageDesc.

public Referenceable fillStorageDesc(StorageDescriptor storageDesc, String tableQualifiedName, String sdQualifiedName, Id tableId) throws AtlasHookException {
    LOG.debug("Filling storage descriptor information for {}", storageDesc);
    Referenceable sdReferenceable = new Referenceable(HiveDataTypes.HIVE_STORAGEDESC.getName());
    sdReferenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, sdQualifiedName);
    SerDeInfo serdeInfo = storageDesc.getSerdeInfo();
    LOG.debug("serdeInfo = {}", serdeInfo);
    // SkewedInfo skewedInfo = storageDesc.getSkewedInfo();
    String serdeInfoName = HiveDataTypes.HIVE_SERDE.getName();
    Struct serdeInfoStruct = new Struct(serdeInfoName);
    serdeInfoStruct.set(AtlasClient.NAME, serdeInfo.getName());
    serdeInfoStruct.set("serializationLib", serdeInfo.getSerializationLib());
    serdeInfoStruct.set(PARAMETERS, serdeInfo.getParameters());
    sdReferenceable.set("serdeInfo", serdeInfoStruct);
    sdReferenceable.set(STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets());
    sdReferenceable.set(STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories());
    List<Struct> sortColsStruct = new ArrayList<>();
    for (Order sortcol : storageDesc.getSortCols()) {
        String hiveOrderName = HiveDataTypes.HIVE_ORDER.getName();
        Struct colStruct = new Struct(hiveOrderName);
        colStruct.set("col", sortcol.getCol());
        colStruct.set("order", sortcol.getOrder());
        sortColsStruct.add(colStruct);
    }
    if (sortColsStruct.size() > 0) {
        sdReferenceable.set("sortCols", sortColsStruct);
    }
    sdReferenceable.set(LOCATION, storageDesc.getLocation());
    sdReferenceable.set("inputFormat", storageDesc.getInputFormat());
    sdReferenceable.set("outputFormat", storageDesc.getOutputFormat());
    sdReferenceable.set("compressed", storageDesc.isCompressed());
    if (storageDesc.getBucketCols().size() > 0) {
        sdReferenceable.set("bucketCols", storageDesc.getBucketCols());
    }
    sdReferenceable.set(PARAMETERS, storageDesc.getParameters());
    sdReferenceable.set("storedAsSubDirectories", storageDesc.isStoredAsSubDirectories());
    sdReferenceable.set(TABLE, tableId);
    return sdReferenceable;
}

Example 8 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class FalconHookIT method getHDFSFeed.

private TypeUtils.Pair<String, Feed> getHDFSFeed(String feedResource, String clusterName) throws Exception {
    Feed feed = loadEntity(EntityType.FEED, feedResource, "feed" + random());
    org.apache.falcon.entity.v0.feed.Cluster feedCluster = feed.getClusters().getClusters().get(0);
    feedCluster.setName(clusterName);
    STORE.publish(EntityType.FEED, feed);
    String feedId = assertFeedIsRegistered(feed, clusterName);
    assertFeedAttributes(feedId);
    String processId = assertEntityIsRegistered(FalconDataTypes.FALCON_FEED_CREATION.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, FalconBridge.getFeedQualifiedName(feed.getName(), clusterName));
    Referenceable processEntity = atlasClient.getEntity(processId);
    assertEquals(((List<Id>) processEntity.get("outputs")).get(0).getId()._getId(), feedId);
    String inputId = ((List<Id>) processEntity.get("inputs")).get(0).getId()._getId();
    Referenceable pathEntity = atlasClient.getEntity(inputId);
    assertEquals(pathEntity.getTypeName(), HiveMetaStoreBridge.HDFS_PATH);
    List<Location> locations = FeedHelper.getLocations(feedCluster, feed);
    Location dataLocation = FileSystemStorage.getLocation(locations, LocationType.DATA);
    assertEquals(pathEntity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), FalconBridge.normalize(dataLocation.getPath()));
    return TypeUtils.Pair.of(feedId, feed);
}

Example 9 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class FalconHookIT method testCreateProcess.

@Test
public void testCreateProcess() throws Exception {
    Cluster cluster = loadEntity(EntityType.CLUSTER, CLUSTER_RESOURCE, "cluster" + random());
    STORE.publish(EntityType.CLUSTER, cluster);
    assertClusterIsRegistered(cluster);
    Feed infeed = getTableFeed(FEED_RESOURCE, cluster.getName(), null);
    String infeedId = atlasClient.getEntity(FalconDataTypes.FALCON_FEED.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, FalconBridge.getFeedQualifiedName(infeed.getName(), cluster.getName())).getId()._getId();
    Feed outfeed = getTableFeed(FEED_RESOURCE, cluster.getName());
    String outFeedId = atlasClient.getEntity(FalconDataTypes.FALCON_FEED.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, FalconBridge.getFeedQualifiedName(outfeed.getName(), cluster.getName())).getId()._getId();
    Process process = loadEntity(EntityType.PROCESS, PROCESS_RESOURCE, "process" + random());
    process.getClusters().getClusters().get(0).setName(cluster.getName());
    process.getInputs().getInputs().get(0).setFeed(infeed.getName());
    process.getOutputs().getOutputs().get(0).setFeed(outfeed.getName());
    STORE.publish(EntityType.PROCESS, process);
    String pid = assertProcessIsRegistered(process, cluster.getName());
    Referenceable processEntity = atlasClient.getEntity(pid);
    assertNotNull(processEntity);
    assertEquals(processEntity.get(AtlasClient.NAME), process.getName());
    assertEquals(((List<Id>) processEntity.get("inputs")).get(0)._getId(), infeedId);
    assertEquals(((List<Id>) processEntity.get("outputs")).get(0)._getId(), outFeedId);
}

Example 10 with Referenceable

use of org.apache.atlas.typesystem.Referenceable in project incubator-atlas by apache.

the class FalconHookIT method assertEntityIsRegistered.

private String assertEntityIsRegistered(final String typeName, final String property, final String value) throws Exception {
    waitFor(80000, new Predicate() {

        @Override
        public void evaluate() throws Exception {
            Referenceable entity = atlasClient.getEntity(typeName, property, value);
            assertNotNull(entity);
        }
    });
    Referenceable entity = atlasClient.getEntity(typeName, property, value);
    return entity.getId()._getId();
}