Examples with Instance org.odpi.openmetadata.connector.sas.event.model.catalog.instance.Instance used on opensource projects

Example 1 with Instance

use of org.odpi.openmetadata.connector.sas.event.model.catalog.instance.Instance in project egeria-connector-sas-viya by odpi.

the class RepositoryEventMapper method processEvent.

/**
 * Method to pass an event received on topic.
 *
 * @param event inbound event
 */
@Override
public void processEvent(String event) {
    log.info("Processing event: {}", event);
    CatalogEventPayload eventPayload;
    try {
        eventPayload = this.mapper.readValue(event, CatalogEventPayload.class);
    } catch (IOException e) {
        log.warn("Could not parse event payload", e);
        return;
    }
    log.info("Received integration event for Catalog operation: {}", eventPayload.getOperation());
    // Convert event payload into SASCatalogObject that rest of application uses
    SASCatalogObject catalogObject = new SASCatalogObject();
    String type;
    if (eventPayload.getType().equals(CatalogEventPayload.TYPE_INSTANCE)) {
        Instance instance = eventPayload.getInstance();
        Definition definition = eventPayload.getDefinition();
        type = instance.getInstanceType();
        catalogObject.guid = instance.getId();
        catalogObject.defId = instance.getDefinitionId();
        catalogObject.addInstance(instance);
        catalogObject.addDefinition(definition);
    } else if (eventPayload.getType().equals(CatalogEventPayload.TYPE_DEFINITION)) {
        Definition definition = eventPayload.getDefinition();
        type = definition.getDefinitionType();
        catalogObject.guid = definition.getId();
        catalogObject.defId = definition.getId();
        catalogObject.addDefinition(definition);
    } else {
        log.warn("Invalid catalog object type: " + eventPayload.getType());
        return;
    }
    if (eventPayload.getOperation().startsWith(CREATE) && type.equals(CatalogType.ENTITY)) {
        processNewEntity(catalogObject);
    } else if (eventPayload.getOperation().startsWith(UPDATE) && type.equals(CatalogType.ENTITY)) {
        processUpdatedEntity(catalogObject);
    } else if (eventPayload.getOperation().startsWith(DELETE) && type.equals(CatalogType.ENTITY)) {
        processRemovedEntity(catalogObject);
    } else if (eventPayload.getOperation().startsWith(CREATE) && type.equals(CatalogType.RELATIONSHIP)) {
        processNewRelationship(catalogObject);
    } else if (eventPayload.getOperation().startsWith(UPDATE) && type.equals(CatalogType.RELATIONSHIP)) {
        processUpdatedRelationship(catalogObject);
    } else if (eventPayload.getOperation().startsWith(DELETE) && type.equals(CatalogType.RELATIONSHIP)) {
        processRemovedRelationship(catalogObject);
    } else {
        log.info("Event processing does not support operation, {}, for type, {}", eventPayload.getOperation(), type);
    }
}

Example 2 with Instance

use of org.odpi.openmetadata.connector.sas.event.model.catalog.instance.Instance in project egeria-connector-sas-viya by odpi.

the class MetadataCollection method getEntityDetailsFromCatalogResults.

/**
 * Retrieves a list of EntityDetail objects given a list of AtlasEntityHeader objects.
 *
 * @param instances the Atlas entities for which to retrieve details
 * @param entityTypeGUID the type of entity that was requested (or null for all)
 * @param userId the user through which to do the retrieval
 * @return {@code List<EntityDetail>}
 * @throws InvalidParameterException the guid is null.
 * @throws RepositoryErrorException there is a problem communicating with the metadata repository where
 *                                  the metadata collection is stored.
 * @throws UserNotAuthorizedException the userId is not permitted to perform this operation.
 */
private List<EntityDetail> getEntityDetailsFromCatalogResults(List<Instance> instances, String entityTypeGUID, String userId) throws InvalidParameterException, RepositoryErrorException, UserNotAuthorizedException {
    List<EntityDetail> entityDetails = new ArrayList<>();
    if (instances != null) {
        for (Instance instance : instances) {
            try {
                // TODO: See if we can do this without making another REST request
                EntityDetail detail = getEntityDetail(userId, instance.getId());
                if (detail != null) {
                    String typeName = detail.getType().getTypeDefName();
                    log.debug("getEntityDetailsFromCatalogResults: typeName {}", typeName);
                    try {
                        TypeDef typeDef = repositoryHelper.getTypeDef(repositoryName, "entityTypeGUID", entityTypeGUID, "getEntityDetailsFromAtlasResults");
                        if (repositoryHelper.isTypeOf(repositoryName, typeName, typeDef.getName())) {
                            entityDetails.add(detail);
                        }
                    } catch (TypeErrorException e) {
                        log.error("Unable to find any TypeDef for entityTypeGUID: {}", entityTypeGUID);
                    }
                } else {
                    log.error("Entity with GUID {} not known -- excluding from results.", instance.getId());
                }
            } catch (EntityNotKnownException e) {
                log.error("Entity with GUID {} not known -- excluding from results.", instance.getId());
            }
        }
    }
    return entityDetails;
}

Example 3 with Instance

use of org.odpi.openmetadata.connector.sas.event.model.catalog.instance.Instance in project egeria-connector-sas-viya by odpi.

the class MetadataCollection method findEntitiesByPropertyValue.

@Override
public List<EntityDetail> findEntitiesByPropertyValue(String userId, String entityTypeGUID, String searchCriteria, int fromEntityElement, List<InstanceStatus> limitResultsByStatus, List<String> limitResultsByClassification, Date asOfTime, String sequencingProperty, SequencingOrder sequencingOrder, int pageSize) throws InvalidParameterException, TypeErrorException, RepositoryErrorException, PropertyErrorException, PagingErrorException, FunctionNotSupportedException, UserNotAuthorizedException {
    final String methodName = "findEntitiesByPropertyValue";
    findEntitiesByPropertyValueParameterValidation(userId, entityTypeGUID, searchCriteria, fromEntityElement, limitResultsByStatus, limitResultsByClassification, asOfTime, sequencingProperty, sequencingOrder, pageSize);
    List<Instance> results = new ArrayList<Instance>();
    // Immediately throw unimplemented exception if trying to retrieve historical view
    if (asOfTime != null) {
        raiseFunctionNotSupportedException(ErrorCode.NO_HISTORY, methodName, repositoryName);
    }
    // Search criteria is not allowed to be empty for this method, so cannot be null or empty string.
    if (!searchCriteria.isEmpty()) {
        // Otherwise we need to do an OR-based search across all string properties in Atlas, using whatever the
        // regex of searchCriteria contains for each property
        // Add all textual properties of the provided entity as matchProperties,
        // for an OR-based search of their values
        Map<String, Map<String, String>> mappingsToSearch = new HashMap<>();
        if (entityTypeGUID != null) {
            // We are searching for a particular entity type, get the associated mappings
            mappingsToSearch = getMappingsToSearch(entityTypeGUID, userId);
        } else {
            // We are searching across all entity types, get all mappings
            // We will need to send the request only once, so we'll only use the first mapping
            mappingsToSearch = typeDefStore.getAllOmrsNameToCatalogNameMappings();
        }
        for (Map.Entry<String, Map<String, String>> entryToSearch : mappingsToSearch.entrySet()) {
            InstanceProperties matchProperties = new InstanceProperties();
            String omrsTypeName = entryToSearch.getKey();
            String omrsTypeGUID = typeDefStore.getTypeDefByName(omrsTypeName).getGUID();
            Map<String, TypeDefAttribute> typeDefAttributeMap = typeDefStore.getAllTypeDefAttributesForName(omrsTypeName);
            if (typeDefAttributeMap != null) {
                // This will look at all OMRS attributes, but buildAndRunDSLSearch (later) should limit to only those mapped to catalog
                for (Map.Entry<String, TypeDefAttribute> attributeEntry : typeDefAttributeMap.entrySet()) {
                    String attributeName = attributeEntry.getKey();
                    // Only supporting search by name value for now
                    if (attributeName.equals("qualifiedName")) {
                        TypeDefAttribute typeDefAttribute = attributeEntry.getValue();
                        // Only need to retain string-based attributes for the full text search
                        AttributeTypeDef attributeTypeDef = typeDefAttribute.getAttributeType();
                        if (attributeTypeDef.getCategory().equals(AttributeTypeDefCategory.PRIMITIVE)) {
                            PrimitiveDefCategory primitiveDefCategory = ((PrimitiveDef) attributeTypeDef).getPrimitiveDefCategory();
                            if (primitiveDefCategory.equals(PrimitiveDefCategory.OM_PRIMITIVE_TYPE_STRING) || primitiveDefCategory.equals(PrimitiveDefCategory.OM_PRIMITIVE_TYPE_BYTE) || primitiveDefCategory.equals(PrimitiveDefCategory.OM_PRIMITIVE_TYPE_CHAR)) {
                                matchProperties = repositoryHelper.addStringPropertyToInstance(repositoryName, matchProperties, attributeName, searchCriteria, methodName);
                            } else {
                                log.debug("Skipping inclusion of non-string attribute: {}", attributeName);
                            }
                        } else {
                            log.debug("Skipping inclusion of non-string attribute: {}", attributeName);
                        }
                    }
                }
            }
            List<Instance> innerResults = new ArrayList<Instance>();
            try {
                innerResults = buildAndRunDSLSearch(methodName, entityTypeGUID, omrsTypeGUID, limitResultsByClassification, matchProperties, MatchCriteria.ANY, fromEntityElement, limitResultsByStatus, sequencingProperty, sequencingOrder, pageSize, userId);
            } catch (Exception e) {
                log.error("Exception from findEntitiesByPropertyValue inner search for omrsTypeName {}: {}", omrsTypeName, e.getMessage());
            }
            if (innerResults != null) {
                results.addAll(innerResults);
            }
            // so can break out of the loop
            if (entityTypeGUID == null) {
                break;
            }
        }
    }
    List<EntityDetail> entityDetails = null;
    if (results != null) {
        entityDetails = sortAndLimitFinalResults(results, entityTypeGUID, fromEntityElement, sequencingProperty, sequencingOrder, pageSize, userId);
    }
    return (entityDetails == null || entityDetails.isEmpty()) ? null : entityDetails;
}

Example 4 with Instance

use of org.odpi.openmetadata.connector.sas.event.model.catalog.instance.Instance in project egeria-connector-sas-viya by odpi.

the class MetadataCollection method buildAndRunDSLSearch.

/**
 * Build an Atlas domain-specific language (DSL) query based on the provided parameters, and return its results.
 *
 * @param methodName the name of the calling method
 * @param entityTypeGUID unique identifier for the type of entity requested.  Null means any type of entity
 *                       (but could be slow so not recommended.
 * @param limitResultsByClassification list of classifications by which to limit the results.
 * @param matchProperties Optional list of entity properties to match (contains wildcards).
 * @param matchCriteria Enum defining how the match properties should be matched to the classifications in the repository.
 * @param fromEntityElement the starting element number of the entities to return.
 *                                This is used when retrieving elements
 *                                beyond the first page of results. Zero means start from the first element.
 * @param limitResultsByStatus By default, entities in all statuses are returned.  However, it is possible
 *                             to specify a list of statuses (eg ACTIVE) to restrict the results to.  Null means all
 *                             status values.
 * @param sequencingProperty String name of the entity property that is to be used to sequence the results.
 *                           Null means do not sequence on a property name (see SequencingOrder).
 * @param sequencingOrder Enum defining how the results should be ordered.
 * @param pageSize the maximum number of result entities that can be returned on this request.  Zero means
 *                 unrestricted return results size.
 * @param userId the user through which to run the search
 * @return {@code List<AtlasEntityHeader>}
 * @throws FunctionNotSupportedException when trying to search using a status that is not supported in Atlas
 * @throws RepositoryErrorException when there is some error running the search against Atlas
 */
private List<Instance> buildAndRunDSLSearch(String methodName, String incomingEntityTypeGUID, String entityTypeGUID, List<String> limitResultsByClassification, InstanceProperties matchProperties, MatchCriteria matchCriteria, int fromEntityElement, List<InstanceStatus> limitResultsByStatus, String sequencingProperty, SequencingOrder sequencingOrder, int pageSize, String userId) throws FunctionNotSupportedException, RepositoryErrorException {
    List<Instance> results = null;
    Map<String, String> queryParams = new HashMap<>();
    Map<String, String> attributeFilter = new HashMap<>();
    String catalogTypeName = "";
    String propertyMatchDelim = "and";
    if (matchCriteria != null && matchCriteria.equals(MatchCriteria.ANY)) {
        propertyMatchDelim = "or";
    }
    // Run multiple searches, if there are multiple types mapped to the OMRS type...
    Map<String, Map<String, String>> mappingsToSearch = getMappingsToSearch(entityTypeGUID, userId);
    for (Map.Entry<String, Map<String, String>> entryToSearch : mappingsToSearch.entrySet()) {
        String filter = "";
        String typeFilter = "";
        String propertyFilter = "";
        String omrsTypeName = entryToSearch.getKey();
        Map<String, String> catalogTypeNamesByPrefix = entryToSearch.getValue();
        int typeCount = catalogTypeNamesByPrefix.size();
        Map<String, InstancePropertyValue> properties = matchProperties.getInstanceProperties();
        Map<String, TypeDefAttribute> omrsAttrTypeDefs = typeDefStore.getAllTypeDefAttributesForName(omrsTypeName);
        for (Map.Entry<String, String> entry : catalogTypeNamesByPrefix.entrySet()) {
            propertyFilter = "";
            String prefix = entry.getKey();
            catalogTypeName = entry.getValue();
            Map<String, String> omrsPropertyMap = typeDefStore.getPropertyMappingsForOMRSTypeDef(omrsTypeName, prefix);
            // Add match properties, if requested
            if (matchProperties != null) {
                // across ALL entity types)//
                if (properties != null) {
                    for (Map.Entry<String, InstancePropertyValue> property : properties.entrySet()) {
                        String omrsPropertyName = property.getKey();
                        String catalogName = omrsPropertyMap.get(omrsPropertyName);
                        String catalogPropertyName = catalogName.substring(catalogName.indexOf(".") + 1);
                        InstancePropertyValue value = property.getValue();
                        if (catalogName.startsWith("attribute.")) {
                            attributeFilter.put(catalogPropertyName, value.valueAsString());
                        } else {
                            if (propertyFilter.isEmpty()) {
                                propertyFilter = String.format("contains(%s,\"%s\")", catalogPropertyName, value.valueAsString());
                            } else {
                                propertyFilter = String.format("%s(%s,contains(%s,\"%s\"))", propertyMatchDelim, propertyFilter, catalogPropertyName, value.valueAsString());
                            }
                        }
                    }
                }
            }
            String typeFilterStr = String.format("eq(type,\"%s\")", catalogTypeName);
            // Handle reference types differently since they all have a type of "reference"
            if (catalogTypeName.startsWith("reference.")) {
                // Extract reference name after "reference."
                String refName = catalogTypeName.substring(catalogTypeName.indexOf(".") + 1);
                typeFilterStr = "eq(type,\"reference\")";
                attributeFilter.put("referencedType", refName);
            }
            if (typeFilter.isEmpty() && propertyFilter.isEmpty()) {
                typeFilter = typeFilterStr;
            } else if (typeFilter.isEmpty() && !propertyFilter.isEmpty()) {
                typeFilter = String.format("and(%s,%s)", typeFilterStr, propertyFilter);
            } else if (!typeFilter.isEmpty() && propertyFilter.isEmpty()) {
                typeFilter = String.format("and(%s,%s)", typeFilter, typeFilter);
            } else {
                typeFilter = String.format("or(and(%s,%s), %s)", typeFilterStr, propertyFilter, typeFilter);
            }
            // If searching by property value across all entity types, we'll only need property filter
            if (incomingEntityTypeGUID == null && methodName.equals("findEntitiesByPropertyValue")) {
                queryParams.put("filter", propertyFilter);
            } else {
                queryParams.put("filter", typeFilter);
            }
        }
        // Add paging criteria, if requested
        if (pageSize > 0) {
            queryParams.put("limit", pageSize + "");
        }
        if (fromEntityElement > 0) {
            queryParams.put("offset", fromEntityElement + "");
        }
        try {
            results = repositoryConnector.getInstancesWithParams(queryParams, attributeFilter);
        } catch (Exception e) {
            raiseRepositoryErrorException(ErrorCode.INVALID_SEARCH, methodName, e, filter);
            log.error("Repository error exception for method {} and filter {} : {}", methodName, filter, e);
        }
    }
    return results;
}

Example 5 with Instance

use of org.odpi.openmetadata.connector.sas.event.model.catalog.instance.Instance in project egeria-connector-sas-viya by odpi.

the class SASCatalogRestClient method getInstancesWithFilter.

private List<Instance> getInstancesWithFilter(Map<String, String> params, Map<String, String> attributeFilter, int retries) throws Exception {
    if (retries > MAX_RETRIES) {
        throw new RuntimeException("Could not complete request after " + retries + " retries.");
    }
    List<Instance> instances = new ArrayList<>();
    URIBuilder builder = new URIBuilder(this.scheme + "://sas-catalog/catalog/instances");
    for (Map.Entry<String, String> param : params.entrySet()) {
        log.info("Param: " + param.getKey() + " : " + param.getValue());
        builder.addParameter(param.getKey(), param.getValue());
    }
    HttpGet httpGet = new HttpGet(builder.build());
    addAuthHeader(httpGet);
    // httpGet.addHeader("Accept", String.format("application/vnd.sas.metadata.instance.%s+json", type));
    try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
        log.info("Get Instances with filter (" + params.toString() + "): " + response.getStatusLine());
        if (response.getStatusLine().getStatusCode() == 401) {
            response.close();
            setAuthToken(username, password);
            return getInstancesWithFilter(params, attributeFilter, retries + 1);
        }
        HttpEntity entity = response.getEntity();
        InputStreamReader reader = new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8);
        Map results = new Gson().fromJson(reader, Map.class);
        List<Map<String, Object>> items = (List<Map<String, Object>>) results.get("items");
        for (Map<String, Object> instance : items) {
            Map<String, String> attributes = (Map<String, String>) instance.get("attributes");
            // Not currently possible to filter on attributes in Catalog, have to do it after the fact
            if (attributeFilter == null || matchesAttributes(attributes, attributeFilter)) {
                Instance inst = new Instance();
                inst.setId((String) instance.get("id"));
                instances.add(inst);
            }
        }
    }
    return instances;
}