Search in sources :

Example 6 with QueryParams

use of org.apache.atlas.query.QueryParams in project incubator-atlas by apache.

the class DefaultMetadataServiceTest method testSpecialCharacters.

@Test
public //See GraphHelper.encodePropertyKey()
void testSpecialCharacters() throws Exception {
    //Verify that type can be created with reserved characters in typename, attribute name
    String strAttrName = randomStrWithReservedChars();
    String arrayAttrName = randomStrWithReservedChars();
    String mapAttrName = randomStrWithReservedChars();
    HierarchicalTypeDefinition<ClassType> typeDefinition = createClassTypeDef("test_type_" + RandomStringUtils.randomAlphanumeric(10), ImmutableSet.<String>of(), createOptionalAttrDef(strAttrName, DataTypes.STRING_TYPE), new AttributeDefinition(arrayAttrName, DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, false, null), new AttributeDefinition(mapAttrName, DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, false, null));
    metadataService.createType(TypesSerialization.toJson(typeDefinition, false));
    //verify that entity can be created with reserved characters in string value, array value and map key and value
    Referenceable entity = new Referenceable(typeDefinition.typeName);
    entity.set(strAttrName, randomStrWithReservedChars());
    entity.set(arrayAttrName, new ArrayList<String>() {

        {
            add(randomStrWithReservedChars());
        }
    });
    entity.set(mapAttrName, new HashMap<String, String>() {

        {
            put(randomStrWithReservedChars(), randomStrWithReservedChars());
        }
    });
    String id = createInstance(metadataService, entity);
    //Verify that get entity definition returns actual values with reserved characters
    Referenceable instance = InstanceSerialization.fromJsonReferenceable(metadataService.getEntityDefinitionJson(id), true);
    assertReferenceableEquals(instance, entity);
    //Verify that search with reserved characters works - for string attribute
    String query = String.format("`%s` where `%s` = '%s'", typeDefinition.typeName, strAttrName, entity.get(strAttrName));
    String responseJson = discoveryService.searchByDSL(query, new QueryParams(1, 0));
    JSONObject response = new JSONObject(responseJson);
    assertEquals(response.getJSONArray("rows").length(), 1);
}
Also used : Referenceable(org.apache.atlas.typesystem.Referenceable) JSONObject(org.codehaus.jettison.json.JSONObject) AttributeDefinition(org.apache.atlas.typesystem.types.AttributeDefinition) QueryParams(org.apache.atlas.query.QueryParams) ClassType(org.apache.atlas.typesystem.types.ClassType) Test(org.testng.annotations.Test) AfterTest(org.testng.annotations.AfterTest) BeforeTest(org.testng.annotations.BeforeTest)

Example 7 with QueryParams

use of org.apache.atlas.query.QueryParams in project incubator-atlas by apache.

the class EntityDiscoveryService method searchUsingBasicQuery.

@Override
public AtlasSearchResult searchUsingBasicQuery(String query, String typeName, String classification, String attrName, String attrValuePrefix, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException {
    AtlasSearchResult ret = new AtlasSearchResult(AtlasQueryType.BASIC);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Executing basic search query: {} with type: {} and classification: {}", query, typeName, classification);
    }
    final QueryParams params = validateSearchParams(limit, offset);
    Set<String> typeNames = null;
    Set<String> classificationNames = null;
    String attrQualifiedName = null;
    if (StringUtils.isNotEmpty(typeName)) {
        AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
        if (entityType == null) {
            throw new AtlasBaseException(UNKNOWN_TYPENAME, typeName);
        }
        typeNames = entityType.getTypeAndAllSubTypes();
        ret.setType(typeName);
    }
    if (StringUtils.isNotEmpty(classification)) {
        AtlasClassificationType classificationType = typeRegistry.getClassificationTypeByName(classification);
        if (classificationType == null) {
            throw new AtlasBaseException(CLASSIFICATION_NOT_FOUND, classification);
        }
        classificationNames = classificationType.getTypeAndAllSubTypes();
        ret.setClassification(classification);
    }
    boolean isAttributeSearch = StringUtils.isNotEmpty(attrName) || StringUtils.isNotEmpty(attrValuePrefix);
    boolean isGuidPrefixSearch = false;
    if (isAttributeSearch) {
        AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
        ret.setQueryType(AtlasQueryType.ATTRIBUTE);
        if (entityType != null) {
            AtlasAttribute attribute = null;
            if (StringUtils.isNotEmpty(attrName)) {
                attribute = entityType.getAttribute(attrName);
                if (attribute == null) {
                    throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, typeName);
                }
            } else {
                // if attrName is null|empty iterate defaultAttrNames to get attribute value
                final List<String> defaultAttrNames = new ArrayList<>(Arrays.asList("qualifiedName", "name"));
                Iterator<String> iter = defaultAttrNames.iterator();
                while (iter.hasNext() && attribute == null) {
                    attrName = iter.next();
                    attribute = entityType.getAttribute(attrName);
                }
            }
            if (attribute == null) {
                // for guid prefix search use gremlin and nullify query to avoid using fulltext
                // (guids cannot be searched in fulltext)
                isGuidPrefixSearch = true;
                query = null;
            } else {
                attrQualifiedName = attribute.getQualifiedName();
                String attrQuery = String.format("%s AND (%s *)", attrName, attrValuePrefix.replaceAll("\\.", " "));
                query = StringUtils.isEmpty(query) ? attrQuery : String.format("(%s) AND (%s)", query, attrQuery);
            }
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("Executing attribute search attrName: {} and attrValue: {}", attrName, attrValuePrefix);
        }
    }
    // results in a faster and accurate results than using CONTAINS/CONTAINS_PREFIX filter on entityText property
    if (StringUtils.isNotEmpty(query)) {
        final String idxQuery = getQueryForFullTextSearch(query, typeName, classification);
        final int startIdx = params.offset();
        final int resultSize = params.limit();
        int resultIdx = 0;
        for (int indexQueryOffset = 0; ; indexQueryOffset += getMaxResultSetSize()) {
            final Iterator<Result<?, ?>> qryResult = graph.indexQuery(Constants.FULLTEXT_INDEX, idxQuery, indexQueryOffset).vertices();
            if (LOG.isDebugEnabled()) {
                LOG.debug("indexQuery: query=" + idxQuery + "; offset=" + indexQueryOffset);
            }
            if (!qryResult.hasNext()) {
                break;
            }
            while (qryResult.hasNext()) {
                AtlasVertex<?, ?> vertex = qryResult.next().getVertex();
                String vertexTypeName = GraphHelper.getTypeName(vertex);
                // skip non-entity vertices
                if (StringUtils.isEmpty(vertexTypeName) || StringUtils.isEmpty(GraphHelper.getGuid(vertex))) {
                    continue;
                }
                if (typeNames != null && !typeNames.contains(vertexTypeName)) {
                    continue;
                }
                if (classificationNames != null) {
                    List<String> traitNames = GraphHelper.getTraitNames(vertex);
                    if (CollectionUtils.isEmpty(traitNames) || !CollectionUtils.containsAny(classificationNames, traitNames)) {
                        continue;
                    }
                }
                if (isAttributeSearch) {
                    String vertexAttrValue = vertex.getProperty(attrQualifiedName, String.class);
                    if (StringUtils.isNotEmpty(vertexAttrValue) && !vertexAttrValue.startsWith(attrValuePrefix)) {
                        continue;
                    }
                }
                if (skipDeletedEntities(excludeDeletedEntities, vertex)) {
                    continue;
                }
                resultIdx++;
                if (resultIdx <= startIdx) {
                    continue;
                }
                AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex);
                ret.addEntity(header);
                if (ret.getEntities().size() == resultSize) {
                    break;
                }
            }
            if (ret.getEntities() != null && ret.getEntities().size() == resultSize) {
                break;
            }
        }
    } else {
        final Map<String, Object> bindings = new HashMap<>();
        String basicQuery = "g.V()";
        if (classificationNames != null) {
            bindings.put("traitNames", classificationNames);
            basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.BASIC_SEARCH_CLASSIFICATION_FILTER);
        }
        if (typeNames != null) {
            bindings.put("typeNames", typeNames);
            basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER);
        }
        if (excludeDeletedEntities) {
            bindings.put("state", Status.ACTIVE.toString());
            basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.BASIC_SEARCH_STATE_FILTER);
        }
        if (isGuidPrefixSearch) {
            bindings.put("guid", attrValuePrefix + ".*");
            basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.GUID_PREFIX_FILTER);
        }
        bindings.put("startIdx", params.offset());
        bindings.put("endIdx", params.offset() + params.limit());
        basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.TO_RANGE_LIST);
        ScriptEngine scriptEngine = graph.getGremlinScriptEngine();
        try {
            Object result = graph.executeGremlinScript(scriptEngine, bindings, basicQuery, false);
            if (result instanceof List && CollectionUtils.isNotEmpty((List) result)) {
                List queryResult = (List) result;
                Object firstElement = queryResult.get(0);
                if (firstElement instanceof AtlasVertex) {
                    for (Object element : queryResult) {
                        if (element instanceof AtlasVertex) {
                            ret.addEntity(entityRetriever.toAtlasEntityHeader((AtlasVertex) element));
                        } else {
                            LOG.warn("searchUsingBasicQuery({}): expected an AtlasVertex; found unexpected entry in result {}", basicQuery, element);
                        }
                    }
                }
            }
        } catch (ScriptException e) {
            throw new AtlasBaseException(DISCOVERY_QUERY_FAILED, basicQuery);
        } finally {
            graph.releaseGremlinScriptEngine(scriptEngine);
        }
    }
    return ret;
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) AtlasClassificationType(org.apache.atlas.type.AtlasClassificationType) ScriptEngine(javax.script.ScriptEngine) AtlasSearchResult(org.apache.atlas.model.discovery.AtlasSearchResult) AtlasFullTextResult(org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult) AtlasSearchResult(org.apache.atlas.model.discovery.AtlasSearchResult) Result(org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result) AttributeSearchResult(org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult) AtlasAttribute(org.apache.atlas.type.AtlasStructType.AtlasAttribute) ScriptException(javax.script.ScriptException) AtlasBaseException(org.apache.atlas.exception.AtlasBaseException) AtlasVertex(org.apache.atlas.repository.graphdb.AtlasVertex) AtlasEntityHeader(org.apache.atlas.model.instance.AtlasEntityHeader) QueryParams(org.apache.atlas.query.QueryParams) List(java.util.List) ArrayList(java.util.ArrayList) AtlasEntityType(org.apache.atlas.type.AtlasEntityType)

Example 8 with QueryParams

use of org.apache.atlas.query.QueryParams in project incubator-atlas by apache.

the class EntityDiscoveryService method searchUsingFullTextQuery.

@Override
public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException {
    AtlasSearchResult ret = new AtlasSearchResult(fullTextQuery, AtlasQueryType.FULL_TEXT);
    QueryParams params = validateSearchParams(limit, offset);
    AtlasIndexQuery idxQuery = toAtlasIndexQuery(fullTextQuery);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Executing Full text query: {}", fullTextQuery);
    }
    ret.setFullTextResult(getIndexQueryResults(idxQuery, params, excludeDeletedEntities));
    return ret;
}
Also used : QueryParams(org.apache.atlas.query.QueryParams) AtlasIndexQuery(org.apache.atlas.repository.graphdb.AtlasIndexQuery) AtlasSearchResult(org.apache.atlas.model.discovery.AtlasSearchResult)

Example 9 with QueryParams

use of org.apache.atlas.query.QueryParams in project incubator-atlas by apache.

the class DataSetLineageServiceTest method testSearchByDSLQueries.

@Test(enabled = false)
public void testSearchByDSLQueries(String dslQuery) throws Exception {
    System.out.println("Executing dslQuery = " + dslQuery);
    String jsonResults = discoveryService.searchByDSL(dslQuery, new QueryParams(100, 0));
    assertNotNull(jsonResults);
    JSONObject results = new JSONObject(jsonResults);
    Assert.assertEquals(results.length(), 3);
    System.out.println("results = " + results);
    Object query = results.get("query");
    assertNotNull(query);
    JSONObject dataType = results.getJSONObject("dataType");
    assertNotNull(dataType);
    String typeName = dataType.getString("typeName");
    assertNotNull(typeName);
    JSONArray rows = results.getJSONArray("rows");
    assertNotNull(rows);
    // some queries may not have any results
    Assert.assertTrue(rows.length() >= 0);
    System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
}
Also used : JSONObject(org.codehaus.jettison.json.JSONObject) JSONArray(org.codehaus.jettison.json.JSONArray) QueryParams(org.apache.atlas.query.QueryParams) JSONObject(org.codehaus.jettison.json.JSONObject) Test(org.testng.annotations.Test) BaseRepositoryTest(org.apache.atlas.BaseRepositoryTest)

Example 10 with QueryParams

use of org.apache.atlas.query.QueryParams in project incubator-atlas by apache.

the class GraphBackedDiscoveryServiceTest method runQuery.

public void runQuery(String dslQuery, Integer expectedNumRows, int limitParam, int offsetParam) throws Exception {
    System.out.println("Executing dslQuery = " + dslQuery);
    String jsonResults = discoveryService.searchByDSL(dslQuery, new QueryParams(limitParam, offsetParam));
    assertNotNull(jsonResults);
    JSONObject results = new JSONObject(jsonResults);
    assertEquals(results.length(), 3);
    System.out.println("results = " + results);
    Object query = results.get("query");
    assertNotNull(query);
    JSONObject dataType = results.getJSONObject("dataType");
    assertNotNull(dataType);
    String typeName = dataType.getString("typeName");
    assertNotNull(typeName);
    JSONArray rows = results.getJSONArray("rows");
    assertNotNull(rows);
    // some queries may not have any results
    assertEquals(rows.length(), expectedNumRows.intValue(), "query [" + dslQuery + "] returned [" + rows.length() + "] rows.  Expected " + expectedNumRows + " rows.");
    System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
}
Also used : JSONObject(org.codehaus.jettison.json.JSONObject) JSONArray(org.codehaus.jettison.json.JSONArray) QueryParams(org.apache.atlas.query.QueryParams) JSONObject(org.codehaus.jettison.json.JSONObject)

Aggregations

QueryParams (org.apache.atlas.query.QueryParams)15 Test (org.testng.annotations.Test)8 JSONObject (org.codehaus.jettison.json.JSONObject)6 JSONArray (org.codehaus.jettison.json.JSONArray)4 ArrayList (java.util.ArrayList)2 Consumes (javax.ws.rs.Consumes)2 GET (javax.ws.rs.GET)2 Path (javax.ws.rs.Path)2 Produces (javax.ws.rs.Produces)2 WebApplicationException (javax.ws.rs.WebApplicationException)2 DiscoveryException (org.apache.atlas.discovery.DiscoveryException)2 AtlasBaseException (org.apache.atlas.exception.AtlasBaseException)2 AtlasSearchResult (org.apache.atlas.model.discovery.AtlasSearchResult)2 Referenceable (org.apache.atlas.typesystem.Referenceable)2 AttributeDefinition (org.apache.atlas.typesystem.types.AttributeDefinition)2 ClassType (org.apache.atlas.typesystem.types.ClassType)2 AtlasPerfTracer (org.apache.atlas.utils.AtlasPerfTracer)2 HashMap (java.util.HashMap)1 List (java.util.List)1 Callable (java.util.concurrent.Callable)1