Examples with ImportInfo org.apache.atlas.bulkimport.BulkImportResponse.ImportInfo used on opensource projects

Example 1 with ImportInfo

use of org.apache.atlas.bulkimport.BulkImportResponse.ImportInfo in project atlas by apache.

the class AtlasEntityStoreV2 method bulkCreateOrUpdateBusinessAttributes.

@Override
@GraphTransaction
public BulkImportResponse bulkCreateOrUpdateBusinessAttributes(InputStream inputStream, String fileName) throws AtlasBaseException {
    BulkImportResponse ret = new BulkImportResponse();
    if (StringUtils.isBlank(fileName)) {
        throw new AtlasBaseException(AtlasErrorCode.FILE_NAME_NOT_FOUND, fileName);
    }
    List<String[]> fileData = FileUtils.readFileData(fileName, inputStream);
    Map<String, AtlasEntity> attributesToAssociate = getBusinessMetadataDefList(fileData, ret);
    for (AtlasEntity entity : attributesToAssociate.values()) {
        Map<String, Map<String, Object>> businessAttributes = entity.getBusinessAttributes();
        String guid = entity.getGuid();
        try {
            addOrUpdateBusinessAttributes(guid, businessAttributes, true);
            ret.addToSuccessImportInfoList(new ImportInfo(guid, businessAttributes.toString()));
        } catch (Exception e) {
            LOG.error("Error occurred while updating BusinessMetadata Attributes for Entity " + guid);
            ret.addToFailedImportInfoList(new ImportInfo(guid, businessAttributes.toString(), FAILED, e.getMessage()));
        }
    }
    return ret;
}

Example 2 with ImportInfo

use of org.apache.atlas.bulkimport.BulkImportResponse.ImportInfo in project atlas by apache.

the class AtlasEntityStoreV2 method getBusinessMetadataDefList.

private Map<String, AtlasEntity> getBusinessMetadataDefList(List<String[]> fileData, BulkImportResponse bulkImportResponse) throws AtlasBaseException {
    Map<String, AtlasEntity> ret = new HashMap<>();
    Map<String, AtlasVertex> vertexCache = new HashMap<>();
    List<String> failedMsgList = new ArrayList<>();
    for (int lineIndex = 0; lineIndex < fileData.size(); lineIndex++) {
        String[] record = fileData.get(lineIndex);
        int lineIndexToLog = lineIndex + 2;
        boolean missingFields = record.length < FileUtils.UNIQUE_ATTR_NAME_COLUMN_INDEX || StringUtils.isBlank(record[FileUtils.TYPENAME_COLUMN_INDEX]) || StringUtils.isBlank(record[FileUtils.UNIQUE_ATTR_VALUE_COLUMN_INDEX]) || StringUtils.isBlank(record[FileUtils.BM_ATTR_NAME_COLUMN_INDEX]) || StringUtils.isBlank(record[FileUtils.BM_ATTR_VALUE_COLUMN_INDEX]);
        if (missingFields) {
            failedMsgList.add("Line #" + lineIndexToLog + ": missing fields. " + Arrays.toString(record));
            continue;
        }
        String typeName = record[FileUtils.TYPENAME_COLUMN_INDEX];
        AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
        if (entityType == null) {
            failedMsgList.add("Line #" + lineIndexToLog + ": invalid entity-type '" + typeName + "'");
            continue;
        }
        String uniqueAttrValue = record[FileUtils.UNIQUE_ATTR_VALUE_COLUMN_INDEX];
        String bmAttribute = record[FileUtils.BM_ATTR_NAME_COLUMN_INDEX];
        String bmAttributeValue = record[FileUtils.BM_ATTR_VALUE_COLUMN_INDEX];
        String uniqueAttrName = AtlasTypeUtil.ATTRIBUTE_QUALIFIED_NAME;
        if (record.length > FileUtils.UNIQUE_ATTR_NAME_COLUMN_INDEX && StringUtils.isNotBlank(record[FileUtils.UNIQUE_ATTR_NAME_COLUMN_INDEX])) {
            uniqueAttrName = record[FileUtils.UNIQUE_ATTR_NAME_COLUMN_INDEX];
        }
        AtlasAttribute uniqueAttribute = entityType.getAttribute(uniqueAttrName);
        if (uniqueAttribute == null) {
            failedMsgList.add("Line #" + lineIndexToLog + ": attribute '" + uniqueAttrName + "' not found in entity-type '" + typeName + "'");
            continue;
        }
        if (!uniqueAttribute.getAttributeDef().getIsUnique()) {
            failedMsgList.add("Line #" + lineIndexToLog + ": attribute '" + uniqueAttrName + "' is not an unique attribute in entity-type '" + typeName + "'");
            continue;
        }
        String vertexKey = uniqueAttribute.getVertexPropertyName() + "_" + uniqueAttrValue;
        AtlasVertex vertex = vertexCache.get(vertexKey);
        if (vertex == null) {
            vertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(graph, typeName, uniqueAttribute.getVertexUniquePropertyName(), uniqueAttrValue);
            if (vertex == null) {
                failedMsgList.add("Line #" + lineIndexToLog + ": no " + typeName + " entity found with " + uniqueAttrName + "=" + uniqueAttrValue);
                continue;
            }
            vertexCache.put(vertexKey, vertex);
        }
        AtlasBusinessAttribute businessAttribute = entityType.getBusinesAAttribute(bmAttribute);
        if (businessAttribute == null) {
            failedMsgList.add("Line #" + lineIndexToLog + ": invalid business-metadata '" + bmAttribute + "' for entity type '" + entityType.getTypeName() + "'");
            continue;
        }
        final Object attrValue;
        if (businessAttribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY) {
            AtlasArrayType arrayType = (AtlasArrayType) businessAttribute.getAttributeType();
            List arrayValue;
            if (arrayType.getElementType() instanceof AtlasEnumType) {
                arrayValue = AtlasGraphUtilsV2.assignEnumValues(bmAttributeValue, (AtlasEnumType) arrayType.getElementType(), failedMsgList, lineIndex + 1);
            } else {
                arrayValue = assignMultipleValues(bmAttributeValue, arrayType.getElementTypeName(), failedMsgList, lineIndex + 1);
            }
            attrValue = arrayValue;
        } else {
            attrValue = bmAttributeValue;
        }
        if (ret.containsKey(vertexKey)) {
            AtlasEntity entity = ret.get(vertexKey);
            entity.setBusinessAttribute(businessAttribute.getDefinedInType().getTypeName(), businessAttribute.getName(), attrValue);
        } else {
            AtlasEntity entity = new AtlasEntity();
            String guid = GraphHelper.getGuid(vertex);
            Map<String, Map<String, Object>> businessAttributes = entityRetriever.getBusinessMetadata(vertex);
            entity.setGuid(guid);
            entity.setTypeName(typeName);
            entity.setAttribute(uniqueAttribute.getName(), uniqueAttrValue);
            if (businessAttributes == null) {
                businessAttributes = new HashMap<>();
            }
            entity.setBusinessAttributes(businessAttributes);
            entity.setBusinessAttribute(businessAttribute.getDefinedInType().getTypeName(), businessAttribute.getName(), attrValue);
            ret.put(vertexKey, entity);
        }
    }
    for (String failedMsg : failedMsgList) {
        LOG.error(failedMsg);
        bulkImportResponse.addToFailedImportInfoList(new ImportInfo(FAILED, failedMsg));
    }
    return ret;
}

Example 3 with ImportInfo

use of org.apache.atlas.bulkimport.BulkImportResponse.ImportInfo in project atlas by apache.

the class GlossaryService method updateGlossaryTermsRelation.

private void updateGlossaryTermsRelation(List<AtlasGlossaryTerm> glossaryTerms, BulkImportResponse bulkImportResponse) {
    for (AtlasGlossaryTerm glossaryTerm : glossaryTerms) {
        glossaryTermUtils.updateGlossaryTermRelations(glossaryTerm);
        if (glossaryTerm.hasTerms()) {
            String glossaryTermName = glossaryTerm.getName();
            String glossaryName = getGlossaryName(glossaryTerm);
            try {
                updateTerm(glossaryTerm, false);
            } catch (AtlasBaseException e) {
                LOG.error(AtlasErrorCode.FAILED_TO_UPDATE_GLOSSARY_TERM.toString(), glossaryTermName, e);
                bulkImportResponse.addToFailedImportInfoList(new ImportInfo(glossaryName, glossaryTermName, FAILED, e.getMessage()));
            }
        }
    }
}

Example 4 with ImportInfo

use of org.apache.atlas.bulkimport.BulkImportResponse.ImportInfo in project atlas by apache.

the class GlossaryService method createGlossaryTerms.

private void createGlossaryTerms(List<AtlasGlossaryTerm> glossaryTerms, BulkImportResponse bulkImportResponse) throws AtlasBaseException {
    for (AtlasGlossaryTerm glossaryTerm : glossaryTerms) {
        String glossaryTermName = glossaryTerm.getName();
        String glossaryName = getGlossaryName(glossaryTerm);
        try {
            AtlasGlossaryTerm createdTerm = createTerm(glossaryTerm);
            bulkImportResponse.addToSuccessImportInfoList(new ImportInfo(glossaryName, glossaryTermName, SUCCESS, AtlasJson.toJson(createdTerm.getGlossaryTermHeader())));
        } catch (AtlasBaseException e) {
            LOG.error(AtlasErrorCode.FAILED_TO_CREATE_GLOSSARY_TERM.toString(), glossaryTermName, e);
            bulkImportResponse.addToFailedImportInfoList(new ImportInfo(glossaryName, glossaryTermName, FAILED, e.getMessage()));
        }
    }
    checkForSuccessImports(bulkImportResponse);
}

Example 5 with ImportInfo

use of org.apache.atlas.bulkimport.BulkImportResponse.ImportInfo in project atlas by apache.

the class GlossaryTermUtils method getGlossaryTermDataWithRelations.

protected List<AtlasGlossaryTerm> getGlossaryTermDataWithRelations(List<String[]> fileData, BulkImportResponse bulkImportResponse) throws AtlasBaseException {
    List<AtlasGlossaryTerm> glossaryTerms = new ArrayList<>();
    int rowCount = 1;
    for (String[] record : fileData) {
        List<String> failedTermMsgs = new ArrayList<>();
        if (ArrayUtils.isNotEmpty(record) && StringUtils.isNotBlank(record[INDEX_FOR_GLOSSARY_AT_RECORD])) {
            AtlasGlossaryTerm glossaryTerm = new AtlasGlossaryTerm();
            String glossaryName = record[INDEX_FOR_GLOSSARY_AT_RECORD];
            String glossaryGuid = glossaryNameGuidCache.get().get(glossaryName);
            if (StringUtils.isNotEmpty(glossaryGuid)) {
                glossaryTerm = populateGlossaryTermObject(failedTermMsgs, record, glossaryGuid, true);
                glossaryTerm.setQualifiedName(getGlossaryTermQualifiedName(glossaryTerm.getName(), glossaryName));
                glossaryTerms.add(glossaryTerm);
            }
            if (failedTermMsgs.size() > 0) {
                String failedTermMsg = StringUtils.join(failedTermMsgs, System.lineSeparator());
                String glossaryTermName = glossaryTerm.getName();
                bulkImportResponse.addToFailedImportInfoList(new ImportInfo(glossaryName, glossaryTermName, FAILED, failedTermMsg, rowCount));
            }
        }
        rowCount++;
    }
    return glossaryTerms;
}