Examples with AtlasModule io.atlasmap.spi.AtlasModule used on opensource projects

Example 1 with AtlasModule

use of io.atlasmap.spi.AtlasModule in project atlasmap by atlasmap.

the class DefaultAtlasContextFactory method loadModules.

/**
 * Loads all modules in the classpath.
 * @param moduleClassProperty module class property
 * @param moduleInterface module interface
 */
protected void loadModules(String moduleClassProperty, Class<?> moduleInterface) {
    Class<?> moduleClass = null;
    String moduleClassName = null;
    Set<String> serviceClasses = new HashSet<>();
    try {
        Enumeration<URL> urls = classLoader.getResources("META-INF/services/atlas/module/atlas.module");
        while (urls.hasMoreElements()) {
            URL tmp = urls.nextElement();
            Properties prop = AtlasUtil.loadPropertiesFromURL(tmp);
            String serviceClassPropertyValue = (String) prop.get(moduleClassProperty);
            String[] splitted = serviceClassPropertyValue != null ? serviceClassPropertyValue.split(",") : new String[0];
            for (String entry : splitted) {
                if (!AtlasUtil.isEmpty(entry)) {
                    serviceClasses.add((entry));
                }
            }
        }
    } catch (Exception e) {
        LOG.warn("Error loading module resources", e);
    }
    for (String clazz : serviceClasses) {
        try {
            moduleClass = classLoader.loadClass(clazz);
            moduleClassName = moduleClass.getName();
            if (isClassAtlasModule(moduleClass, moduleInterface)) {
                @SuppressWarnings("unchecked") Class<AtlasModule> atlasModuleClass = (Class<AtlasModule>) moduleClass;
                Constructor<AtlasModule> constructor = atlasModuleClass.getDeclaredConstructor();
                if (constructor != null) {
                    AtlasModuleInfo module = new DefaultAtlasModuleInfo(getModuleName(moduleClass), getModuleUri(moduleClass), atlasModuleClass, constructor, getSupportedDataFormats(moduleClass), getConfigPackages(moduleClass));
                    getModuleInfoRegistry().register(module);
                } else {
                    LOG.warn("Invalid module class {}: constructor is not present", moduleClassName);
                }
            } else {
                LOG.warn("Invalid module class {}: unsupported AtlasModule", moduleClassName);
            }
        } catch (NoSuchMethodException e) {
            LOG.warn(String.format("Invalid module class %s: constructor is not present.", moduleClassName), e);
        } catch (ClassNotFoundException e) {
            LOG.warn(String.format("Invalid module class %s: not found in classLoader.", moduleClassName), e);
        } catch (Exception e) {
            LOG.warn(String.format("Invalid module class %s: unknown error.", moduleClassName), e);
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Loaded: {} of {} detected modules", getModuleInfoRegistry().size(), serviceClasses.size());
    }
}

Example 2 with AtlasModule

use of io.atlasmap.spi.AtlasModule in project atlasmap by atlasmap.

the class DefaultAtlasContext method processSourceFields.

private void processSourceFields(DefaultAtlasSession session, List<Field> sourceFields) throws AtlasException {
    for (int i = 0; i < sourceFields.size(); i++) {
        Field sourceField = sourceFields.get(i);
        session.head().setSourceField(sourceField);
        if (sourceField instanceof FieldGroup) {
            processSourceFields(session, ((FieldGroup) sourceField).getField());
            Field processed = applyFieldActions(session, sourceField);
            session.head().setSourceField(processed);
            continue;
        }
        AtlasModule module = resolveModule(FieldDirection.SOURCE, sourceField);
        if (module == null) {
            AtlasUtil.addAudit(session, sourceField, String.format("Module not found for docId '%s'", sourceField.getDocId()), AuditStatus.ERROR, null);
            return;
        }
        if (!module.isSupportedField(sourceField)) {
            AtlasUtil.addAudit(session, sourceField, String.format("Unsupported source field type '%s' for DataSource '%s'", sourceField.getClass().getName(), module.getUri()), AuditStatus.ERROR, null);
            return;
        }
        module.readSourceValue(session);
        Field processed = applyFieldActions(session, session.head().getSourceField());
        session.head().setSourceField(processed);
        sourceFields.set(i, processed);
    }
}

Example 3 with AtlasModule

use of io.atlasmap.spi.AtlasModule in project atlasmap by atlasmap.

the class DefaultAtlasExpressionProcessor method processExpression.

/**
 * Processes the expression.
 * @param session session
 * @param expression expression
 */
public static void processExpression(DefaultAtlasSession session, String expression) {
    if (expression == null || expression.trim().isEmpty()) {
        return;
    }
    try {
        Map<String, Field> sourceFieldMap = new HashMap<>();
        Field parent = session.head().getSourceField();
        if (parent != null && !AtlasUtil.isEmpty(parent.getDocId()) && !AtlasUtil.isEmpty(parent.getPath())) {
            sourceFieldMap.put(parent.getDocId() + ":" + parent.getPath(), parent);
        }
        // Anonymous FieldGroup is just a wrapping, peel it off
        if (parent instanceof FieldGroup && AtlasUtil.isEmpty(parent.getPath())) {
            FieldGroup parentGroup = FieldGroup.class.cast(parent);
            for (Field child : parentGroup.getField()) {
                if (!(AtlasUtil.isEmpty(child.getDocId()) && AtlasUtil.isEmpty(child.getPath()))) {
                    sourceFieldMap.put(child.getDocId() + ":" + child.getPath(), child);
                }
            }
        }
        Expression parsedExpression = Expression.parse(expression, DefaultAtlasFunctionResolver.getInstance());
        Object answer = parsedExpression.evaluate((path) -> {
            if (path == null || path.isEmpty()) {
                return null;
            }
            try {
                Field f = sourceFieldMap.get(path);
                if (f == null) {
                    return null;
                }
                AtlasModule sourceModule;
                Map<String, AtlasModule> sourceModules = session.getAtlasContext().getSourceModules();
                if (f instanceof ConstantField) {
                    sourceModule = sourceModules.get(AtlasConstants.CONSTANTS_DOCUMENT_ID);
                } else if (f instanceof PropertyField) {
                    sourceModule = sourceModules.get(AtlasConstants.PROPERTIES_SOURCE_DOCUMENT_ID);
                } else {
                    String[] splitted = path.split(":", 2);
                    sourceModule = sourceModules.get(splitted[0]);
                }
                if (sourceModule == null) {
                    throw new ExpressionException(String.format("Module for the path '%s' is not found", path));
                }
                session.head().setSourceField(f);
                sourceModule.readSourceValue(session);
                return session.head().getSourceField();
            } catch (Exception e) {
                throw new ExpressionException(e);
            }
        });
        if (answer instanceof Field) {
            session.head().setSourceField((Field) answer);
        } else {
            Field from = session.head().getSourceField();
            SimpleField to = new SimpleField();
            AtlasModelFactory.copyField(from, to, false);
            to.setValue(answer);
            session.head().setSourceField(to);
        }
    } catch (Exception e) {
        AtlasUtil.addAudit(session, expression, String.format("Expression processing error [%s]: %s", expression, e.getMessage()), AuditStatus.ERROR, null);
        if (LOG.isDebugEnabled()) {
            LOG.debug("", e);
        }
    }
}

Example 4 with AtlasModule

use of io.atlasmap.spi.AtlasModule in project atlasmap by atlasmap.

the class DefaultAtlasContext method init.

/**
 * TODO: For dynamic re-load. This needs lock()
 *
 * @throws AtlasException failed to initialize
 */
protected synchronized void init() throws AtlasException {
    if (this.initialized) {
        return;
    }
    registerJmx(this);
    if (this.atlasMappingUri != null) {
        this.admHandler = new ADMArchiveHandler(factory.getClassLoader());
        this.admHandler.setIgnoreLibrary(true);
        this.admHandler.load(Paths.get(this.atlasMappingUri));
        this.dataSourceMetadataMap = this.admHandler.getDataSourceMetadataMap();
    }
    if (this.admHandler == null || this.admHandler.getMappingDefinition() == null) {
        LOG.warn("AtlasMap context cannot initialize without mapping definition, ignoring:" + " Mapping URI={}", this.atlasMappingUri);
        return;
    }
    sourceModules.clear();
    ConstantModule constant = new ConstantModule();
    constant.setConversionService(factory.getConversionService());
    constant.setFieldActionService(factory.getFieldActionService());
    sourceModules.put(AtlasConstants.CONSTANTS_DOCUMENT_ID, constant);
    PropertyModule property = new PropertyModule(factory.getPropertyStrategy());
    property.setConversionService(factory.getConversionService());
    property.setFieldActionService(factory.getFieldActionService());
    property.setMode(AtlasModuleMode.SOURCE);
    sourceModules.put(AtlasConstants.PROPERTIES_SOURCE_DOCUMENT_ID, property);
    targetModules.clear();
    property = new PropertyModule(factory.getPropertyStrategy());
    property.setConversionService(factory.getConversionService());
    property.setFieldActionService(factory.getFieldActionService());
    property.setMode(AtlasModuleMode.TARGET);
    targetModules.put(AtlasConstants.PROPERTIES_TARGET_DOCUMENT_ID, property);
    lookupTables.clear();
    if (admHandler.getMappingDefinition().getLookupTables() != null && admHandler.getMappingDefinition().getLookupTables().getLookupTable() != null) {
        for (LookupTable table : admHandler.getMappingDefinition().getLookupTables().getLookupTable()) {
            lookupTables.put(table.getName(), table);
        }
    }
    AtlasModuleInfoRegistry moduleInfoRegistry = factory.getModuleInfoRegistry();
    for (DataSource ds : admHandler.getMappingDefinition().getDataSource()) {
        AtlasModuleInfo moduleInfo = moduleInfoRegistry.lookupByUri(ds.getUri());
        if (moduleInfo == null) {
            LOG.error("Cannot find module info for the DataSource uri '{}'", ds.getUri());
            continue;
        }
        if (ds.getDataSourceType() != DataSourceType.SOURCE && ds.getDataSourceType() != DataSourceType.TARGET) {
            LOG.error("Unsupported DataSource type '{}'", ds.getDataSourceType());
            continue;
        }
        String docId = ds.getId();
        if (docId == null || docId.isEmpty()) {
            docId = ds.getDataSourceType() == DataSourceType.SOURCE ? AtlasConstants.DEFAULT_SOURCE_DOCUMENT_ID : AtlasConstants.DEFAULT_TARGET_DOCUMENT_ID;
        }
        if (ds.getDataSourceType() == DataSourceType.SOURCE && sourceModules.containsKey(docId)) {
            LOG.error("Duplicated {} DataSource ID '{}' was detected, ignoring...", ds.getDataSourceType(), ds.getId());
            continue;
        }
        if (ds.getDataSourceType() == DataSourceType.TARGET && targetModules.containsKey(docId)) {
            LOG.error("Duplicated {} DataSource ID '{}' was detected, ignoring...", ds.getDataSourceType(), docId);
            continue;
        }
        try {
            AtlasModule module = moduleInfo.getModuleClass().getDeclaredConstructor().newInstance();
            module.setClassLoader(factory.getClassLoader());
            module.setConversionService(factory.getConversionService());
            module.setFieldActionService(factory.getFieldActionService());
            module.setDataSource(ds);
            if (ds.getDataSourceType() == DataSourceType.SOURCE) {
                getSourceModules().put(docId, module);
            } else if (ds.getDataSourceType() == DataSourceType.TARGET) {
                getTargetModules().put(docId, module);
            }
            if (this.dataSourceMetadataMap != null) {
                DataSourceKey dskey = new DataSourceKey(ds.getDataSourceType() == DataSourceType.SOURCE, docId);
                DataSourceMetadata meta = this.dataSourceMetadataMap.get(dskey);
                if (meta != null) {
                    module.setDataSourceMetadata(meta);
                }
            }
            module.init();
        } catch (Exception t) {
            LOG.error("Unable to initialize {} module: {}", ds.getDataSourceType(), moduleInfo);
            LOG.error(t.getMessage(), t);
            throw new AtlasException(String.format("Unable to initialize %s module: %s", ds.getDataSourceType(), moduleInfo.toString()), t);
        }
    }
    initialized = true;
}

Example 5 with AtlasModule

use of io.atlasmap.spi.AtlasModule in project atlasmap by atlasmap.

the class DefaultAtlasContext method process.

@Override
public void process(AtlasSession userSession) throws AtlasException {
    if (!(userSession instanceof DefaultAtlasSession)) {
        throw new AtlasException(String.format("Unsupported session class '%s'", userSession.getClass().getName()));
    }
    if (!this.equals(userSession.getAtlasContext())) {
        throw new AtlasException("Cannot execute AtlasSession created by the other AtlasContext");
    }
    DefaultAtlasSession session = (DefaultAtlasSession) userSession;
    if (LOG.isDebugEnabled()) {
        LOG.debug("Begin process {}", (session == null ? null : session.toString()));
    }
    session.head().unset();
    session.getAudits().getAudit().clear();
    session.getValidations().getValidation().clear();
    // TODO https://github.com/atlasmap/atlasmap/issues/863 - Add an option to enable/disable runtime validation
    processValidation(session);
    for (Validation v : session.getValidations().getValidation()) {
        AtlasUtil.addAudit(session, v);
    }
    // Additional runtime only audit
    Mappings mappings = session.getMapping().getMappings();
    if (mappings != null && mappings.getMapping().isEmpty()) {
        AtlasUtil.addAudit(session, (String) null, String.format("Field mappings should not be empty"), AuditStatus.WARN, null);
    }
    session.getValidations().getValidation().clear();
    if (session.hasErrors()) {
        if (LOG.isDebugEnabled()) {
            LOG.error("Aborting due to {} errors in pre-validation", session.errorCount());
        }
        return;
    }
    for (AtlasModule module : getSourceModules().values()) {
        module.processPreSourceExecution(session);
    }
    for (AtlasModule module : getTargetModules().values()) {
        module.processPreTargetExecution(session);
    }
    if (session.hasErrors()) {
        if (LOG.isDebugEnabled()) {
            LOG.error("Aborting due to {} errors in pre-execution", session.errorCount());
        }
        return;
    }
    for (BaseMapping baseMapping : session.getMapping().getMappings().getMapping()) {
        for (BaseMapping innerMapping : unwrapCollectionMappings(session, baseMapping)) {
            if (innerMapping instanceof CustomMapping) {
                DefaultAtlasCustomMappingProcessor.getInstance().process(session, (CustomMapping) innerMapping);
                continue;
            }
            Mapping mapping = (Mapping) innerMapping;
            session.head().setMapping(mapping).setLookupTable(lookupTables.get(mapping.getLookupTableName()));
            if (mapping.getOutputField() == null || mapping.getOutputField().isEmpty()) {
                AtlasUtil.addAudit(session, (String) null, String.format("Mapping does not contain at least one target field: alias=%s desc=%s", mapping.getAlias(), mapping.getDescription()), AuditStatus.WARN, null);
                continue;
            }
            processSourceFieldMapping(session);
            if (!session.head().hasError()) {
                processTargetFieldMapping(session, mapping);
            }
            session.getAudits().getAudit().addAll(session.head().getAudits());
            session.head().unset();
        }
    }
    for (AtlasModule module : getSourceModules().values()) {
        module.processPostValidation(session);
    }
    for (AtlasModule module : getTargetModules().values()) {
        module.processPostValidation(session);
    }
    for (AtlasModule module : getSourceModules().values()) {
        module.processPostSourceExecution(session);
    }
    for (AtlasModule module : getTargetModules().values()) {
        module.processPostTargetExecution(session);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("End process {}", session == null ? null : session.toString());
    }
}