Examples with DataSourceMetadata io.atlasmap.v2.DataSourceMetadata used on opensource projects

Example 1 with DataSourceMetadata

use of io.atlasmap.v2.DataSourceMetadata in project atlasmap by atlasmap.

the class DefaultAtlasContext method init.

/**
 * TODO: For dynamic re-load. This needs lock()
 *
 * @throws AtlasException failed to initialize
 */
protected synchronized void init() throws AtlasException {
    if (this.initialized) {
        return;
    }
    registerJmx(this);
    if (this.atlasMappingUri != null) {
        this.admHandler = new ADMArchiveHandler(factory.getClassLoader());
        this.admHandler.setIgnoreLibrary(true);
        this.admHandler.load(Paths.get(this.atlasMappingUri));
        this.dataSourceMetadataMap = this.admHandler.getDataSourceMetadataMap();
    }
    if (this.admHandler == null || this.admHandler.getMappingDefinition() == null) {
        LOG.warn("AtlasMap context cannot initialize without mapping definition, ignoring:" + " Mapping URI={}", this.atlasMappingUri);
        return;
    }
    sourceModules.clear();
    ConstantModule constant = new ConstantModule();
    constant.setConversionService(factory.getConversionService());
    constant.setFieldActionService(factory.getFieldActionService());
    sourceModules.put(AtlasConstants.CONSTANTS_DOCUMENT_ID, constant);
    PropertyModule property = new PropertyModule(factory.getPropertyStrategy());
    property.setConversionService(factory.getConversionService());
    property.setFieldActionService(factory.getFieldActionService());
    property.setMode(AtlasModuleMode.SOURCE);
    sourceModules.put(AtlasConstants.PROPERTIES_SOURCE_DOCUMENT_ID, property);
    targetModules.clear();
    property = new PropertyModule(factory.getPropertyStrategy());
    property.setConversionService(factory.getConversionService());
    property.setFieldActionService(factory.getFieldActionService());
    property.setMode(AtlasModuleMode.TARGET);
    targetModules.put(AtlasConstants.PROPERTIES_TARGET_DOCUMENT_ID, property);
    lookupTables.clear();
    if (admHandler.getMappingDefinition().getLookupTables() != null && admHandler.getMappingDefinition().getLookupTables().getLookupTable() != null) {
        for (LookupTable table : admHandler.getMappingDefinition().getLookupTables().getLookupTable()) {
            lookupTables.put(table.getName(), table);
        }
    }
    AtlasModuleInfoRegistry moduleInfoRegistry = factory.getModuleInfoRegistry();
    for (DataSource ds : admHandler.getMappingDefinition().getDataSource()) {
        AtlasModuleInfo moduleInfo = moduleInfoRegistry.lookupByUri(ds.getUri());
        if (moduleInfo == null) {
            LOG.error("Cannot find module info for the DataSource uri '{}'", ds.getUri());
            continue;
        }
        if (ds.getDataSourceType() != DataSourceType.SOURCE && ds.getDataSourceType() != DataSourceType.TARGET) {
            LOG.error("Unsupported DataSource type '{}'", ds.getDataSourceType());
            continue;
        }
        String docId = ds.getId();
        if (docId == null || docId.isEmpty()) {
            docId = ds.getDataSourceType() == DataSourceType.SOURCE ? AtlasConstants.DEFAULT_SOURCE_DOCUMENT_ID : AtlasConstants.DEFAULT_TARGET_DOCUMENT_ID;
        }
        if (ds.getDataSourceType() == DataSourceType.SOURCE && sourceModules.containsKey(docId)) {
            LOG.error("Duplicated {} DataSource ID '{}' was detected, ignoring...", ds.getDataSourceType(), ds.getId());
            continue;
        }
        if (ds.getDataSourceType() == DataSourceType.TARGET && targetModules.containsKey(docId)) {
            LOG.error("Duplicated {} DataSource ID '{}' was detected, ignoring...", ds.getDataSourceType(), docId);
            continue;
        }
        try {
            AtlasModule module = moduleInfo.getModuleClass().getDeclaredConstructor().newInstance();
            module.setClassLoader(factory.getClassLoader());
            module.setConversionService(factory.getConversionService());
            module.setFieldActionService(factory.getFieldActionService());
            module.setDataSource(ds);
            if (ds.getDataSourceType() == DataSourceType.SOURCE) {
                getSourceModules().put(docId, module);
            } else if (ds.getDataSourceType() == DataSourceType.TARGET) {
                getTargetModules().put(docId, module);
            }
            if (this.dataSourceMetadataMap != null) {
                DataSourceKey dskey = new DataSourceKey(ds.getDataSourceType() == DataSourceType.SOURCE, docId);
                DataSourceMetadata meta = this.dataSourceMetadataMap.get(dskey);
                if (meta != null) {
                    module.setDataSourceMetadata(meta);
                }
            }
            module.init();
        } catch (Exception t) {
            LOG.error("Unable to initialize {} module: {}", ds.getDataSourceType(), moduleInfo);
            LOG.error(t.getMessage(), t);
            throw new AtlasException(String.format("Unable to initialize %s module: %s", ds.getDataSourceType(), moduleInfo.toString()), t);
        }
    }
    initialized = true;
}

Example 2 with DataSourceMetadata

use of io.atlasmap.v2.DataSourceMetadata in project atlasmap by atlasmap.

the class E2ETest method test.

@Test
public void test() throws Exception {
    driver.get("http://127.0.0.1:" + port);
    WebDriverWait waitForLoad = new WebDriverWait(driver, Duration.ofSeconds(30));
    waitForLoad.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//article[@aria-label='Properties']")));
    WebElement atlasmapMenuBtn = driver.findElement(By.xpath("//button[@data-testid='atlasmap-menu-button']"));
    atlasmapMenuBtn.click();
    waitForLoad.until(ExpectedConditions.elementToBeClickable(By.xpath("//a[@data-testid='import-mappings-button']")));
    WebElement importBtn = driver.findElement(By.xpath("//a[@data-testid='import-mappings-button']"));
    importBtn.click();
    WebElement fileInput = driver.findElement(By.xpath("//div[@id='data-toolbar']//input[@type='file']"));
    String cwd = System.getProperty("user.dir");
    fileInput.sendKeys(cwd + "/src/test/resources/json-schema-source-to-xml-schema-target.adm");
    WebElement confirmBtn = driver.findElement(By.xpath("//button[@data-testid='confirmation-dialog-confirm-button']"));
    confirmBtn.click();
    waitForLoad.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//article[@aria-label='JSONSchemaSource']")));
    try {
        Thread.sleep(1000);
    } catch (InterruptedException e1) {
    }
    // Check custom action param
    WebElement orderBtn = driver.findElement(By.xpath("//button[@id='sources-field-atlas:json:JSONSchemaSource:source:/order-toggle']"));
    orderBtn.click();
    By addressToggle = By.xpath("//button[@id='sources-field-atlas:json:JSONSchemaSource:source:/order/address-toggle']");
    waitForLoad.until(ExpectedConditions.visibilityOfElementLocated(addressToggle));
    WebElement addrBtn = driver.findElement(addressToggle);
    addrBtn.click();
    By addrDivPath = By.xpath("//button[@id='sources-field-atlas:json:JSONSchemaSource:source:/order/address-toggle']/../..");
    waitForLoad.until(ExpectedConditions.visibilityOfElementLocated(addrDivPath));
    WebElement addrDiv = driver.findElement(addrDivPath);
    By cityDivPath = By.xpath(".//button[@data-testid='grip-city-button']/../../../..");
    waitForLoad.until(ExpectedConditions.visibilityOfElementLocated(cityDivPath));
    WebElement cityDiv = addrDiv.findElement(cityDivPath);
    Actions action = new Actions(driver);
    action.moveToElement(cityDiv).perform();
    waitForLoad.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//button[@data-testid='show-mapping-details-button']")));
    WebElement showDetailsBtn = cityDiv.findElement(By.xpath(".//button[@data-testid='show-mapping-details-button']"));
    showDetailsBtn.click();
    WebElement detailsCity = waitForLoad.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//div[@role='dialog']//div[@aria-labelledby='mapping-field-city']")));
    assertNotNull(detailsCity);
    WebElement customActionParamInput = driver.findElement(By.id("user-field-action-io.atlasmap.service.my.MyFieldActionsModel-transformation-0"));
    assertEquals("testparam", customActionParamInput.getAttribute("value"));
    // Check custom source class mapping
    WebElement customClassDoc = driver.findElement(By.xpath("//article[@aria-label='MyFieldActionsModel']"));
    WebElement paramDiv = customClassDoc.findElement(By.xpath(".//button[@data-testid='grip-param-button']/../../../.."));
    action = new Actions(driver);
    action.moveToElement(paramDiv).perform();
    waitForLoad.until(ExpectedConditions.elementToBeClickable(By.xpath("//button[@data-testid='show-mapping-details-button']")));
    showDetailsBtn = paramDiv.findElement(By.xpath(".//button[@data-testid='show-mapping-details-button']"));
    showDetailsBtn.click();
    WebElement detailsPhotoUrl = waitForLoad.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//div[@data-testid='column-mapping-details-area']" + "//div[@data-testid='mapping-field-photoUrl']")));
    assertNotNull(detailsPhotoUrl);
    atlasmapMenuBtn = driver.findElement(By.xpath("//button[@data-testid='atlasmap-menu-button']"));
    atlasmapMenuBtn.click();
    waitForLoad.until(ExpectedConditions.elementToBeClickable(By.xpath("//a[@data-testid='export-mappings-button']")));
    WebElement exportBtn = driver.findElement(By.xpath("//a[@data-testid='export-mappings-button']"));
    exportBtn.click();
    WebElement dialogDiv = driver.findElement(By.xpath("//div[@data-testid='export-catalog-dialog']/.."));
    WebElement exportInput = dialogDiv.findElement(By.id("filename"));
    String exportAdmFileName = UUID.randomUUID().toString() + "-exported.adm";
    exportInput.clear();
    exportInput.sendKeys(exportAdmFileName);
    confirmBtn = dialogDiv.findElement(By.xpath(".//button[@data-testid='confirmation-dialog-confirm-button']"));
    WatchService watcher = FileSystems.getDefault().newWatchService();
    Executors.newSingleThreadExecutor().execute(() -> {
        long start = System.currentTimeMillis();
        while ((System.currentTimeMillis() - start) < 300000) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                break;
            }
        }
        try {
            watcher.close();
        } catch (Exception e) {
            fail("Failed to close file watcher");
        }
    });
    Path dirPath = Paths.get(DLDIR);
    dirPath.register(watcher, StandardWatchEventKinds.ENTRY_CREATE);
    confirmBtn.click();
    WatchKey key = watcher.take();
    while (key.isValid()) {
        List<WatchEvent<?>> events = key.pollEvents();
        if (events.isEmpty()) {
            Thread.sleep(1000);
            continue;
        }
        for (WatchEvent<?> event : events) {
            if (!StandardWatchEventKinds.ENTRY_CREATE.name().equals(event.kind().name())) {
                continue;
            }
            ;
            Path eventPath = (Path) event.context();
            LOG.info("File '{}' is created", eventPath.getFileName().toString());
            if (!exportAdmFileName.equals(eventPath.getFileName().toString())) {
                continue;
            }
            ADMArchiveHandler handler = new ADMArchiveHandler(getClass().getClassLoader());
            handler.setLibraryDirectory(Paths.get(DLDIR + File.separator + "lib"));
            handler.load(Paths.get(DLDIR + File.separator + exportAdmFileName));
            assertEquals("UI.0", handler.getMappingDefinition().getName());
            DataSourceMetadata sourceMeta = handler.getDataSourceMetadata(true, "JSONSchemaSource");
            assertEquals(true, sourceMeta.getIsSource());
            assertEquals("JSONSchemaSource", sourceMeta.getName());
            assertEquals("JSON", sourceMeta.getDataSourceType());
            DataSourceMetadata targetMeta = handler.getDataSourceMetadata(false, "XMLSchemaSource");
            assertEquals(false, targetMeta.getIsSource());
            assertEquals("XMLSchemaSource", targetMeta.getName());
            assertEquals("XML", targetMeta.getDataSourceType());
            return;
        }
        ;
    }
    fail("exported.adm was not created");
}

Example 3 with DataSourceMetadata

use of io.atlasmap.v2.DataSourceMetadata in project atlasmap by atlasmap.

the class ADMArchiveHandler method getDataSourceMetadataMap.

/**
 * Gets a map of DataSource metadata.
 * @return a map of DataSource metadata.
 * @throws AtlasException unexpected error
 */
public Map<DataSourceKey, DataSourceMetadata> getDataSourceMetadataMap() throws AtlasException {
    if (this.dataSourceMetadata == null) {
        if (this.gzippedAdmDigestBytes == null) {
            return null;
        }
        try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(this.gzippedAdmDigestBytes))) {
            ADMDigest digest = jsonMapperForDigest.readValue(in, ADMDigest.class);
            this.dataSourceMetadata = new HashMap<>();
            for (int i = 0; i < digest.getExportMeta().length; i++) {
                DataSourceMetadata meta = digest.getExportMeta()[i];
                String spec = digest.getExportBlockData()[i].getValue();
                if (meta.getId() == null) {
                    meta.setId(meta.getName());
                }
                meta.setSpecification(spec != null ? spec.getBytes() : null);
                this.dataSourceMetadata.put(new DataSourceKey(meta.getIsSource(), meta.getId()), meta);
            }
        } catch (Exception e) {
            throw new AtlasException(e);
        }
    }
    return Collections.unmodifiableMap(this.dataSourceMetadata);
}