Examples with DataxReader com.qlangtech.tis.datax.impl.DataxReader used on opensource projects

Example 21 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project tis by qlangtech.

the class PluginItems method save.

public List<Describable> save(Context context) {
    Objects.requireNonNull(this.pluginContext, "pluginContext can not be null");
    if (items == null) {
        throw new IllegalStateException("prop items can not be null");
    }
    Descriptor.ParseDescribable describable = null;
    AttrValMap attrValMap = null;
    List<Descriptor.ParseDescribable<?>> dlist = Lists.newArrayList();
    List<Describable> describableList = Lists.newArrayList();
    if (this.pluginMeta.isAppend()) {
        IPluginStore pluginStore = heteroEnum.getPluginStore(this.pluginContext, this.pluginMeta);
        if (pluginStore != null) {
            List<Describable> plugins = pluginStore.getPlugins();
            boolean firstSkip = false;
            for (Describable p : plugins) {
                if (!firstSkip) {
                    firstSkip = true;
                    Descriptor.ParseDescribable describablesWithMeta = PluginStore.getDescribablesWithMeta(pluginStore, p);
                    dlist.add(describablesWithMeta);
                } else {
                    dlist.add(new Descriptor.ParseDescribable(p));
                }
            }
        }
    }
    for (int i = 0; i < this.items.size(); i++) {
        attrValMap = this.items.get(i);
        /**
         *====================================================
         * 将客户端post数据包装
         *       ======================================================
         */
        describable = attrValMap.createDescribable(pluginContext);
        dlist.add(describable);
        describableList.add((Describable) describable.instance);
    }
    IPluginStoreSave<?> store = null;
    if (heteroEnum == HeteroEnum.APP_SOURCE) {
        for (Descriptor.ParseDescribable<?> d : dlist) {
            if (d.instance instanceof IdentityName) {
                store = IAppSource.getPluginStore(pluginContext, ((IdentityName) d.instance).identityValue());
                break;
            }
        }
        Objects.requireNonNull(store, "plugin type:" + heteroEnum.getIdentity() + " can not find relevant Store");
    } else if (this.pluginContext.isDataSourceAware()) {
        store = new IPluginStoreSave<DataSourceFactory>() {

            @Override
            public boolean setPlugins(IPluginContext pluginContext, Optional<Context> context, List<Descriptor.ParseDescribable<DataSourceFactory>> dlist, boolean update) {
                for (Descriptor.ParseDescribable<DataSourceFactory> plugin : dlist) {
                    PostedDSProp dbExtraProps = PostedDSProp.parse(pluginMeta);
                    if (DbScope.DETAILED == dbExtraProps.getDbType()) {
                        dbExtraProps.setDbname(plugin.instance.identityValue());
                    }
                    boolean success = TIS.getDataBasePluginStore(dbExtraProps).setPlugins(pluginContext, context, Collections.singletonList(plugin), dbExtraProps.isUpdate());
                    if (!success) {
                        return false;
                    }
                }
                return true;
            }
        };
    } else if (heteroEnum == HeteroEnum.DATAX_WRITER || heteroEnum == HeteroEnum.DATAX_READER) {
        // final String dataxName = pluginMeta.getExtraParam(DataxUtils.DATAX_NAME);
        // if (StringUtils.isEmpty(dataxName)) {
        // String saveDbName = pluginMeta.getExtraParam(DataxUtils.DATAX_DB_NAME);
        // if (StringUtils.isNotBlank(saveDbName)) {
        // store = DataxReader.getPluginStore(this.pluginContext, true, saveDbName);
        // } else {
        // throw new IllegalArgumentException("plugin extra param " + DataxUtils.DATAX_NAME + " can not be null");
        // }
        // } else {
        // KeyedPluginStore<?> keyStore = (heteroEnum == HeteroEnum.DATAX_READER)
        // ? DataxReader.getPluginStore(this.pluginContext, dataxName) : DataxWriter.getPluginStore(this.pluginContext, dataxName);
        // store = keyStore;
        // }
        // if ((heteroEnum == HeteroEnum.DATAX_READER)) {
        // for (Descriptor.ParseDescribable<?> dataXReader : dlist) {
        // DataSourceMeta sourceMeta = (DataSourceMeta) dataXReader.instance;
        // pluginContext.setBizResult(context, sourceMeta.getTablesInDB());
        // }
        // }
        store = HeteroEnum.getDataXReaderAndWriterStore(this.pluginContext, this.heteroEnum == HeteroEnum.DATAX_READER, this.pluginMeta);
        Optional<IPropertyType.SubFormFilter> subFormFilter = pluginMeta.getSubFormFilter();
        if (subFormFilter.isPresent()) {
            IPropertyType.SubFormFilter filter = subFormFilter.get();
            Optional<Descriptor> firstDesc = heteroEnum.descriptors().stream().filter((des) -> filter.match((Descriptor) des)).map((des) -> (Descriptor) des).findFirst();
            if (!firstDesc.isPresent()) {
                throw new IllegalStateException("can not find relevant descriptor:" + filter.uploadPluginMeta.toString());
            }
            // Describable p = keyStore.getPlugin();
            // Descriptor des = p.getDescriptor();
            PluginFormProperties pluginProps = firstDesc.get().getPluginFormPropertyTypes(subFormFilter);
            store = pluginProps.accept(new PluginFormProperties.IVisitor() {

                @Override
                public IPluginStoreSave<?> visit(SuFormProperties props) {
                    // 为了在更新插件时候不把plugin上的@SubForm标记的属性覆盖掉，需要先将老的plugin上的值覆盖到新http post过来的反序列化之后的plugin上
                    // IPluginContext pluginContext, String appname, SuFormProperties subfieldForm, Class<TT> clazz
                    Class<Describable> clazz = (Class<Describable>) heteroEnum.getExtensionPoint();
                    // DataxReader.SubFieldFormAppKey<Describable> key
                    // = new DataxReader.SubFieldFormAppKey<>(pluginContext, dataxName, props, clazz);
                    // 
                    DataxReader.SubFieldFormAppKey<Describable> key = HeteroEnum.createDataXReaderAndWriterRelevant(pluginContext, pluginMeta, new HeteroEnum.DataXReaderAndWriterRelevantCreator<DataxReader.SubFieldFormAppKey<Describable>>() {

                        @Override
                        public DataxReader.SubFieldFormAppKey<Describable> dbRelevant(IPluginContext pluginContext, String saveDbName) {
                            return new DataxReader.SubFieldFormAppKey<>(pluginContext, true, saveDbName, props, clazz);
                        }

                        @Override
                        public DataxReader.SubFieldFormAppKey<Describable> appRelevant(IPluginContext pluginContext, String dataxName) {
                            return new DataxReader.SubFieldFormAppKey<>(pluginContext, false, dataxName, props, clazz);
                        }
                    });
                    return KeyedPluginStore.getPluginStore(key);
                }
            });
        }
    } else if (heteroEnum == HeteroEnum.PARAMS_CONFIG) {
        store = new ParamsConfigPluginStore(this.pluginMeta);
    } else if (heteroEnum == HeteroEnum.DATAX_WORKER) {
        if (!this.pluginContext.isCollectionAware()) {
            throw new IllegalStateException("must be collection aware");
        }
        store = DataXJobWorker.getJobWorkerStore(new TargetResName(this.pluginContext.getCollectionName()));
    // } else if (this.pluginContext.isCollectionAware()) {
    // store = TIS.getPluginStore(this.pluginContext.getCollectionName(), heteroEnum.getExtensionPoint());
    } else {
        if (heteroEnum.isAppNameAware()) {
            if (!this.pluginContext.isCollectionAware()) {
                throw new IllegalStateException(heteroEnum.getExtensionPoint().getName() + " must be collection aware");
            }
            store = TIS.getPluginStore(this.pluginContext.getCollectionName(), heteroEnum.getExtensionPoint());
        } else {
            store = TIS.getPluginStore(heteroEnum.getExtensionPoint());
        }
    // if (heteroEnum.isAppNameAware()) {
    // throw new IllegalStateException(heteroEnum.getExtensionPoint().getName() + " must be app name aware");
    // }
    }
    // dlist
    if (!store.setPlugins(pluginContext, Optional.of(context), convert(dlist))) {
        return Collections.emptyList();
    }
    observable.notifyObservers(new PluginItemsSaveEvent(this.pluginContext, this.heteroEnum, describableList));
    return describableList;
}

Example 22 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project tis by qlangtech.

the class TestPluginItems method validatePluginValue.

private void validatePluginValue() throws Exception {
    DataxReader reader = readerStore.getPlugin();
    assertNotNull(reader);
    PluginFormProperties rootPropertyTypes = reader.getDescriptor().getPluginFormPropertyTypes();
    assertTrue("get RootFormProperties process result", rootPropertyTypes.accept(new PluginFormProperties.IVisitor() {

        @Override
        public Boolean visit(RootFormProperties props) {
            Map<String, PropertyType> propertiesType = props.propertiesType;
            validatePropertyValue(propertiesType, "dbName", "order1", reader);
            validatePropertyValue(propertiesType, "splitPk", true, reader);
            // validatePropertyValue(propertiesType, "template"
            // , IOUtils.loadResourceFromClasspath(TestPluginItems.class, "datax_reader_mysql_prop_template.json"), reader);
            assertEquals(3, propertiesType.size());
            return true;
        }
    }));
    Optional<IPropertyType.SubFormFilter> subFormFilter = subFieldPluginMeta.getSubFormFilter();
    assertTrue(subFormFilter.isPresent());
    PluginFormProperties pluginFormPropertyTypes = reader.getDescriptor().getPluginFormPropertyTypes(subFormFilter);
    assertTrue("get SuFormProperties process result", pluginFormPropertyTypes.accept(new PluginFormProperties.IVisitor() {

        @Override
        public Boolean visit(SuFormProperties props) {
            assertEquals("selectedTabs", props.getSubFormFieldName());
            Map<String, PropertyType> fieldsType = props.fieldsType;
            assertEquals(3, fieldsType.size());
            List<ISelectedTab> selectedTabs = reader.getSelectedTabs();
            assertNotNull("selectedTabs can not be null", selectedTabs);
            assertEquals(1, selectedTabs.size());
            ISelectedTab selectedTab = selectedTabs.get(0);
            assertEquals("customer_order_relation", selectedTab.getName());
            assertEquals("1=999", selectedTab.getWhere());
            List<ISelectedTab.ColMeta> cols = selectedTab.getCols();
            assertEquals(5, cols.size());
            Set<String> selectedCols = Sets.newHashSet("customerregister_id", "waitingorder_id", "kind", "create_time", "last_ver");
            cols.forEach((c) -> assertTrue(selectedCols.contains(c.getName())));
            return true;
        }
    }));
}