Examples with IHttpToken com.qlangtech.tis.config.aliyun.IHttpToken used on opensource projects

Example 1 with IHttpToken

use of com.qlangtech.tis.config.aliyun.IHttpToken in project plugins by qlangtech.

the class AliyunOSSFileSystemFactory method getFileSystem.

@Override
public ITISFileSystem getFileSystem() {
    if (ossFs == null) {
        IHttpToken aliyunToken = ParamsConfig.getItem(this.aliyunToken, IHttpToken.KEY_DISPLAY_NAME);
        ossFs = new AliyunOSSFileSystem(aliyunToken, this.endpoint, this.bucketName, this.rootDir);
    }
    return ossFs;
}

Example 2 with IHttpToken

use of com.qlangtech.tis.config.aliyun.IHttpToken in project plugins by qlangtech.

the class ElasticSearchSinkFactory method createSinkFunction.

@Override
public Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> createSinkFunction(IDataxProcessor dataxProcessor) {
    DataXElasticsearchWriter dataXWriter = (DataXElasticsearchWriter) dataxProcessor.getWriter(null);
    Objects.requireNonNull(dataXWriter, "dataXWriter can not be null");
    IHttpToken token = dataXWriter.getToken();
    ESTableAlias esSchema = null;
    for (Map.Entry<String, IDataxProcessor.TableAlias> e : dataxProcessor.getTabAlias().entrySet()) {
        IDataxProcessor.TableAlias value = e.getValue();
        if (!(value instanceof ESTableAlias)) {
            throw new IllegalStateException("value must be type of 'ESTableAlias',but now is :" + value.getClass());
        }
        esSchema = (ESTableAlias) value;
        break;
    }
    Objects.requireNonNull(esSchema, "esSchema can not be null");
    List<ISelectedTab.ColMeta> cols = esSchema.getSourceCols();
    if (CollectionUtils.isEmpty(cols)) {
        throw new IllegalStateException("cols can not be null");
    }
    Optional<ISelectedTab.ColMeta> firstPK = cols.stream().filter((c) -> c.isPk()).findFirst();
    if (!firstPK.isPresent()) {
        throw new IllegalStateException("has not set PK col");
    }
    /**
     ******************************************************
     * 初始化索引Schema
     ******************************************************
     */
    dataXWriter.initialIndex(esSchema);
    // JSONArray schemaCols = esSchema.getSchemaCols();
    // ESClient esClient = new ESClient();
    // esClient.createClient(token.getEndpoint(),
    // token.getAccessKeyId(),
    // token.getAccessKeySecret(),
    // false,
    // 300000,
    // false,
    // false);
    // try {
    // esClient.createIndex(dataXWriter.getIndexName()
    // , dataXWriter.type
    // , esClient.genMappings(schemaCols, dataXWriter.type, (columnList) -> {
    // }), dataXWriter.settings, false);
    // } catch (Exception e) {
    // throw new RuntimeException(e);
    // } finally {
    // try {
    // esClient.closeJestClient();
    // } catch (Throwable e) {
    // 
    // }
    // }
    // if (!) {
    // throw new IllegalStateException("create index or mapping failed indexName:" + dataXWriter.getIndexName());
    // }
    // Map<String, String> config = new HashMap<>();
    // config.put("cluster.name", "my-cluster-name");
    // // This instructs the sink to emit after every element, otherwise they would be buffered
    // config.put("bulk.flush.max.actions", "1");
    List<HttpHost> transportAddresses = new ArrayList<>();
    transportAddresses.add(HttpHost.create(token.getEndpoint()));
    ElasticsearchSink.Builder<DTO> sinkBuilder = new ElasticsearchSink.Builder<>(transportAddresses, new DefaultElasticsearchSinkFunction(cols.stream().map((c) -> c.getName()).collect(Collectors.toSet()), firstPK.get().getName(), dataXWriter.getIndexName()));
    if (this.bulkFlushMaxActions != null) {
        sinkBuilder.setBulkFlushMaxActions(this.bulkFlushMaxActions);
    }
    if (this.bulkFlushMaxSizeMb != null) {
        sinkBuilder.setBulkFlushMaxSizeMb(bulkFlushMaxSizeMb);
    }
    if (this.bulkFlushIntervalMs != null) {
        sinkBuilder.setBulkFlushInterval(this.bulkFlushIntervalMs);
    }
    // new RestClientBuilder.HttpClientConfigCallback() {
    // @Override
    // public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
    // return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
    // }
    // }
    sinkBuilder.setFailureHandler(new DefaultActionRequestFailureHandler());
    if (StringUtils.isNotEmpty(token.getAccessKeyId()) || StringUtils.isNotEmpty(token.getAccessKeySecret())) {
        // 如果用户设置了accessKey 或者accessSecret
        sinkBuilder.setRestClientFactory(new TISElasticRestClientFactory(token.getAccessKeyId(), token.getAccessKeySecret()));
    // sinkBuilder.setRestClientFactory(new RestClientFactory() {
    // @Override
    // public void configureRestClientBuilder(RestClientBuilder restClientBuilder) {
    // final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    // credentialsProvider.setCredentials(AuthScope.ANY,
    // new UsernamePasswordCredentials(token.getAccessKeyId(), token.getAccessKeySecret()));
    // restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
    // @Override
    // public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
    // return httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
    // }
    // });
    // }
    // });
    }
    IDataxProcessor.TableAlias tableMapper = new IDataxProcessor.TableAlias();
    tableMapper.setTo(dataXWriter.getIndexName());
    IDataxReader reader = dataxProcessor.getReader(null);
    for (ISelectedTab selectedTab : reader.getSelectedTabs()) {
        tableMapper.setFrom(selectedTab.getName());
    }
    return Collections.singletonMap(tableMapper, sinkBuilder.build());
}

Example 3 with IHttpToken

use of com.qlangtech.tis.config.aliyun.IHttpToken in project plugins by qlangtech.

the class DataXElasticsearchWriter method initialIndex.

/**
 * 当增量开始执行前，先需要初始化一下索引实例
 *
 * @param esSchema
 */
public void initialIndex(ESTableAlias esSchema) {
    if (esSchema == null) {
        throw new IllegalArgumentException("param esSchema can not be null");
    }
    IHttpToken token = this.getToken();
    /**
     ******************************************************
     * 初始化索引Schema
     ******************************************************
     */
    JSONArray schemaCols = esSchema.getSchemaCols();
    ESClient esClient = new ESClient(ESInitialization.create(token.getEndpoint(), token.getAccessKeyId(), token.getAccessKeySecret(), false, 300000, false, false));
    try {
        esClient.createIndex(this.getIndexName(), this.type, esClient.genMappings(schemaCols, this.type, (columnList) -> {
        }), this.settings, false);
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        try {
            esClient.closeJestClient();
        } catch (Throwable e) {
        }
    }
}