Examples with HiveSink org.apache.inlong.manager.client.api.sink.HiveSink used on opensource projects

Example 1 with HiveSink

use of org.apache.inlong.manager.client.api.sink.HiveSink in project incubator-inlong by apache.

the class InlongStreamSinkTransfer method parseHiveSink.

private static HiveSink parseHiveSink(HiveSinkResponse sinkResponse, StreamSink sink) {
    HiveSink hiveSink = new HiveSink();
    if (sink != null) {
        AssertUtil.isTrue(sinkResponse.getSinkName().equals(sink.getSinkName()), String.format("SinkName is not equal: %s != %s", sinkResponse, sink));
        HiveSink snapshot = (HiveSink) sink;
        hiveSink.setSinkName(snapshot.getSinkName());
        hiveSink.setDataSeparator(snapshot.getDataSeparator());
        hiveSink.setCharset(snapshot.getCharset());
        hiveSink.setAuthentication(snapshot.getAuthentication());
        hiveSink.setWarehouseDir(snapshot.getWarehouseDir());
        hiveSink.setFileFormat(snapshot.getFileFormat());
        hiveSink.setJdbcUrl(snapshot.getJdbcUrl());
        hiveSink.setTableName(snapshot.getTableName());
        hiveSink.setDbName(snapshot.getDbName());
        hiveSink.setHdfsDefaultFs(snapshot.getHdfsDefaultFs());
        hiveSink.setSecondaryPartition(snapshot.getSecondaryPartition());
        hiveSink.setPrimaryPartition(snapshot.getPrimaryPartition());
    } else {
        hiveSink.setSinkName(sinkResponse.getSinkName());
        hiveSink.setDataSeparator(DataSeparator.forAscii(Integer.parseInt(sinkResponse.getDataSeparator())));
        hiveSink.setCharset(Charset.forName(sinkResponse.getDataEncoding()));
        String password = sinkResponse.getPassword();
        String uname = sinkResponse.getUsername();
        hiveSink.setAuthentication(new DefaultAuthentication(uname, password));
        hiveSink.setWarehouseDir(sinkResponse.getWarehouseDir());
        hiveSink.setFileFormat(FileFormat.forName(sinkResponse.getFileFormat()));
        hiveSink.setJdbcUrl(sinkResponse.getJdbcUrl());
        hiveSink.setTableName(sinkResponse.getTableName());
        hiveSink.setDbName(sinkResponse.getDbName());
        hiveSink.setHdfsDefaultFs(sinkResponse.getHdfsDefaultFs());
        hiveSink.setSecondaryPartition(sinkResponse.getSecondaryPartition());
        hiveSink.setPrimaryPartition(sinkResponse.getPrimaryPartition());
    }
    hiveSink.setProperties(sinkResponse.getProperties());
    hiveSink.setSinkType(SinkType.HIVE);
    hiveSink.setNeedCreated(sinkResponse.getEnableCreateResource() == 1);
    if (CollectionUtils.isNotEmpty(sinkResponse.getFieldList())) {
        hiveSink.setSinkFields(convertToSinkFields(sinkResponse.getFieldList()));
    }
    return hiveSink;
}

Example 2 with HiveSink

use of org.apache.inlong.manager.client.api.sink.HiveSink in project incubator-inlong by apache.

the class InlongStreamSinkTransfer method createHiveRequest.

private static HiveSinkRequest createHiveRequest(StreamSink streamSink, InlongStreamInfo streamInfo) {
    HiveSinkRequest hiveSinkRequest = new HiveSinkRequest();
    HiveSink hiveSink = (HiveSink) streamSink;
    hiveSinkRequest.setSinkName(streamSink.getSinkName());
    hiveSinkRequest.setInlongGroupId(streamInfo.getInlongGroupId());
    hiveSinkRequest.setInlongStreamId(streamInfo.getInlongStreamId());
    hiveSinkRequest.setDataEncoding(hiveSink.getCharset().name());
    hiveSinkRequest.setEnableCreateTable(hiveSink.isNeedCreated() ? 1 : 0);
    hiveSinkRequest.setDataSeparator(String.valueOf(hiveSink.getDataSeparator().getAsciiCode()));
    hiveSinkRequest.setDbName(hiveSink.getDbName());
    hiveSinkRequest.setTableName(hiveSink.getTableName());
    hiveSinkRequest.setHdfsDefaultFs(hiveSink.getHdfsDefaultFs());
    hiveSinkRequest.setJdbcUrl(hiveSink.getJdbcUrl());
    hiveSinkRequest.setWarehouseDir(hiveSink.getWarehouseDir());
    hiveSinkRequest.setFileFormat(hiveSink.getFileFormat().name());
    hiveSinkRequest.setSinkType(hiveSink.getSinkType().name());
    DefaultAuthentication defaultAuthentication = hiveSink.getAuthentication();
    AssertUtil.notNull(defaultAuthentication, String.format("Hive storage:%s must be authenticated", hiveSink.getDbName()));
    hiveSinkRequest.setUsername(defaultAuthentication.getUserName());
    hiveSinkRequest.setPassword(defaultAuthentication.getPassword());
    hiveSinkRequest.setPrimaryPartition(hiveSink.getPrimaryPartition());
    hiveSinkRequest.setSecondaryPartition(hiveSink.getSecondaryPartition());
    hiveSinkRequest.setProperties(hiveSink.getProperties());
    if (CollectionUtils.isNotEmpty(hiveSink.getSinkFields())) {
        List<SinkFieldRequest> fieldRequests = createSinkFieldRequests(streamSink.getSinkFields());
        hiveSinkRequest.setFieldList(fieldRequests);
    }
    return hiveSinkRequest;
}