Examples with SourceConfig com.google.api.services.datastream.v1alpha1.model.SourceConfig used on opensource projects

Example 6 with SourceConfig

use of com.google.api.services.datastream.v1alpha1.model.SourceConfig in project DataflowTemplates by GoogleCloudPlatform.

the class DataStreamClient method discoverMysqlTableSchema.

/**
 * Return a {@link MysqlTable} object with schema and PK information.
 *
 * @param streamName A fully qualified Stream name (ie. projects/my-project/stream/my-stream)
 * @param schemaName The name of the schema for the table being discovered.
 * @param tableName The name of the table to discover.
 * @param sourceConnProfile The SourceConfig connection profile to be discovered.
 */
public MysqlTable discoverMysqlTableSchema(String streamName, String schemaName, String tableName, SourceConfig sourceConnProfile) throws IOException {
    DataStream.Projects.Locations.ConnectionProfiles.Discover discoverConnProfile = getDiscoverTableRequest(streamName, schemaName, tableName, sourceConnProfile);
    MysqlRdbms tableResponse = discoverConnProfile.execute().getMysqlRdbms();
    MysqlDatabase schema = tableResponse.getMysqlDatabases().get(0);
    MysqlTable table = schema.getMysqlTables().get(0);
    return table;
}

Example 7 with SourceConfig

use of com.google.api.services.datastream.v1alpha1.model.SourceConfig in project DataflowTemplates by GoogleCloudPlatform.

the class DataStreamClientTest method testDataStreamClientGetSourceConnectionProfileName.

/**
 * Test whether {@link DataStreamClient#getSourceConnectionProfileName(String)} gets a Streams
 * source connection profile name.
 */
@Ignore
@Test
public void testDataStreamClientGetSourceConnectionProfileName() throws IOException, GeneralSecurityException {
    String projectId = "dataflow-bigstream-integration";
    String streamName = "projects/dataflow-bigstream-integration/locations/us-central1/streams/dfstream1";
    String connProfileName = "projects/402074789819/locations/us-central1/connectionProfiles/cp-1";
    DataStreamClient datastream = new DataStreamClient(null);
    SourceConfig sourceConnProfile = datastream.getSourceConnectionProfile(streamName);
    String sourceConnProfileName = sourceConnProfile.getSourceConnectionProfileName();
    assertEquals(sourceConnProfileName, connProfileName);
}

Example 8 with SourceConfig

use of com.google.api.services.datastream.v1alpha1.model.SourceConfig in project DataflowTemplates by GoogleCloudPlatform.

the class DataStreamClientTest method testDataStreamClientDiscoverOracleTableSchema.

/**
 * Test whether {@link DataStreamClient#discoverTableSchema(...)} gets a Streams source connection
 * profile name.
 */
@Ignore
@Test
public void testDataStreamClientDiscoverOracleTableSchema() throws IOException, GeneralSecurityException {
    String projectId = "dataflow-bigstream-integration";
    String streamName = "projects/dataflow-bigstream-integration/locations/us-central1/streams/dfstream1";
    String schemaName = "HR";
    String tableName = "JOBS";
    DataStreamClient datastream = new DataStreamClient(null);
    SourceConfig sourceConnProfile = datastream.getSourceConnectionProfile(streamName);
    OracleTable table = datastream.discoverOracleTableSchema(streamName, schemaName, tableName, sourceConnProfile);
    String columnName = table.getOracleColumns().get(0).getColumnName();
    Boolean isPrimaryKey = table.getOracleColumns().get(0).getPrimaryKey();
    assertEquals(columnName, "JOB_TITLE");
    assertEquals(isPrimaryKey, null);
}

Example 9 with SourceConfig

use of com.google.api.services.datastream.v1alpha1.model.SourceConfig in project DataflowTemplates by GoogleCloudPlatform.

the class DataStreamToSpanner method getSourceType.

private static String getSourceType(Options options) {
    if (options.getDatastreamSourceType() != null) {
        return options.getDatastreamSourceType();
    }
    if (options.getStreamName() == null) {
        throw new IllegalArgumentException("Stream name cannot be empty. ");
    }
    GcpOptions gcpOptions = options.as(GcpOptions.class);
    DataStreamClient datastreamClient;
    SourceConfig sourceConfig;
    try {
        datastreamClient = new DataStreamClient(gcpOptions.getGcpCredential());
        sourceConfig = datastreamClient.getSourceConnectionProfile(options.getStreamName());
    } catch (IOException e) {
        LOG.error("IOException Occurred: DataStreamClient failed initialization.");
        throw new IllegalArgumentException("Unable to initialize DatastreamClient: " + e);
    }
    if (sourceConfig.getMysqlSourceConfig() != null) {
        return DatastreamConstants.MYSQL_SOURCE_TYPE;
    } else if (sourceConfig.getOracleSourceConfig() != null) {
        return DatastreamConstants.ORACLE_SOURCE_TYPE;
    }
    LOG.error("Source Connection Profile Type Not Supported");
    throw new IllegalArgumentException("Unsupported source connection profile type in Datastream");
}

Example 10 with SourceConfig

use of com.google.api.services.datastream.v1alpha1.model.SourceConfig in project DataflowTemplates by GoogleCloudPlatform.

the class DataStreamClient method getOracleObjectSchema.

private Map<String, StandardSQLTypeName> getOracleObjectSchema(String streamName, String schemaName, String tableName, SourceConfig sourceConnProfile) throws IOException {
    Map<String, StandardSQLTypeName> objectSchema = new HashMap<String, StandardSQLTypeName>();
    OracleTable table = discoverOracleTableSchema(streamName, schemaName, tableName, sourceConnProfile);
    for (OracleColumn column : table.getOracleColumns()) {
        StandardSQLTypeName bqType = convertOracleToBigQueryColumnType(column);
        objectSchema.put(column.getColumnName(), bqType);
    }
    return objectSchema;
}