Examples with StandardSQLTypeName com.google.cloud.bigquery.StandardSQLTypeName used on opensource projects

Example 1 with StandardSQLTypeName

use of com.google.cloud.bigquery.StandardSQLTypeName in project DataflowTemplates by GoogleCloudPlatform.

the class DataStreamClient method getMysqlObjectSchema.

private Map<String, StandardSQLTypeName> getMysqlObjectSchema(String streamName, String schemaName, String tableName, SourceConfig sourceConnProfile) throws IOException {
    Map<String, StandardSQLTypeName> objectSchema = new HashMap<String, StandardSQLTypeName>();
    MysqlTable table = discoverMysqlTableSchema(streamName, schemaName, tableName, sourceConnProfile);
    for (MysqlColumn column : table.getMysqlColumns()) {
        StandardSQLTypeName bqType = convertMysqlToBigQueryColumnType(column);
        objectSchema.put(column.getColumnName(), bqType);
    }
    return objectSchema;
}

Example 2 with StandardSQLTypeName

use of com.google.cloud.bigquery.StandardSQLTypeName in project DataflowTemplates by GoogleCloudPlatform.

the class BigQueryMapper method updateTableIfRequired.

/**
 * Extracts and applies new column information to BigQuery by comparing the TableRow against the
 * BigQuery Table.
 *
 * @param tableId a TableId referencing the BigQuery table to be loaded to.
 * @param row a TableRow with the raw data to be loaded into BigQuery.
 * @param inputSchema The source schema lookup to be used in mapping.
 */
private void updateTableIfRequired(TableId tableId, TableRow row) {
    Table table = this.tableCache.getOrCreateBigQueryTable(tableId, this.dayPartitioning);
    Map<String, StandardSQLTypeName> inputSchema = new HashMap<String, StandardSQLTypeName>();
    List<Field> newFieldList = getNewTableFields(row, table, inputSchema, this.ignoreFields);
    if (newFieldList.size() > 0) {
        LOG.info("Updating Table: {}", tableId.toString());
        updateBigQueryTable(tableId, row, this.ignoreFields);
    }
}

Example 3 with StandardSQLTypeName

use of com.google.cloud.bigquery.StandardSQLTypeName in project DataflowTemplates by GoogleCloudPlatform.

the class BigQueryMapper method withDefaultSchemaFromGCS.

/**
 * The function {@link withDefaultSchemaFromGCS} reads a BigQuery Schema file from GCS and stores
 * it to be used in the Mapper schema logic.
 *
 * @param filePath path to file in GCS
 */
public BigQueryMapper<InputT, OutputT> withDefaultSchemaFromGCS(String filePath) {
    if (filePath == null) {
        return this;
    }
    // TODO: A supplier that reloads the GCS file regularly would allow
    // a user to change the file w/o tearing down the pipeline.
    String schemaStr = SchemaUtils.getGcsFileAsString(filePath);
    List<Field> schemaFields = BigQueryConverters.SchemaUtils.schemaFromString(schemaStr);
    Map<String, StandardSQLTypeName> schema = new HashMap<String, StandardSQLTypeName>();
    for (Field field : schemaFields) {
        schema.put(field.getName(), field.getType().getStandardType());
    }
    this.defaultSchema = schema;
    return this;
}

Example 4 with StandardSQLTypeName

use of com.google.cloud.bigquery.StandardSQLTypeName in project bigquery-utils by GoogleCloudPlatform.

the class BigQuerySchemaJsonDeserializer method deserialize.

@Override
public FieldList deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) throws JsonParseException {
    JsonArray fieldsArray = jsonElement.getAsJsonArray();
    for (JsonElement fieldElement : fieldsArray) {
        JsonObject fieldObject = fieldElement.getAsJsonObject();
        // Field class stores types as LegacySQLTypeName instead of StandardSQLTypeName
        if (fieldObject.has("type")) {
            // Convert standard type to legacy type
            StandardSQLTypeName standardType = StandardSQLTypeName.valueOf(fieldObject.get("type").getAsString());
            LegacySQLTypeName legacyType = LegacySQLTypeName.legacySQLTypeName(standardType);
            // Insert LegacySQLTypeName object so it can be used for type
            JsonObject typeObject = new JsonObject();
            typeObject.addProperty("constant", legacyType.name());
            fieldObject.add("type", typeObject);
        }
        // Field class uses subFields instead of fields for STRUCT/RECORD type
        if (fieldObject.has("fields")) {
            fieldObject.add("subFields", fieldObject.get("fields"));
            fieldObject.remove("fields");
        }
    }
    Field[] fields = jsonDeserializationContext.deserialize(fieldsArray, Field[].class);
    return FieldList.of(fields);
}

Example 5 with StandardSQLTypeName

use of com.google.cloud.bigquery.StandardSQLTypeName in project DataflowTemplates by GoogleCloudPlatform.

the class DataStreamClient method getOracleObjectSchema.

private Map<String, StandardSQLTypeName> getOracleObjectSchema(String streamName, String schemaName, String tableName, SourceConfig sourceConnProfile) throws IOException {
    Map<String, StandardSQLTypeName> objectSchema = new HashMap<String, StandardSQLTypeName>();
    OracleTable table = discoverOracleTableSchema(streamName, schemaName, tableName, sourceConnProfile);
    for (OracleColumn column : table.getOracleColumns()) {
        StandardSQLTypeName bqType = convertOracleToBigQueryColumnType(column);
        objectSchema.put(column.getColumnName(), bqType);
    }
    return objectSchema;
}