Examples with IOMessageImpl eu.esdihumboldt.hale.common.core.io.report.impl.IOMessageImpl used on opensource projects

Example 51 with IOMessageImpl

use of eu.esdihumboldt.hale.common.core.io.report.impl.IOMessageImpl in project hale by halestudio.

the class JDBCInstanceWriter method setStatementParameter.

/**
 * Set a prepared statement parameter value.
 *
 * @param statement the prepared statement
 * @param index the parameter index
 * @param value the value, not <code>null</code>
 * @param propertyDef the associated property definition
 * @param sqlType the SQL type
 * @param reporter the reporter
 * @param conn Connection (currently used for geometry conversion for oracle
 *            database
 * @throws SQLException if setting the parameter fails
 */
@SuppressWarnings("unchecked")
private void setStatementParameter(PreparedStatement statement, int index, Object value, PropertyDefinition propertyDef, int sqlType, IOReporter reporter, Connection conn) throws SQLException {
    if (propertyDef.getPropertyType().getConstraint(GeometryType.class).isGeometry()) {
        // is a geometry column
        // get the geometry advisor
        @SuppressWarnings("rawtypes") GeometryAdvisor advisor = propertyDef.getPropertyType().getConstraint(GeometryAdvisorConstraint.class).getAdvisor();
        if (advisor != null) {
            // use the advisor to convert the geometry
            if (value instanceof GeometryProperty<?>) {
                // XXX JTS geometry conversion needed beforehand?
                try {
                    value = advisor.convertGeometry((GeometryProperty<?>) value, propertyDef.getPropertyType(), conn);
                } catch (Exception e) {
                    reporter.error(new IOMessageImpl("Something went wrong during conversion", e));
                }
            } else {
                reporter.error(new IOMessageImpl("Geometry value is not of type GeometryProperty and could thus not be converted for the database", null));
            }
        }
    }
    SQLArray arrayInfo = propertyDef.getPropertyType().getConstraint(SQLArray.class);
    if (arrayInfo.isArray()) {
        // is an array column
        Object[] values;
        if (value.getClass().isArray()) {
            values = (Object[]) value;
        } else {
            values = new Object[] { value };
        }
        // FIXME for multi-dimensional arrays, make sure internal structures
        // are arrays?
        // use SQL array as value
        Array array = conn.createArrayOf(arrayInfo.getElementTypeName(), values);
        value = array;
    // FIXME collect arrays to allow them to be freed after the
    // statement is executed?
    }
    // TODO handling of other types?
    // set the value
    statement.setObject(index, value, sqlType);
}

Example 52 with IOMessageImpl

use of eu.esdihumboldt.hale.common.core.io.report.impl.IOMessageImpl in project hale by halestudio.

the class JDBCSchemaReader method loadFromSource.

@Override
protected Schema loadFromSource(ProgressIndicator progress, IOReporter reporter) throws IOProviderConfigurationException, IOException {
    DefaultSchema typeIndex = null;
    progress.begin("Read database schema", ProgressIndicator.UNKNOWN);
    Connection connection = null;
    try {
        // connect to the database
        try {
            connection = getConnection();
        } catch (Exception e) {
            reporter.error(new IOMessageImpl(e.getLocalizedMessage(), e));
            reporter.setSuccess(false);
            reporter.setSummary("Failed to connect to database.");
            return null;
        }
        // connection has been created), report a warning message instead
        try {
            connection.setReadOnly(true);
        } catch (SQLException e) {
        // ignore
        // reporter.warn(new IOMessageImpl(e.getLocalizedMessage(), e));
        }
        URI jdbcURI = getSource().getLocation();
        final SchemaCrawlerOptions options = new SchemaCrawlerOptions();
        SchemaInfoLevel level = new SchemaInfoLevel();
        level.setTag("hale");
        // these are enabled by default, we don't need them (yet)
        level.setRetrieveSchemaCrawlerInfo(false);
        level.setRetrieveJdbcDriverInfo(false);
        level.setRetrieveDatabaseInfo(false);
        // set what we need
        level.setRetrieveTables(true);
        level.setRetrieveColumnDataTypes(true);
        level.setRetrieveUserDefinedColumnDataTypes(true);
        // to get table columns
        level.setRetrieveTableColumns(true);
        // information, also
        // includes primary key
        // to get linking information
        level.setRetrieveForeignKeys(true);
        // level.setRetrieveIndices(true); // to get info about UNIQUE indices for validation
        // XXX For some advanced info / DBMS specific info we'll need a
        // properties file. See Config & InformationSchemaViews.
        level.setTag("hale");
        if (getParameter(SCHEMAS).as(String.class) != null) {
            String schemas = getParameter(SCHEMAS).as(String.class).replace(',', '|');
            options.setSchemaInclusionRule(new RegularExpressionInclusionRule(schemas));
        }
        if (SchemaSpaceID.SOURCE.equals(getSchemaSpace())) {
            // show views and tables
            final List<String> tableTypesWanted = Arrays.asList("TABLE", "VIEW", "MATERIALIZED VIEW");
            // try to determine table types supported by the JDBC connection
            final List<String> tableTypeSupported = new ArrayList<>();
            try {
                ResultSet rs = connection.getMetaData().getTableTypes();
                while (rs.next()) {
                    String tableType = rs.getString(1);
                    tableTypeSupported.add(tableType);
                }
            } catch (Throwable t) {
                // Ignore, try with wanted list
                reporter.warn(new IOMessageImpl(MessageFormat.format("Could not determine supported table types for connection: {0}", t.getMessage()), t));
                tableTypeSupported.addAll(tableTypesWanted);
            }
            options.setTableTypes(tableTypesWanted.stream().filter(tt -> tableTypeSupported.contains(tt)).collect(Collectors.toList()));
        } else {
            // only show tables
            options.setTableTypes(Arrays.asList("TABLE"));
        }
        options.setSchemaInfoLevel(level);
        // get advisor
        // XXX should be created once, and used in other places if
        // applicable
        JDBCSchemaReaderAdvisor advisor = SchemaReaderAdvisorExtension.getInstance().getAdvisor(connection);
        if (advisor != null) {
            advisor.configureSchemaCrawler(options);
        }
        final Catalog database = SchemaCrawlerUtility.getCatalog(connection, options);
        @SuppressWarnings("unused") String quotes = JDBCUtil.determineQuoteString(connection);
        // FIXME not actually used here or in SQL schema reader
        String overallNamespace = JDBCUtil.determineNamespace(jdbcURI, advisor);
        // create the type index
        typeIndex = new DefaultSchema(overallNamespace, jdbcURI);
        for (final schemacrawler.schema.Schema schema : database.getSchemas()) {
            // each schema represents a namespace
            String namespace;
            if (overallNamespace.isEmpty()) {
                namespace = unquote(schema.getName());
            } else {
                namespace = overallNamespace;
                if (schema.getName() != null) {
                    namespace += ":" + unquote(schema.getName());
                }
            }
            for (final Table table : database.getTables(schema)) {
                // each table is a type
                // get the type definition
                TypeDefinition type = getOrCreateTableType(schema, table, overallNamespace, namespace, typeIndex, connection, reporter, database);
                // get ResultSetMetaData for extra info about columns (e. g.
                // auto increment)
                ResultsColumns additionalInfo = null;
                Statement stmt = null;
                try {
                    stmt = connection.createStatement();
                    // get if in table name, quotation required or not.
                    String fullTableName = getQuotedValue(table.getName());
                    if (schema.getName() != null) {
                        fullTableName = getQuotedValue(schema.getName()) + "." + fullTableName;
                    }
                    ResultSet rs = stmt.executeQuery("SELECT * FROM " + fullTableName + " WHERE 1 = 0");
                    additionalInfo = SchemaCrawlerUtility.getResultColumns(rs);
                } catch (SQLException sqle) {
                    reporter.warn(new IOMessageImpl("Couldn't retrieve additional column meta data.", sqle));
                } finally {
                    if (stmt != null)
                        try {
                            stmt.close();
                        } catch (SQLException e) {
                        // ignore
                        }
                }
                // create property definitions for each column
                for (final Column column : table.getColumns()) {
                    DefaultPropertyDefinition property = getOrCreateProperty(schema, type, column, overallNamespace, namespace, typeIndex, connection, reporter, database);
                    // XXX does not work for example for PostgreSQL
                    if (additionalInfo != null) {
                        // ResultColumns does not quote the column namen in
                        // contrast to every other place
                        ResultsColumn rc = additionalInfo.getColumn(unquote(column.getName()));
                        if (rc != null && rc.isAutoIncrement())
                            property.setConstraint(AutoIncrementFlag.get(true));
                    }
                }
            }
        }
        reporter.setSuccess(true);
    } catch (SchemaCrawlerException e) {
        throw new IOProviderConfigurationException("Failed to read database schema", e);
    } finally {
        if (connection != null) {
            try {
                connection.close();
            } catch (SQLException e) {
            // ignore
            }
        }
        progress.end();
    }
    return typeIndex;
}

Example 53 with IOMessageImpl

use of eu.esdihumboldt.hale.common.core.io.report.impl.IOMessageImpl in project hale by halestudio.

the class JDBCSchemaReader method getOrCreateTableType.

/**
 * Get or create the type definition for the given table.
 *
 * @param schema the schema in which the table exists
 * @param table the table
 * @param overallNamespace the database namespace
 * @param namespace the schema namespace
 * @param types the type index
 * @param connection the database connection
 * @param reporter the reporter
 * @param catalog the catalog for access to other column types
 * @return the type definition for the given table
 */
private TypeDefinition getOrCreateTableType(schemacrawler.schema.Schema schema, Table table, String overallNamespace, String namespace, DefaultSchema types, Connection connection, IOReporter reporter, Catalog catalog) {
    QName typeName = new QName(namespace, unquote(table.getName()));
    // check for existing type
    TypeDefinition existingType = types.getType(typeName);
    if (existingType != null)
        return existingType;
    // create new type
    DefaultTypeDefinition type = new DefaultTypeDefinition(typeName);
    // set description if available
    if (table.getRemarks() != null && !table.getRemarks().isEmpty())
        type.setDescription(table.getRemarks());
    // configure type
    type.setConstraint(MappableFlag.ENABLED);
    type.setConstraint(MappingRelevantFlag.ENABLED);
    type.setConstraint(HasValueFlag.DISABLED);
    // set schema and table name
    DatabaseTable tableConstraint = new DatabaseTable(unquote(schema.getName()), unquote(table.getName()), useQuote);
    type.setConstraint(tableConstraint);
    // set SQL query constraint
    String query = "SELECT * FROM " + tableConstraint.getFullTableName();
    type.setConstraint(new SQLQuery(query));
    // set primary key if possible
    PrimaryKey key = null;
    try {
        key = table.getPrimaryKey();
    } catch (Exception e) {
        reporter.warn(new IOMessageImpl("Could not read primary key metadata for table: " + table.getFullName(), e));
    }
    if (key != null) {
        List<IndexColumn> columns = key.getColumns();
        if (columns.size() > 1) {
            reporter.warn(new IOMessageImpl("Primary keys over multiple columns are not yet supported.", null));
        } else if (columns.size() == 1) {
            // create constraint, get property definition for original table
            // column (maybe could use index column, too)
            type.setConstraint(new eu.esdihumboldt.hale.common.schema.model.constraint.type.PrimaryKey(Collections.<QName>singletonList(getOrCreateProperty(schema, type, table.getColumn(columns.get(0).getName()), overallNamespace, namespace, types, connection, reporter, catalog).getName())));
        }
    }
    // TODO validation of constraints? Most are about several instances (i.
    // e. UNIQUE)
    types.addType(type);
    return type;
}

Example 54 with IOMessageImpl

use of eu.esdihumboldt.hale.common.core.io.report.impl.IOMessageImpl in project hale by halestudio.

the class JDBCSchemaReader method getOrCreateColumnType.

/**
 * Get or create the type definition for the given column.
 *
 * @param column the column
 * @param overallNamespace the database namespace
 * @param types the type index
 * @param connection the database connection
 * @param tableType the type definition of the table the column is part of
 * @param reporter the reporter
 * @param catalog the catalog for access to other column types
 * @return the type definition for the column type
 */
public static TypeDefinition getOrCreateColumnType(BaseColumn<?> column, final String overallNamespace, DefaultSchema types, Connection connection, TypeDefinition tableType, IOReporter reporter, Catalog catalog) {
    // XXX what about shared types?
    // TODO the size/width info (VARCHAR(_30_)) is in column, the
    // columntype/-name is not sufficient
    // getType();
    ColumnDataType columnType = column.getColumnDataType();
    String localName = columnType.getName();
    QName typeName = new QName(overallNamespace, localName);
    // check for geometry type
    GeometryTypeInfo geomType = GeometryTypeExtension.getInstance().getTypeInfo(columnType.getName(), connection);
    @SuppressWarnings("rawtypes") GeometryAdvisor geomAdvisor = null;
    if (geomType != null) {
        geomAdvisor = geomType.getGeometryAdvisor();
        // determine if a type specifically for this column is needed
        if (!geomAdvisor.isFixedType(columnType)) {
            // must use a specific type definition for this column
            // -> use a type name based on the column
            // new namespace is the table and column name
            String ns = tableType.getName().getNamespaceURI() + '/' + tableType.getName().getLocalPart() + '/' + unquote(column.getName());
            typeName = new QName(ns, localName);
        }
    }
    // check for existing type
    TypeDefinition existing = types.getType(typeName);
    if (existing != null)
        return existing;
    // create new type
    DefaultTypeDefinition type = new DefaultTypeDefinition(typeName);
    type.setConstraint(HasValueFlag.ENABLED);
    CustomType cust = CustomTypeExtension.getInstance().getCustomType(localName, connection);
    /*
		 * Oracle jdbc was returning sqltype -6 for NUMBER data type, but it is
		 * bound to boolean by Types.java class. Configured the sqltype 6 for
		 * NUMBER data type.
		 */
    if (cust != null) {
        type.setConstraint(SQLType.get(cust.getSQLType()));
    } else {
        type.setConstraint(SQLType.get(columnType.getJavaSqlType().getJavaSqlType()));
    }
    if (geomType != null && geomAdvisor != null) {
        // configure geometry type
        @SuppressWarnings("unchecked") Class<? extends Geometry> geomClass = geomAdvisor.configureGeometryColumnType(connection, column, type);
        type.setConstraint(GeometryType.get(geomClass));
        // always a single geometry
        type.setConstraint(Binding.get(GeometryProperty.class));
        // remember advisor for type (used in instance writer)
        type.setConstraint(geomType.getConstraint());
    } else {
        // configure type
        Class<?> binding = null;
        if (cust != null) {
            binding = cust.getBinding();
        } else {
            if (column.getColumnDataType().getJavaSqlType().getJavaSqlType() == Types.ARRAY) {
                // TODO let this be handled by a possible advisor?
                /*
					 * Special handling for arrays.
					 * 
					 * Challenges:
					 * 
					 * - find the type of contained items
					 * 
					 * - determine dimensions and size
					 */
                Class<?> elementBinding;
                // determine type/binding of contained items
                ColumnDataType cdt = column.getColumnDataType();
                ColumnDataType itemType = null;
                String dbTypeName = cdt.getDatabaseSpecificTypeName();
                // prefix and look up type
                if (dbTypeName.startsWith("_")) {
                    String testName = dbTypeName.substring(1);
                    itemType = catalog.getSystemColumnDataType(testName);
                }
                if (itemType == null) {
                    // generic binding
                    elementBinding = Object.class;
                    reporter.error(new IOMessageImpl("Could not determine element type for array column", null));
                } else {
                    elementBinding = itemType.getTypeMappedClass();
                // TODO support custom bindings?
                // XXX probably needed for Oracle?
                }
                // dimensions and size cannot be determined from schema
                // crawler it seems - would need database specific queries
                // (if the info is available at all)
                /*
					 * Postgres:
					 * 
					 * Dimensions and size are not part of the schema, they can
					 * only be determined for a value.
					 */
                // XXX for now, stick to what we can determine
                int dimension = SQLArray.UNKNOWN_DIMENSION;
                String specificTypeName = (itemType != null) ? (itemType.getDatabaseSpecificTypeName()) : (null);
                type.setConstraint(new SQLArray(elementBinding, specificTypeName, dimension, null));
                // set binding
                if (dimension <= 1) {
                    // XXX for now, use this representation also if
                    // dimension is not known
                    // 1-dimensional -> as multiple occurrences
                    binding = elementBinding;
                } else {
                    // XXX use collection or something similar instead?
                    binding = Object.class;
                }
            } else {
                binding = column.getColumnDataType().getTypeMappedClass();
            }
        }
        type.setConstraint(Binding.get(binding));
        type.setConstraint(HasValueFlag.ENABLED);
    }
    if (columnType.getRemarks() != null && !columnType.getRemarks().isEmpty())
        type.setDescription(columnType.getRemarks());
    types.addType(type);
    return type;
}

Example 55 with IOMessageImpl

use of eu.esdihumboldt.hale.common.core.io.report.impl.IOMessageImpl in project hale by halestudio.

the class JacksonMapper method streamWriteInstanceValue.

/**
 * Writes a single instance into JSON
 *
 * @param instance the instance to write
 * @param reporter the reporter
 * @throws JsonGenerationException if there was a problem generating the
 *             JSON
 * @throws IOException if writing the file failed
 */
private void streamWriteInstanceValue(Instance instance, IOReporter reporter) throws JsonGenerationException, IOException {
    jsonGen.writeStartObject();
    // check if the instance contains a value and write it down
    Object value = instance.getValue();
    if (value != null) {
        // detect geometry collections (as occurring in XML schemas)
        if (value instanceof Collection) {
            Collection<?> col = (Collection<?>) value;
            if (!col.isEmpty()) {
                Object element = col.iterator().next();
                if (element instanceof GeometryProperty<?> || element instanceof Geometry) {
                    // extract geometry
                    value = element;
                    if (col.size() > 1) {
                        // there are more geometry values
                        // XXX can we handle multiple here?
                        // XXX for now warn
                        reporter.warn(new IOMessageImpl("Ignoring multiple geometries in instance value", null));
                    }
                }
            }
        }
        jsonGen.writeFieldName("_value");
        streamWriteValue(value);
    }
    streamWriteProperties(instance, reporter);
    jsonGen.writeEndObject();
}