Examples with AbstractExecutionSource org.jkiss.dbeaver.model.impl.AbstractExecutionSource used on opensource projects

Example 6 with AbstractExecutionSource

use of org.jkiss.dbeaver.model.impl.AbstractExecutionSource in project dbeaver by serge-rider.

the class DatabaseTransferConsumer method fetchEnd.

@Override
public void fetchEnd(DBCSession session, DBCResultSet resultSet) throws DBCException {
    try {
        if (rowsExported > 0) {
            insertBatch(true);
        }
        if (executeBatch != null) {
            executeBatch.close();
            executeBatch = null;
        }
    } finally {
        DBSDataManipulator targetObject = getTargetObject();
        if (!isPreview && targetObject instanceof DBSDataManipulatorExt) {
            ((DBSDataManipulatorExt) targetObject).afterDataChange(targetSession, DBSManipulationType.INSERT, targetAttributes.toArray(new DBSAttributeBase[0]), new AbstractExecutionSource(getSourceObject(), targetContext, this));
        }
    }
}

Example 7 with AbstractExecutionSource

use of org.jkiss.dbeaver.model.impl.AbstractExecutionSource in project dbeaver by serge-rider.

the class DatabaseTransferProducer method transferData.

@Override
public void transferData(@NotNull DBRProgressMonitor monitor1, @NotNull IDataTransferConsumer consumer, @Nullable IDataTransferProcessor processor, @NotNull DatabaseProducerSettings settings, @Nullable DBTTask task) throws DBException {
    String contextTask = DTMessages.data_transfer_wizard_job_task_export;
    DBSDataContainer databaseObject = getDatabaseObject();
    if (databaseObject == null) {
        throw new DBException("No input database object found");
    }
    DBPDataSource dataSource = databaseObject.getDataSource();
    assert (dataSource != null);
    DBExecUtils.tryExecuteRecover(monitor1, dataSource, monitor -> {
        long readFlags = DBSDataContainer.FLAG_NONE;
        if (settings.isSelectedColumnsOnly()) {
            readFlags |= DBSDataContainer.FLAG_USE_SELECTED_COLUMNS;
        }
        if (settings.isSelectedRowsOnly()) {
            readFlags |= DBSDataContainer.FLAG_USE_SELECTED_ROWS;
        }
        boolean newConnection = settings.isOpenNewConnections() && !getDatabaseObject().getDataSource().getContainer().getDriver().isEmbedded();
        boolean forceDataReadTransactions = Boolean.TRUE.equals(dataSource.getDataSourceFeature(DBConstants.FEATURE_LOB_REQUIRE_TRANSACTIONS));
        boolean selectiveExportFromUI = settings.isSelectedColumnsOnly() || settings.isSelectedRowsOnly();
        try {
            DBCExecutionContext context;
            if (dataContainer instanceof DBPContextProvider) {
                context = ((DBPContextProvider) dataContainer).getExecutionContext();
            } else {
                context = DBUtils.getDefaultContext(dataContainer, false);
            }
            if (context == null) {
                throw new DBCException("Can't retrieve execution context from data container " + dataContainer);
            }
            if (!selectiveExportFromUI && newConnection) {
                context = DBUtils.getObjectOwnerInstance(getDatabaseObject()).openIsolatedContext(monitor, "Data transfer producer", context);
                DBExecUtils.setExecutionContextDefaults(monitor, dataSource, context, defaultCatalog, null, defaultSchema);
            }
            if (task != null) {
                DBTaskUtils.initFromContext(monitor, task, context);
            }
            try (DBCSession session = context.openSession(monitor, DBCExecutionPurpose.UTIL, contextTask)) {
                Boolean oldAutoCommit = null;
                try {
                    AbstractExecutionSource transferSource = new AbstractExecutionSource(dataContainer, context, consumer);
                    session.enableLogging(false);
                    if (!selectiveExportFromUI && (newConnection || forceDataReadTransactions)) {
                        // other complex structures only in transactional mode
                        try {
                            DBCTransactionManager txnManager = DBUtils.getTransactionManager(context);
                            if (txnManager != null && txnManager.isSupportsTransactions()) {
                                oldAutoCommit = txnManager.isAutoCommit();
                                txnManager.setAutoCommit(monitor, false);
                            }
                        } catch (DBCException e) {
                            log.warn("Can't change auto-commit", e);
                        }
                    }
                    long totalRows = 0;
                    if (settings.isQueryRowCount() && (dataContainer.getSupportedFeatures() & DBSDataContainer.DATA_COUNT) != 0) {
                        monitor.beginTask(DTMessages.data_transfer_wizard_job_task_retrieve, 1);
                        try {
                            totalRows = dataContainer.countData(transferSource, session, dataFilter, readFlags);
                        } catch (Throwable e) {
                            log.warn("Can't retrieve row count from '" + dataContainer.getName() + "'", e);
                            try {
                                DBCTransactionManager txnManager = DBUtils.getTransactionManager(session.getExecutionContext());
                                if (txnManager != null && !txnManager.isAutoCommit()) {
                                    txnManager.rollback(session, null);
                                }
                            } catch (Throwable e1) {
                                log.warn("Error rolling back transaction", e1);
                            }
                        } finally {
                            monitor.done();
                        }
                    }
                    monitor.beginTask(DTMessages.data_transfer_wizard_job_task_export_table_data, (int) totalRows);
                    try {
                        monitor.subTask("Read data");
                        // Perform export
                        if (settings.getExtractType() == DatabaseProducerSettings.ExtractType.SINGLE_QUERY) {
                            // Just do it in single query
                            dataContainer.readData(transferSource, session, consumer, dataFilter, -1, -1, readFlags, settings.getFetchSize());
                        } else {
                            // Read all data by segments
                            long offset = 0;
                            int segmentSize = settings.getSegmentSize();
                            for (; ; ) {
                                DBCStatistics statistics = dataContainer.readData(transferSource, session, consumer, dataFilter, offset, segmentSize, readFlags, settings.getFetchSize());
                                if (statistics == null || statistics.getRowsFetched() < segmentSize) {
                                    // Done
                                    break;
                                }
                                offset += statistics.getRowsFetched();
                            }
                        }
                    } finally {
                        monitor.done();
                    }
                } finally {
                    if (!selectiveExportFromUI && (newConnection || forceDataReadTransactions)) {
                        DBCTransactionManager txnManager = DBUtils.getTransactionManager(context);
                        if (txnManager != null && txnManager.isSupportsTransactions() && !txnManager.isAutoCommit()) {
                            try {
                                txnManager.commit(session);
                            } catch (Exception e) {
                                log.error("Can't finish transaction in data producer connection", e);
                            }
                            if (oldAutoCommit != null) {
                                try {
                                    txnManager.setAutoCommit(session.getProgressMonitor(), oldAutoCommit);
                                } catch (Exception e) {
                                    log.error("Can't finish transaction in data producer connection", e);
                                }
                            }
                        }
                    }
                    if (!selectiveExportFromUI && newConnection) {
                        context.close();
                    }
                }
            }
        } catch (DBException e) {
            throw new InvocationTargetException(e);
        }
    });
}

Example 8 with AbstractExecutionSource

use of org.jkiss.dbeaver.model.impl.AbstractExecutionSource in project dbeaver by dbeaver.

the class DatabaseTransferConsumer method fetchStart.

@Override
public void fetchStart(DBCSession session, DBCResultSet resultSet, long offset, long maxRows) throws DBCException {
    try {
        initExporter(session.getProgressMonitor());
    } catch (DBException e) {
        throw new DBCException("Error initializing exporter", e);
    }
    if (containerMapping == null) {
        throw new DBCException("Internal error: consumer mappings not set");
    }
    AbstractExecutionSource executionSource = new AbstractExecutionSource(containerMapping.getSource(), targetContext, this);
    DBSDataManipulator targetObject = getTargetObject();
    if (!isPreview && offset <= 0 && settings.isTruncateBeforeLoad() && (containerMapping == null || containerMapping.getMappingType() == DatabaseMappingType.existing)) {
        // Truncate target tables
        if ((targetObject.getSupportedFeatures() & DBSDataManipulator.DATA_TRUNCATE) != 0) {
            targetObject.truncateData(targetSession, executionSource);
        } else {
            log.error("Table '" + targetObject.getName() + "' doesn't support truncate operation");
        }
    }
    DBDAttributeBinding[] rsAttributes;
    boolean dynamicTarget = targetContext.getDataSource().getInfo().isDynamicMetadata();
    DBSDataContainer sourceObject = getSourceObject();
    if (dynamicTarget) {
        // Document-based datasource
        rsAttributes = DBUtils.getAttributeBindings(session, sourceObject, resultSet.getMeta());
    } else {
        rsAttributes = DBUtils.makeLeafAttributeBindings(session, sourceObject, resultSet);
    }
    columnMappings = new ColumnMapping[rsAttributes.length];
    sourceBindings = rsAttributes;
    targetAttributes = new ArrayList<>(columnMappings.length);
    for (int i = 0; i < rsAttributes.length; i++) {
        if (isSkipColumn(rsAttributes[i])) {
            continue;
        }
        ColumnMapping columnMapping = new ColumnMapping(rsAttributes[i]);
        if (containerMapping == null) {
            // Map all attributes directly.
            if (targetObject instanceof DBSEntity) {
                try {
                    DBSEntityAttribute attribute = ((DBSEntity) targetObject).getAttribute(session.getProgressMonitor(), columnMapping.sourceAttr.getName());
                    if (attribute != null) {
                        columnMapping.targetAttr = new DatabaseMappingAttribute(null, columnMapping.sourceAttr);
                        columnMapping.targetAttr.setTarget(attribute);
                        columnMapping.targetAttr.setMappingType(DatabaseMappingType.existing);
                    }
                } catch (DBException e) {
                    log.error("Error getting target attribute");
                }
            }
            if (columnMapping.targetAttr == null) {
                throw new DBCException("Can't resolve target attribute for [" + columnMapping.sourceAttr.getName() + "]");
            }
        } else if (sourceObject instanceof DBSDocumentContainer && dynamicTarget) {
            try {
                DBSDocumentContainer docContainer = (DBSDocumentContainer) (targetObject instanceof DBSDocumentContainer ? targetObject : sourceObject);
                DBSEntityAttribute docAttribute = docContainer.getDocumentAttribute(session.getProgressMonitor());
                if (docAttribute != null) {
                    columnMapping.targetAttr = new DatabaseMappingAttribute(containerMapping, columnMapping.sourceAttr);
                    columnMapping.targetAttr.setTarget(docAttribute);
                    columnMapping.targetAttr.setMappingType(DatabaseMappingType.existing);
                }
            } catch (DBException e) {
                throw new DBCException("Error getting document attribute", e);
            }
        } else {
            columnMapping.targetAttr = containerMapping.getAttributeMapping(columnMapping.sourceAttr);
            if (columnMapping.targetAttr == null) {
                throw new DBCException("Can't find target attribute [" + columnMapping.sourceAttr.getName() + "]");
            }
        }
        if (columnMapping.targetAttr.getMappingType() == DatabaseMappingType.skip) {
            continue;
        }
        DBSEntityAttribute targetAttr = columnMapping.targetAttr.getTarget();
        if (targetAttr == null) {
            if (isPreview) {
                targetAttr = new PreviewColumnInfo(null, columnMapping.sourceAttr, columnMapping.targetIndex);
            } else if (columnMapping.targetAttr.getSource() instanceof DBSEntityAttribute) {
                // Use source attr. Some datasource (e.g. document oriented do not have strict set of attributes)
                targetAttr = (DBSEntityAttribute) columnMapping.targetAttr.getSource();
            } else {
                throw new DBCException("Target attribute for [" + columnMapping.sourceAttr.getName() + "] wasn't resolved");
            }
        }
        columnMapping.sourceValueHandler = columnMapping.sourceAttr.getValueHandler();
        columnMapping.targetValueHandler = DBUtils.findValueHandler(targetContext.getDataSource(), targetAttr);
        columnMapping.targetIndex = targetAttributes.size();
        columnMappings[i] = columnMapping;
        targetAttributes.add(targetAttr);
    }
    DBSAttributeBase[] attributes = targetAttributes.toArray(new DBSAttributeBase[0]);
    if (!isPreview) {
        if (targetObject instanceof DBSDataManipulatorExt) {
            ((DBSDataManipulatorExt) targetObject).beforeDataChange(targetSession, DBSManipulationType.INSERT, attributes, executionSource);
        }
        executeBatch = targetObject.insertData(targetSession, attributes, null, executionSource);
    } else {
        previewRows = new ArrayList<>();
        executeBatch = new PreviewBatch();
    }
}

Example 9 with AbstractExecutionSource

use of org.jkiss.dbeaver.model.impl.AbstractExecutionSource in project dbeaver by dbeaver.

the class DatabaseTransferConsumer method fetchEnd.

@Override
public void fetchEnd(DBCSession session, DBCResultSet resultSet) throws DBCException {
    try {
        if (rowsExported > 0) {
            insertBatch(true);
        }
        if (executeBatch != null) {
            executeBatch.close();
            executeBatch = null;
        }
    } finally {
        DBSDataManipulator targetObject = getTargetObject();
        if (!isPreview && targetObject instanceof DBSDataManipulatorExt) {
            ((DBSDataManipulatorExt) targetObject).afterDataChange(targetSession, DBSManipulationType.INSERT, targetAttributes.toArray(new DBSAttributeBase[0]), new AbstractExecutionSource(getSourceObject(), targetContext, this));
        }
    }
}

Example 10 with AbstractExecutionSource

use of org.jkiss.dbeaver.model.impl.AbstractExecutionSource in project dbeaver by dbeaver.

the class DatabaseTransferProducer method transferData.

@Override
public void transferData(@NotNull DBRProgressMonitor monitor1, @NotNull IDataTransferConsumer consumer, @Nullable IDataTransferProcessor processor, @NotNull DatabaseProducerSettings settings, @Nullable DBTTask task) throws DBException {
    String contextTask = DTMessages.data_transfer_wizard_job_task_export;
    DBSDataContainer databaseObject = getDatabaseObject();
    if (databaseObject == null) {
        throw new DBException("No input database object found");
    }
    DBPDataSource dataSource = databaseObject.getDataSource();
    assert (dataSource != null);
    DBExecUtils.tryExecuteRecover(monitor1, dataSource, monitor -> {
        long readFlags = DBSDataContainer.FLAG_NONE;
        if (settings.isSelectedColumnsOnly()) {
            readFlags |= DBSDataContainer.FLAG_USE_SELECTED_COLUMNS;
        }
        if (settings.isSelectedRowsOnly()) {
            readFlags |= DBSDataContainer.FLAG_USE_SELECTED_ROWS;
        }
        boolean newConnection = settings.isOpenNewConnections() && !getDatabaseObject().getDataSource().getContainer().getDriver().isEmbedded();
        boolean forceDataReadTransactions = Boolean.TRUE.equals(dataSource.getDataSourceFeature(DBConstants.FEATURE_LOB_REQUIRE_TRANSACTIONS));
        boolean selectiveExportFromUI = settings.isSelectedColumnsOnly() || settings.isSelectedRowsOnly();
        try {
            DBCExecutionContext context;
            if (dataContainer instanceof DBPContextProvider) {
                context = ((DBPContextProvider) dataContainer).getExecutionContext();
            } else {
                context = DBUtils.getDefaultContext(dataContainer, false);
            }
            if (context == null) {
                throw new DBCException("Can't retrieve execution context from data container " + dataContainer);
            }
            if (!selectiveExportFromUI && newConnection) {
                context = DBUtils.getObjectOwnerInstance(getDatabaseObject()).openIsolatedContext(monitor, "Data transfer producer", context);
                DBExecUtils.setExecutionContextDefaults(monitor, dataSource, context, defaultCatalog, null, defaultSchema);
            }
            if (task != null) {
                DBTaskUtils.initFromContext(monitor, task, context);
            }
            try (DBCSession session = context.openSession(monitor, DBCExecutionPurpose.UTIL, contextTask)) {
                Boolean oldAutoCommit = null;
                try {
                    AbstractExecutionSource transferSource = new AbstractExecutionSource(dataContainer, context, consumer);
                    session.enableLogging(false);
                    if (!selectiveExportFromUI && (newConnection || forceDataReadTransactions)) {
                        // other complex structures only in transactional mode
                        try {
                            DBCTransactionManager txnManager = DBUtils.getTransactionManager(context);
                            if (txnManager != null && txnManager.isSupportsTransactions()) {
                                oldAutoCommit = txnManager.isAutoCommit();
                                txnManager.setAutoCommit(monitor, false);
                            }
                        } catch (DBCException e) {
                            log.warn("Can't change auto-commit", e);
                        }
                    }
                    long totalRows = 0;
                    if (settings.isQueryRowCount() && (dataContainer.getSupportedFeatures() & DBSDataContainer.DATA_COUNT) != 0) {
                        monitor.beginTask(DTMessages.data_transfer_wizard_job_task_retrieve, 1);
                        try {
                            totalRows = dataContainer.countData(transferSource, session, dataFilter, readFlags);
                        } catch (Throwable e) {
                            log.warn("Can't retrieve row count from '" + dataContainer.getName() + "'", e);
                            try {
                                DBCTransactionManager txnManager = DBUtils.getTransactionManager(session.getExecutionContext());
                                if (txnManager != null && !txnManager.isAutoCommit()) {
                                    txnManager.rollback(session, null);
                                }
                            } catch (Throwable e1) {
                                log.warn("Error rolling back transaction", e1);
                            }
                        } finally {
                            monitor.done();
                        }
                    }
                    monitor.beginTask(DTMessages.data_transfer_wizard_job_task_export_table_data, (int) totalRows);
                    try {
                        monitor.subTask("Read data");
                        // Perform export
                        if (settings.getExtractType() == DatabaseProducerSettings.ExtractType.SINGLE_QUERY) {
                            // Just do it in single query
                            dataContainer.readData(transferSource, session, consumer, dataFilter, -1, -1, readFlags, settings.getFetchSize());
                        } else {
                            // Read all data by segments
                            long offset = 0;
                            int segmentSize = settings.getSegmentSize();
                            for (; ; ) {
                                DBCStatistics statistics = dataContainer.readData(transferSource, session, consumer, dataFilter, offset, segmentSize, readFlags, settings.getFetchSize());
                                if (statistics == null || statistics.getRowsFetched() < segmentSize) {
                                    // Done
                                    break;
                                }
                                offset += statistics.getRowsFetched();
                            }
                        }
                    } finally {
                        monitor.done();
                    }
                } finally {
                    if (!selectiveExportFromUI && (newConnection || forceDataReadTransactions)) {
                        DBCTransactionManager txnManager = DBUtils.getTransactionManager(context);
                        if (txnManager != null && txnManager.isSupportsTransactions() && !txnManager.isAutoCommit()) {
                            try {
                                txnManager.commit(session);
                            } catch (Exception e) {
                                log.error("Can't finish transaction in data producer connection", e);
                            }
                            if (oldAutoCommit != null) {
                                try {
                                    txnManager.setAutoCommit(session.getProgressMonitor(), oldAutoCommit);
                                } catch (Exception e) {
                                    log.error("Can't finish transaction in data producer connection", e);
                                }
                            }
                        }
                    }
                    if (!selectiveExportFromUI && newConnection) {
                        context.close();
                    }
                }
            }
        } catch (DBException e) {
            throw new InvocationTargetException(e);
        }
    });
}