Examples with ConnectorTableInfo org.voltdb.catalog.ConnectorTableInfo used on opensource projects

Example 1 with ConnectorTableInfo

use of org.voltdb.catalog.ConnectorTableInfo in project voltdb by VoltDB.

the class ExportManager method updateProcessorConfig.

private void updateProcessorConfig(final CatalogMap<Connector> connectors) {
    Map<String, Pair<Properties, Set<String>>> config = new HashMap<>();
    // If the export source changes before the previous generation drains
    // then the outstanding exports will go to the new source when export resumes.
    int connCount = 0;
    int tableCount = 0;
    for (Connector conn : connectors) {
        // skip disabled connectors
        if (!conn.getEnabled() || conn.getTableinfo().isEmpty()) {
            continue;
        }
        connCount++;
        Properties properties = new Properties();
        Set<String> tables = new HashSet<>();
        String targetName = conn.getTypeName();
        for (ConnectorTableInfo ti : conn.getTableinfo()) {
            tables.add(ti.getTable().getTypeName());
            tableCount++;
        }
        if (conn.getConfig() != null) {
            Iterator<ConnectorProperty> connPropIt = conn.getConfig().iterator();
            while (connPropIt.hasNext()) {
                ConnectorProperty prop = connPropIt.next();
                properties.put(prop.getName(), prop.getValue().trim());
                if (!prop.getName().toLowerCase().contains("password")) {
                    properties.put(prop.getName(), prop.getValue().trim());
                } else {
                    //Dont trim passwords
                    properties.put(prop.getName(), prop.getValue());
                }
            }
        }
        Pair<Properties, Set<String>> connConfig = new Pair<>(properties, tables);
        config.put(targetName, connConfig);
    }
    m_connCount = connCount;
    m_exportTablesCount = tableCount;
    m_processorConfig = config;
}

Example 2 with ConnectorTableInfo

use of org.voltdb.catalog.ConnectorTableInfo in project voltdb by VoltDB.

the class ExportGeneration method initializeGenerationFromCatalog.

void initializeGenerationFromCatalog(final CatalogMap<Connector> connectors, int hostId, HostMessenger messenger, List<Integer> partitions) {
    //Only populate partitions in use if export is actually happening
    Set<Integer> partitionsInUse = new HashSet<Integer>();
    /*
         * Now create datasources based on the catalog
         */
    for (Connector conn : connectors) {
        if (conn.getEnabled()) {
            for (ConnectorTableInfo ti : conn.getTableinfo()) {
                Table table = ti.getTable();
                addDataSources(table, hostId, partitions);
                partitionsInUse.addAll(partitions);
            }
        }
    }
    createAndRegisterAckMailboxes(partitionsInUse, messenger);
}

Example 3 with ConnectorTableInfo

use of org.voltdb.catalog.ConnectorTableInfo in project voltdb by VoltDB.

the class CatalogUtil method setExportInfo.

/**
     * Set deployment time settings for export
     * @param catalog The catalog to be updated.
     * @param exportsType A reference to the <exports> element of the deployment.xml file.
     */
private static void setExportInfo(Catalog catalog, ExportType exportType) {
    final Cluster cluster = catalog.getClusters().get("cluster");
    Database db = cluster.getDatabases().get("database");
    if (DrRoleType.XDCR.value().equals(cluster.getDrrole())) {
        // add default export configuration to DR conflict table
        exportType = addExportConfigToDRConflictsTable(catalog, exportType);
    }
    if (exportType == null) {
        return;
    }
    List<String> targetList = new ArrayList<>();
    for (ExportConfigurationType exportConfiguration : exportType.getConfiguration()) {
        boolean connectorEnabled = exportConfiguration.isEnabled();
        String targetName = exportConfiguration.getTarget();
        if (connectorEnabled) {
            m_exportEnabled = true;
            if (targetList.contains(targetName)) {
                throw new RuntimeException("Multiple connectors can not be assigned to single export target: " + targetName + ".");
            } else {
                targetList.add(targetName);
            }
        }
        Properties processorProperties = checkExportProcessorConfiguration(exportConfiguration);
        org.voltdb.catalog.Connector catconn = db.getConnectors().get(targetName);
        if (catconn == null) {
            if (connectorEnabled) {
                hostLog.info("Export configuration enabled and provided for export target " + targetName + " in deployment file however no export " + "tables are assigned to the this target. " + "Export target " + targetName + " will be disabled.");
            }
            continue;
        }
        // checking rowLengthLimit
        int rowLengthLimit = Integer.parseInt(processorProperties.getProperty(ROW_LENGTH_LIMIT, "0"));
        if (rowLengthLimit > 0) {
            for (ConnectorTableInfo catTableinfo : catconn.getTableinfo()) {
                Table tableref = catTableinfo.getTable();
                int rowLength = Boolean.parseBoolean(processorProperties.getProperty("skipinternals", "false")) ? 0 : EXPORT_INTERNAL_FIELD_Length;
                for (Column catColumn : tableref.getColumns()) {
                    rowLength += catColumn.getSize();
                }
                if (rowLength > rowLengthLimit) {
                    hostLog.error("Export configuration for export target " + targetName + " has" + "configured to has row length limit " + rowLengthLimit + ". But the export table " + tableref.getTypeName() + " has estimated row length " + rowLength + ".");
                    throw new RuntimeException("Export table " + tableref.getTypeName() + " row length is " + rowLength + ", exceeding configurated limitation " + rowLengthLimit + ".");
                }
            }
        }
        for (String name : processorProperties.stringPropertyNames()) {
            ConnectorProperty prop = catconn.getConfig().add(name);
            prop.setName(name);
            prop.setValue(processorProperties.getProperty(name));
        }
        // on-server export always uses the guest processor
        catconn.setLoaderclass(ExportManager.PROCESSOR_CLASS);
        catconn.setEnabled(connectorEnabled);
        if (!connectorEnabled) {
            hostLog.info("Export configuration for export target " + targetName + " is present and is " + "configured to be disabled. Export target " + targetName + " will be disabled.");
        } else {
            hostLog.info("Export target " + targetName + " is configured and enabled with type=" + exportConfiguration.getType());
            if (exportConfiguration.getProperty() != null) {
                hostLog.info("Export target " + targetName + " configuration properties are: ");
                for (PropertyType configProp : exportConfiguration.getProperty()) {
                    if (!configProp.getName().toLowerCase().contains("password")) {
                        hostLog.info("Export Configuration Property NAME=" + configProp.getName() + " VALUE=" + configProp.getValue());
                    }
                }
            }
        }
    }
}