Examples with PropertyType org.voltdb.compiler.deploymentfile.PropertyType used on opensource projects

Example 1 with PropertyType

use of org.voltdb.compiler.deploymentfile.PropertyType in project voltdb by VoltDB.

the class StandaloneExportManager method updateProcessorConfig.

public void updateProcessorConfig(String exportClassName, List<PropertyType> exportConfiguration) {
    Properties newConfig = new Properties();
    for (PropertyType prop : exportConfiguration) {
        newConfig.put(prop.getName(), prop.getValue());
    }
    newConfig.put(EXPORT_TO_TYPE, exportClassName);
    m_processorConfig = newConfig;
}

Example 2 with PropertyType

use of org.voltdb.compiler.deploymentfile.PropertyType in project voltdb by VoltDB.

the class CatalogUtil method addExportConfigToDRConflictsTable.

/**
     * Add default configuration to DR conflicts export target if deployment file doesn't have the configuration
     *
     * @param catalog  current catalog
     * @param export   list of export configuration
     */
public static ExportType addExportConfigToDRConflictsTable(Catalog catalog, ExportType export) {
    if (export == null) {
        export = new ExportType();
    }
    boolean userDefineStream = false;
    for (ExportConfigurationType exportConfiguration : export.getConfiguration()) {
        if (exportConfiguration.getTarget().equals(DR_CONFLICTS_TABLE_EXPORT_GROUP)) {
            userDefineStream = true;
        }
    }
    if (!userDefineStream) {
        ExportConfigurationType defaultConfiguration = new ExportConfigurationType();
        defaultConfiguration.setEnabled(true);
        defaultConfiguration.setTarget(DR_CONFLICTS_TABLE_EXPORT_GROUP);
        defaultConfiguration.setType(ServerExportEnum.FILE);
        // type
        PropertyType type = new PropertyType();
        type.setName("type");
        type.setValue(DEFAULT_DR_CONFLICTS_EXPORT_TYPE);
        defaultConfiguration.getProperty().add(type);
        // nonce
        PropertyType nonce = new PropertyType();
        nonce.setName("nonce");
        nonce.setValue(DEFAULT_DR_CONFLICTS_NONCE);
        defaultConfiguration.getProperty().add(nonce);
        // outdir
        PropertyType outdir = new PropertyType();
        outdir.setName("outdir");
        outdir.setValue(DEFAULT_DR_CONFLICTS_DIR);
        defaultConfiguration.getProperty().add(outdir);
        // k-safe file export
        PropertyType ksafe = new PropertyType();
        ksafe.setName("replicated");
        ksafe.setValue("true");
        defaultConfiguration.getProperty().add(ksafe);
        // skip internal export columns
        PropertyType skipinternal = new PropertyType();
        skipinternal.setName("skipinternals");
        skipinternal.setValue("true");
        defaultConfiguration.getProperty().add(skipinternal);
        export.getConfiguration().add(defaultConfiguration);
    }
    return export;
}

Example 3 with PropertyType

use of org.voltdb.compiler.deploymentfile.PropertyType in project voltdb by VoltDB.

the class CatalogUtil method checkImportProcessorConfiguration.

private static ImportConfiguration checkImportProcessorConfiguration(ImportConfigurationType importConfiguration) {
    String importBundleUrl = importConfiguration.getModule();
    if (!importConfiguration.isEnabled()) {
        return null;
    }
    switch(importConfiguration.getType()) {
        case CUSTOM:
            break;
        case KAFKA:
            importBundleUrl = "kafkastream.jar";
            break;
        case KINESIS:
            importBundleUrl = "kinesisstream.jar";
            break;
        default:
            throw new DeploymentCheckException("Import Configuration type must be specified.");
    }
    Properties moduleProps = new Properties();
    Properties formatterProps = new Properties();
    String formatBundle = importConfiguration.getFormat();
    String formatName = null;
    if (formatBundle != null && formatBundle.trim().length() > 0) {
        if ("csv".equalsIgnoreCase(formatBundle) || "tsv".equalsIgnoreCase(formatBundle)) {
            formatName = formatBundle;
            formatBundle = "voltcsvformatter.jar";
        } else if (JAR_EXTENSION_RE.matcher(formatBundle).matches()) {
            int typeIndex = formatBundle.lastIndexOf("/");
            formatName = formatBundle.substring(typeIndex + 1);
            formatBundle = formatBundle.substring(0, typeIndex);
        } else {
            throw new DeploymentCheckException("Import format " + formatBundle + " not valid.");
        }
        formatterProps.setProperty(ImportDataProcessor.IMPORT_FORMATTER, buildBundleURL(formatBundle, true));
    }
    if (importBundleUrl != null && importBundleUrl.trim().length() > 0) {
        moduleProps.setProperty(ImportDataProcessor.IMPORT_MODULE, buildBundleURL(importBundleUrl, false));
    }
    List<PropertyType> importProperties = importConfiguration.getProperty();
    if (importProperties != null && !importProperties.isEmpty()) {
        for (PropertyType prop : importProperties) {
            String key = prop.getName();
            String value = prop.getValue();
            if (!key.toLowerCase().contains("passw")) {
                moduleProps.setProperty(key, value.trim());
            } else {
                //Don't trim passwords
                moduleProps.setProperty(key, value);
            }
        }
    }
    List<PropertyType> formatProperties = importConfiguration.getFormatProperty();
    if (formatProperties != null && !formatProperties.isEmpty()) {
        for (PropertyType prop : formatProperties) {
            formatterProps.setProperty(prop.getName(), prop.getValue());
        }
    }
    return new ImportConfiguration(formatName, moduleProps, formatterProps);
}

Example 4 with PropertyType

use of org.voltdb.compiler.deploymentfile.PropertyType in project voltdb by VoltDB.

the class CatalogUtil method checkExportProcessorConfiguration.

private static Properties checkExportProcessorConfiguration(ExportConfigurationType exportConfiguration) {
    // on-server export always uses the guest processor
    String exportClientClassName = null;
    switch(exportConfiguration.getType()) {
        case FILE:
            exportClientClassName = "org.voltdb.exportclient.ExportToFileClient";
            break;
        case JDBC:
            exportClientClassName = "org.voltdb.exportclient.JDBCExportClient";
            break;
        case KAFKA:
            exportClientClassName = "org.voltdb.exportclient.kafka.KafkaExportClient";
            break;
        case RABBITMQ:
            exportClientClassName = "org.voltdb.exportclient.RabbitMQExportClient";
            break;
        case HTTP:
            exportClientClassName = "org.voltdb.exportclient.HttpExportClient";
            break;
        case ELASTICSEARCH:
            exportClientClassName = "org.voltdb.exportclient.ElasticSearchHttpExportClient";
            break;
        //Validate that we can load the class.
        case CUSTOM:
            exportClientClassName = exportConfiguration.getExportconnectorclass();
            if (exportConfiguration.isEnabled()) {
                try {
                    CatalogUtil.class.getClassLoader().loadClass(exportClientClassName);
                } catch (ClassNotFoundException ex) {
                    String msg = "Custom Export failed to configure, failed to load" + " export plugin class: " + exportConfiguration.getExportconnectorclass() + " Disabling export.";
                    hostLog.error(msg);
                    throw new DeploymentCheckException(msg);
                }
            }
            break;
    }
    Properties processorProperties = new Properties();
    if (exportClientClassName != null && exportClientClassName.trim().length() > 0) {
        String dexportClientClassName = System.getProperty(ExportDataProcessor.EXPORT_TO_TYPE, exportClientClassName);
        //Override for tests
        if (dexportClientClassName != null && dexportClientClassName.trim().length() > 0 && exportConfiguration.getType().equals(ServerExportEnum.CUSTOM)) {
            processorProperties.setProperty(ExportDataProcessor.EXPORT_TO_TYPE, dexportClientClassName);
        } else {
            processorProperties.setProperty(ExportDataProcessor.EXPORT_TO_TYPE, exportClientClassName);
        }
    }
    if (exportConfiguration != null) {
        List<PropertyType> configProperties = exportConfiguration.getProperty();
        if (configProperties != null && !configProperties.isEmpty()) {
            for (PropertyType configProp : configProperties) {
                String key = configProp.getName();
                String value = configProp.getValue();
                if (key.toLowerCase().contains("passw")) {
                    // Don't trim password
                    processorProperties.setProperty(key, value);
                } else if (key.toLowerCase().contains("delim")) {
                    // Don't trim \n in delimiters
                    String trimmedDelimiters = value.replaceAll("^(\r|\f|\t| )+", "").replaceAll("(\r|\f|\t| )+$", "");
                    processorProperties.setProperty(key, StringEscapeUtils.escapeJava(trimmedDelimiters));
                } else {
                    processorProperties.setProperty(key, value.trim());
                }
            }
        }
    }
    if (!exportConfiguration.isEnabled()) {
        return processorProperties;
    }
    // Instantiate the Guest Processor
    Class<?> processorClazz = null;
    try {
        processorClazz = Class.forName(ExportManager.PROCESSOR_CLASS);
    } catch (ClassNotFoundException e) {
        throw new DeploymentCheckException("Export is a PRO version only feature");
    }
    ExportDataProcessor processor = null;
    try {
        processor = (ExportDataProcessor) processorClazz.newInstance();
    } catch (InstantiationException | IllegalAccessException e) {
        hostLog.error("Unable to instantiate export processor", e);
        throw new DeploymentCheckException("Unable to instantiate export processor", e);
    }
    try {
        processor.addLogger(hostLog);
        processorProperties.put(ExportManager.CONFIG_CHECK_ONLY, "true");
        processor.checkProcessorConfig(processorProperties);
        processor.shutdown();
    } catch (Exception e) {
        hostLog.error("Export processor failed its configuration check", e);
        throw new DeploymentCheckException("Export processor failed its configuration check: " + e.getMessage(), e);
    }
    processorProperties.remove(ExportManager.CONFIG_CHECK_ONLY);
    return processorProperties;
}

Example 5 with PropertyType

use of org.voltdb.compiler.deploymentfile.PropertyType in project voltdb by VoltDB.

the class CatalogUtil method setExportInfo.

/**
     * Set deployment time settings for export
     * @param catalog The catalog to be updated.
     * @param exportsType A reference to the <exports> element of the deployment.xml file.
     */
private static void setExportInfo(Catalog catalog, ExportType exportType) {
    final Cluster cluster = catalog.getClusters().get("cluster");
    Database db = cluster.getDatabases().get("database");
    if (DrRoleType.XDCR.value().equals(cluster.getDrrole())) {
        // add default export configuration to DR conflict table
        exportType = addExportConfigToDRConflictsTable(catalog, exportType);
    }
    if (exportType == null) {
        return;
    }
    List<String> targetList = new ArrayList<>();
    for (ExportConfigurationType exportConfiguration : exportType.getConfiguration()) {
        boolean connectorEnabled = exportConfiguration.isEnabled();
        String targetName = exportConfiguration.getTarget();
        if (connectorEnabled) {
            m_exportEnabled = true;
            if (targetList.contains(targetName)) {
                throw new RuntimeException("Multiple connectors can not be assigned to single export target: " + targetName + ".");
            } else {
                targetList.add(targetName);
            }
        }
        Properties processorProperties = checkExportProcessorConfiguration(exportConfiguration);
        org.voltdb.catalog.Connector catconn = db.getConnectors().get(targetName);
        if (catconn == null) {
            if (connectorEnabled) {
                hostLog.info("Export configuration enabled and provided for export target " + targetName + " in deployment file however no export " + "tables are assigned to the this target. " + "Export target " + targetName + " will be disabled.");
            }
            continue;
        }
        // checking rowLengthLimit
        int rowLengthLimit = Integer.parseInt(processorProperties.getProperty(ROW_LENGTH_LIMIT, "0"));
        if (rowLengthLimit > 0) {
            for (ConnectorTableInfo catTableinfo : catconn.getTableinfo()) {
                Table tableref = catTableinfo.getTable();
                int rowLength = Boolean.parseBoolean(processorProperties.getProperty("skipinternals", "false")) ? 0 : EXPORT_INTERNAL_FIELD_Length;
                for (Column catColumn : tableref.getColumns()) {
                    rowLength += catColumn.getSize();
                }
                if (rowLength > rowLengthLimit) {
                    hostLog.error("Export configuration for export target " + targetName + " has" + "configured to has row length limit " + rowLengthLimit + ". But the export table " + tableref.getTypeName() + " has estimated row length " + rowLength + ".");
                    throw new RuntimeException("Export table " + tableref.getTypeName() + " row length is " + rowLength + ", exceeding configurated limitation " + rowLengthLimit + ".");
                }
            }
        }
        for (String name : processorProperties.stringPropertyNames()) {
            ConnectorProperty prop = catconn.getConfig().add(name);
            prop.setName(name);
            prop.setValue(processorProperties.getProperty(name));
        }
        // on-server export always uses the guest processor
        catconn.setLoaderclass(ExportManager.PROCESSOR_CLASS);
        catconn.setEnabled(connectorEnabled);
        if (!connectorEnabled) {
            hostLog.info("Export configuration for export target " + targetName + " is present and is " + "configured to be disabled. Export target " + targetName + " will be disabled.");
        } else {
            hostLog.info("Export target " + targetName + " is configured and enabled with type=" + exportConfiguration.getType());
            if (exportConfiguration.getProperty() != null) {
                hostLog.info("Export target " + targetName + " configuration properties are: ");
                for (PropertyType configProp : exportConfiguration.getProperty()) {
                    if (!configProp.getName().toLowerCase().contains("password")) {
                        hostLog.info("Export Configuration Property NAME=" + configProp.getName() + " VALUE=" + configProp.getValue());
                    }
                }
            }
        }
    }
}