Examples with ExportConfigurationType org.voltdb.compiler.deploymentfile.ExportConfigurationType used on opensource projects

Example 1 with ExportConfigurationType

use of org.voltdb.compiler.deploymentfile.ExportConfigurationType in project voltdb by VoltDB.

the class PushSpecificGeneration method main.

/**
     * @param args the command line arguments
     */
public static void main(String[] args) {
    // Arguments deployment, directory
    try {
        if (args.length != 2) {
            System.out.println("Usage: draingen deployment.xml dir-where-where-generations-are");
            System.exit(1);
        }
        DeploymentType dep = CatalogUtil.getDeployment(new FileInputStream(new File(args[0])));
        ExportConfigurationType exportConfiguration = dep.getExport().getConfiguration().get(0);
        String exportClientClassName = null;
        switch(exportConfiguration.getType()) {
            case FILE:
                exportClientClassName = "org.voltdb.exportclient.ExportToFileClient";
                break;
            case JDBC:
                exportClientClassName = "org.voltdb.exportclient.JDBCExportClient";
                break;
            case KAFKA:
                exportClientClassName = "org.voltdb.exportclient.kafka.KafkaExportClient";
                break;
            case RABBITMQ:
                exportClientClassName = "org.voltdb.exportclient.RabbitMQExportClient";
                break;
            case HTTP:
                exportClientClassName = "org.voltdb.exportclient.HttpExportClient";
                break;
            case ELASTICSEARCH:
                exportClientClassName = "org.voltdb.exportclient.ElasticSearchHttpExportClient";
                break;
            //Validate that we can load the class.
            case CUSTOM:
                exportClientClassName = exportConfiguration.getExportconnectorclass();
                if (exportConfiguration.isEnabled()) {
                    try {
                        CatalogUtil.class.getClassLoader().loadClass(exportClientClassName);
                    } catch (ClassNotFoundException ex) {
                        String msg = "Custom Export failed to configure, failed to load" + " export plugin class: " + exportConfiguration.getExportconnectorclass() + " Disabling export.";
                        throw new CatalogUtil.DeploymentCheckException(msg);
                    }
                }
                break;
        }
        StandaloneExportManager.initialize(0, args[1], exportClientClassName, dep.getExport().getConfiguration().get(0).getProperty());
        int maxPart = dep.getCluster().getSitesperhost();
        System.out.println("Please wait...|");
        while (true) {
            System.out.print(".");
            Thread.yield();
            Thread.sleep(1000);
            long sz = 0;
            for (int i = 0; i < maxPart; i++) {
                sz += StandaloneExportManager.getQueuedExportBytes(i, "");
            }
            if (sz <= 0 && StandaloneExportManager.shouldExit()) {
                break;
            }
        }
    } catch (Exception ex) {
        ex.printStackTrace();
    } finally {
        StandaloneExportManager.instance().shutdown(null);
    }
}

Example 2 with ExportConfigurationType

use of org.voltdb.compiler.deploymentfile.ExportConfigurationType in project voltdb by VoltDB.

the class CatalogUtil method addExportConfigToDRConflictsTable.

/**
     * Add default configuration to DR conflicts export target if deployment file doesn't have the configuration
     *
     * @param catalog  current catalog
     * @param export   list of export configuration
     */
public static ExportType addExportConfigToDRConflictsTable(Catalog catalog, ExportType export) {
    if (export == null) {
        export = new ExportType();
    }
    boolean userDefineStream = false;
    for (ExportConfigurationType exportConfiguration : export.getConfiguration()) {
        if (exportConfiguration.getTarget().equals(DR_CONFLICTS_TABLE_EXPORT_GROUP)) {
            userDefineStream = true;
        }
    }
    if (!userDefineStream) {
        ExportConfigurationType defaultConfiguration = new ExportConfigurationType();
        defaultConfiguration.setEnabled(true);
        defaultConfiguration.setTarget(DR_CONFLICTS_TABLE_EXPORT_GROUP);
        defaultConfiguration.setType(ServerExportEnum.FILE);
        // type
        PropertyType type = new PropertyType();
        type.setName("type");
        type.setValue(DEFAULT_DR_CONFLICTS_EXPORT_TYPE);
        defaultConfiguration.getProperty().add(type);
        // nonce
        PropertyType nonce = new PropertyType();
        nonce.setName("nonce");
        nonce.setValue(DEFAULT_DR_CONFLICTS_NONCE);
        defaultConfiguration.getProperty().add(nonce);
        // outdir
        PropertyType outdir = new PropertyType();
        outdir.setName("outdir");
        outdir.setValue(DEFAULT_DR_CONFLICTS_DIR);
        defaultConfiguration.getProperty().add(outdir);
        // k-safe file export
        PropertyType ksafe = new PropertyType();
        ksafe.setName("replicated");
        ksafe.setValue("true");
        defaultConfiguration.getProperty().add(ksafe);
        // skip internal export columns
        PropertyType skipinternal = new PropertyType();
        skipinternal.setName("skipinternals");
        skipinternal.setValue("true");
        defaultConfiguration.getProperty().add(skipinternal);
        export.getConfiguration().add(defaultConfiguration);
    }
    return export;
}

Example 3 with ExportConfigurationType

use of org.voltdb.compiler.deploymentfile.ExportConfigurationType in project voltdb by VoltDB.

the class VoltProjectBuilder method writeDeploymentFile.

/**
     * Writes deployment.xml file to a temporary file. It is constructed from the passed parameters and the m_users
     * field.
     *
     * @param voltRoot
     * @param dinfo an instance {@link DeploymentInfo}
     * @return deployment path
     * @throws IOException
     * @throws JAXBException
     */
private String writeDeploymentFile(String voltRoot, DeploymentInfo dinfo) throws IOException, JAXBException {
    org.voltdb.compiler.deploymentfile.ObjectFactory factory = new org.voltdb.compiler.deploymentfile.ObjectFactory();
    // <deployment>
    DeploymentType deployment = factory.createDeploymentType();
    JAXBElement<DeploymentType> doc = factory.createDeployment(deployment);
    // <cluster>
    ClusterType cluster = factory.createClusterType();
    deployment.setCluster(cluster);
    cluster.setHostcount(dinfo.hostCount);
    cluster.setSitesperhost(dinfo.sitesPerHost);
    cluster.setKfactor(dinfo.replication);
    cluster.setId(dinfo.clusterId);
    cluster.setSchema(m_useDDLSchema ? SchemaType.DDL : SchemaType.CATALOG);
    // <paths>
    PathsType paths = factory.createPathsType();
    deployment.setPaths(paths);
    if ((voltRoot != null) && !voltRoot.trim().isEmpty()) {
        Voltdbroot voltdbroot = factory.createPathsTypeVoltdbroot();
        paths.setVoltdbroot(voltdbroot);
        voltdbroot.setPath(voltRoot);
    }
    if (m_snapshotPath != null) {
        PathsType.Snapshots snapshotPathElement = factory.createPathsTypeSnapshots();
        snapshotPathElement.setPath(m_snapshotPath);
        paths.setSnapshots(snapshotPathElement);
    }
    if (m_deadHostTimeout != null) {
        HeartbeatType heartbeat = factory.createHeartbeatType();
        heartbeat.setTimeout(m_deadHostTimeout);
        deployment.setHeartbeat(heartbeat);
    }
    if (m_commandLogPath != null) {
        PathsType.Commandlog commandLogPathElement = factory.createPathsTypeCommandlog();
        commandLogPathElement.setPath(m_commandLogPath);
        paths.setCommandlog(commandLogPathElement);
    }
    if (m_internalSnapshotPath != null) {
        PathsType.Commandlogsnapshot commandLogSnapshotPathElement = factory.createPathsTypeCommandlogsnapshot();
        commandLogSnapshotPathElement.setPath(m_internalSnapshotPath);
        paths.setCommandlogsnapshot(commandLogSnapshotPathElement);
    }
    if (m_snapshotPrefix != null) {
        SnapshotType snapshot = factory.createSnapshotType();
        deployment.setSnapshot(snapshot);
        snapshot.setFrequency(m_snapshotFrequency);
        snapshot.setPrefix(m_snapshotPrefix);
        snapshot.setRetain(m_snapshotRetain);
    }
    SecurityType security = factory.createSecurityType();
    deployment.setSecurity(security);
    security.setEnabled(m_securityEnabled);
    SecurityProviderString provider = SecurityProviderString.HASH;
    if (m_securityEnabled)
        try {
            provider = SecurityProviderString.fromValue(m_securityProvider);
        } catch (IllegalArgumentException shouldNotHappenSeeSetter) {
        }
    security.setProvider(provider);
    // set the command log (which defaults to off)
    CommandLogType commandLogType = factory.createCommandLogType();
    commandLogType.setEnabled(m_commandLogEnabled);
    if (m_commandLogSync != null) {
        commandLogType.setSynchronous(m_commandLogSync.booleanValue());
    }
    if (m_commandLogSize != null) {
        commandLogType.setLogsize(m_commandLogSize);
    }
    if (m_commandLogFsyncInterval != null || m_commandLogMaxTxnsBeforeFsync != null) {
        CommandLogType.Frequency frequency = factory.createCommandLogTypeFrequency();
        if (m_commandLogFsyncInterval != null) {
            frequency.setTime(m_commandLogFsyncInterval);
        }
        if (m_commandLogMaxTxnsBeforeFsync != null) {
            frequency.setTransactions(m_commandLogMaxTxnsBeforeFsync);
        }
        commandLogType.setFrequency(frequency);
    }
    deployment.setCommandlog(commandLogType);
    // <partition-detection>/<snapshot>
    PartitionDetectionType ppd = factory.createPartitionDetectionType();
    deployment.setPartitionDetection(ppd);
    ppd.setEnabled(m_ppdEnabled);
    // don't include this element if not explicitly set
    if (m_heartbeatTimeout != null) {
        HeartbeatType hb = factory.createHeartbeatType();
        deployment.setHeartbeat(hb);
        hb.setTimeout((int) m_heartbeatTimeout);
    }
    // don't include this element if not explicitly set
    if (m_consistencyReadLevel != null) {
        ConsistencyType ct = factory.createConsistencyType();
        deployment.setConsistency(ct);
        ct.setReadlevel(m_consistencyReadLevel.toReadLevelType());
    }
    deployment.setSystemsettings(createSystemSettingsType(factory));
    // <users>
    if (m_users.size() > 0) {
        UsersType users = factory.createUsersType();
        deployment.setUsers(users);
        // <user>
        for (final UserInfo info : m_users) {
            User user = factory.createUsersTypeUser();
            users.getUser().add(user);
            user.setName(info.name);
            user.setPassword(info.password);
            user.setPlaintext(info.plaintext);
            // build up user/roles.
            if (info.roles.length > 0) {
                final StringBuilder roles = new StringBuilder();
                for (final String role : info.roles) {
                    if (roles.length() > 0)
                        roles.append(",");
                    roles.append(role);
                }
                user.setRoles(roles.toString());
            }
        }
    }
    SslType ssl = factory.createSslType();
    deployment.setSsl(ssl);
    ssl.setEnabled(m_sslEnabled);
    ssl.setExternal(m_sslExternal);
    if (m_keystore != null) {
        KeyOrTrustStoreType store = factory.createKeyOrTrustStoreType();
        store.setPath(m_keystore);
        store.setPassword(m_keystorePassword);
        ssl.setKeystore(store);
    }
    if (m_certstore != null) {
        KeyOrTrustStoreType store = factory.createKeyOrTrustStoreType();
        store.setPath(m_certstore);
        store.setPassword(m_certstorePassword);
        ssl.setTruststore(store);
    }
    // <httpd>. Disabled unless port # is configured by a testcase
    // Omit element(s) when null.
    HttpdType httpd = factory.createHttpdType();
    deployment.setHttpd(httpd);
    httpd.setEnabled(m_httpdPortNo != -1);
    httpd.setPort(m_httpdPortNo);
    Jsonapi json = factory.createHttpdTypeJsonapi();
    httpd.setJsonapi(json);
    json.setEnabled(m_jsonApiEnabled);
    //SNMP
    SnmpType snmpType = factory.createSnmpType();
    if (m_snmpEnabled) {
        snmpType.setEnabled(true);
        snmpType.setTarget(m_snmpTarget);
        deployment.setSnmp(snmpType);
    }
    // <export>
    ExportType export = factory.createExportType();
    deployment.setExport(export);
    for (HashMap<String, Object> exportConnector : m_elExportConnectors) {
        ExportConfigurationType exportConfig = factory.createExportConfigurationType();
        exportConfig.setEnabled((boolean) exportConnector.get("elEnabled") && exportConnector.get("elLoader") != null && !((String) exportConnector.get("elLoader")).trim().isEmpty());
        ServerExportEnum exportTarget = ServerExportEnum.fromValue(((String) exportConnector.get("elExportTarget")).toLowerCase());
        exportConfig.setType(exportTarget);
        if (exportTarget.equals(ServerExportEnum.CUSTOM)) {
            exportConfig.setExportconnectorclass(System.getProperty(ExportDataProcessor.EXPORT_TO_TYPE));
        }
        exportConfig.setTarget((String) exportConnector.get("elGroup"));
        Properties config = (Properties) exportConnector.get("elConfig");
        if ((config != null) && (config.size() > 0)) {
            List<PropertyType> configProperties = exportConfig.getProperty();
            for (Object nameObj : config.keySet()) {
                String name = String.class.cast(nameObj);
                PropertyType prop = factory.createPropertyType();
                prop.setName(name);
                prop.setValue(config.getProperty(name));
                configProperties.add(prop);
            }
        }
        export.getConfiguration().add(exportConfig);
    }
    // <import>
    ImportType importt = factory.createImportType();
    deployment.setImport(importt);
    for (HashMap<String, Object> importConnector : m_ilImportConnectors) {
        ImportConfigurationType importConfig = factory.createImportConfigurationType();
        importConfig.setEnabled((boolean) importConnector.get("ilEnabled"));
        ServerImportEnum importType = ServerImportEnum.fromValue(((String) importConnector.get("ilImportType")).toLowerCase());
        importConfig.setType(importType);
        importConfig.setModule((String) importConnector.get("ilModule"));
        String formatter = (String) importConnector.get("ilFormatter");
        if (formatter != null) {
            importConfig.setFormat(formatter);
        }
        Properties config = (Properties) importConnector.get("ilConfig");
        if ((config != null) && (config.size() > 0)) {
            List<PropertyType> configProperties = importConfig.getProperty();
            for (Object nameObj : config.keySet()) {
                String name = String.class.cast(nameObj);
                PropertyType prop = factory.createPropertyType();
                prop.setName(name);
                prop.setValue(config.getProperty(name));
                configProperties.add(prop);
            }
        }
        Properties formatConfig = (Properties) importConnector.get("ilFormatterConfig");
        if ((formatConfig != null) && (formatConfig.size() > 0)) {
            List<PropertyType> configProperties = importConfig.getFormatProperty();
            for (Object nameObj : formatConfig.keySet()) {
                String name = String.class.cast(nameObj);
                PropertyType prop = factory.createPropertyType();
                prop.setName(name);
                prop.setValue(formatConfig.getProperty(name));
                configProperties.add(prop);
            }
        }
        importt.getConfiguration().add(importConfig);
    }
    DrType dr = factory.createDrType();
    deployment.setDr(dr);
    dr.setListen(m_drProducerEnabled);
    dr.setRole(m_drRole);
    if (m_drMasterHost != null && !m_drMasterHost.isEmpty()) {
        ConnectionType conn = factory.createConnectionType();
        dr.setConnection(conn);
        conn.setSource(m_drMasterHost);
        conn.setPreferredSource(m_preferredSource);
        conn.setEnabled(m_drConsumerConnectionEnabled);
    }
    // Have some yummy boilerplate!
    File file = File.createTempFile("myAppDeployment", ".tmp");
    JAXBContext context = JAXBContext.newInstance(DeploymentType.class);
    Marshaller marshaller = context.createMarshaller();
    marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
    marshaller.marshal(doc, file);
    final String deploymentPath = file.getPath();
    return deploymentPath;
}

Example 4 with ExportConfigurationType

use of org.voltdb.compiler.deploymentfile.ExportConfigurationType in project voltdb by VoltDB.

the class CatalogUtil method setExportInfo.

/**
     * Set deployment time settings for export
     * @param catalog The catalog to be updated.
     * @param exportsType A reference to the <exports> element of the deployment.xml file.
     */
private static void setExportInfo(Catalog catalog, ExportType exportType) {
    final Cluster cluster = catalog.getClusters().get("cluster");
    Database db = cluster.getDatabases().get("database");
    if (DrRoleType.XDCR.value().equals(cluster.getDrrole())) {
        // add default export configuration to DR conflict table
        exportType = addExportConfigToDRConflictsTable(catalog, exportType);
    }
    if (exportType == null) {
        return;
    }
    List<String> targetList = new ArrayList<>();
    for (ExportConfigurationType exportConfiguration : exportType.getConfiguration()) {
        boolean connectorEnabled = exportConfiguration.isEnabled();
        String targetName = exportConfiguration.getTarget();
        if (connectorEnabled) {
            m_exportEnabled = true;
            if (targetList.contains(targetName)) {
                throw new RuntimeException("Multiple connectors can not be assigned to single export target: " + targetName + ".");
            } else {
                targetList.add(targetName);
            }
        }
        Properties processorProperties = checkExportProcessorConfiguration(exportConfiguration);
        org.voltdb.catalog.Connector catconn = db.getConnectors().get(targetName);
        if (catconn == null) {
            if (connectorEnabled) {
                hostLog.info("Export configuration enabled and provided for export target " + targetName + " in deployment file however no export " + "tables are assigned to the this target. " + "Export target " + targetName + " will be disabled.");
            }
            continue;
        }
        // checking rowLengthLimit
        int rowLengthLimit = Integer.parseInt(processorProperties.getProperty(ROW_LENGTH_LIMIT, "0"));
        if (rowLengthLimit > 0) {
            for (ConnectorTableInfo catTableinfo : catconn.getTableinfo()) {
                Table tableref = catTableinfo.getTable();
                int rowLength = Boolean.parseBoolean(processorProperties.getProperty("skipinternals", "false")) ? 0 : EXPORT_INTERNAL_FIELD_Length;
                for (Column catColumn : tableref.getColumns()) {
                    rowLength += catColumn.getSize();
                }
                if (rowLength > rowLengthLimit) {
                    hostLog.error("Export configuration for export target " + targetName + " has" + "configured to has row length limit " + rowLengthLimit + ". But the export table " + tableref.getTypeName() + " has estimated row length " + rowLength + ".");
                    throw new RuntimeException("Export table " + tableref.getTypeName() + " row length is " + rowLength + ", exceeding configurated limitation " + rowLengthLimit + ".");
                }
            }
        }
        for (String name : processorProperties.stringPropertyNames()) {
            ConnectorProperty prop = catconn.getConfig().add(name);
            prop.setName(name);
            prop.setValue(processorProperties.getProperty(name));
        }
        // on-server export always uses the guest processor
        catconn.setLoaderclass(ExportManager.PROCESSOR_CLASS);
        catconn.setEnabled(connectorEnabled);
        if (!connectorEnabled) {
            hostLog.info("Export configuration for export target " + targetName + " is present and is " + "configured to be disabled. Export target " + targetName + " will be disabled.");
        } else {
            hostLog.info("Export target " + targetName + " is configured and enabled with type=" + exportConfiguration.getType());
            if (exportConfiguration.getProperty() != null) {
                hostLog.info("Export target " + targetName + " configuration properties are: ");
                for (PropertyType configProp : exportConfiguration.getProperty()) {
                    if (!configProp.getName().toLowerCase().contains("password")) {
                        hostLog.info("Export Configuration Property NAME=" + configProp.getName() + " VALUE=" + configProp.getValue());
                    }
                }
            }
        }
    }
}