Examples with DeploymentType org.voltdb.compiler.deploymentfile.DeploymentType used on opensource projects

Example 16 with DeploymentType

use of org.voltdb.compiler.deploymentfile.DeploymentType in project voltdb by VoltDB.

the class UpdateApplicationBase method prepareApplicationCatalogDiff.

/**
     *
     * @param operationBytes The bytes for the catalog operation, if any. May be null in all cases.
     * For UpdateApplicationCatalog, this will contain the compiled catalog jarfile bytes
     * For UpdateClasses, this will contain the class jarfile bytes
     * For AdHoc DDL work, this will be null
     * @param operationString The string for the catalog operation, if any. May be null in all cases.
     * For UpdateApplicationCatalog, this will contain the deployment string to apply
     * For UpdateClasses, this will contain the class deletion patterns
     * For AdHoc DDL work, this will be null
     */
public static CatalogChangeResult prepareApplicationCatalogDiff(String invocationName, final byte[] operationBytes, final String operationString, final String[] adhocDDLStmts, final byte[] replayHashOverride, final boolean isPromotion, final DrRoleType drRole, final boolean useAdhocDDL, boolean adminConnection, String hostname, String user) throws PrepareDiffFailureException {
    // create the change result and set up all the boiler plate
    CatalogChangeResult retval = new CatalogChangeResult();
    // ensure non-null
    retval.tablesThatMustBeEmpty = new String[0];
    retval.hasSchemaChange = true;
    if (replayHashOverride != null) {
        retval.isForReplay = true;
    }
    try {
        // catalog change specific boiler plate
        CatalogContext context = VoltDB.instance().getCatalogContext();
        // Start by assuming we're doing an @UpdateApplicationCatalog.  If-ladder below
        // will complete with newCatalogBytes actually containing the bytes of the
        // catalog to be applied, and deploymentString will contain an actual deployment string,
        // or null if it still needs to be filled in.
        InMemoryJarfile newCatalogJar = null;
        InMemoryJarfile oldJar = context.getCatalogJar().deepCopy();
        boolean updatedClass = false;
        String deploymentString = operationString;
        if ("@UpdateApplicationCatalog".equals(invocationName)) {
            // Grab the current catalog bytes if @UAC had a null catalog from deployment-only update
            if ((operationBytes == null) || (operationBytes.length == 0)) {
                newCatalogJar = oldJar;
            } else {
                newCatalogJar = CatalogUtil.loadInMemoryJarFile(operationBytes);
            }
        // If the deploymentString is null, we'll fill it in with current deployment later
        // Otherwise, deploymentString has the right contents, don't need to touch it
        } else if ("@UpdateClasses".equals(invocationName)) {
            // provided newCatalogJar is the jarfile with the new classes
            if (operationBytes != null) {
                newCatalogJar = new InMemoryJarfile(operationBytes);
            }
            try {
                InMemoryJarfile modifiedJar = modifyCatalogClasses(context.catalog, oldJar, operationString, newCatalogJar, drRole == DrRoleType.XDCR);
                if (modifiedJar == null) {
                    newCatalogJar = oldJar;
                } else {
                    newCatalogJar = modifiedJar;
                    updatedClass = true;
                }
            } catch (ClassNotFoundException e) {
                throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, "Unexpected error in @UpdateClasses modifying classes from catalog: " + e.getMessage());
            }
            // Real deploymentString should be the current deployment, just set it to null
            // here and let it get filled in correctly later.
            deploymentString = null;
            // mark it as non-schema change
            retval.hasSchemaChange = false;
        } else if ("@AdHoc".equals(invocationName)) {
            // work.adhocDDLStmts should be applied to the current catalog
            try {
                newCatalogJar = addDDLToCatalog(context.catalog, oldJar, adhocDDLStmts, drRole == DrRoleType.XDCR);
            } catch (VoltCompilerException vce) {
                throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, vce.getMessage());
            } catch (IOException ioe) {
                throw new PrepareDiffFailureException(ClientResponse.UNEXPECTED_FAILURE, ioe.getMessage());
            } catch (Throwable t) {
                String msg = "Unexpected condition occurred applying DDL statements: " + t.toString();
                compilerLog.error(msg);
                throw new PrepareDiffFailureException(ClientResponse.UNEXPECTED_FAILURE, msg);
            }
            assert (newCatalogJar != null);
            if (newCatalogJar == null) {
                // Shouldn't ever get here
                String msg = "Unexpected failure in applying DDL statements to original catalog";
                compilerLog.error(msg);
                throw new PrepareDiffFailureException(ClientResponse.UNEXPECTED_FAILURE, msg);
            }
            // Real deploymentString should be the current deployment, just set it to null
            // here and let it get filled in correctly later.
            deploymentString = null;
        } else {
            // TODO: this if-chain doesn't feel like it even should exist
            assert (false);
        }
        // get the diff between catalogs
        // try to get the new catalog from the params
        Pair<InMemoryJarfile, String> loadResults = null;
        try {
            loadResults = CatalogUtil.loadAndUpgradeCatalogFromJar(newCatalogJar, drRole == DrRoleType.XDCR);
        } catch (IOException ioe) {
            // falling through to the ZOMG message in the big catch
            throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, ioe.getMessage());
        }
        retval.catalogBytes = loadResults.getFirst().getFullJarBytes();
        if (!retval.isForReplay) {
            retval.catalogHash = loadResults.getFirst().getSha1Hash();
        } else {
            retval.catalogHash = replayHashOverride;
        }
        String newCatalogCommands = CatalogUtil.getSerializedCatalogStringFromJar(loadResults.getFirst());
        retval.upgradedFromVersion = loadResults.getSecond();
        if (newCatalogCommands == null) {
            throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, "Unable to read from catalog bytes");
        }
        Catalog newCatalog = new Catalog();
        newCatalog.execute(newCatalogCommands);
        // Retrieve the original deployment string, if necessary
        if (deploymentString == null) {
            // Go get the deployment string from the current catalog context
            byte[] deploymentBytes = context.getDeploymentBytes();
            if (deploymentBytes != null) {
                deploymentString = new String(deploymentBytes, Constants.UTF8ENCODING);
            }
            if (deploymentBytes == null || deploymentString == null) {
                throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, "No deployment file provided and unable to recover previous deployment settings.");
            }
        }
        DeploymentType dt = CatalogUtil.parseDeploymentFromString(deploymentString);
        if (dt == null) {
            throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, "Unable to update deployment configuration: Error parsing deployment string");
        }
        if (isPromotion && drRole == DrRoleType.REPLICA) {
            assert dt.getDr().getRole() == DrRoleType.REPLICA;
            dt.getDr().setRole(DrRoleType.MASTER);
        }
        String result = CatalogUtil.compileDeployment(newCatalog, dt, false);
        if (result != null) {
            throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, "Unable to update deployment configuration: " + result);
        }
        //In non legacy mode discard the path element.
        if (!VoltDB.instance().isRunningWithOldVerbs()) {
            dt.setPaths(null);
        }
        //Always get deployment after its adjusted.
        retval.deploymentString = CatalogUtil.getDeployment(dt, true);
        retval.deploymentHash = CatalogUtil.makeDeploymentHash(retval.deploymentString.getBytes(Constants.UTF8ENCODING));
        // store the version of the catalog the diffs were created against.
        // verified when / if the update procedure runs in order to verify
        // catalogs only move forward
        retval.expectedCatalogVersion = context.catalogVersion;
        // compute the diff in StringBuilder
        CatalogDiffEngine diff = new CatalogDiffEngine(context.catalog, newCatalog);
        if (!diff.supported()) {
            throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, "The requested catalog change(s) are not supported:\n" + diff.errors());
        }
        String commands = diff.commands();
        compilerLog.info(diff.getDescriptionOfChanges(updatedClass));
        retval.requireCatalogDiffCmdsApplyToEE = diff.requiresCatalogDiffCmdsApplyToEE();
        // since diff commands can be stupidly big, compress them here
        retval.encodedDiffCommands = Encoder.compressAndBase64Encode(commands);
        retval.diffCommandsLength = commands.length();
        String[][] emptyTablesAndReasons = diff.tablesThatMustBeEmpty();
        assert (emptyTablesAndReasons.length == 2);
        assert (emptyTablesAndReasons[0].length == emptyTablesAndReasons[1].length);
        retval.tablesThatMustBeEmpty = emptyTablesAndReasons[0];
        retval.reasonsForEmptyTables = emptyTablesAndReasons[1];
        retval.requiresSnapshotIsolation = diff.requiresSnapshotIsolation();
        retval.requiresNewExportGeneration = diff.requiresNewExportGeneration();
        retval.worksWithElastic = diff.worksWithElastic();
    } catch (PrepareDiffFailureException e) {
        throw e;
    } catch (Exception e) {
        String msg = "Unexpected error in adhoc or catalog update: " + e.getClass() + ", " + e.getMessage();
        compilerLog.warn(msg, e);
        throw new PrepareDiffFailureException(ClientResponse.UNEXPECTED_FAILURE, msg);
    }
    return retval;
}

Example 17 with DeploymentType

use of org.voltdb.compiler.deploymentfile.DeploymentType in project voltdb by VoltDB.

the class TestJSONInterface method testUpdateDeploymentMemoryLimit.

public void testUpdateDeploymentMemoryLimit() throws Exception {
    try {
        String simpleSchema = "CREATE TABLE foo (\n" + "    bar BIGINT NOT NULL,\n" + "    PRIMARY KEY (bar)\n" + ");";
        File schemaFile = VoltProjectBuilder.writeStringToTempFile(simpleSchema);
        String schemaPath = schemaFile.getPath();
        schemaPath = URLEncoder.encode(schemaPath, "UTF-8");
        VoltProjectBuilder builder = new VoltProjectBuilder();
        builder.addSchema(schemaPath);
        builder.addPartitionInfo("foo", "bar");
        builder.addProcedures(DelayProc.class);
        builder.setHTTPDPort(8095);
        boolean success = builder.compile(Configuration.getPathToCatalogForTest("json.jar"));
        assertTrue(success);
        VoltDB.Configuration config = new VoltDB.Configuration();
        config.m_pathToCatalog = config.setPathToCatalogForTest("json.jar");
        config.m_pathToDeployment = builder.getPathToDeployment();
        server = new ServerThread(config);
        server.start();
        server.waitForInitialization();
        //Get deployment
        String jdep = getUrlOverJSON(protocolPrefix + "localhost:8095/deployment", null, null, null, 200, "application/json");
        Map<String, String> params = new HashMap<>();
        ObjectMapper mapper = new ObjectMapper();
        DeploymentType deptype = mapper.readValue(jdep, DeploymentType.class);
        SystemSettingsType ss = deptype.getSystemsettings();
        if (ss == null) {
            ss = new SystemSettingsType();
            deptype.setSystemsettings(ss);
        }
        ResourceMonitorType resourceMonitor = new ResourceMonitorType();
        Memorylimit memLimit = new Memorylimit();
        memLimit.setSize("10");
        memLimit.setAlert("5");
        resourceMonitor.setMemorylimit(memLimit);
        ss.setResourcemonitor(resourceMonitor);
        String ndeptype = mapper.writeValueAsString(deptype);
        params.put("deployment", ndeptype);
        String pdep = postUrlOverJSON(protocolPrefix + "localhost:8095/deployment/", null, null, null, 200, "application/json", params);
        assertTrue(pdep.contains("Deployment Updated"));
        jdep = getUrlOverJSON(protocolPrefix + "localhost:8095/deployment", null, null, null, 200, "application/json");
        DeploymentType gotValue = mapper.readValue(jdep, DeploymentType.class);
        assertEquals("10", gotValue.getSystemsettings().getResourcemonitor().getMemorylimit().getSize());
        memLimit.setSize("90%");
        ndeptype = mapper.writeValueAsString(deptype);
        params.put("deployment", ndeptype);
        pdep = postUrlOverJSON(protocolPrefix + "localhost:8095/deployment/", null, null, null, 200, "application/json", params);
        assertTrue(pdep.contains("Deployment Updated"));
        jdep = getUrlOverJSON(protocolPrefix + "localhost:8095/deployment", null, null, null, 200, "application/json");
        gotValue = mapper.readValue(jdep, DeploymentType.class);
        assertEquals("90%", gotValue.getSystemsettings().getResourcemonitor().getMemorylimit().getSize());
        // decimal values not allowed for percentages
        memLimit.setSize("90.5%25");
        ndeptype = mapper.writeValueAsString(deptype);
        params.put("deployment", ndeptype);
        pdep = postUrlOverJSON(protocolPrefix + "localhost:8095/deployment/", null, null, null, 200, "application/json", params);
        jdep = getUrlOverJSON(protocolPrefix + "localhost:8095/deployment", null, null, null, 200, "application/json");
        gotValue = mapper.readValue(jdep, DeploymentType.class);
        // must be still the old value
        assertEquals("90%", gotValue.getSystemsettings().getResourcemonitor().getMemorylimit().getSize());
    } finally {
        if (server != null) {
            server.shutdown();
            server.join();
        }
        server = null;
    }
}

Example 18 with DeploymentType

use of org.voltdb.compiler.deploymentfile.DeploymentType in project voltdb by VoltDB.

the class TestJSONInterface method testaUpdateDeploymentSnmpConfig.

public void testaUpdateDeploymentSnmpConfig() throws Exception {
    try {
        String simpleSchema = "CREATE TABLE foo (\n" + "    bar BIGINT NOT NULL,\n" + "    PRIMARY KEY (bar)\n" + ");";
        File schemaFile = VoltProjectBuilder.writeStringToTempFile(simpleSchema);
        String schemaPath = schemaFile.getPath();
        schemaPath = URLEncoder.encode(schemaPath, "UTF-8");
        VoltProjectBuilder builder = new VoltProjectBuilder();
        builder.addSchema(schemaPath);
        builder.addPartitionInfo("foo", "bar");
        builder.addProcedures(DelayProc.class);
        builder.setHTTPDPort(8095);
        boolean success = builder.compile(Configuration.getPathToCatalogForTest("json.jar"));
        assertTrue(success);
        VoltDB.Configuration config = new VoltDB.Configuration();
        config.m_pathToCatalog = config.setPathToCatalogForTest("json.jar");
        config.m_pathToDeployment = builder.getPathToDeployment();
        server = new ServerThread(config);
        server.start();
        server.waitForInitialization();
        //Get deployment
        String jdep = getUrlOverJSON(protocolPrefix + "localhost:8095/deployment", null, null, null, 200, "application/json");
        Map<String, String> params = new HashMap<>();
        ObjectMapper mapper = new ObjectMapper();
        DeploymentType deptype = mapper.readValue(jdep, DeploymentType.class);
        SnmpType snmpConfig = new SnmpType();
        snmpConfig.setTarget("localhost");
        deptype.setSnmp(snmpConfig);
        String ndeptype = mapper.writeValueAsString(deptype);
        params.put("deployment", ndeptype);
        String pdep = postUrlOverJSON(protocolPrefix + "localhost:8095/deployment/", null, null, null, 200, "application/json", params);
        assertTrue(pdep.contains("Deployment Updated"));
        jdep = getUrlOverJSON(protocolPrefix + "localhost:8095/deployment", null, null, null, 200, "application/json");
        DeploymentType gotValue = mapper.readValue(jdep, DeploymentType.class);
        assertEquals("public", gotValue.getSnmp().getCommunity());
        snmpConfig.setCommunity("foobar");
        deptype.setSnmp(snmpConfig);
        ndeptype = mapper.writeValueAsString(deptype);
        params.put("deployment", ndeptype);
        pdep = postUrlOverJSON(protocolPrefix + "localhost:8095/deployment/", null, null, null, 200, "application/json", params);
        assertTrue(pdep.contains("Deployment Updated"));
        jdep = getUrlOverJSON(protocolPrefix + "localhost:8095/deployment", null, null, null, 200, "application/json");
        gotValue = mapper.readValue(jdep, DeploymentType.class);
        assertEquals("foobar", gotValue.getSnmp().getCommunity());
    } finally {
        if (server != null) {
            server.shutdown();
            server.join();
        }
        server = null;
    }
}

Example 19 with DeploymentType

use of org.voltdb.compiler.deploymentfile.DeploymentType in project voltdb by VoltDB.

the class CatalogPasswordScrambler method writeOutMaskedDeploymentFile.

public static void writeOutMaskedDeploymentFile(DeploymentType depl, File maskedFH) {
    try {
        org.voltdb.compiler.deploymentfile.ObjectFactory factory = new org.voltdb.compiler.deploymentfile.ObjectFactory();
        JAXBContext context = JAXBContext.newInstance(DeploymentType.class);
        JAXBElement<DeploymentType> doc = factory.createDeployment(depl);
        Marshaller marshaller = context.createMarshaller();
        marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
        marshaller.marshal(doc, maskedFH);
    } catch (JAXBException e) {
        String msg = e.getMessage();
        if (msg == null)
            msg = e.getLinkedException().getMessage();
        System.err.println("Failed to write masked deployemnt: " + msg);
        e.printStackTrace();
    }
}

Example 20 with DeploymentType

use of org.voltdb.compiler.deploymentfile.DeploymentType in project voltdb by VoltDB.

the class CatalogPasswordScrambler method main.

public static void main(String[] args) {
    if (args.length == 1)
        args = new String[] { args[0], args[0] };
    if (args.length != 2) {
        System.out.println("Usage: CatalogPasswordScrambler [deployment-file-spec] [masked-deployment-file-spec]");
    }
    File deployFH = new File(args[0]);
    if (!deployFH.exists() || !deployFH.isFile() || !deployFH.canRead()) {
        System.err.println("cannot access: " + deployFH);
    }
    File maskedFH = new File(args[1]);
    try {
        File canonMaskedFH = maskedFH.getCanonicalFile();
        File parentFH = canonMaskedFH.getParentFile();
        if (!parentFH.exists() || !parentFH.isDirectory() || !parentFH.canWrite()) {
            System.err.println("do not have write access to " + parentFH);
            return;
        }
    } catch (IOException e) {
        System.err.println("could not stat specified files: " + e.getMessage());
        return;
    }
    DeploymentType depl = getDeployment(deployFH);
    if (depl == null)
        return;
    try {
        scramblePasswords(depl);
    } catch (UnsupportedOperationException e) {
        System.err.println(e.getMessage());
        return;
    }
    writeOutMaskedDeploymentFile(depl, maskedFH);
}