Examples with InMemoryJarfile org.voltdb.utils.InMemoryJarfile used on opensource projects

Example 26 with InMemoryJarfile

use of org.voltdb.utils.InMemoryJarfile in project voltdb by VoltDB.

the class TestUpdateClasses method testDeleteClasses.

// Delete tests:
// single file match
// * match
// ** match
// comma-separated matches
// combine new jarfile with deleted stuff
// deleting inner classes
@Test
public void testDeleteClasses() throws Exception {
    System.out.println("\n\n-----\n testCollidingProc \n-----\n\n");
    String pathToCatalog = Configuration.getPathToCatalogForTest("updateclasses.jar");
    String pathToDeployment = Configuration.getPathToCatalogForTest("updateclasses.xml");
    VoltProjectBuilder builder = new VoltProjectBuilder();
    builder.addLiteralSchema("-- Don't care");
    builder.setUseDDLSchema(true);
    boolean success = builder.compile(pathToCatalog, 2, 1, 0);
    assertTrue("Schema compilation failed", success);
    MiscUtils.copyFile(builder.getPathToDeployment(), pathToDeployment);
    // This is maybe cheating a little bit?
    InMemoryJarfile jarfile = new InMemoryJarfile();
    for (Class<?> clazz : PROC_CLASSES) {
        VoltCompiler comp = new VoltCompiler(false);
        comp.addClassToJar(jarfile, clazz);
    }
    for (Class<?> clazz : EXTRA_CLASSES) {
        VoltCompiler comp = new VoltCompiler(false);
        comp.addClassToJar(jarfile, clazz);
    }
    for (Class<?> clazz : COLLIDING_CLASSES) {
        VoltCompiler comp = new VoltCompiler(false);
        comp.addClassToJar(jarfile, clazz);
    }
    try {
        VoltDB.Configuration config = new VoltDB.Configuration();
        config.m_pathToCatalog = pathToCatalog;
        config.m_pathToDeployment = pathToDeployment;
        startSystem(config);
        ClientResponse resp;
        // Make sure we're clean
        resp = m_client.callProcedure("@SystemCatalog", "CLASSES");
        assertEquals(0, resp.getResults()[0].getRowCount());
        // Add the jarfile we built
        resp = m_client.callProcedure("@UpdateClasses", jarfile.getFullJarBytes(), null);
        resp = m_client.callProcedure("@SystemCatalog", "CLASSES");
        assertEquals(PROC_CLASSES.length + EXTRA_CLASSES.length + COLLIDING_CLASSES.length, resp.getResults()[0].getRowCount());
        // remove one class
        assertTrue(findClassInSystemCatalog(PROC_CLASSES[0].getCanonicalName()));
        resp = m_client.callProcedure("@UpdateClasses", null, PROC_CLASSES[0].getCanonicalName());
        assertFalse(findClassInSystemCatalog(PROC_CLASSES[0].getCanonicalName()));
        // remove everything under fullddlfeatures
        assertTrue(findClassInSystemCatalog(COLLIDING_CLASSES[0].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(COLLIDING_CLASSES[1].getCanonicalName()));
        resp = m_client.callProcedure("@UpdateClasses", null, "org.voltdb_testprocs.fullddlfeatures.*");
        assertFalse(findClassInSystemCatalog(COLLIDING_CLASSES[0].getCanonicalName()));
        assertFalse(findClassInSystemCatalog(COLLIDING_CLASSES[1].getCanonicalName()));
        // Remove everything left
        assertTrue(findClassInSystemCatalog(PROC_CLASSES[1].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(EXTRA_CLASSES[0].getCanonicalName()));
        resp = m_client.callProcedure("@UpdateClasses", null, "org.voltdb**");
        assertFalse(findClassInSystemCatalog(PROC_CLASSES[1].getCanonicalName()));
        assertFalse(findClassInSystemCatalog(EXTRA_CLASSES[0].getCanonicalName()));
        resp = m_client.callProcedure("@SystemCatalog", "CLASSES");
        assertEquals(0, resp.getResults()[0].getRowCount());
        // put everything back
        // Add the jarfile we built
        resp = m_client.callProcedure("@UpdateClasses", jarfile.getFullJarBytes(), null);
        resp = m_client.callProcedure("@SystemCatalog", "CLASSES");
        assertEquals(PROC_CLASSES.length + EXTRA_CLASSES.length + COLLIDING_CLASSES.length, resp.getResults()[0].getRowCount());
        // delete the common simple names from both packages simultaneously
        resp = m_client.callProcedure("@UpdateClasses", null, "**testImportProc   , **testCreateProcFromClassProc");
        resp = m_client.callProcedure("@SystemCatalog", "CLASSES");
        // should be the only thing left
        assertEquals(1, resp.getResults()[0].getRowCount());
        assertTrue(findClassInSystemCatalog(EXTRA_CLASSES[0].getCanonicalName()));
        // make a jar without the extra
        InMemoryJarfile jarfile2 = new InMemoryJarfile();
        for (Class<?> clazz : PROC_CLASSES) {
            VoltCompiler comp = new VoltCompiler(false);
            comp.addClassToJar(jarfile2, clazz);
        }
        for (Class<?> clazz : COLLIDING_CLASSES) {
            VoltCompiler comp = new VoltCompiler(false);
            comp.addClassToJar(jarfile2, clazz);
        }
        // finally, delete what's left and put the new jar in simultaneously
        resp = m_client.callProcedure("@UpdateClasses", jarfile2.getFullJarBytes(), "**updateclasses.*");
        resp = m_client.callProcedure("@SystemCatalog", "CLASSES");
        // extra class should be gone, others installed
        assertEquals(PROC_CLASSES.length + COLLIDING_CLASSES.length, resp.getResults()[0].getRowCount());
        assertFalse(findClassInSystemCatalog(EXTRA_CLASSES[0].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(PROC_CLASSES[0].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(PROC_CLASSES[1].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(COLLIDING_CLASSES[0].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(COLLIDING_CLASSES[1].getCanonicalName()));
        // now add a class with inner classes
        InMemoryJarfile inner = new InMemoryJarfile();
        VoltCompiler comp = new VoltCompiler(false);
        comp.addClassToJar(inner, org.voltdb_testprocs.updateclasses.InnerClassesTestProc.class);
        resp = m_client.callProcedure("@UpdateClasses", inner.getFullJarBytes(), null);
        // old stuff should have survived
        assertTrue(findClassInSystemCatalog(PROC_CLASSES[0].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(PROC_CLASSES[1].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(COLLIDING_CLASSES[0].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(COLLIDING_CLASSES[1].getCanonicalName()));
        // Did we get the new class and inner classes too?
        assertTrue(findClassInSystemCatalog("org.voltdb_testprocs.updateclasses.InnerClassesTestProc"));
        assertTrue(findClassInSystemCatalog("org.voltdb_testprocs.updateclasses.InnerClassesTestProc$InnerNotPublic"));
        assertTrue(findClassInSystemCatalog("org.voltdb_testprocs.updateclasses.InnerClassesTestProc$InnerWithConstructorArgs"));
        assertTrue(findClassInSystemCatalog("org.voltdb_testprocs.updateclasses.InnerClassesTestProc$InnerWithEasyConstructor"));
        assertTrue(findClassInSystemCatalog("org.voltdb_testprocs.updateclasses.InnerClassesTestProc$InnerWithNoConstructor"));
        // now just delete the parent class
        resp = m_client.callProcedure("@UpdateClasses", null, "org.voltdb_testprocs.updateclasses.InnerClassesTestProc");
        // old stuff should have survived
        resp = m_client.callProcedure("@SystemCatalog", "CLASSES");
        // Non-inner stuff should have survived
        assertEquals(PROC_CLASSES.length + COLLIDING_CLASSES.length, resp.getResults()[0].getRowCount());
        assertTrue(findClassInSystemCatalog(PROC_CLASSES[0].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(PROC_CLASSES[1].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(COLLIDING_CLASSES[0].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(COLLIDING_CLASSES[1].getCanonicalName()));
        // Inner classes and parent gone
        assertFalse(findClassInSystemCatalog("org.voltdb_testprocs.updateclasses.InnerClassesTestProc"));
        assertFalse(findClassInSystemCatalog("org.voltdb_testprocs.updateclasses.InnerClassesTestProc$InnerNotPublic"));
        assertFalse(findClassInSystemCatalog("org.voltdb_testprocs.updateclasses.InnerClassesTestProc$InnerWithConstructorArgs"));
        assertFalse(findClassInSystemCatalog("org.voltdb_testprocs.updateclasses.InnerClassesTestProc$InnerWithEasyConstructor"));
        assertFalse(findClassInSystemCatalog("org.voltdb_testprocs.updateclasses.InnerClassesTestProc$InnerWithNoConstructor"));
        // Empty string has no effect
        resp = m_client.callProcedure("@UpdateClasses", null, "");
        // old stuff should have survived
        resp = m_client.callProcedure("@SystemCatalog", "CLASSES");
        // Non-inner stuff should have survived
        assertEquals(PROC_CLASSES.length + COLLIDING_CLASSES.length, resp.getResults()[0].getRowCount());
        assertTrue(findClassInSystemCatalog(PROC_CLASSES[0].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(PROC_CLASSES[1].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(COLLIDING_CLASSES[0].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(COLLIDING_CLASSES[1].getCanonicalName()));
        // pattern that matches nothing has no effect
        resp = m_client.callProcedure("@UpdateClasses", null, "com.voltdb.*");
        // old stuff should have survived
        resp = m_client.callProcedure("@SystemCatalog", "CLASSES");
        // Non-inner stuff should have survived
        assertEquals(PROC_CLASSES.length + COLLIDING_CLASSES.length, resp.getResults()[0].getRowCount());
        assertTrue(findClassInSystemCatalog(PROC_CLASSES[0].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(PROC_CLASSES[1].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(COLLIDING_CLASSES[0].getCanonicalName()));
        assertTrue(findClassInSystemCatalog(COLLIDING_CLASSES[1].getCanonicalName()));
    } finally {
        teardownSystem();
    }
}

Example 27 with InMemoryJarfile

use of org.voltdb.utils.InMemoryJarfile in project voltdb by VoltDB.

the class VoltCompiler method compileInMemoryJarfile.

/**
     * Compile the provided jarfile.  Basically, treat the jarfile as a staging area
     * for the artifacts to be included in the compile, and then compile it in place.
     *
     * *NOTE*: Does *NOT* work with project.xml jarfiles.
     *
     * @return the compiled catalog is contained in the provided jarfile.
     *
     */
public void compileInMemoryJarfile(InMemoryJarfile jarfile) throws IOException {
    // Gather DDL files for recompilation
    List<VoltCompilerReader> ddlReaderList = new ArrayList<>();
    Entry<String, byte[]> entry = jarfile.firstEntry();
    while (entry != null) {
        String path = entry.getKey();
        // ddl files instead of using a brute force *.sql glob.
        if (path.toLowerCase().endsWith(".sql")) {
            ddlReaderList.add(new VoltCompilerJarFileReader(jarfile, path));
            compilerLog.trace("Added SQL file from jarfile to compilation: " + path);
        }
        entry = jarfile.higherEntry(entry.getKey());
    }
    // Use the in-memory jarfile-provided class loader so that procedure
    // classes can be found and copied to the new file that gets written.
    ClassLoader originalClassLoader = m_classLoader;
    try {
        m_classLoader = jarfile.getLoader();
        // Do the compilation work.
        InMemoryJarfile jarOut = compileInternal(null, null, ddlReaderList, jarfile);
        // explanation
        if (jarOut != null) {
            compilerLog.debug("Successfully recompiled InMemoryJarfile");
        } else {
            String errString = "Adhoc DDL failed";
            if (m_errors.size() > 0) {
                errString = m_errors.get(m_errors.size() - 1).getLogString();
            }
            int fronttrim = errString.indexOf("DDL Error");
            if (fronttrim < 0) {
                fronttrim = 0;
            }
            int endtrim = errString.indexOf(" in statement starting");
            if (endtrim < 0) {
                endtrim = errString.length();
            }
            String trimmed = errString.substring(fronttrim, endtrim);
            throw new IOException(trimmed);
        }
    } finally {
        // Restore the original class loader
        m_classLoader = originalClassLoader;
    }
}

Example 28 with InMemoryJarfile

use of org.voltdb.utils.InMemoryJarfile in project voltdb by VoltDB.

the class RealVoltDB method outputProcedures.

private int outputProcedures(Configuration config) {
    File outputFile = new File(config.m_getOutput);
    if (outputFile.exists() && !config.m_forceGetCreate) {
        consoleLog.fatal("Failed to save classes, file already exists: " + config.m_getOutput);
        return -1;
    }
    try {
        InMemoryJarfile catalogJar = CatalogUtil.loadInMemoryJarFile(MiscUtils.fileToBytes(new File(config.m_pathToCatalog)));
        InMemoryJarfile filteredJar = CatalogUtil.getCatalogJarWithoutDefaultArtifacts(catalogJar);
        filteredJar.writeToFile(outputFile);
        consoleLog.info("Classes saved in " + outputFile.getPath());
    } catch (IOException e) {
        consoleLog.fatal("Failed to read classes " + config.m_pathToCatalog + " : " + e.getMessage());
        return -1;
    }
    return 0;
}

Example 29 with InMemoryJarfile

use of org.voltdb.utils.InMemoryJarfile in project voltdb by VoltDB.

the class RestoreAgent method checkSnapshotIsComplete.

private SnapshotInfo checkSnapshotIsComplete(Long key, Snapshot s) {
    int partitionCount = -1;
    for (TableFiles tf : s.m_tableFiles.values()) {
        // Check if the snapshot is complete
        if (tf.m_completed.stream().anyMatch(b -> !b)) {
            m_snapshotErrLogStr.append("\nRejected snapshot ").append(s.getNonce()).append(" because it was not completed.");
            return null;
        }
        // Replicated table doesn't check partition count
        if (tf.m_isReplicated) {
            continue;
        }
        // Everyone has to agree on the total partition count
        for (int count : tf.m_totalPartitionCounts) {
            if (partitionCount == -1) {
                partitionCount = count;
            } else if (count != partitionCount) {
                m_snapshotErrLogStr.append("\nRejected snapshot ").append(s.getNonce()).append(" because it had the wrong partition count ").append(count).append(", expecting ").append(partitionCount);
                return null;
            }
        }
    }
    if (s.m_digests.isEmpty()) {
        m_snapshotErrLogStr.append("\nRejected snapshot ").append(s.getNonce()).append(" because it had no valid digest file.");
        return null;
    }
    File digest = s.m_digests.get(0);
    Long catalog_crc = null;
    Map<Integer, Long> pidToTxnMap = new TreeMap<Integer, Long>();
    Set<String> digestTableNames = new HashSet<String>();
    // Create a valid but meaningless InstanceId to support pre-instanceId checking versions
    InstanceId instanceId = new InstanceId(0, 0);
    int newParitionCount = -1;
    try {
        JSONObject digest_detail = SnapshotUtil.CRCCheck(digest, LOG);
        if (digest_detail == null)
            throw new IOException();
        catalog_crc = digest_detail.getLong("catalogCRC");
        if (digest_detail.has("partitionTransactionIds")) {
            JSONObject pidToTxnId = digest_detail.getJSONObject("partitionTransactionIds");
            Iterator<String> it = pidToTxnId.keys();
            while (it.hasNext()) {
                String pidkey = it.next();
                Long txnidval = pidToTxnId.getLong(pidkey);
                pidToTxnMap.put(Integer.valueOf(pidkey), txnidval);
            }
        }
        if (digest_detail.has("instanceId")) {
            instanceId = new InstanceId(digest_detail.getJSONObject("instanceId"));
        }
        if (digest_detail.has("newPartitionCount")) {
            newParitionCount = digest_detail.getInt("newPartitionCount");
        }
        if (digest_detail.has("tables")) {
            JSONArray tableObj = digest_detail.getJSONArray("tables");
            for (int i = 0; i < tableObj.length(); i++) {
                digestTableNames.add(tableObj.getString(i));
            }
        }
    } catch (IOException ioe) {
        m_snapshotErrLogStr.append("\nUnable to read digest file: ").append(digest.getAbsolutePath()).append(" due to: ").append(ioe.getMessage());
        return null;
    } catch (JSONException je) {
        m_snapshotErrLogStr.append("\nUnable to extract catalog CRC from digest: ").append(digest.getAbsolutePath()).append(" due to: ").append(je.getMessage());
        return null;
    }
    if (s.m_catalogFile == null) {
        m_snapshotErrLogStr.append("\nRejected snapshot ").append(s.getNonce()).append(" because it had no catalog.");
        return null;
    }
    try {
        byte[] bytes = MiscUtils.fileToBytes(s.m_catalogFile);
        InMemoryJarfile jarfile = CatalogUtil.loadInMemoryJarFile(bytes);
        if (jarfile.getCRC() != catalog_crc) {
            m_snapshotErrLogStr.append("\nRejected snapshot ").append(s.getNonce()).append(" because catalog CRC did not match digest.");
            return null;
        }
        // Make sure this is not a partial snapshot.
        // Compare digestTableNames with all normal table names in catalog file.
        // A normal table is one that's NOT a materialized view, nor an export table.
        Set<String> catalogNormalTableNames = CatalogUtil.getNormalTableNamesFromInMemoryJar(jarfile);
        if (!catalogNormalTableNames.equals(digestTableNames)) {
            m_snapshotErrLogStr.append("\nRejected snapshot ").append(s.getNonce()).append(" because this is a partial snapshot.");
            return null;
        }
    } catch (IOException ioe) {
        m_snapshotErrLogStr.append("\nRejected snapshot ").append(s.getNonce()).append(" because catalog file could not be validated");
        return null;
    }
    SnapshotInfo info = new SnapshotInfo(key, digest.getParent(), SnapshotUtil.parseNonceFromDigestFilename(digest.getName()), partitionCount, newParitionCount, catalog_crc, m_hostId, instanceId, digestTableNames, s.m_stype);
    // populate table to partition map.
    for (Entry<String, TableFiles> te : s.m_tableFiles.entrySet()) {
        TableFiles tableFile = te.getValue();
        HashSet<Integer> ids = new HashSet<Integer>();
        for (Set<Integer> idSet : tableFile.m_validPartitionIds) {
            ids.addAll(idSet);
        }
        if (!tableFile.m_isReplicated) {
            info.partitions.put(te.getKey(), ids);
        }
        // keep track of tables for which we've seen files while we're here
        info.fileTables.add(te.getKey());
    }
    info.setPidToTxnIdMap(pidToTxnMap);
    return info;
}

Example 30 with InMemoryJarfile

use of org.voltdb.utils.InMemoryJarfile in project voltdb by VoltDB.

the class RealVoltDB method outputSchema.

private int outputSchema(Configuration config) {
    if ((new File(config.m_getOutput)).exists() && !config.m_forceGetCreate) {
        consoleLog.fatal("Failed to save schema file, file already exists: " + config.m_getOutput);
        return -1;
    }
    try {
        InMemoryJarfile catalogJar = CatalogUtil.loadInMemoryJarFile(MiscUtils.fileToBytes(new File(config.m_pathToCatalog)));
        String ddl = CatalogUtil.getAutoGenDDLFromJar(catalogJar);
        try (FileOutputStream fos = new FileOutputStream(config.m_getOutput.trim())) {
            fos.write(ddl.getBytes());
        } catch (IOException e) {
            consoleLog.fatal("Failed to write schema to " + config.m_getOutput + " : " + e.getMessage());
            return -1;
        }
        consoleLog.info("Schema saved in " + config.m_getOutput.trim());
    } catch (IOException e) {
        consoleLog.fatal("Failed to load the catalog jar from " + config.m_pathToCatalog + " : " + e.getMessage());
        return -1;
    }
    return 0;
}