Examples with TimestampType org.voltdb.types.TimestampType used on opensource projects

Example 36 with TimestampType

use of org.voltdb.types.TimestampType in project voltdb by VoltDB.

the class ContinuousDeleter method run.

@Override
public void run() {
    // Will be set to true if any partition needs to delete more rows to meet the goals.
    AtomicBoolean unfinished = new AtomicBoolean(false);
    // Default yield time between deletes. May be shortened if there are a ton of tuples that need deleting.
    long yieldTimeMs = app.config.deleteyieldtime;
    try {
        // Get the targets from the main app class. Only one will be used, depending on whether the user is deleting
        // old rows by date or by row count.
        TimestampType dateTarget = app.getTargetDate();
        long rowTarget = app.getTargetRowsPerPartition();
        // Send the procedure call to all partitions and get results from each partitions.
        ClientResponseWithPartitionKey[] responses;
        if (app.config.historyseconds > 0) {
            responses = app.client.callAllPartitionProcedure("DeleteAfterDate", dateTarget, app.config.deletechunksize);
        } else /* if (app.config.maxrows > 0) */
        {
            responses = app.client.callAllPartitionProcedure("DeleteOldestToTarget", rowTarget, app.config.deletechunksize);
        }
        app.updatePartitionCount(responses.length);
        for (ClientResponseWithPartitionKey resp : responses) {
            if (resp.response.getStatus() == ClientResponse.SUCCESS) {
                long tuplesDeleted = resp.response.getResults()[0].asScalarLong();
                app.addToDeletedTuples(tuplesDeleted);
                // If the procedure deleted up to its limit, reduce the time before the deletes process runs again.
                if (tuplesDeleted >= app.config.deletechunksize) {
                    unfinished.set(true);
                }
            } else {
                failureCount.incrementAndGet();
            }
        }
        // WindowingApp and schedule two of them to run concurrently.
        if (unfinished.get()) {
            yieldTimeMs = 0;
        }
    } catch (Exception e) {
        failureCount.incrementAndGet();
    } catch (Throwable t) {
        t.printStackTrace();
    // Live dangerously and ignore this.
    } finally {
        // Use a finally block to ensure this task is re-scheduled.
        app.scheduler.schedule(this, yieldTimeMs, TimeUnit.MILLISECONDS);
    }
}

Example 37 with TimestampType

use of org.voltdb.types.TimestampType in project voltdb by VoltDB.

the class InsertAndDeleteOldestToTarget method run.

/**
     * Procedure main logic.
     *
     * @param uuid Column value for tuple insertion and partitioning key for this procedure.
     * @param val Column value for tuple insertion.
     * @param update_ts Column value for tuple insertion.
     * @param maxTotalRows The desired number of rows per partition.
     * @param targetMaxRowsToDelete The upper limit on the number of rows to delete per transaction.
     * @return The number of deleted rows.
     * @throws VoltAbortException on bad input.
     */
public long run(String uuid, long val, TimestampType update_ts, long maxTotalRows, long targetMaxRowsToDelete) {
    if (targetMaxRowsToDelete <= 0) {
        throw new VoltAbortException("maxRowsToDeletePerProc must be > 0.");
    }
    if (maxTotalRows < 0) {
        throw new VoltAbortException("maxTotalRows must be >= 0.");
    }
    // This line inserts the row.
    voltQueueSQL(insert, EXPECT_SCALAR_MATCH(1), uuid, val, update_ts);
    // In the same round trip to the storage engine, count the rows.
    voltQueueSQL(countRows, EXPECT_SCALAR_LONG);
    // Can assume insert worked because of EXPECT_SCALAR_MATCH(1)
    // Note that the index into the set of results tables below is the second table.
    long count = voltExecuteSQL()[1].asScalarLong();
    // If partition is smaller than desired, return
    if (count < maxTotalRows) {
        return 0;
    }
    // If asked to remove all rows, go ahead
    if ((maxTotalRows == 0) && (count < targetMaxRowsToDelete)) {
        voltQueueSQL(deleteAll, EXPECT_SCALAR_MATCH(count));
        voltExecuteSQL(true);
        return count;
    }
    // Figure out how many rows to try to delete.
    long rowsToConsider = Math.min(count - maxTotalRows, targetMaxRowsToDelete);
    // Find the timestamp of the row at position N in the sorted order, where N is the chunk size
    voltQueueSQL(getNthOldestTimestamp, EXPECT_SCALAR, rowsToConsider);
    TimestampType newestToDiscard = voltExecuteSQL()[0].fetchRow(0).getTimestampAsTimestamp(0);
    // Delete all rows >= the timestamp found in the previous statement.
    // This will delete AT LEAST N rows, but since timestamps may be non-unique,
    //  it might delete more than N. In the worst case, it could delete all rows
    //  if every row has an identical timestamp value. It is guaranteed to make
    //  progress. If we used strictly less than, it might not make progress.
    // This is why the max rows to delete number is a target, not always a perfect max.
    voltQueueSQL(deleteOlderThanDate, EXPECT_SCALAR_LONG, newestToDiscard);
    long deletedCount = voltExecuteSQL(true)[0].asScalarLong();
    // Return the number of rows deleted.
    return deletedCount;
}

Example 38 with TimestampType

use of org.voltdb.types.TimestampType in project voltdb by VoltDB.

the class ProcedureRunner method getCleanParams.

private final ParameterSet getCleanParams(SQLStmt stmt, boolean verifyTypeConv, Object... inArgs) {
    final byte[] stmtParamTypes = stmt.statementParamTypes;
    final int numParamTypes = stmtParamTypes.length;
    final Object[] args = new Object[numParamTypes];
    if (inArgs.length != numParamTypes) {
        throw new VoltAbortException("Number of arguments provided was " + inArgs.length + " where " + numParamTypes + " was expected for statement " + stmt.getText());
    }
    for (int ii = 0; ii < numParamTypes; ii++) {
        VoltType type = VoltType.get(stmtParamTypes[ii]);
        // handle non-null values
        if (inArgs[ii] != null) {
            args[ii] = inArgs[ii];
            assert (type != VoltType.INVALID);
            if (verifyTypeConv && type != VoltType.INVALID) {
                throwIfInfeasibleTypeConversion(stmt, args[ii].getClass(), ii, type);
            }
            continue;
        }
        // handle null values
        switch(type) {
            case TINYINT:
                args[ii] = Byte.MIN_VALUE;
                break;
            case SMALLINT:
                args[ii] = Short.MIN_VALUE;
                break;
            case INTEGER:
                args[ii] = Integer.MIN_VALUE;
                break;
            case BIGINT:
                args[ii] = Long.MIN_VALUE;
                break;
            case FLOAT:
                args[ii] = VoltType.NULL_FLOAT;
                break;
            case TIMESTAMP:
                args[ii] = new TimestampType(Long.MIN_VALUE);
                break;
            case STRING:
                args[ii] = VoltType.NULL_STRING_OR_VARBINARY;
                break;
            case VARBINARY:
                args[ii] = VoltType.NULL_STRING_OR_VARBINARY;
                break;
            case DECIMAL:
                args[ii] = VoltType.NULL_DECIMAL;
                break;
            case GEOGRAPHY_POINT:
                args[ii] = VoltType.NULL_POINT;
                break;
            case GEOGRAPHY:
                args[ii] = VoltType.NULL_GEOGRAPHY;
                break;
            default:
                throw new VoltAbortException("Unknown type " + type + " can not be converted to NULL representation for arg " + ii + " for SQL stmt: " + stmt.getText());
        }
    }
    return ParameterSet.fromArrayNoCopy(args);
}

Example 39 with TimestampType

use of org.voltdb.types.TimestampType in project voltdb by VoltDB.

the class NonVoltDBBackend method runSQLWithSubstitutions.

VoltTable runSQLWithSubstitutions(final SQLStmt stmt, ParameterSet params, byte[] paramJavaTypes) {
    //HSQLProcedureWrapper does nothing smart. it just implements this interface with runStatement()
    StringBuilder sqlOut = new StringBuilder(stmt.getText().length() * 2);
    assert (paramJavaTypes != null);
    int lastIndex = 0;
    String sql = stmt.getText();
    // if there's no ? in the statmemt, then zero out any auto-parameterization
    int paramCount = StringUtils.countMatches(sql, "?");
    if (paramCount == 0) {
        params = ParameterSet.emptyParameterSet();
        paramJavaTypes = new byte[0];
    }
    Object[] paramObjs = params.toArray();
    for (int i = 0; i < paramObjs.length; i++) {
        int nextIndex = sql.indexOf('?', lastIndex);
        if (nextIndex == -1)
            throw new RuntimeException("SQL Statement has more arguments than params.");
        sqlOut.append(sql, lastIndex, nextIndex);
        lastIndex = nextIndex + 1;
        VoltType type = VoltType.get(paramJavaTypes[i]);
        if (VoltType.isVoltNullValue(paramObjs[i])) {
            sqlOut.append("NULL");
        } else if (paramObjs[i] instanceof TimestampType) {
            if (type != VoltType.TIMESTAMP)
                throw new RuntimeException("Inserting date into mismatched column type in HSQL.");
            TimestampType d = (TimestampType) paramObjs[i];
            // convert VoltDB's microsecond granularity to millis.
            Timestamp t = new Timestamp(d.getTime() / 1000);
            sqlOut.append('\'').append(t.toString()).append('\'');
        } else if (paramObjs[i] instanceof byte[]) {
            if (type == VoltType.STRING) {
                // Convert from byte[] -> String; escape single quotes
                try {
                    sqlOut.append(sqlEscape(new String((byte[]) paramObjs[i], "UTF-8")));
                } catch (UnsupportedEncodingException e) {
                    // should NEVER HAPPEN
                    System.err.println("FATAL: Your JVM doens't support UTF-8");
                    System.exit(-1);
                }
            } else if (type == VoltType.VARBINARY) {
                // Convert from byte[] -> String; using hex
                sqlOut.append(sqlEscape(Encoder.hexEncode((byte[]) paramObjs[i])));
            } else {
                throw new RuntimeException("Inserting string/varbinary (bytes) into mismatched column type in HSQL.");
            }
        } else if (paramObjs[i] instanceof String) {
            if (type != VoltType.STRING)
                throw new RuntimeException("Inserting string into mismatched column type in HSQL.");
            // Escape single quotes
            sqlOut.append(sqlEscape((String) paramObjs[i]));
        } else {
            if (type == VoltType.TIMESTAMP) {
                long t = Long.parseLong(paramObjs[i].toString());
                TimestampType d = new TimestampType(t);
                // convert VoltDB's microsecond granularity to millis
                Timestamp ts = new Timestamp(d.getTime() * 1000);
                sqlOut.append('\'').append(ts.toString()).append('\'');
            } else
                sqlOut.append(paramObjs[i].toString());
        }
    }
    sqlOut.append(sql, lastIndex, sql.length());
    return runDML(sqlOut.toString());
}

Example 40 with TimestampType

use of org.voltdb.types.TimestampType in project voltdb by VoltDB.

the class TestJSONInterface method testSimple.

public void testSimple() throws Exception {
    try {
        String simpleSchema = "create table blah (" + "ival bigint default 23 not null, " + "sval varchar(200) default 'foo', " + "dateval timestamp, " + "fval float, " + "decval decimal, " + "PRIMARY KEY(ival));";
        File schemaFile = VoltProjectBuilder.writeStringToTempFile(simpleSchema);
        String schemaPath = schemaFile.getPath();
        schemaPath = URLEncoder.encode(schemaPath, "UTF-8");
        VoltDB.Configuration config = new VoltDB.Configuration();
        VoltProjectBuilder builder = new VoltProjectBuilder();
        builder.addSchema(schemaPath);
        builder.addPartitionInfo("blah", "ival");
        builder.addStmtProcedure("Insert", "insert into blah values (?,?,?,?,?);");
        builder.addProcedures(CrazyBlahProc.class);
        builder.setHTTPDPort(8095);
        boolean success = builder.compile(Configuration.getPathToCatalogForTest("json.jar"), 1, 1, 0, 0) != null;
        assertTrue(success);
        config.m_pathToCatalog = config.setPathToCatalogForTest("json.jar");
        config.m_pathToDeployment = builder.getPathToDeployment();
        server = new ServerThread(config);
        server.start();
        server.waitForInitialization();
        ParameterSet pset;
        String responseJSON;
        Response response;
        // Call insert
        pset = ParameterSet.fromArrayNoCopy(1, "hello", new TimestampType(System.currentTimeMillis()), 5.0, "5.0");
        responseJSON = callProcOverJSON("Insert", pset, null, null, false);
        System.out.println(responseJSON);
        response = responseFromJSON(responseJSON);
        assertTrue(response.status == ClientResponse.SUCCESS);
        // Call insert again (with failure expected)
        responseJSON = callProcOverJSON("Insert", pset, null, null, false);
        System.out.println(responseJSON);
        response = responseFromJSON(responseJSON);
        assertTrue(response.status != ClientResponse.SUCCESS);
        // Call proc with complex params
        pset = ParameterSet.fromArrayNoCopy(1, 5, new double[] { 1.5, 6.0, 4 }, new VoltTable(new VoltTable.ColumnInfo("foo", VoltType.BIGINT)), new BigDecimal(5), new BigDecimal[] {}, new TimestampType(System.currentTimeMillis()));
        responseJSON = callProcOverJSON("CrazyBlahProc", pset, null, null, false);
        System.out.println(responseJSON);
        response = responseFromJSON(responseJSON);
        assertEquals(ClientResponse.SUCCESS, response.status);
        // check the JSON itself makes sense
        JSONObject jsonObj = new JSONObject(responseJSON);
        JSONArray results = jsonObj.getJSONArray("results");
        assertEquals(4, response.results.length);
        JSONObject table = results.getJSONObject(0);
        JSONArray data = table.getJSONArray("data");
        assertEquals(1, data.length());
        JSONArray row = data.getJSONArray(0);
        assertEquals(1, row.length());
        long value = row.getLong(0);
        assertEquals(1, value);
        // try to pass a string as a date
        java.sql.Timestamp ts = new java.sql.Timestamp(System.currentTimeMillis());
        ts.setNanos(123456000);
        pset = ParameterSet.fromArrayNoCopy(1, 5, new double[] { 1.5, 6.0, 4 }, new VoltTable(new VoltTable.ColumnInfo("foo", VoltType.BIGINT)), new BigDecimal(5), new BigDecimal[] {}, ts.toString());
        responseJSON = callProcOverJSON("CrazyBlahProc", pset, null, null, false);
        System.out.println(responseJSON);
        response = responseFromJSON(responseJSON);
        assertEquals(ClientResponse.SUCCESS, response.status);
        response.results[3].advanceRow();
        System.out.println(response.results[3].getTimestampAsTimestamp(0).getTime());
        assertEquals(123456, response.results[3].getTimestampAsTimestamp(0).getTime() % 1000000);
        // now try a null short value sent as a int  (param expects short)
        pset = ParameterSet.fromArrayNoCopy(1, VoltType.NULL_SMALLINT, new double[] { 1.5, 6.0, 4 }, new VoltTable(new VoltTable.ColumnInfo("foo", VoltType.BIGINT)), new BigDecimal(5), new BigDecimal[] {}, new TimestampType(System.currentTimeMillis()));
        responseJSON = callProcOverJSON("CrazyBlahProc", pset, null, null, false);
        System.out.println(responseJSON);
        response = responseFromJSON(responseJSON);
        assertFalse(response.status == ClientResponse.SUCCESS);
        // now try an out of range long value (param expects short)
        pset = ParameterSet.fromArrayNoCopy(1, Long.MAX_VALUE - 100, new double[] { 1.5, 6.0, 4 }, new VoltTable(new VoltTable.ColumnInfo("foo", VoltType.BIGINT)), new BigDecimal(5), new BigDecimal[] {}, new TimestampType(System.currentTimeMillis()));
        responseJSON = callProcOverJSON("CrazyBlahProc", pset, null, null, false);
        System.out.println(responseJSON);
        response = responseFromJSON(responseJSON);
        assertFalse(response.status == ClientResponse.SUCCESS);
        // now try bigdecimal with small value
        pset = ParameterSet.fromArrayNoCopy(1, 4, new double[] { 1.5, 6.0, 4 }, new VoltTable(new VoltTable.ColumnInfo("foo", VoltType.BIGINT)), 5, new BigDecimal[] {}, new TimestampType(System.currentTimeMillis()));
        responseJSON = callProcOverJSON("CrazyBlahProc", pset, null, null, false);
        System.out.println(responseJSON);
        response = responseFromJSON(responseJSON);
        System.out.println(response.statusString);
        assertEquals(ClientResponse.SUCCESS, response.status);
        // now try null
        pset = ParameterSet.fromArrayNoCopy(1, 4, new double[] { 1.5, 6.0, 4 }, new VoltTable(new VoltTable.ColumnInfo("foo", VoltType.BIGINT)), 5, new BigDecimal[] {}, null);
        responseJSON = callProcOverJSON("CrazyBlahProc", pset, null, null, false);
        System.out.println(responseJSON);
        response = responseFromJSON(responseJSON);
        System.out.println(response.statusString);
        assertEquals(ClientResponse.SUCCESS, response.status);
        // now try jsonp
        responseJSON = callProcOverJSONRaw("Procedure=@Statistics&Parameters=[TABLE]&jsonp=fooBar", 200);
        System.out.println(responseJSON);
        assertTrue(responseJSON.startsWith("fooBar("));
        // now try adhoc
        pset = ParameterSet.fromArrayNoCopy("select * from blah");
        responseJSON = callProcOverJSON("@AdHoc", pset, null, null, false);
        System.out.println(responseJSON);
        response = responseFromJSON(responseJSON);
        System.out.println(response.statusString);
        assertEquals(ClientResponse.SUCCESS, response.status);
        // now try adhoc insert with a huge bigint
        pset = ParameterSet.fromArrayNoCopy("insert into blah values (974599638818488300, NULL, NULL, NULL, NULL);");
        responseJSON = callProcOverJSON("@AdHoc", pset, null, null, false);
        System.out.println(responseJSON);
        response = responseFromJSON(responseJSON);
        System.out.println(response.statusString);
        assertEquals(ClientResponse.SUCCESS, response.status);
        pset = ParameterSet.fromArrayNoCopy("select * from blah where ival = 974599638818488300;");
        responseJSON = callProcOverJSON("@AdHoc", pset, null, null, false);
        System.out.println(responseJSON);
        response = responseFromJSON(responseJSON);
        System.out.println(response.statusString);
        assertEquals(ClientResponse.SUCCESS, response.status);
        assertEquals(1, response.results.length);
        assertEquals(1, response.results[0].getRowCount());
        // Call @AdHoc with zero parameters
        pset = ParameterSet.emptyParameterSet();
        responseJSON = callProcOverJSON("@AdHoc", pset, null, null, false);
        assertTrue(responseJSON.contains("Adhoc system procedure requires at least the query parameter."));
        // Call @AdHoc with many parameters (more than 2)
        pset = ParameterSet.fromArrayNoCopy("select * from blah", "foo", "bar");
        responseJSON = callProcOverJSON("@AdHoc", pset, null, null, false);
        System.err.println(responseJSON);
        assertTrue(responseJSON.contains("Too many actual arguments were passed for the parameters in the sql " + "statement(s): (2 vs. 0)"));
    } finally {
        if (server != null) {
            server.shutdown();
            server.join();
        }
        server = null;
    }
}