Examples with IntHolder herddb.utils.IntHolder used on opensource projects

Example 1 with IntHolder

use of herddb.utils.IntHolder in project herddb by diennea.

the class HerdDBCLI method executeSqlFile.

private static void executeSqlFile(int autotransactionbatchsize, final Connection connection, String file, final boolean verbose, final boolean ignoreerrors, final boolean frommysqldump, final boolean rewritestatements, final Statement statement, TableSpaceMapper tableSpaceMapper, boolean pretty) throws Exception, SQLException {
    if (autotransactionbatchsize > 0) {
        connection.setAutoCommit(false);
    }
    long _start = System.currentTimeMillis();
    final IntHolder doneCount = new IntHolder();
    final IntHolder totalDoneCount = new IntHolder();
    File f = new File(file);
    long fileSize = f.length();
    try (FileInputStream rawStream = new FileInputStream(file);
        BufferedInputStream buffer = new BufferedInputStream(rawStream);
        CounterInputStream counter = new CounterInputStream(buffer);
        InputStream fIn = wrapStream(f.getName(), counter);
        CounterInputStream counterUnzipped = new CounterInputStream(fIn);
        InputStreamReader ii = new InputStreamReader(counterUnzipped, "utf-8")) {
        int _autotransactionbatchsize = autotransactionbatchsize;
        SQLFileParser.parseSQLFile(ii, (st) -> {
            if (!st.comment) {
                ExecuteStatementResult res = executeStatement(verbose, ignoreerrors, frommysqldump, rewritestatements, st.content, statement, tableSpaceMapper, true, pretty);
                int count = res.updateCount;
                doneCount.value += count;
                totalDoneCount.value += count;
                if (_autotransactionbatchsize > 0 && doneCount.value > _autotransactionbatchsize) {
                    long _now = System.currentTimeMillis();
                    long countZipped = counter.count;
                    int percent = (int) (counter.count * 100.0 / fileSize);
                    long delta = (_now - _start);
                    long countUnzipped = counterUnzipped.count;
                    double speed = ((countUnzipped * 60000.0) / (1.0 * delta));
                    double speedZipped = ((countZipped * 60000.0) / (1.0 * delta));
                    if (countUnzipped != counter.count) {
                        System.out.println(new java.sql.Timestamp(System.currentTimeMillis()) + " COMMIT after " + totalDoneCount.value + " records, read " + formatBytes(counter.count) + " (" + formatBytes(countUnzipped) + " unzipped) over " + formatBytes(fileSize) + ". " + percent + "%, " + formatBytes(speedZipped) + "/min (UNZIPPED " + formatBytes(speed) + "/min)");
                    } else {
                        System.out.println(new java.sql.Timestamp(System.currentTimeMillis()) + " COMMIT after " + totalDoneCount.value + " records, read " + formatBytes(counter.count) + " over " + formatBytes(fileSize) + ". " + percent + "%, " + formatBytes(speed) + " /min");
                    }
                    connection.commit();
                    doneCount.value = 0;
                }
            }
        });
    }
    if (!connection.getAutoCommit()) {
        System.out.println("final COMMIT after " + totalDoneCount.value + " records");
        connection.commit();
    }
}

Example 2 with IntHolder

use of herddb.utils.IntHolder in project herddb by diennea.

the class SQLPlanner method buildInsertStatement.

private ExecutionPlan buildInsertStatement(String defaultTableSpace, Insert s, boolean returnValues) throws StatementExecutionException {
    String tableSpace = s.getTable().getSchemaName();
    String tableName = s.getTable().getName();
    if (tableSpace == null) {
        tableSpace = defaultTableSpace;
    }
    TableSpaceManager tableSpaceManager = manager.getTableSpaceManager(tableSpace);
    if (tableSpaceManager == null) {
        throw new StatementExecutionException("no such tablespace " + tableSpace + " here at " + manager.getNodeId());
    }
    AbstractTableManager tableManager = tableSpaceManager.getTableManager(tableName);
    if (tableManager == null) {
        throw new StatementExecutionException("no such table " + tableName + " in tablespace " + tableSpace);
    }
    Table table = tableManager.getTable();
    ItemsList itemlist = s.getItemsList();
    if (itemlist instanceof ExpressionList) {
        int index = 0;
        List<Expression> keyValueExpression = new ArrayList<>();
        List<String> keyExpressionToColumn = new ArrayList<>();
        List<CompiledSQLExpression> valuesExpressions = new ArrayList<>();
        List<net.sf.jsqlparser.schema.Column> valuesColumns = new ArrayList<>();
        ExpressionList list = (ExpressionList) itemlist;
        if (s.getColumns() != null) {
            for (net.sf.jsqlparser.schema.Column c : s.getColumns()) {
                Column column = table.getColumn(c.getColumnName());
                if (column == null) {
                    throw new StatementExecutionException("no such column " + c.getColumnName() + " in table " + tableName + " in tablespace " + tableSpace);
                }
                Expression expression;
                try {
                    expression = list.getExpressions().get(index++);
                } catch (IndexOutOfBoundsException badQuery) {
                    throw new StatementExecutionException("bad number of VALUES in INSERT clause");
                }
                if (table.isPrimaryKeyColumn(column.name)) {
                    keyExpressionToColumn.add(column.name);
                    keyValueExpression.add(expression);
                }
                valuesColumns.add(c);
                valuesExpressions.add(SQLExpressionCompiler.compileExpression(null, expression));
            }
        } else {
            for (Column column : table.columns) {
                Expression expression = list.getExpressions().get(index++);
                if (table.isPrimaryKeyColumn(column.name)) {
                    keyExpressionToColumn.add(column.name);
                    keyValueExpression.add(expression);
                }
                valuesColumns.add(new net.sf.jsqlparser.schema.Column(column.name));
                valuesExpressions.add(SQLExpressionCompiler.compileExpression(null, expression));
            }
        }
        RecordFunction keyfunction;
        if (keyValueExpression.isEmpty() && table.auto_increment) {
            keyfunction = new AutoIncrementPrimaryKeyRecordFunction();
        } else {
            if (keyValueExpression.size() != table.primaryKey.length) {
                throw new StatementExecutionException("you must set a value for the primary key (expressions=" + keyValueExpression.size() + ")");
            }
            keyfunction = new SQLRecordKeyFunction(table, keyExpressionToColumn, keyValueExpression);
        }
        RecordFunction valuesfunction = new SQLRecordFunction(table, valuesColumns, valuesExpressions);
        try {
            return ExecutionPlan.simple(new InsertStatement(tableSpace, tableName, keyfunction, valuesfunction).setReturnValues(returnValues));
        } catch (IllegalArgumentException err) {
            throw new StatementExecutionException(err);
        }
    } else if (itemlist instanceof MultiExpressionList) {
        if (returnValues) {
            throw new StatementExecutionException("cannot 'return values' on multi-values insert");
        }
        MultiExpressionList multilist = (MultiExpressionList) itemlist;
        List<InsertStatement> inserts = new ArrayList<>();
        for (ExpressionList list : multilist.getExprList()) {
            List<Expression> keyValueExpression = new ArrayList<>();
            List<String> keyExpressionToColumn = new ArrayList<>();
            List<CompiledSQLExpression> valuesExpressions = new ArrayList<>();
            List<net.sf.jsqlparser.schema.Column> valuesColumns = new ArrayList<>();
            int index = 0;
            if (s.getColumns() != null) {
                for (net.sf.jsqlparser.schema.Column c : s.getColumns()) {
                    Column column = table.getColumn(c.getColumnName());
                    if (column == null) {
                        throw new StatementExecutionException("no such column " + c.getColumnName() + " in table " + tableName + " in tablespace " + tableSpace);
                    }
                    Expression expression;
                    try {
                        expression = list.getExpressions().get(index++);
                    } catch (IndexOutOfBoundsException badQuery) {
                        throw new StatementExecutionException("bad number of VALUES in INSERT clause");
                    }
                    if (table.isPrimaryKeyColumn(column.name)) {
                        keyExpressionToColumn.add(column.name);
                        keyValueExpression.add(expression);
                    }
                    valuesColumns.add(c);
                    valuesExpressions.add(SQLExpressionCompiler.compileExpression(null, expression));
                }
            } else {
                for (Column column : table.columns) {
                    Expression expression = list.getExpressions().get(index++);
                    if (table.isPrimaryKeyColumn(column.name)) {
                        keyExpressionToColumn.add(column.name);
                        keyValueExpression.add(expression);
                    }
                    valuesColumns.add(new net.sf.jsqlparser.schema.Column(column.name));
                    valuesExpressions.add(SQLExpressionCompiler.compileExpression(null, expression));
                }
            }
            RecordFunction keyfunction;
            if (keyValueExpression.isEmpty() && table.auto_increment) {
                keyfunction = new AutoIncrementPrimaryKeyRecordFunction();
            } else {
                if (keyValueExpression.size() != table.primaryKey.length) {
                    throw new StatementExecutionException("you must set a value for the primary key (expressions=" + keyValueExpression.size() + ")");
                }
                keyfunction = new SQLRecordKeyFunction(table, keyExpressionToColumn, keyValueExpression);
            }
            RecordFunction valuesfunction = new SQLRecordFunction(table, valuesColumns, valuesExpressions);
            InsertStatement insert = new InsertStatement(tableSpace, tableName, keyfunction, valuesfunction);
            inserts.add(insert);
        }
        try {
            return ExecutionPlan.multiInsert(inserts);
        } catch (IllegalArgumentException err) {
            throw new StatementExecutionException(err);
        }
    } else {
        List<Expression> keyValueExpression = new ArrayList<>();
        List<String> keyExpressionToColumn = new ArrayList<>();
        List<CompiledSQLExpression> valuesExpressions = new ArrayList<>();
        List<net.sf.jsqlparser.schema.Column> valuesColumns = new ArrayList<>();
        Select select = s.getSelect();
        ExecutionPlan datasource = buildSelectStatement(defaultTableSpace, select, true, -1);
        if (s.getColumns() == null) {
            throw new StatementExecutionException("for INSERT ... SELECT you have to declare the columns to be filled in (use INSERT INTO TABLE(c,c,c,) SELECT .....)");
        }
        IntHolder holder = new IntHolder(1);
        for (net.sf.jsqlparser.schema.Column c : s.getColumns()) {
            Column column = table.getColumn(c.getColumnName());
            if (column == null) {
                throw new StatementExecutionException("no such column " + c.getColumnName() + " in table " + tableName + " in tablespace " + tableSpace);
            }
            JdbcParameter readFromResultSetAsJdbcParameter = new JdbcParameter();
            readFromResultSetAsJdbcParameter.setIndex(holder.value++);
            if (table.isPrimaryKeyColumn(column.name)) {
                keyExpressionToColumn.add(column.name);
                keyValueExpression.add(readFromResultSetAsJdbcParameter);
            }
            valuesColumns.add(c);
            valuesExpressions.add(SQLExpressionCompiler.compileExpression(null, readFromResultSetAsJdbcParameter));
        }
        RecordFunction keyfunction;
        if (keyValueExpression.isEmpty() && table.auto_increment) {
            keyfunction = new AutoIncrementPrimaryKeyRecordFunction();
        } else {
            if (keyValueExpression.size() != table.primaryKey.length) {
                throw new StatementExecutionException("you must set a value for the primary key (expressions=" + keyValueExpression.size() + ")");
            }
            keyfunction = new SQLRecordKeyFunction(table, keyExpressionToColumn, keyValueExpression);
        }
        RecordFunction valuesfunction = new SQLRecordFunction(table, valuesColumns, valuesExpressions);
        try {
            return ExecutionPlan.dataManipulationFromSelect(new InsertStatement(tableSpace, tableName, keyfunction, valuesfunction).setReturnValues(returnValues), datasource);
        } catch (IllegalArgumentException err) {
            throw new StatementExecutionException(err);
        }
    }
}

Example 3 with IntHolder

use of herddb.utils.IntHolder in project herddb by diennea.

the class MessageUtils method writeEncodedSimpleValue.

@SuppressWarnings("rawtypes")
private static void writeEncodedSimpleValue(ByteBuf output, Object o) {
    if (o == null) {
        output.writeByte(OPCODE_NULL_VALUE);
    } else if (o instanceof String) {
        output.writeByte(OPCODE_STRING_VALUE);
        writeUTF8String(output, (String) o);
    } else if (o instanceof RawString) {
        output.writeByte(OPCODE_STRING_VALUE);
        writeUTF8String(output, (RawString) o);
    } else if (o instanceof java.sql.Timestamp) {
        output.writeByte(OPCODE_TIMESTAMP_VALUE);
        ByteBufUtils.writeVLong(output, ((java.sql.Timestamp) o).getTime());
    } else if (o instanceof java.lang.Byte) {
        output.writeByte(OPCODE_BYTE_VALUE);
        output.writeByte(((Byte) o));
    } else if (o instanceof KeyValue) {
        KeyValue kv = (KeyValue) o;
        output.writeByte(OPCODE_KEYVALUE_VALUE);
        ByteBufUtils.writeArray(output, kv.key);
        ByteBufUtils.writeArray(output, kv.value);
    } else if (o instanceof Integer) {
        int i = (int) o;
        if (i < 0) {
            if (i < WRITE_MIN_Z_INT_LIMIT) {
                output.writeByte(OPCODE_INT_VALUE);
                output.writeInt(i);
            } else {
                output.writeByte(OPCODE_Z_INT_VALUE);
                ByteBufUtils.writeZInt(output, i);
            }
        } else {
            if (i > WRITE_MAX_V_INT_LIMIT) {
                output.writeByte(OPCODE_INT_VALUE);
                output.writeInt(i);
            } else {
                output.writeByte(OPCODE_V_INT_VALUE);
                ByteBufUtils.writeVInt(output, i);
            }
        }
    } else if (o instanceof Long) {
        long l = (long) o;
        if (l < 0) {
            if (l < WRITE_MIN_Z_LONG_LIMIT) {
                output.writeByte(OPCODE_LONG_VALUE);
                output.writeLong(l);
            } else {
                output.writeByte(OPCODE_Z_LONG_VALUE);
                ByteBufUtils.writeZLong(output, l);
            }
        } else {
            if (l > WRITE_MAX_V_LONG_LIMIT) {
                output.writeByte(OPCODE_LONG_VALUE);
                output.writeLong(l);
            } else {
                output.writeByte(OPCODE_V_LONG_VALUE);
                ByteBufUtils.writeVLong(output, l);
            }
        }
    } else if (o instanceof Boolean) {
        output.writeByte(OPCODE_BOOLEAN_VALUE);
        output.writeByte(((Boolean) o).booleanValue() ? 1 : 0);
    } else if (o instanceof Double) {
        output.writeByte(OPCODE_DOUBLE_VALUE);
        ByteBufUtils.writeDouble(output, ((Double) o).doubleValue());
    } else if (o instanceof Set) {
        Set set = (Set) o;
        output.writeByte(OPCODE_SET_VALUE);
        ByteBufUtils.writeVInt(output, set.size());
        for (Object o2 : set) {
            writeEncodedSimpleValue(output, o2);
        }
    } else if (o instanceof List) {
        List set = (List) o;
        output.writeByte(OPCODE_LIST_VALUE);
        ByteBufUtils.writeVInt(output, set.size());
        for (Object o2 : set) {
            writeEncodedSimpleValue(output, o2);
        }
    } else if (o instanceof Map) {
        Map set = (Map) o;
        output.writeByte(OPCODE_MAP_VALUE);
        ByteBufUtils.writeVInt(output, set.size());
        for (Map.Entry entry : (Iterable<Map.Entry>) set.entrySet()) {
            writeEncodedSimpleValue(output, entry.getKey());
            writeEncodedSimpleValue(output, entry.getValue());
        }
    } else if (o instanceof DataAccessor) {
        DataAccessor set = (DataAccessor) o;
        output.writeByte(OPCODE_MAP2_VALUE);
        // number of entries is not known
        set.forEach((key, value) -> {
            writeEncodedSimpleValue(output, key);
            writeEncodedSimpleValue(output, value);
        });
        output.writeByte(OPCODE_MAP2_VALUE_END);
    } else if (o instanceof TuplesList) {
        TuplesList set = (TuplesList) o;
        output.writeByte(OPCODE_TUPLELIST_VALUE);
        final int numColumns = set.columnNames.length;
        ByteBufUtils.writeVInt(output, numColumns);
        for (String columnName : set.columnNames) {
            writeUTF8String(output, columnName);
        }
        ByteBufUtils.writeVInt(output, set.tuples.size());
        for (DataAccessor da : set.tuples) {
            IntHolder currentColumn = new IntHolder();
            da.forEach((String key, Object value) -> {
                String expectedColumnName = set.columnNames[currentColumn.value];
                while (!key.equals(expectedColumnName)) {
                    // nulls are not returned for some special accessors, lie DataAccessorForFullRecord
                    writeEncodedSimpleValue(output, null);
                    currentColumn.value++;
                    expectedColumnName = set.columnNames[currentColumn.value];
                }
                writeEncodedSimpleValue(output, value);
                currentColumn.value++;
            });
            // fill with nulls
            while (currentColumn.value < numColumns) {
                writeEncodedSimpleValue(output, null);
                currentColumn.value++;
            }
            if (currentColumn.value > numColumns) {
                throw new RuntimeException("unexpected number of columns " + currentColumn.value + " > " + numColumns);
            }
        }
    } else if (o instanceof byte[]) {
        byte[] set = (byte[]) o;
        output.writeByte(OPCODE_BYTEARRAY_VALUE);
        ByteBufUtils.writeArray(output, set);
    } else {
        throw new RuntimeException("unsupported class " + o.getClass());
    }
}