Examples with BulkLoadContextCursor org.apache.ignite.cache.query.BulkLoadContextCursor used on opensource projects

Example 1 with BulkLoadContextCursor

use of org.apache.ignite.cache.query.BulkLoadContextCursor in project ignite by apache.

the class DmlStatementsProcessor method processBulkLoadCommand.

/**
 * Process bulk load COPY command.
 *
 * @param cmd The command.
 * @return The context (which is the result of the first request/response).
 * @throws IgniteCheckedException If something failed.
 */
public FieldsQueryCursor<List<?>> processBulkLoadCommand(SqlBulkLoadCommand cmd) throws IgniteCheckedException {
    if (cmd.packetSize() == null)
        cmd.packetSize(BulkLoadAckClientParameters.DFLT_PACKET_SIZE);
    GridH2Table tbl = idx.dataTable(cmd.schemaName(), cmd.tableName());
    if (tbl == null) {
        idx.kernalContext().cache().createMissingQueryCaches();
        tbl = idx.dataTable(cmd.schemaName(), cmd.tableName());
    }
    if (tbl == null) {
        throw new IgniteSQLException("Table does not exist: " + cmd.tableName(), IgniteQueryErrorCode.TABLE_NOT_FOUND);
    }
    UpdatePlan plan = UpdatePlanBuilder.planForBulkLoad(cmd, tbl);
    IgniteClosureX<List<?>, IgniteBiTuple<?, ?>> dataConverter = new BulkLoadDataConverter(plan);
    GridCacheContext cache = tbl.cache();
    IgniteDataStreamer<Object, Object> streamer = cache.grid().dataStreamer(cache.name());
    BulkLoadCacheWriter outputWriter = new BulkLoadStreamerWriter(streamer);
    BulkLoadParser inputParser = BulkLoadParser.createParser(cmd.inputFormat());
    BulkLoadProcessor processor = new BulkLoadProcessor(inputParser, dataConverter, outputWriter);
    BulkLoadAckClientParameters params = new BulkLoadAckClientParameters(cmd.localFileName(), cmd.packetSize());
    return new BulkLoadContextCursor(processor, params);
}

Example 2 with BulkLoadContextCursor

use of org.apache.ignite.cache.query.BulkLoadContextCursor in project ignite by apache.

the class JdbcQueryTask method call.

/**
 * {@inheritDoc}
 */
@Override
public JdbcQueryTaskResult call() throws Exception {
    Cursor cursor = CURSORS.get(uuid);
    List<String> tbls = null;
    List<String> cols = null;
    List<String> types = null;
    boolean first;
    if (first = (cursor == null)) {
        IgniteCache<?, ?> cache = ignite.cache(cacheName);
        // Don't create caches on server nodes in order to avoid of data rebalancing.
        boolean start = ignite.configuration().isClientMode();
        if (cache == null && cacheName == null)
            cache = ((IgniteKernal) ignite).context().cache().getOrStartPublicCache(start, !loc && locQry);
        if (cache == null) {
            if (cacheName == null)
                throw new SQLException("Failed to execute query. No suitable caches found.");
            else
                throw new SQLException("Cache not found [cacheName=" + cacheName + ']');
        }
        SqlFieldsQuery qry = (isQry != null ? new SqlFieldsQueryEx(sql, isQry) : new SqlFieldsQuery(sql)).setArgs(args);
        qry.setPageSize(fetchSize);
        qry.setLocal(locQry);
        qry.setCollocated(collocatedQry);
        qry.setDistributedJoins(distributedJoins);
        qry.setEnforceJoinOrder(enforceJoinOrder());
        qry.setLazy(lazy());
        qry.setSchema(schemaName);
        FieldsQueryCursor<List<?>> fldQryCursor = cache.withKeepBinary().query(qry);
        if (fldQryCursor instanceof BulkLoadContextCursor) {
            fldQryCursor.close();
            throw new SQLException("COPY command is currently supported only in thin JDBC driver.");
        }
        QueryCursorImpl<List<?>> qryCursor = (QueryCursorImpl<List<?>>) fldQryCursor;
        if (isQry == null)
            isQry = qryCursor.isQuery();
        CURSORS.put(uuid, cursor = new Cursor(qryCursor, qryCursor.iterator()));
    }
    if (first || updateMetadata()) {
        Collection<GridQueryFieldMetadata> meta = cursor.queryCursor().fieldsMeta();
        tbls = new ArrayList<>(meta.size());
        cols = new ArrayList<>(meta.size());
        types = new ArrayList<>(meta.size());
        for (GridQueryFieldMetadata desc : meta) {
            tbls.add(desc.typeName());
            cols.add(desc.fieldName().toUpperCase());
            types.add(desc.fieldTypeName());
        }
    }
    List<List<?>> rows = new ArrayList<>();
    for (List<?> row : cursor) {
        List<Object> row0 = new ArrayList<>(row.size());
        for (Object val : row) row0.add(val == null || JdbcUtils.isSqlType(val.getClass()) ? val : val.toString());
        rows.add(row0);
        if (// If fetchSize is 0 then unlimited
        rows.size() == fetchSize)
            break;
    }
    boolean finished = !cursor.hasNext();
    if (finished)
        remove(uuid, cursor);
    else if (first) {
        if (!loc)
            scheduleRemoval(uuid);
    } else if (!loc && !CURSORS.replace(uuid, cursor, new Cursor(cursor.cursor, cursor.iter)))
        assert !CURSORS.containsKey(uuid) : "Concurrent cursor modification.";
    assert isQry != null : "Query flag must be set prior to returning result";
    return new JdbcQueryTaskResult(uuid, finished, isQry, rows, cols, tbls, types);
}

Example 3 with BulkLoadContextCursor

use of org.apache.ignite.cache.query.BulkLoadContextCursor in project ignite by apache.

the class JdbcRequestHandler method executeBatchedQuery.

/**
 * Executes query and updates result counters.
 *
 * @param qry Query.
 * @param updCntsAcc Per query rows updates counter.
 * @param firstErr First error data - code and message.
 */
@SuppressWarnings("ForLoopReplaceableByForEach")
private void executeBatchedQuery(SqlFieldsQueryEx qry, List<Integer> updCntsAcc, IgniteBiTuple<Integer, String> firstErr) {
    try {
        if (cliCtx.isStream()) {
            List<Long> cnt = ctx.query().streamBatchedUpdateQuery(qry.getSchema(), cliCtx, qry.getSql(), qry.batchedArguments());
            for (int i = 0; i < cnt.size(); i++) updCntsAcc.add(cnt.get(i).intValue());
            return;
        }
        List<FieldsQueryCursor<List<?>>> qryRes = ctx.query().querySqlFields(null, qry, cliCtx, true, true);
        for (FieldsQueryCursor<List<?>> cur : qryRes) {
            if (cur instanceof BulkLoadContextCursor)
                throw new IgniteSQLException("COPY command cannot be executed in batch mode.");
            assert !((QueryCursorImpl) cur).isQuery();
            Iterator<List<?>> it = cur.iterator();
            if (it.hasNext()) {
                int val = ((Long) it.next().get(0)).intValue();
                updCntsAcc.add(val);
            }
        }
    } catch (Exception e) {
        int code;
        String msg;
        if (e instanceof IgniteSQLException) {
            BatchUpdateException batchCause = X.cause(e, BatchUpdateException.class);
            if (batchCause != null) {
                int[] updCntsOnErr = batchCause.getUpdateCounts();
                for (int i = 0; i < updCntsOnErr.length; i++) updCntsAcc.add(updCntsOnErr[i]);
                msg = batchCause.getMessage();
                code = batchCause.getErrorCode();
            } else {
                for (int i = 0; i < qry.batchedArguments().size(); i++) updCntsAcc.add(Statement.EXECUTE_FAILED);
                msg = e.getMessage();
                code = ((IgniteSQLException) e).statusCode();
            }
        } else {
            for (int i = 0; i < qry.batchedArguments().size(); i++) updCntsAcc.add(Statement.EXECUTE_FAILED);
            msg = e.getMessage();
            code = IgniteQueryErrorCode.UNKNOWN;
        }
        if (firstErr.isEmpty())
            firstErr.set(code, msg);
        else
            U.error(log, "Failed to execute batch query [qry=" + qry + ']', e);
    }
}

Example 4 with BulkLoadContextCursor

use of org.apache.ignite.cache.query.BulkLoadContextCursor in project ignite by apache.

the class JdbcRequestHandler method executeQuery.

/**
 * {@link JdbcQueryExecuteRequest} command handler.
 *
 * @param req Execute query request.
 * @return Response.
 */
@SuppressWarnings("unchecked")
private JdbcResponse executeQuery(JdbcQueryExecuteRequest req) {
    int cursorCnt = qryCursors.size();
    if (maxCursors > 0 && cursorCnt >= maxCursors)
        return new JdbcResponse(IgniteQueryErrorCode.UNKNOWN, "Too many open cursors (either close other " + "open cursors or increase the limit through " + "ClientConnectorConfiguration.maxOpenCursorsPerConnection) [maximum=" + maxCursors + ", current=" + cursorCnt + ']');
    long qryId = QRY_ID_GEN.getAndIncrement();
    assert !cliCtx.isStream();
    try {
        String sql = req.sqlQuery();
        SqlFieldsQuery qry;
        switch(req.expectedStatementType()) {
            case ANY_STATEMENT_TYPE:
                qry = new SqlFieldsQuery(sql);
                break;
            case SELECT_STATEMENT_TYPE:
                qry = new SqlFieldsQueryEx(sql, true);
                break;
            default:
                assert req.expectedStatementType() == JdbcStatementType.UPDATE_STMT_TYPE;
                qry = new SqlFieldsQueryEx(sql, false);
                if (cliCtx.isSkipReducerOnUpdate())
                    ((SqlFieldsQueryEx) qry).setSkipReducerOnUpdate(true);
        }
        qry.setArgs(req.arguments());
        qry.setDistributedJoins(cliCtx.isDistributedJoins());
        qry.setEnforceJoinOrder(cliCtx.isEnforceJoinOrder());
        qry.setCollocated(cliCtx.isCollocated());
        qry.setReplicatedOnly(cliCtx.isReplicatedOnly());
        qry.setLazy(cliCtx.isLazy());
        if (req.pageSize() <= 0)
            return new JdbcResponse(IgniteQueryErrorCode.UNKNOWN, "Invalid fetch size: " + req.pageSize());
        qry.setPageSize(req.pageSize());
        String schemaName = req.schemaName();
        if (F.isEmpty(schemaName))
            schemaName = QueryUtils.DFLT_SCHEMA;
        qry.setSchema(schemaName);
        List<FieldsQueryCursor<List<?>>> results = ctx.query().querySqlFields(null, qry, cliCtx, true, protocolVer.compareTo(VER_2_3_0) < 0);
        FieldsQueryCursor<List<?>> fieldsCur = results.get(0);
        if (fieldsCur instanceof BulkLoadContextCursor) {
            BulkLoadContextCursor blCur = (BulkLoadContextCursor) fieldsCur;
            BulkLoadProcessor blProcessor = blCur.bulkLoadProcessor();
            BulkLoadAckClientParameters clientParams = blCur.clientParams();
            bulkLoadRequests.put(qryId, new JdbcBulkLoadProcessor(blProcessor));
            return new JdbcResponse(new JdbcBulkLoadAckResult(qryId, clientParams));
        }
        if (results.size() == 1) {
            JdbcQueryCursor cur = new JdbcQueryCursor(qryId, req.pageSize(), req.maxRows(), (QueryCursorImpl) fieldsCur);
            JdbcQueryExecuteResult res;
            if (cur.isQuery())
                res = new JdbcQueryExecuteResult(qryId, cur.fetchRows(), !cur.hasNext());
            else {
                List<List<Object>> items = cur.fetchRows();
                assert items != null && items.size() == 1 && items.get(0).size() == 1 && items.get(0).get(0) instanceof Long : "Invalid result set for not-SELECT query. [qry=" + sql + ", res=" + S.toString(List.class, items) + ']';
                res = new JdbcQueryExecuteResult(qryId, (Long) items.get(0).get(0));
            }
            if (res.last() && (!res.isQuery() || autoCloseCursors))
                cur.close();
            else
                qryCursors.put(qryId, cur);
            return new JdbcResponse(res);
        } else {
            List<JdbcResultInfo> jdbcResults = new ArrayList<>(results.size());
            List<List<Object>> items = null;
            boolean last = true;
            for (FieldsQueryCursor<List<?>> c : results) {
                QueryCursorImpl qryCur = (QueryCursorImpl) c;
                JdbcResultInfo jdbcRes;
                if (qryCur.isQuery()) {
                    jdbcRes = new JdbcResultInfo(true, -1, qryId);
                    JdbcQueryCursor cur = new JdbcQueryCursor(qryId, req.pageSize(), req.maxRows(), qryCur);
                    qryCursors.put(qryId, cur);
                    qryId = QRY_ID_GEN.getAndIncrement();
                    if (items == null) {
                        items = cur.fetchRows();
                        last = cur.hasNext();
                    }
                } else
                    jdbcRes = new JdbcResultInfo(false, (Long) ((List<?>) qryCur.getAll().get(0)).get(0), -1);
                jdbcResults.add(jdbcRes);
            }
            return new JdbcResponse(new JdbcQueryExecuteMultipleStatementsResult(jdbcResults, items, last));
        }
    } catch (Exception e) {
        qryCursors.remove(qryId);
        U.error(log, "Failed to execute SQL query [reqId=" + req.requestId() + ", req=" + req + ']', e);
        return exceptionToResult(e);
    }
}