Examples with Dataset org.apache.asterix.metadata.entities.Dataset used on opensource projects

Example 46 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class InvertedIndexAccessMethod method createSecondaryToPrimaryPlan.

@Override
public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException {
    IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
    Dataset dataset = indexSubTree.getDataset();
    ARecordType recordType = indexSubTree.getRecordType();
    ARecordType metaRecordType = indexSubTree.getMetaRecordType();
    // we made sure indexSubTree has datasource scan
    DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) indexSubTree.getDataSourceRef().getValue();
    InvertedIndexJobGenParams jobGenParams = new InvertedIndexJobGenParams(chosenIndex.getIndexName(), chosenIndex.getIndexType(), dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);
    // Add function-specific args such as search modifier, and possibly a similarity threshold.
    addFunctionSpecificArgs(optFuncExpr, jobGenParams);
    // Add the type of search key from the optFuncExpr.
    addSearchKeyType(optFuncExpr, indexSubTree, context, jobGenParams);
    // Operator that feeds the secondary-index search.
    AbstractLogicalOperator inputOp = null;
    // Here we generate vars and funcs for assigning the secondary-index keys to be fed into the secondary-index search.
    // List of variables for the assign.
    ArrayList<LogicalVariable> keyVarList = new ArrayList<LogicalVariable>();
    // probeSubTree is null if we are dealing with a selection query, and non-null for join queries.
    if (probeSubTree == null) {
        // List of expressions for the assign.
        ArrayList<Mutable<ILogicalExpression>> keyExprList = new ArrayList<Mutable<ILogicalExpression>>();
        // Add key vars and exprs to argument list.
        addKeyVarsAndExprs(optFuncExpr, keyVarList, keyExprList, context);
        // Assign operator that sets the secondary-index search-key fields.
        inputOp = new AssignOperator(keyVarList, keyExprList);
        // Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
        inputOp.getInputs().add(new MutableObject<>(OperatorManipulationUtil.deepCopy(dataSourceScan.getInputs().get(0).getValue())));
        inputOp.setExecutionMode(dataSourceScan.getExecutionMode());
    } else {
        // We are optimizing a join. Add the input variable to the secondaryIndexFuncArgs.
        LogicalVariable inputSearchVariable = getInputSearchVar(optFuncExpr, indexSubTree);
        keyVarList.add(inputSearchVariable);
        inputOp = (AbstractLogicalOperator) probeSubTree.getRoot();
    }
    jobGenParams.setKeyVarList(keyVarList);
    ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType, metaRecordType, chosenIndex, inputOp, jobGenParams, context, true, retainInput, retainNull);
    // Generate the rest of the upstream plan which feeds the search results into the primary index.
    AbstractUnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceScan, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
    return primaryIndexUnnestOp;
}

Example 47 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class IntroduceLSMComponentFilterRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    if (!checkIfRuleIsApplicable(opRef, context)) {
        return false;
    }
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    typeEnvironment = context.getOutputTypeEnvironment(op);
    ILogicalExpression condExpr = ((SelectOperator) op).getCondition().getValue();
    AccessMethodAnalysisContext analysisCtx = analyzeCondition(condExpr, context, typeEnvironment);
    if (analysisCtx.getMatchedFuncExprs().isEmpty()) {
        return false;
    }
    Dataset dataset = getDataset(op, context);
    List<String> filterFieldName = null;
    ARecordType recType = null;
    if (dataset != null && dataset.getDatasetType() == DatasetType.INTERNAL) {
        filterFieldName = DatasetUtil.getFilterField(dataset);
        IAType itemType = ((MetadataProvider) context.getMetadataProvider()).findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
        if (itemType.getTypeTag() == ATypeTag.OBJECT) {
            recType = (ARecordType) itemType;
        }
    }
    if (filterFieldName == null || recType == null) {
        return false;
    }
    List<Index> datasetIndexes = ((MetadataProvider) context.getMetadataProvider()).getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName());
    List<IOptimizableFuncExpr> optFuncExprs = new ArrayList<>();
    for (int i = 0; i < analysisCtx.getMatchedFuncExprs().size(); i++) {
        IOptimizableFuncExpr optFuncExpr = analysisCtx.getMatchedFuncExpr(i);
        boolean found = findMacthedExprFieldName(optFuncExpr, op, dataset, recType, datasetIndexes, context);
        if (found && optFuncExpr.getFieldName(0).equals(filterFieldName)) {
            optFuncExprs.add(optFuncExpr);
        }
    }
    if (optFuncExprs.isEmpty()) {
        return false;
    }
    changePlan(optFuncExprs, op, dataset, context);
    OperatorPropertiesUtil.typeOpRec(opRef, context);
    context.addToDontApplySet(this, op);
    return true;
}

Example 48 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class FilePartition method get.

@Override
protected void get(IServletRequest request, IServletResponse response) {
    response.setStatus(HttpResponseStatus.OK);
    try {
        HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
    } catch (IOException e) {
        LOGGER.log(Level.WARNING, "Failure setting content type", e);
        response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        response.writer().write(e.toString());
        return;
    }
    PrintWriter out = response.writer();
    try {
        ObjectMapper om = new ObjectMapper();
        ObjectNode jsonResponse = om.createObjectNode();
        String dataverseName = request.getParameter("dataverseName");
        String datasetName = request.getParameter("datasetName");
        if (dataverseName == null || datasetName == null) {
            jsonResponse.put("error", "Parameter dataverseName or datasetName is null,");
            out.write(jsonResponse.toString());
            return;
        }
        IHyracksClientConnection hcc = (IHyracksClientConnection) ctx.get(HYRACKS_CONNECTION_ATTR);
        // Metadata transaction begins.
        MetadataManager.INSTANCE.init();
        MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
        // Retrieves file splits of the dataset.
        MetadataProvider metadataProvider = new MetadataProvider(appCtx, null, new StorageComponentProvider());
        try {
            metadataProvider.setMetadataTxnContext(mdTxnCtx);
            Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
            if (dataset == null) {
                jsonResponse.put("error", "Dataset " + datasetName + " does not exist in " + "dataverse " + dataverseName);
                out.write(jsonResponse.toString());
                out.flush();
                return;
            }
            boolean temp = dataset.getDatasetDetails().isTemp();
            FileSplit[] fileSplits = metadataProvider.splitsForIndex(mdTxnCtx, dataset, datasetName);
            ARecordType recordType = (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
            List<List<String>> primaryKeys = dataset.getPrimaryKeys();
            StringBuilder pkStrBuf = new StringBuilder();
            for (List<String> keys : primaryKeys) {
                for (String key : keys) {
                    pkStrBuf.append(key).append(",");
                }
            }
            pkStrBuf.delete(pkStrBuf.length() - 1, pkStrBuf.length());
            // Constructs the returned json object.
            formResponseObject(jsonResponse, fileSplits, recordType, pkStrBuf.toString(), temp, hcc.getNodeControllerInfos());
            // Flush the cached contents of the dataset to file system.
            FlushDatasetUtil.flushDataset(hcc, metadataProvider, dataverseName, datasetName, datasetName);
            // Metadata transaction commits.
            MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
            // Writes file splits.
            out.write(jsonResponse.toString());
        } finally {
            metadataProvider.getLocks().unlock();
        }
    } catch (Exception e) {
        LOGGER.log(Level.WARNING, "Failure handling a request", e);
        response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        out.write(e.toString());
    } finally {
        out.flush();
    }
}

Example 49 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class RebalanceApiServlet method post.

@Override
protected void post(IServletRequest request, IServletResponse response) {
    PrintWriter out = response.writer();
    ObjectMapper om = new ObjectMapper();
    ObjectNode jsonResponse = om.createObjectNode();
    try {
        // Sets the content type.
        HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
        // Gets dataverse, dataset, and target nodes for rebalance.
        String dataverseName = request.getParameter("dataverseName");
        String datasetName = request.getParameter("datasetName");
        String nodes = request.getParameter("nodes");
        // Parses and check target nodes.
        if (nodes == null) {
            sendResponse(out, jsonResponse, response, HttpResponseStatus.BAD_REQUEST, "nodes are not given");
            return;
        }
        String nodesString = StringUtils.strip(nodes, "\"'").trim();
        String[] targetNodes = nodesString.split(",");
        if ("".equals(nodesString)) {
            sendResponse(out, jsonResponse, response, HttpResponseStatus.BAD_REQUEST, "target nodes should not be empty");
            return;
        }
        // If a user gives parameter datasetName, she should give dataverseName as well.
        if (dataverseName == null && datasetName != null) {
            sendResponse(out, jsonResponse, response, HttpResponseStatus.BAD_REQUEST, "to rebalance a particular dataset, the parameter dataverseName must be given");
            return;
        }
        // Does not allow rebalancing a metadata dataset.
        if (METADATA.equals(dataverseName)) {
            sendResponse(out, jsonResponse, response, HttpResponseStatus.BAD_REQUEST, "cannot rebalance a metadata dataset");
            return;
        }
        if (datasetName == null) {
            // Rebalances datasets in a given dataverse or all non-metadata datasets.
            List<Dataset> datasets = dataverseName == null ? getAllDatasetsForRebalance() : getAllDatasetsForRebalance(dataverseName);
            for (Dataset dataset : datasets) {
                // By the time rebalanceDataset(...) is called, the dataset could have been dropped.
                // If that's the case, rebalanceDataset(...) would be a no-op.
                rebalanceDataset(dataset.getDataverseName(), dataset.getDatasetName(), targetNodes);
            }
        } else {
            // Rebalances a given dataset from its current locations to the target nodes.
            rebalanceDataset(dataverseName, datasetName, targetNodes);
        }
        // Sends response.
        sendResponse(out, jsonResponse, response, HttpResponseStatus.OK, "successful");
    } catch (Exception e) {
        sendResponse(out, jsonResponse, response, HttpResponseStatus.INTERNAL_SERVER_ERROR, e.getMessage());
        LOGGER.log(Level.WARNING, e.getMessage(), e);
    }
}

Example 50 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class RebalanceApiServlet method getAllDatasetsForRebalance.

// Lists all datasets that should be rebalanced.
private List<Dataset> getAllDatasetsForRebalance() throws Exception {
    List<Dataset> datasets = new ArrayList<>();
    MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
    try {
        List<Dataverse> dataverses = MetadataManager.INSTANCE.getDataverses(mdTxnCtx);
        for (Dataverse dv : dataverses) {
            datasets.addAll(getDatasetsInDataverseForRebalance(dv.getDataverseName(), mdTxnCtx));
        }
        MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
    } catch (Exception e) {
        MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
        throw e;
    }
    return datasets;
}