Examples with AsterixException org.apache.asterix.common.exceptions.AsterixException used on opensource projects

Example 56 with AsterixException

use of org.apache.asterix.common.exceptions.AsterixException in project asterixdb by apache.

the class ParserFactoryProvider method initFactories.

protected static Map<String, Class> initFactories() throws AsterixException {
    Map<String, Class> factories = new HashMap<>();
    ClassLoader cl = ParserFactoryProvider.class.getClassLoader();
    try {
        Enumeration<URL> urls = cl.getResources(RESOURCE);
        for (URL url : Collections.list(urls)) {
            List<String> classNames;
            try (InputStream is = url.openStream()) {
                classNames = IOUtils.readLines(is, StandardCharsets.UTF_8);
            }
            for (String className : classNames) {
                if (className.startsWith("#")) {
                    continue;
                }
                final Class<?> clazz = Class.forName(className);
                List<String> formats = ((IDataParserFactory) clazz.newInstance()).getParserFormats();
                for (String format : formats) {
                    if (factories.containsKey(format)) {
                        throw new AsterixException("Duplicate format " + format);
                    }
                    factories.put(format, clazz);
                }
            }
        }
    } catch (IOException | ClassNotFoundException | InstantiationException | IllegalAccessException e) {
        throw new AsterixException(e);
    }
    return factories;
}

Example 57 with AsterixException

use of org.apache.asterix.common.exceptions.AsterixException in project asterixdb by apache.

the class FeedMetadataUtil method getPrimaryFeedFactoryAndOutput.

@SuppressWarnings("rawtypes")
public static Triple<IAdapterFactory, RecordDescriptor, AdapterType> getPrimaryFeedFactoryAndOutput(Feed feed, FeedPolicyAccessor policyAccessor, MetadataTransactionContext mdTxnCtx, ICcApplicationContext appCtx) throws AlgebricksException {
    // This method needs to be re-visited
    String adapterName = null;
    DatasourceAdapter adapterEntity = null;
    String adapterFactoryClassname = null;
    IAdapterFactory adapterFactory = null;
    ARecordType adapterOutputType = null;
    ARecordType metaType = null;
    Triple<IAdapterFactory, RecordDescriptor, IDataSourceAdapter.AdapterType> feedProps = null;
    IDataSourceAdapter.AdapterType adapterType = null;
    try {
        adapterName = feed.getAdapterName();
        Map<String, String> configuration = feed.getAdapterConfiguration();
        configuration.putAll(policyAccessor.getFeedPolicy());
        adapterOutputType = getOutputType(feed, configuration, ExternalDataConstants.KEY_TYPE_NAME);
        metaType = getOutputType(feed, configuration, ExternalDataConstants.KEY_META_TYPE_NAME);
        ExternalDataUtils.prepareFeed(configuration, feed.getDataverseName(), feed.getFeedName());
        // Get adapter from metadata dataset <Metadata dataverse>
        adapterEntity = MetadataManager.INSTANCE.getAdapter(mdTxnCtx, MetadataConstants.METADATA_DATAVERSE_NAME, adapterName);
        // Get adapter from metadata dataset <The feed dataverse>
        if (adapterEntity == null) {
            adapterEntity = MetadataManager.INSTANCE.getAdapter(mdTxnCtx, feed.getDataverseName(), adapterName);
        }
        if (adapterEntity != null) {
            adapterType = adapterEntity.getType();
            adapterFactoryClassname = adapterEntity.getClassname();
            switch(adapterType) {
                case INTERNAL:
                    adapterFactory = (IAdapterFactory) Class.forName(adapterFactoryClassname).newInstance();
                    break;
                case EXTERNAL:
                    String[] anameComponents = adapterName.split("#");
                    String libraryName = anameComponents[0];
                    ClassLoader cl = appCtx.getLibraryManager().getLibraryClassLoader(feed.getDataverseName(), libraryName);
                    adapterFactory = (IAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance();
                    break;
                default:
                    throw new AsterixException("Unknown Adapter type " + adapterType);
            }
            adapterFactory.setOutputType(adapterOutputType);
            adapterFactory.setMetaType(metaType);
            adapterFactory.configure(appCtx.getServiceContext(), configuration);
        } else {
            adapterFactory = AdapterFactoryProvider.getAdapterFactory(appCtx.getServiceContext(), adapterName, configuration, adapterOutputType, metaType);
            adapterType = IDataSourceAdapter.AdapterType.INTERNAL;
        }
        if (metaType == null) {
            metaType = getOutputType(feed, configuration, ExternalDataConstants.KEY_META_TYPE_NAME);
        }
        if (adapterOutputType == null) {
            if (!configuration.containsKey(ExternalDataConstants.KEY_TYPE_NAME)) {
                throw new AsterixException("Unspecified feed output data type");
            }
            adapterOutputType = getOutputType(feed, configuration, ExternalDataConstants.KEY_TYPE_NAME);
        }
        int numOfOutputs = 1;
        if (metaType != null) {
            numOfOutputs++;
        }
        if (ExternalDataUtils.isChangeFeed(configuration)) {
            // get number of PKs
            numOfOutputs += ExternalDataUtils.getNumberOfKeys(configuration);
        }
        ISerializerDeserializer[] serdes = new ISerializerDeserializer[numOfOutputs];
        int i = 0;
        serdes[i++] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(adapterOutputType);
        if (metaType != null) {
            serdes[i++] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(metaType);
        }
        if (ExternalDataUtils.isChangeFeed(configuration)) {
            getSerdesForPKs(serdes, configuration, metaType, adapterOutputType, i);
        }
        feedProps = new Triple<>(adapterFactory, new RecordDescriptor(serdes), adapterType);
    } catch (Exception e) {
        throw new AlgebricksException("unable to create adapter", e);
    }
    return feedProps;
}

Example 58 with AsterixException

use of org.apache.asterix.common.exceptions.AsterixException in project asterixdb by apache.

the class DatasetUtil method compactDatasetJobSpec.

public static JobSpecification compactDatasetJobSpec(Dataverse dataverse, String datasetName, MetadataProvider metadataProvider) throws AlgebricksException {
    String dataverseName = dataverse.getDataverseName();
    Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
    if (dataset == null) {
        throw new AsterixException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
    }
    JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
    Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset);
    IIndexDataflowHelperFactory indexHelperFactory = new IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(), splitsAndConstraint.first);
    LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec, indexHelperFactory);
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp, splitsAndConstraint.second);
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp, splitsAndConstraint.second);
    spec.addRoot(compactOp);
    return spec;
}

Example 59 with AsterixException

use of org.apache.asterix.common.exceptions.AsterixException in project asterixdb by apache.

the class InvertedIndexResourceFactoryProvider method getResourceFactory.

@Override
public IResourceFactory getResourceFactory(MetadataProvider mdProvider, Dataset dataset, Index index, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories) throws AlgebricksException {
    // Get basic info
    List<List<String>> primaryKeys = dataset.getPrimaryKeys();
    List<List<String>> secondaryKeys = index.getKeyFieldNames();
    List<String> filterFieldName = DatasetUtil.getFilterField(dataset);
    int numPrimaryKeys = primaryKeys.size();
    int numSecondaryKeys = secondaryKeys.size();
    // Validate
    if (dataset.getDatasetType() != DatasetType.INTERNAL) {
        throw new CompilationException(ErrorCode.COMPILATION_INDEX_TYPE_NOT_SUPPORTED_FOR_DATASET_TYPE, index.getIndexType().name(), dataset.getDatasetType());
    }
    if (numPrimaryKeys > 1) {
        throw new AsterixException("Cannot create inverted index on dataset with composite primary key.");
    }
    if (numSecondaryKeys > 1) {
        throw new AsterixException("Cannot create composite inverted index on multiple fields.");
    }
    boolean isPartitioned = index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
    int numTokenKeyPairFields = (!isPartitioned) ? 1 + numPrimaryKeys : 2 + numPrimaryKeys;
    int[] invertedIndexFields = null;
    int[] secondaryFilterFieldsForNonBulkLoadOps = null;
    int[] invertedIndexFieldsForNonBulkLoadOps = null;
    int[] secondaryFilterFields = null;
    if (filterFieldName != null) {
        invertedIndexFields = new int[numTokenKeyPairFields];
        for (int i = 0; i < invertedIndexFields.length; i++) {
            invertedIndexFields[i] = i;
        }
        secondaryFilterFieldsForNonBulkLoadOps = new int[filterFieldName.size()];
        secondaryFilterFieldsForNonBulkLoadOps[0] = numSecondaryKeys + numPrimaryKeys;
        invertedIndexFieldsForNonBulkLoadOps = new int[numSecondaryKeys + numPrimaryKeys];
        for (int i = 0; i < invertedIndexFieldsForNonBulkLoadOps.length; i++) {
            invertedIndexFieldsForNonBulkLoadOps[i] = i;
        }
        secondaryFilterFields = new int[filterFieldName.size()];
        secondaryFilterFields[0] = numTokenKeyPairFields - numPrimaryKeys + numPrimaryKeys;
    }
    IStorageComponentProvider storageComponentProvider = mdProvider.getStorageComponentProvider();
    IStorageManager storageManager = storageComponentProvider.getStorageManager();
    ILSMOperationTrackerFactory opTrackerFactory = dataset.getIndexOperationTrackerFactory(index);
    ILSMIOOperationCallbackFactory ioOpCallbackFactory = dataset.getIoOperationCallbackFactory(index);
    IMetadataPageManagerFactory metadataPageManagerFactory = storageComponentProvider.getMetadataPageManagerFactory();
    AsterixVirtualBufferCacheProvider vbcProvider = new AsterixVirtualBufferCacheProvider(dataset.getDatasetId());
    ILSMIOOperationSchedulerProvider ioSchedulerProvider = storageComponentProvider.getIoOperationSchedulerProvider();
    boolean durable = !dataset.isTemp();
    double bloomFilterFalsePositiveRate = mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate();
    ITypeTraits[] typeTraits = getInvListTypeTraits(mdProvider, dataset, recordType, metaType);
    IBinaryComparatorFactory[] cmpFactories = getInvListComparatorFactories(mdProvider, dataset, recordType, metaType);
    ITypeTraits[] tokenTypeTraits = getTokenTypeTraits(dataset, index, recordType, metaType);
    IBinaryComparatorFactory[] tokenCmpFactories = getTokenComparatorFactories(dataset, index, recordType, metaType);
    IBinaryTokenizerFactory tokenizerFactory = getTokenizerFactory(dataset, index, recordType, metaType);
    return new LSMInvertedIndexLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, secondaryFilterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, isPartitioned, invertedIndexFields, secondaryFilterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, bloomFilterFalsePositiveRate);
}

Example 60 with AsterixException

use of org.apache.asterix.common.exceptions.AsterixException in project asterixdb by apache.

the class FuzzyJoinRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    // current opperator is join
    if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
        return false;
    }
    // Find GET_ITEM function.
    AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) op;
    Mutable<ILogicalExpression> expRef = joinOp.getCondition();
    Mutable<ILogicalExpression> getItemExprRef = getSimilarityExpression(expRef);
    if (getItemExprRef == null) {
        return false;
    }
    // Check if the GET_ITEM function is on one of the supported similarity-check functions.
    AbstractFunctionCallExpression getItemFuncExpr = (AbstractFunctionCallExpression) getItemExprRef.getValue();
    Mutable<ILogicalExpression> argRef = getItemFuncExpr.getArguments().get(0);
    AbstractFunctionCallExpression simFuncExpr = (AbstractFunctionCallExpression) argRef.getValue();
    if (!simFuncs.contains(simFuncExpr.getFunctionIdentifier())) {
        return false;
    }
    // Skip this rule based on annotations.
    if (simFuncExpr.getAnnotations().containsKey(IndexedNLJoinExpressionAnnotation.INSTANCE)) {
        return false;
    }
    List<Mutable<ILogicalOperator>> inputOps = joinOp.getInputs();
    ILogicalOperator leftInputOp = inputOps.get(0).getValue();
    ILogicalOperator rightInputOp = inputOps.get(1).getValue();
    List<Mutable<ILogicalExpression>> inputExps = simFuncExpr.getArguments();
    ILogicalExpression inputExp0 = inputExps.get(0).getValue();
    ILogicalExpression inputExp1 = inputExps.get(1).getValue();
    // left and right expressions are variables
    if (inputExp0.getExpressionTag() != LogicalExpressionTag.VARIABLE || inputExp1.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
        return false;
    }
    LogicalVariable inputVar0 = ((VariableReferenceExpression) inputExp0).getVariableReference();
    LogicalVariable inputVar1 = ((VariableReferenceExpression) inputExp1).getVariableReference();
    LogicalVariable leftInputVar;
    LogicalVariable rightInputVar;
    liveVars.clear();
    VariableUtilities.getLiveVariables(leftInputOp, liveVars);
    if (liveVars.contains(inputVar0)) {
        leftInputVar = inputVar0;
        rightInputVar = inputVar1;
    } else {
        leftInputVar = inputVar1;
        rightInputVar = inputVar0;
    }
    List<LogicalVariable> leftInputPKs = context.findPrimaryKey(leftInputVar);
    List<LogicalVariable> rightInputPKs = context.findPrimaryKey(rightInputVar);
    // Bail if primary keys could not be inferred.
    if (leftInputPKs == null || rightInputPKs == null) {
        return false;
    }
    // primary key has only one variable
    if (leftInputPKs.size() != 1 || rightInputPKs.size() != 1) {
        return false;
    }
    IAType leftType = (IAType) context.getOutputTypeEnvironment(leftInputOp).getVarType(leftInputVar);
    IAType rightType = (IAType) context.getOutputTypeEnvironment(rightInputOp).getVarType(rightInputVar);
    // left-hand side and right-hand side of "~=" has the same type
    IAType left2 = TypeComputeUtils.getActualType(leftType);
    IAType right2 = TypeComputeUtils.getActualType(rightType);
    if (!left2.deepEqual(right2)) {
        return false;
    }
    //
    // -- - FIRE - --
    //
    MetadataProvider metadataProvider = ((MetadataProvider) context.getMetadataProvider());
    FunctionIdentifier funcId = FuzzyUtils.getTokenizer(leftType.getTypeTag());
    String tokenizer;
    if (funcId == null) {
        tokenizer = "";
    } else {
        tokenizer = funcId.getName();
    }
    float simThreshold = FuzzyUtils.getSimThreshold(metadataProvider);
    String simFunction = FuzzyUtils.getSimFunction(metadataProvider);
    // finalize AQL+ query
    String prepareJoin;
    switch(joinOp.getJoinKind()) {
        case INNER:
            {
                prepareJoin = "join" + AQLPLUS;
                break;
            }
        case LEFT_OUTER:
            {
                // other sort of bug.
                return false;
            // prepareJoin = "loj" + AQLPLUS;
            // break;
            }
        default:
            {
                throw new IllegalStateException();
            }
    }
    String aqlPlus = String.format(Locale.US, prepareJoin, tokenizer, tokenizer, simFunction, simThreshold, tokenizer, tokenizer, simFunction, simThreshold, simFunction, simThreshold, simThreshold);
    LogicalVariable leftPKVar = leftInputPKs.get(0);
    LogicalVariable rightPKVar = rightInputPKs.get(0);
    Counter counter = new Counter(context.getVarCounter());
    AQLPlusParser parser = new AQLPlusParser(new StringReader(aqlPlus));
    parser.initScope();
    parser.setVarCounter(counter);
    List<Clause> clauses;
    try {
        clauses = parser.Clauses();
    } catch (ParseException e) {
        throw new AlgebricksException(e);
    }
    // The translator will compile metadata internally. Run this compilation
    // under the same transaction id as the "outer" compilation.
    AqlPlusExpressionToPlanTranslator translator = new AqlPlusExpressionToPlanTranslator(metadataProvider, counter);
    context.setVarCounter(counter.get());
    LogicalOperatorDeepCopyWithNewVariablesVisitor deepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(context, context);
    translator.addOperatorToMetaScope(new Identifier("#LEFT"), leftInputOp);
    translator.addVariableToMetaScope(new Identifier("$$LEFT"), leftInputVar);
    translator.addVariableToMetaScope(new Identifier("$$LEFTPK"), leftPKVar);
    translator.addOperatorToMetaScope(new Identifier("#RIGHT"), rightInputOp);
    translator.addVariableToMetaScope(new Identifier("$$RIGHT"), rightInputVar);
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK"), rightPKVar);
    translator.addOperatorToMetaScope(new Identifier("#LEFT_1"), deepCopyVisitor.deepCopy(leftInputOp));
    translator.addVariableToMetaScope(new Identifier("$$LEFT_1"), deepCopyVisitor.varCopy(leftInputVar));
    translator.addVariableToMetaScope(new Identifier("$$LEFTPK_1"), deepCopyVisitor.varCopy(leftPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    // translator.addOperatorToMetaScope(new Identifier("#LEFT_2"),
    // deepCopyVisitor.deepCopy(leftInputOp, null));
    // translator.addVariableToMetaScope(new Identifier("$$LEFT_2"),
    // deepCopyVisitor.varCopy(leftInputVar));
    // translator.addVariableToMetaScope(new Identifier("$$LEFTPK_2"),
    // deepCopyVisitor.varCopy(leftPKVar));
    // deepCopyVisitor.updatePrimaryKeys(context);
    // deepCopyVisitor.reset();
    //
    // translator.addOperatorToMetaScope(new Identifier("#LEFT_3"),
    // deepCopyVisitor.deepCopy(leftInputOp, null));
    // translator.addVariableToMetaScope(new Identifier("$$LEFT_3"),
    // deepCopyVisitor.varCopy(leftInputVar));
    // translator.addVariableToMetaScope(new Identifier("$$LEFTPK_3"),
    // deepCopyVisitor.varCopy(leftPKVar));
    // deepCopyVisitor.updatePrimaryKeys(context);
    // deepCopyVisitor.reset();
    translator.addOperatorToMetaScope(new Identifier("#RIGHT_1"), deepCopyVisitor.deepCopy(rightInputOp));
    translator.addVariableToMetaScope(new Identifier("$$RIGHT_1"), deepCopyVisitor.varCopy(rightInputVar));
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_1"), deepCopyVisitor.varCopy(rightPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    // TODO pick side to run Stage 1, currently always picks RIGHT side
    translator.addOperatorToMetaScope(new Identifier("#RIGHT_2"), deepCopyVisitor.deepCopy(rightInputOp));
    translator.addVariableToMetaScope(new Identifier("$$RIGHT_2"), deepCopyVisitor.varCopy(rightInputVar));
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_2"), deepCopyVisitor.varCopy(rightPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    translator.addOperatorToMetaScope(new Identifier("#RIGHT_3"), deepCopyVisitor.deepCopy(rightInputOp));
    translator.addVariableToMetaScope(new Identifier("$$RIGHT_3"), deepCopyVisitor.varCopy(rightInputVar));
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_3"), deepCopyVisitor.varCopy(rightPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    ILogicalPlan plan;
    try {
        plan = translator.translate(clauses);
    } catch (AsterixException e) {
        throw new AlgebricksException(e);
    }
    context.setVarCounter(counter.get());
    ILogicalOperator outputOp = plan.getRoots().get(0).getValue();
    SelectOperator extraSelect = null;
    if (getItemExprRef != expRef) {
        // more than one join condition
        getItemExprRef.setValue(ConstantExpression.TRUE);
        switch(joinOp.getJoinKind()) {
            case INNER:
                {
                    extraSelect = new SelectOperator(expRef, false, null);
                    extraSelect.getInputs().add(new MutableObject<ILogicalOperator>(outputOp));
                    outputOp = extraSelect;
                    break;
                }
            case LEFT_OUTER:
                {
                    if (((AbstractLogicalOperator) outputOp).getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
                        throw new IllegalStateException();
                    }
                    LeftOuterJoinOperator topJoin = (LeftOuterJoinOperator) outputOp;
                    topJoin.getCondition().setValue(expRef.getValue());
                    break;
                }
            default:
                {
                    throw new IllegalStateException();
                }
        }
    }
    opRef.setValue(outputOp);
    OperatorPropertiesUtil.typeOpRec(opRef, context);
    return true;
}