Examples with StorageComponentProvider org.apache.asterix.file.StorageComponentProvider used on opensource projects

Example 1 with StorageComponentProvider

use of org.apache.asterix.file.StorageComponentProvider in project asterixdb by apache.

the class CCApplication method start.

@Override
public void start(IServiceContext serviceCtx, String[] args) throws Exception {
    if (args.length > 0) {
        throw new IllegalArgumentException("Unrecognized argument(s): " + Arrays.toString(args));
    }
    final ClusterControllerService controllerService = (ClusterControllerService) serviceCtx.getControllerService();
    this.ccServiceCtx = (ICCServiceContext) serviceCtx;
    ccServiceCtx.setMessageBroker(new CCMessageBroker(controllerService));
    configureLoggingLevel(ccServiceCtx.getAppConfig().getLoggingLevel(ExternalProperties.Option.LOG_LEVEL));
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Starting Asterix cluster controller");
    }
    ccServiceCtx.setThreadFactory(new AsterixThreadFactory(ccServiceCtx.getThreadFactory(), new LifeCycleComponentManager()));
    ILibraryManager libraryManager = new ExternalLibraryManager();
    ResourceIdManager resourceIdManager = new ResourceIdManager();
    IReplicationStrategy repStrategy = ClusterProperties.INSTANCE.getReplicationStrategy();
    IFaultToleranceStrategy ftStrategy = FaultToleranceStrategyFactory.create(ClusterProperties.INSTANCE.getCluster(), repStrategy, ccServiceCtx);
    ExternalLibraryUtils.setUpExternaLibraries(libraryManager, false);
    componentProvider = new StorageComponentProvider();
    GlobalRecoveryManager.instantiate(ccServiceCtx, getHcc(), componentProvider);
    statementExecutorCtx = new StatementExecutorContext();
    appCtx = new CcApplicationContext(ccServiceCtx, getHcc(), libraryManager, resourceIdManager, () -> MetadataManager.INSTANCE, GlobalRecoveryManager.instance(), ftStrategy, new ActiveLifecycleListener(), componentProvider);
    ClusterStateManager.INSTANCE.setCcAppCtx(appCtx);
    ccExtensionManager = new CCExtensionManager(getExtensions());
    appCtx.setExtensionManager(ccExtensionManager);
    final CCConfig ccConfig = controllerService.getCCConfig();
    if (System.getProperty("java.rmi.server.hostname") == null) {
        System.setProperty("java.rmi.server.hostname", ccConfig.getClusterListenAddress());
    }
    MetadataProperties metadataProperties = appCtx.getMetadataProperties();
    setAsterixStateProxy(AsterixStateProxy.registerRemoteObject(metadataProperties.getMetadataCallbackPort()));
    ccServiceCtx.setDistributedState(proxy);
    MetadataManager.initialize(proxy, metadataProperties);
    ccServiceCtx.addJobLifecycleListener(appCtx.getActiveLifecycleListener());
    // create event loop groups
    webManager = new WebManager();
    configureServers();
    webManager.start();
    ClusterManagerProvider.getClusterManager().registerSubscriber(GlobalRecoveryManager.instance());
    ccServiceCtx.addClusterLifecycleListener(new ClusterLifecycleListener(appCtx));
    jobCapacityController = new JobCapacityController(controllerService.getResourceManager());
}

Example 2 with StorageComponentProvider

use of org.apache.asterix.file.StorageComponentProvider in project asterixdb by apache.

the class QueryTranslatorTest method test.

@Test
public void test() throws Exception {
    List<Statement> statements = new ArrayList<>();
    SessionOutput mockSessionOutput = mock(SessionOutput.class);
    RunStatement mockRunStatement = mock(RunStatement.class);
    // Mocks AppContextInfo.
    CcApplicationContext mockAsterixAppContextInfo = mock(CcApplicationContext.class);
    ExternalProperties mockAsterixExternalProperties = mock(ExternalProperties.class);
    when(mockAsterixAppContextInfo.getExternalProperties()).thenReturn(mockAsterixExternalProperties);
    when(mockAsterixExternalProperties.getAPIServerPort()).thenReturn(19002);
    // Mocks AsterixClusterProperties.
    Cluster mockCluster = mock(Cluster.class);
    MasterNode mockMasterNode = mock(MasterNode.class);
    ClusterProperties mockClusterProperties = mock(ClusterProperties.class);
    setFinalStaticField(ClusterProperties.class.getDeclaredField("INSTANCE"), mockClusterProperties);
    when(mockClusterProperties.getCluster()).thenReturn(mockCluster);
    when(mockCluster.getMasterNode()).thenReturn(mockMasterNode);
    when(mockMasterNode.getClientIp()).thenReturn("127.0.0.1");
    IStatementExecutor aqlTranslator = new DefaultStatementExecutorFactory().create(mockAsterixAppContextInfo, statements, mockSessionOutput, new AqlCompilationProvider(), new StorageComponentProvider());
    List<String> parameters = new ArrayList<>();
    parameters.add("examples/pregelix-example-jar-with-dependencies.jar");
    parameters.add("org.apache.pregelix.example.PageRankVertex");
    parameters.add("-ip 10.0.2.15 -port 3199");
    when(mockRunStatement.getParameters()).thenReturn(parameters);
    // Test a customer command without "-cust-prop".
    List<String> cmds = (List<String>) PA.invokeMethod(aqlTranslator, "constructPregelixCommand(org.apache.asterix.lang.common.statement.RunStatement," + "String,String,String,String)", mockRunStatement, "fromDataverse", "fromDataset", "toDataverse", "toDataset");
    List<String> expectedCmds = Arrays.asList(new String[] { "bin/pregelix", "examples/pregelix-example-jar-with-dependencies.jar", "org.apache.pregelix.example.PageRankVertex", "-ip", "10.0.2.15", "-port", "3199", "-cust-prop", "pregelix.asterixdb.url=http://127.0.0.1:19002,pregelix.asterixdb.source=true,pregelix.asterixdb.sink=true,pregelix.asterixdb.input.dataverse=fromDataverse,pregelix.asterixdb.input.dataset=fromDataset,pregelix.asterixdb.output.dataverse=toDataverse,pregelix.asterixdb.output.dataset=toDataset,pregelix.asterixdb.output.cleanup=false,pregelix.asterixdb.input.converterclass=org.apache.pregelix.example.converter.VLongIdInputVertexConverter,pregelix.asterixdb.output.converterclass=org.apache.pregelix.example.converter.VLongIdOutputVertexConverter" });
    Assert.assertEquals(cmds, expectedCmds);
    parameters.remove(parameters.size() - 1);
    parameters.add("-ip 10.0.2.15 -port 3199 -cust-prop " + "pregelix.asterixdb.input.converterclass=org.apache.pregelix.example.converter.TestInputVertexConverter," + "pregelix.asterixdb.output.converterclass=org.apache.pregelix.example.converter.TestOutputVertexConverter");
    // Test a customer command with "-cust-prop".
    cmds = (List<String>) PA.invokeMethod(aqlTranslator, "constructPregelixCommand(org.apache.asterix.lang.common.statement.RunStatement," + "String,String,String,String)", mockRunStatement, "fromDataverse", "fromDataset", "toDataverse", "toDataset");
    expectedCmds = Arrays.asList(new String[] { "bin/pregelix", "examples/pregelix-example-jar-with-dependencies.jar", "org.apache.pregelix.example.PageRankVertex", "-ip", "10.0.2.15", "-port", "3199", "-cust-prop", "pregelix.asterixdb.url=http://127.0.0.1:19002,pregelix.asterixdb.source=true,pregelix.asterixdb.sink=true,pregelix.asterixdb.input.dataverse=fromDataverse,pregelix.asterixdb.input.dataset=fromDataset,pregelix.asterixdb.output.dataverse=toDataverse,pregelix.asterixdb.output.dataset=toDataset,pregelix.asterixdb.output.cleanup=false,pregelix.asterixdb.input.converterclass=org.apache.pregelix.example.converter.TestInputVertexConverter,pregelix.asterixdb.output.converterclass=org.apache.pregelix.example.converter.TestOutputVertexConverter" });
    Assert.assertEquals(cmds, expectedCmds);
}

Example 3 with StorageComponentProvider

use of org.apache.asterix.file.StorageComponentProvider in project asterixdb by apache.

the class ConnectorApiServletTest method getMetadataRecordType.

private ARecordType getMetadataRecordType(String dataverseName, String datasetName) throws Exception {
    MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
    // Retrieves file splits of the dataset.
    MetadataProvider metadataProvider = new MetadataProvider((ICcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext(), null, new StorageComponentProvider());
    try {
        metadataProvider.setMetadataTxnContext(mdTxnCtx);
        Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
        ARecordType recordType = (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
        // Metadata transaction commits.
        MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
        return recordType;
    } finally {
        metadataProvider.getLocks().unlock();
    }
}

Example 4 with StorageComponentProvider

use of org.apache.asterix.file.StorageComponentProvider in project asterixdb by apache.

the class LogMarkerTest method testInsertWithSnapshot.

@Test
public void testInsertWithSnapshot() {
    try {
        TestNodeController nc = new TestNodeController(null, false);
        nc.init();
        StorageComponentProvider storageManager = new StorageComponentProvider();
        List<List<String>> partitioningKeys = new ArrayList<>();
        partitioningKeys.add(Collections.singletonList("key"));
        Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME, null, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH, partitioningKeys, null, null, null, false, null, false), null, DatasetType.INTERNAL, DATASET_ID, 0);
        try {
            nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST);
            IHyracksTaskContext ctx = nc.createTestContext(true);
            nc.newJobId();
            ITransactionContext txnCtx = nc.getTransactionManager().getTransactionContext(nc.getTxnJobId(), true);
            LSMInsertDeleteOperatorNodePushable insertOp = nc.getInsertPipeline(ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATORS_LIST, storageManager).getLeft();
            insertOp.open();
            TupleGenerator tupleGenerator = new TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATORS, RECORD_GEN_FUNCTION, UNIQUE_RECORD_FIELDS, META_GEN_FUNCTION, UNIQUE_META_FIELDS);
            VSizeFrame frame = new VSizeFrame(ctx);
            VSizeFrame marker = new VSizeFrame(ctx);
            FrameTupleAppender tupleAppender = new FrameTupleAppender(frame);
            long markerId = 0L;
            for (int j = 0; j < NUM_OF_RECORDS; j++) {
                if (j % SNAPSHOT_SIZE == 0) {
                    marker.reset();
                    marker.getBuffer().put(MessagingFrameTupleAppender.MARKER_MESSAGE);
                    marker.getBuffer().putLong(markerId);
                    marker.getBuffer().flip();
                    markerId++;
                    TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, marker, ctx);
                    tupleAppender.flush(insertOp);
                }
                ITupleReference tuple = tupleGenerator.next();
                DataflowUtils.addTupleToFrame(tupleAppender, tuple, insertOp);
            }
            if (tupleAppender.getTupleCount() > 0) {
                tupleAppender.write(insertOp, true);
            }
            insertOp.close();
            nc.getTransactionManager().completedTransaction(txnCtx, DatasetId.NULL, -1, true);
            IIndexDataflowHelper dataflowHelper = nc.getPrimaryIndexDataflowHelper(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST);
            dataflowHelper.open();
            LSMBTree btree = (LSMBTree) dataflowHelper.getIndexInstance();
            LongPointable longPointable = LongPointable.FACTORY.createPointable();
            ComponentMetadataUtil.get(btree, ComponentMetadataUtil.MARKER_LSN_KEY, longPointable);
            long lsn = longPointable.getLong();
            int numOfMarkers = 0;
            LogReader logReader = (LogReader) nc.getTransactionSubsystem().getLogManager().getLogReader(false);
            long expectedMarkerId = markerId - 1;
            while (lsn >= 0) {
                numOfMarkers++;
                ILogRecord logRecord = logReader.read(lsn);
                lsn = logRecord.getPreviousMarkerLSN();
                long logMarkerId = logRecord.getMarker().getLong();
                Assert.assertEquals(expectedMarkerId, logMarkerId);
                expectedMarkerId--;
            }
            logReader.close();
            dataflowHelper.close();
            Assert.assertEquals(markerId, numOfMarkers);
            nc.newJobId();
            TestTupleCounterFrameWriter countOp = create(nc.getSearchOutputDesc(KEY_TYPES, RECORD_TYPE, META_TYPE), Collections.emptyList(), Collections.emptyList(), false);
            IPushRuntime emptyTupleOp = nc.getFullScanPipeline(countOp, ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATORS_LIST, storageManager);
            emptyTupleOp.open();
            emptyTupleOp.close();
            Assert.assertEquals(NUM_OF_RECORDS, countOp.getCount());
        } finally {
            nc.deInit();
        }
    } catch (Throwable e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 5 with StorageComponentProvider

use of org.apache.asterix.file.StorageComponentProvider in project asterixdb by apache.

the class DmlTest method enlistTest.

@Test
public void enlistTest() throws Exception {
    File outdir = new File(PATH_ACTUAL);
    if (outdir.exists()) {
        AsterixTestHelper.deleteRec(outdir);
    }
    outdir.mkdirs();
    integrationUtil.init(true);
    Reader loadReader = new BufferedReader(new InputStreamReader(new FileInputStream(LOAD_FOR_ENLIST_FILE), "UTF-8"));
    AsterixJavaClient asterixLoad = new AsterixJavaClient((ICcApplicationContext) integrationUtil.cc.getApplicationContext(), integrationUtil.getHyracksClientConnection(), loadReader, ERR, new AqlCompilationProvider(), new DefaultStatementExecutorFactory(), new StorageComponentProvider());
    try {
        asterixLoad.compile(true, false, false, false, false, true, false);
    } catch (AsterixException e) {
        throw new Exception("Compile ERROR for " + LOAD_FOR_ENLIST_FILE + ": " + e.getMessage(), e);
    } finally {
        loadReader.close();
    }
    asterixLoad.execute();
    integrationUtil.deinit(true);
    for (String d : ASTERIX_DATA_DIRS) {
        testExecutor.deleteRec(new File(d));
    }
    outdir.delete();
}