Search in sources :

Example 1 with Scanner

use of io.datarouter.scanner.Scanner in project datarouter by hotpads.

the class JobletHandler method list.

@Handler
private Mav list(@Param(PARAM_whereStatus) OptionalString pStatus, @Param(PARAM_type) OptionalString pType) {
    Scanner<JobletRequest> requests = jobletRequestDao.scan();
    if (pStatus.isPresent() && pType.isPresent()) {
        JobletStatus status = JobletStatus.fromPersistentStringStatic(pStatus.get());
        requests = requests.include(request -> status == request.getStatus()).include(request -> request.getKey().getType().equals(pType.get()));
    } else if (pStatus.isPresent() && pType.isEmpty()) {
        JobletStatus status = JobletStatus.fromPersistentStringStatic(pStatus.get());
        requests = requests.include(request -> status == request.getStatus());
    } else if (pStatus.isEmpty() && pType.isPresent()) {
        requests = requests.include(request -> request.getKey().getType().equals(pType.get()));
    }
    Collection<JobletSummary> summaries = JobletSummary.summarizeByTypeExecutionOrderStatus(requests);
    return pageFactory.startBuilder(request).withTitle(TITLE).withRequires(DatarouterWebRequireJsV2.SORTTABLE).withContent(makeContent(summaries)).buildMav();
}
Also used : TagCreator.text(j2html.TagCreator.text) Scanner(io.datarouter.scanner.Scanner) JobletSummary(io.datarouter.joblet.dto.JobletSummary) JobletLocalLinkBuilder(io.datarouter.joblet.nav.JobletLocalLinkBuilder) TagCreator.h4(j2html.TagCreator.h4) OptionalString(io.datarouter.web.handler.types.optional.OptionalString) JobletStatus(io.datarouter.joblet.enums.JobletStatus) Inject(javax.inject.Inject) NumberFormatter(io.datarouter.util.number.NumberFormatter) DatarouterWebRequireJsV2(io.datarouter.web.requirejs.DatarouterWebRequireJsV2) JobletExternalLinkBuilderSupplier(io.datarouter.joblet.nav.JobletExternalLinkBuilder.JobletExternalLinkBuilderSupplier) DatarouterJobletRequestDao(io.datarouter.joblet.storage.jobletrequest.DatarouterJobletRequestDao) Param(io.datarouter.web.handler.types.Param) J2HtmlTable(io.datarouter.web.html.j2html.J2HtmlTable) JobletRequest(io.datarouter.joblet.storage.jobletrequest.JobletRequest) JobletPageFactory(io.datarouter.joblet.JobletPageFactory) DatarouterJobletCounters(io.datarouter.joblet.DatarouterJobletCounters) TagCreator.i(j2html.TagCreator.i) Collection(java.util.Collection) Mav(io.datarouter.web.handler.mav.Mav) TagCreator.a(j2html.TagCreator.a) ContainerTag(j2html.tags.ContainerTag) DomContent(j2html.tags.DomContent) TagCreator.td(j2html.TagCreator.td) BaseHandler(io.datarouter.web.handler.BaseHandler) TagCreator.div(j2html.TagCreator.div) MetricLinkBuilder(io.datarouter.instrumentation.metric.MetricLinkBuilder) JobletStatus(io.datarouter.joblet.enums.JobletStatus) JobletSummary(io.datarouter.joblet.dto.JobletSummary) JobletRequest(io.datarouter.joblet.storage.jobletrequest.JobletRequest) BaseHandler(io.datarouter.web.handler.BaseHandler)

Example 2 with Scanner

use of io.datarouter.scanner.Scanner in project datarouter by hotpads.

the class BaseDatarouterServletContextListener method buildExecuteOnActionsLists.

private void buildExecuteOnActionsLists(ServletContextEvent event) {
    DatarouterInjector injector = getInjector(event.getServletContext());
    Scanner.of(listenerClasses).map(injector::getInstance).forEach(allListeners::add);
    Scanner.of(webListenerClasses).map(injector::getInstance).each(listener -> listener.setServletContext(event.getServletContext())).forEach(allListeners::add);
    Scanner.of(allListeners).splitBy(DatarouterAppListener::safeToExecuteInParallel).map(Scanner::list).map(listeners -> new Pair<>(listeners.get(0).safeToExecuteInParallel() ? ExecutionMode.PARALLEL : ExecutionMode.SYNCHRONOUS, listeners)).forEach(listenersByExecutionMods::add);
}
Also used : DatarouterInjector(io.datarouter.inject.DatarouterInjector) Scanner(io.datarouter.scanner.Scanner) Logger(org.slf4j.Logger) ParallelScannerContext(io.datarouter.scanner.ParallelScannerContext) LoggerFactory(org.slf4j.LoggerFactory) Function(java.util.function.Function) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) ArrayList(java.util.ArrayList) NamedThreadFactory(io.datarouter.util.concurrent.NamedThreadFactory) List(java.util.List) ServletContextEvent(javax.servlet.ServletContextEvent) ExecutorServiceTool(io.datarouter.util.concurrent.ExecutorServiceTool) InjectorRetriever(io.datarouter.web.inject.InjectorRetriever) Duration(java.time.Duration) ThreadFactory(java.util.concurrent.ThreadFactory) Pair(io.datarouter.util.tuple.Pair) Collections(java.util.Collections) ServletContextListener(javax.servlet.ServletContextListener) ExecutorService(java.util.concurrent.ExecutorService) PhaseTimer(io.datarouter.util.timer.PhaseTimer) Scanner(io.datarouter.scanner.Scanner) DatarouterInjector(io.datarouter.inject.DatarouterInjector) Pair(io.datarouter.util.tuple.Pair)

Example 3 with Scanner

use of io.datarouter.scanner.Scanner in project datarouter by hotpads.

the class SpannerSingleTableSchemaUpdateService method performSchemaUpdate.

public Optional<SchemaUpdateResult> performSchemaUpdate(ClientId clientId, Supplier<List<String>> existingTableNames, PhysicalNode<?, ?, ?> physicalNode) {
    String tableName = physicalNode.getFieldInfo().getTableName();
    List<Field<?>> primaryKeyFields = physicalNode.getFieldInfo().getSamplePrimaryKey().getFields();
    List<? extends SpannerBaseFieldCodec<?, ?>> primaryKeyCodecs = fieldCodecRegistry.createCodecs(primaryKeyFields);
    for (SpannerBaseFieldCodec<?, ?> codec : primaryKeyCodecs) {
        if (codec.getSpannerColumnType().isArray()) {
            throw new RuntimeException("Invalid field type used for primary key: " + codec.getField().getKey().getName());
        }
    }
    List<SpannerIndex> indexes = new ArrayList<>();
    List<SpannerIndex> uniqueIndexes = Scanner.of(physicalNode.getFieldInfo().getUniqueIndexes().entrySet()).map(entry -> new SpannerIndex(tableName, entry.getKey(), entry.getValue(), Collections.emptyList(), true)).list();
    var statements = new SpannerUpdateStatements();
    String entityTableName = null;
    if (physicalNode instanceof IndexedStorage) {
        IndexedStorage<?, ?> indexedStorage = (IndexedStorage<?, ?>) physicalNode;
        indexes = Scanner.of(indexedStorage.getManagedNodes()).map(node -> new SpannerIndex(tableName, node.getName(), node.getIndexEntryFieldInfo().getPrimaryKeyFields(), node.getIndexEntryFieldInfo().getFields(), false)).list();
    }
    List<SpannerColumn> primaryKeyColumns = Scanner.of(primaryKeyCodecs).map(codec -> codec.getSpannerColumn(false)).list();
    List<SpannerColumn> nonKeyColumns = Scanner.of(fieldCodecRegistry.createCodecs(physicalNode.getFieldInfo().getNonKeyFields())).map(codec -> codec.getSpannerColumn(true)).list();
    if (!existingTableNames.get().contains(tableName)) {
        statements.updateFunction(tableOperationsGenerator.createTable(tableName, primaryKeyColumns, nonKeyColumns, entityTableName), updateOptions::getCreateTables, true);
        Scanner.of(indexes, uniqueIndexes).concat(Scanner::of).map(index -> createIndex(index, primaryKeyColumns)).forEach(statement -> statements.updateFunction(statement, updateOptions::getCreateTables, true));
    } else {
        DatabaseClient databaseClient = clientsHolder.getDatabaseClient(clientId);
        List<SpannerColumn> allColumns = Scanner.of(primaryKeyColumns, nonKeyColumns).concat(Scanner::of).list();
        ResultSet columnRs = databaseClient.singleUse().executeQuery(Statement.of(tableOperationsGenerator.getTableSchema(tableName)));
        ResultSet primaryKeyRs = databaseClient.singleUse().executeQuery(Statement.of(tableOperationsGenerator.getTableIndexColumnsSchema(tableName, "PRIMARY_KEY")));
        tableAlterSchemaService.generateUpdateStatementColumns(tableName, allColumns, primaryKeyColumns, columnRs, primaryKeyRs, statements);
        ResultSet indexesRs = databaseClient.singleUse().executeQuery(Statement.of(tableOperationsGenerator.getTableIndexSchema(tableName)));
        Set<String> currentIndexes = tableAlterSchemaService.getIndexes(indexesRs);
        Scanner.of(indexes, uniqueIndexes).concat(Scanner::of).forEach(index -> {
            Statement tableIndexColumnsSchema = Statement.of(tableOperationsGenerator.getTableIndexColumnsSchema(tableName, index.getIndexName()));
            ResultSet indexRs = databaseClient.singleUse().executeQuery(tableIndexColumnsSchema);
            if (!tableAlterSchemaService.indexEqual(index, indexRs)) {
                if (currentIndexes.contains(index.getIndexName())) {
                    statements.updateFunction(tableOperationsGenerator.dropIndex(index.getIndexName()), updateOptions::getDropIndexes, false);
                }
                statements.updateFunction(createIndex(index, primaryKeyColumns), updateOptions::getAddIndexes, true);
            }
            currentIndexes.remove(index.getIndexName());
        });
        currentIndexes.forEach(name -> statements.updateFunction(tableOperationsGenerator.dropIndex(name), updateOptions::getDropIndexes, false));
    }
    String errorMessage = null;
    if (!statements.getExecuteStatements().isEmpty()) {
        logger.info(SchemaUpdateTool.generateFullWidthMessage("Executing Spanner " + getClass().getSimpleName() + " SchemaUpdate"));
        logger.info(String.join("\n\n", statements.getExecuteStatements()));
        Database database = clientsHolder.getDatabase(clientId);
        OperationFuture<Void, UpdateDatabaseDdlMetadata> future = database.updateDdl(statements.getExecuteStatements(), null);
        errorMessage = FutureTool.get(future.getPollingFuture().getAttemptResult()).getErrorMessage();
        if (StringTool.notNullNorEmptyNorWhitespace(errorMessage)) {
            logger.error(errorMessage);
        }
    }
    if (statements.getPreventStartUp()) {
        errorMessage = "an alter on Spanner table " + tableName + " is required";
    }
    if (statements.getPrintStatements().isEmpty()) {
        return Optional.empty();
    }
    String printStatement = statements.getPrintStatements().stream().map(statement -> statement + ";").collect(Collectors.joining("\n"));
    SchemaUpdateTool.printSchemaUpdate(logger, printStatement);
    return Optional.of(new SchemaUpdateResult(printStatement, errorMessage, clientId));
}
Also used : Scanner(io.datarouter.scanner.Scanner) SchemaUpdateTool(io.datarouter.storage.config.schema.SchemaUpdateTool) OperationFuture(com.google.api.gax.longrunning.OperationFuture) SpannerDatabaseClientsHolder(io.datarouter.gcp.spanner.connection.SpannerDatabaseClientsHolder) LoggerFactory(org.slf4j.LoggerFactory) PhysicalNode(io.datarouter.storage.node.type.physical.PhysicalNode) DatabaseClient(com.google.cloud.spanner.DatabaseClient) Singleton(javax.inject.Singleton) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Inject(javax.inject.Inject) ResultSet(com.google.cloud.spanner.ResultSet) ClientId(io.datarouter.storage.client.ClientId) Logger(org.slf4j.Logger) Database(com.google.cloud.spanner.Database) Set(java.util.Set) StringTool(io.datarouter.util.string.StringTool) Collectors(java.util.stream.Collectors) SchemaUpdateOptions(io.datarouter.storage.config.schema.SchemaUpdateOptions) UpdateDatabaseDdlMetadata(com.google.spanner.admin.database.v1.UpdateDatabaseDdlMetadata) Statement(com.google.cloud.spanner.Statement) List(java.util.List) Field(io.datarouter.model.field.Field) SpannerFieldCodecRegistry(io.datarouter.gcp.spanner.field.SpannerFieldCodecRegistry) Optional(java.util.Optional) IndexedStorage(io.datarouter.storage.node.op.raw.IndexedStorage) Collections(java.util.Collections) SpannerBaseFieldCodec(io.datarouter.gcp.spanner.field.SpannerBaseFieldCodec) SchemaUpdateResult(io.datarouter.storage.config.schema.SchemaUpdateResult) FutureTool(io.datarouter.util.concurrent.FutureTool) Scanner(io.datarouter.scanner.Scanner) ArrayList(java.util.ArrayList) Field(io.datarouter.model.field.Field) UpdateDatabaseDdlMetadata(com.google.spanner.admin.database.v1.UpdateDatabaseDdlMetadata) ResultSet(com.google.cloud.spanner.ResultSet) Database(com.google.cloud.spanner.Database) IndexedStorage(io.datarouter.storage.node.op.raw.IndexedStorage) Statement(com.google.cloud.spanner.Statement) DatabaseClient(com.google.cloud.spanner.DatabaseClient) SchemaUpdateResult(io.datarouter.storage.config.schema.SchemaUpdateResult)

Example 4 with Scanner

use of io.datarouter.scanner.Scanner in project datarouter by hotpads.

the class ClusterSettingService method scanClusterSettingAndValidityWithPrefix.

public Scanner<ClusterSettingAndValidityJspDto> scanClusterSettingAndValidityWithPrefix(String prefix) {
    WebappInstance currentWebappInstance = webappInstanceDao.get(webappInstanceService.buildCurrentWebappInstanceKey());
    Range<ClusterSettingKey> range = prefix == null ? Range.everything() : KeyRangeTool.forPrefixWithWildcard(prefix, name -> new ClusterSettingKey(name, null, null, null));
    return clusterSettingDao.scan(range).map(setting -> {
        ClusterSettingValidity validity = getValidityForWebappInstance(setting, currentWebappInstance);
        return new ClusterSettingAndValidityJspDto(setting, validity);
    });
}
Also used : Scanner(io.datarouter.scanner.Scanner) DatarouterServerTypeSupplier(io.datarouter.storage.config.properties.DatarouterServerTypeSupplier) ServerTypes(io.datarouter.storage.servertype.ServerTypes) ServerType(io.datarouter.storage.servertype.ServerType) ClusterSettingFinder(io.datarouter.clustersetting.ClusterSettingFinder) CachedSetting(io.datarouter.storage.setting.cached.CachedSetting) DatarouterClusterSettingLogDao(io.datarouter.clustersetting.storage.clustersettinglog.DatarouterClusterSettingLogDao) Singleton(javax.inject.Singleton) DatarouterClusterSettingRoot(io.datarouter.clustersetting.config.DatarouterClusterSettingRoot) ServerName(io.datarouter.storage.config.properties.ServerName) Function(java.util.function.Function) DatarouterSettingTag(io.datarouter.storage.setting.DatarouterSettingTag) ClusterSettingScope(io.datarouter.clustersetting.ClusterSettingScope) ClusterSettingAndValidityJspDto(io.datarouter.clustersetting.web.dto.ClusterSettingAndValidityJspDto) Inject(javax.inject.Inject) ClusterSetting(io.datarouter.clustersetting.storage.clustersetting.ClusterSetting) CachedClusterSettingTags(io.datarouter.storage.setting.cached.CachedClusterSettingTags) Map(java.util.Map) KeyRangeTool(io.datarouter.storage.util.KeyRangeTool) SettingRootFinder(io.datarouter.storage.setting.SettingRoot.SettingRootFinder) ClusterSettingScopeComparator(io.datarouter.clustersetting.ClusterSettingScopeComparator) DefaultSettingValue(io.datarouter.storage.setting.DefaultSettingValue) WebappInstanceService(io.datarouter.webappinstance.service.WebappInstanceService) ClusterSettingKey(io.datarouter.clustersetting.storage.clustersetting.ClusterSettingKey) ClusterSettingValidity(io.datarouter.clustersetting.ClusterSettingValidity) WebappInstance(io.datarouter.webappinstance.storage.webappinstance.WebappInstance) StringTool(io.datarouter.util.string.StringTool) Objects(java.util.Objects) Range(io.datarouter.util.tuple.Range) List(java.util.List) EnvironmentName(io.datarouter.storage.config.properties.EnvironmentName) ClusterSettingComparisonTool(io.datarouter.clustersetting.ClusterSettingComparisonTool) DatarouterClusterSettingDao(io.datarouter.clustersetting.storage.clustersetting.DatarouterClusterSettingDao) DatarouterEnvironmentTypeSupplier(io.datarouter.storage.config.properties.DatarouterEnvironmentTypeSupplier) BooleanTool(io.datarouter.util.BooleanTool) Optional(java.util.Optional) DatarouterWebappInstanceDao(io.datarouter.webappinstance.storage.webappinstance.DatarouterWebappInstanceDao) ClusterSettingKey(io.datarouter.clustersetting.storage.clustersetting.ClusterSettingKey) ClusterSettingAndValidityJspDto(io.datarouter.clustersetting.web.dto.ClusterSettingAndValidityJspDto) ClusterSettingValidity(io.datarouter.clustersetting.ClusterSettingValidity) WebappInstance(io.datarouter.webappinstance.storage.webappinstance.WebappInstance)

Example 5 with Scanner

use of io.datarouter.scanner.Scanner in project datarouter by hotpads.

the class ScanningBlockReader method scanLeafBlockKeys.

private Scanner<BlockKey> scanLeafBlockKeys(long fromRecordIdInclusive) {
    BlockKey topBranchBlockKey = rootBlock.rootBranchBlockKey(snapshotKey);
    BranchBlock topBranchBlock = blockLoader.branch(topBranchBlockKey);
    return scanDescendantBranchBlocks(topBranchBlock, fromRecordIdInclusive).include(branchBlock -> branchBlock.level() == 0).concat(branchBlock -> {
        return Scanner.iterate(0, i -> i + 1).limit(branchBlock.numRecords()).include(index -> branchBlock.recordId(index) >= fromRecordIdInclusive).map(branchBlock::childBlock).map(leafBlockId -> branchBlock.leafBlockKey(snapshotKey, leafBlockId));
    });
}
Also used : Scanner(io.datarouter.scanner.Scanner) BlockKey(io.datarouter.filesystem.snapshot.block.BlockKey) BranchBlock(io.datarouter.filesystem.snapshot.block.branch.BranchBlock) ParallelScannerContext(io.datarouter.scanner.ParallelScannerContext) RootBlock(io.datarouter.filesystem.snapshot.block.root.RootBlock) SnapshotKey(io.datarouter.filesystem.snapshot.key.SnapshotKey) Function(java.util.function.Function) LeafBlock(io.datarouter.filesystem.snapshot.block.leaf.LeafBlock) ExecutorService(java.util.concurrent.ExecutorService) BranchBlock(io.datarouter.filesystem.snapshot.block.branch.BranchBlock) BlockKey(io.datarouter.filesystem.snapshot.block.BlockKey)

Aggregations

Scanner (io.datarouter.scanner.Scanner)11 List (java.util.List)9 ArrayList (java.util.ArrayList)6 Map (java.util.Map)6 Inject (javax.inject.Inject)6 Logger (org.slf4j.Logger)6 LoggerFactory (org.slf4j.LoggerFactory)6 Optional (java.util.Optional)5 Collectors (java.util.stream.Collectors)5 StringTool (io.datarouter.util.string.StringTool)4 BaseHandler (io.datarouter.web.handler.BaseHandler)3 Mav (io.datarouter.web.handler.mav.Mav)3 OptionalString (io.datarouter.web.handler.types.optional.OptionalString)3 Comparator (java.util.Comparator)3 ExecutorService (java.util.concurrent.ExecutorService)3 Function (java.util.function.Function)3 ClusterSettingScope (io.datarouter.clustersetting.ClusterSettingScope)2 ClusterSettingValidity (io.datarouter.clustersetting.ClusterSettingValidity)2 SnapshotKey (io.datarouter.filesystem.snapshot.key.SnapshotKey)2 ChangelogRecorder (io.datarouter.instrumentation.changelog.ChangelogRecorder)2