Search in sources :

Example 1 with JobletPackage

use of io.datarouter.joblet.model.JobletPackage in project datarouter by hotpads.

the class JobletService method getJobletPackagesForJobletRequests.

private List<JobletPackage> getJobletPackagesForJobletRequests(Collection<JobletRequest> jobletRequests) {
    List<JobletDataKey> keys = Scanner.of(jobletRequests).map(JobletRequest::getJobletDataKey).list();
    Map<Long, JobletData> dataKeyToJobletData = jobletDataDao.getMulti(keys).stream().collect(Collectors.toMap(jobletData -> jobletData.getKey().getId(), Function.identity()));
    return jobletRequests.stream().map(jobletRequest -> {
        Long dataKey = jobletRequest.getJobletDataId();
        JobletData jobletData = dataKeyToJobletData.get(dataKey);
        return new JobletPackage(jobletRequest, jobletData);
    }).collect(Collectors.toList());
}
Also used : IntStream(java.util.stream.IntStream) Scanner(io.datarouter.scanner.Scanner) DatarouterJobletSettingRoot(io.datarouter.joblet.setting.DatarouterJobletSettingRoot) JobletTypeFactory(io.datarouter.joblet.type.JobletTypeFactory) JobletPackage(io.datarouter.joblet.model.JobletPackage) JobletRequestSelector(io.datarouter.joblet.queue.JobletRequestSelector) ExceptionRecorder(io.datarouter.web.exception.ExceptionRecorder) LoggerFactory(org.slf4j.LoggerFactory) JobletRequestQueueManager(io.datarouter.joblet.queue.JobletRequestQueueManager) Singleton(javax.inject.Singleton) ServerName(io.datarouter.storage.config.properties.ServerName) Function(java.util.function.Function) JobletStatus(io.datarouter.joblet.enums.JobletStatus) Inject(javax.inject.Inject) JobletType(io.datarouter.joblet.type.JobletType) JobletRequestKey(io.datarouter.joblet.storage.jobletrequest.JobletRequestKey) JobletRequestSelectorFactory(io.datarouter.joblet.queue.JobletRequestSelectorFactory) Map(java.util.Map) Config(io.datarouter.storage.config.Config) HashMethods(io.datarouter.util.HashMethods) DatarouterJobletRequestDao(io.datarouter.joblet.storage.jobletrequest.DatarouterJobletRequestDao) JobletDataKey(io.datarouter.joblet.storage.jobletdata.JobletDataKey) PhaseTimer(io.datarouter.util.timer.PhaseTimer) JobletRequest(io.datarouter.joblet.storage.jobletrequest.JobletRequest) JobletExceptionCategory(io.datarouter.joblet.JobletExceptionCategory) JobletRequestQueueKey(io.datarouter.joblet.storage.jobletrequestqueue.JobletRequestQueueKey) Logger(org.slf4j.Logger) DatarouterJobletCounters(io.datarouter.joblet.DatarouterJobletCounters) DatarouterJobletDataDao(io.datarouter.joblet.storage.jobletdata.DatarouterJobletDataDao) Collection(java.util.Collection) DatarouterJobletQueueDao(io.datarouter.joblet.storage.jobletrequestqueue.DatarouterJobletQueueDao) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) Range(io.datarouter.util.tuple.Range) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) Databean(io.datarouter.model.databean.Databean) CachedWebappInstancesOfThisServerType(io.datarouter.webappinstance.service.CachedWebappInstancesOfThisServerType) Entry(java.util.Map.Entry) Optional(java.util.Optional) JobletData(io.datarouter.joblet.storage.jobletdata.JobletData) PutMethod(io.datarouter.storage.config.PutMethod) JobletQueueMechanism(io.datarouter.joblet.enums.JobletQueueMechanism) JobletDataKey(io.datarouter.joblet.storage.jobletdata.JobletDataKey) AtomicLong(java.util.concurrent.atomic.AtomicLong) JobletPackage(io.datarouter.joblet.model.JobletPackage) JobletData(io.datarouter.joblet.storage.jobletdata.JobletData)

Example 2 with JobletPackage

use of io.datarouter.joblet.model.JobletPackage in project datarouter by hotpads.

the class JobletCallable method dequeueJobletPackage.

private final Optional<JobletPackage> dequeueJobletPackage(PhaseTimer timer) {
    String reservedBy = getReservedByString();
    Optional<JobletRequest> optJobletRequest = jobletService.getJobletRequestForProcessing(timer, jobletType, reservedBy);
    if (optJobletRequest.isEmpty()) {
        timer.add("no JobletRequest found");
        return Optional.empty();
    }
    JobletRequest jobletRequest = optJobletRequest.get();
    timer.add("dequeued " + jobletRequest.getKey());
    JobletPackage jobletPackage = jobletService.getJobletPackageForJobletRequest(jobletRequest);
    if (jobletPackage.getJobletData() == null) {
        datarouterJobletCounters.ignoredDataMissingFromDb(jobletType);
        jobletService.handleMissingJobletData(jobletRequest);
        timer.add("deleted, missing JobletData");
        return Optional.empty();
    }
    timer.add("getJobletData");
    jobletRequest.setShutdownRequested(shutdownRequested);
    return Optional.of(jobletPackage);
}
Also used : JobletPackage(io.datarouter.joblet.model.JobletPackage) JobletRequest(io.datarouter.joblet.storage.jobletrequest.JobletRequest)

Example 3 with JobletPackage

use of io.datarouter.joblet.model.JobletPackage in project datarouter by hotpads.

the class WebSocketSessionVacuumJob method run.

@Override
public void run(TaskTracker tracker) {
    JobletPackage jobletPackage = JobletPackage.create(WebSocketSessionVacuumJoblet.JOBLET_TYPE, JobletPriority.DEFAULT, true, null, null, Instant.now());
    jobletService.submitJobletPackages(List.of(jobletPackage));
}
Also used : JobletPackage(io.datarouter.joblet.model.JobletPackage)

Example 4 with JobletPackage

use of io.datarouter.joblet.model.JobletPackage in project datarouter by hotpads.

the class JobletCopyTableHandler method defaultHandler.

@Handler(defaultHandler = true)
private <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> Mav defaultHandler(@Param(P_sourceNodeName) OptionalString sourceNodeName, @Param(P_targetNodeName) OptionalString targetNodeName, @Param(P_putBatchSize) OptionalString putBatchSize, @Param(P_submitAction) OptionalString submitAction) {
    String errorPutBatchSize = null;
    if (submitAction.isPresent()) {
        try {
            if (putBatchSize.map(StringTool::nullIfEmpty).isPresent()) {
                Integer.valueOf(putBatchSize.get());
            }
        } catch (Exception e) {
            errorPutBatchSize = "Please specify an integer";
        }
    }
    List<String> possibleNodes = tableSamplerService.scanCountableNodes().map(node -> node.getClientId().getName() + "." + node.getFieldInfo().getTableName()).append("").sort().list();
    var form = new HtmlForm().withMethod("post");
    form.addSelectField().withDisplay("Source Node Name").withName(P_sourceNodeName).withValues(possibleNodes);
    form.addSelectField().withDisplay("Target Node Name").withName(P_targetNodeName).withValues(possibleNodes);
    form.addTextField().withDisplay("Batch Size").withError(errorPutBatchSize).withName(P_putBatchSize).withPlaceholder(DEFAULT_BATCH_SIZE + "").withValue(putBatchSize.orElse(null));
    form.addButton().withDisplay("Create Joblets").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle("Copy Table - Joblets").withContent(Html.makeContent(form)).buildMav();
    }
    @SuppressWarnings("unchecked") PhysicalSortedStorageNode<PK, D, ?> sourceNode = (PhysicalSortedStorageNode<PK, D, ?>) nodes.getNode(sourceNodeName.get());
    String tableName = sourceNode.getFieldInfo().getTableName();
    List<TableSample> samples = tableSamplerService.scanSamplesForNode(sourceNode).list();
    TableSampleKey previousSampleKey = null;
    List<JobletPackage> jobletPackages = new ArrayList<>();
    // +1 for databeans beyond the final sample
    long numJoblets = samples.size() + 1;
    long counter = 1;
    int batchSize = putBatchSize.map(StringTool::nullIfEmpty).map(Integer::valueOf).orElse(DEFAULT_BATCH_SIZE);
    for (TableSample sample : samples) {
        PK fromKeyExclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, previousSampleKey);
        PK toKeyInclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, sample.getKey());
        jobletPackages.add(createJobletPackage(tableName, sourceNodeName.get(), targetNodeName.get(), fromKeyExclusive, toKeyInclusive, batchSize, sample.getNumRows(), counter, numJoblets));
        ++counter;
        previousSampleKey = sample.getKey();
    }
    // include any rows created since the last sample
    PK fromKeyExclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, previousSampleKey);
    jobletPackages.add(createJobletPackage(tableName, sourceNodeName.get(), targetNodeName.get(), fromKeyExclusive, // open-ended
    null, batchSize, // we have no idea about the true estNumDatabeans
    1, counter, numJoblets));
    ++counter;
    // shuffle as optimization to spread write load.  could be optional
    Scanner.of(jobletPackages).shuffle().flush(jobletService::submitJobletPackages);
    changelogRecorderService.recordChangelog(getSessionInfo(), "Joblet", sourceNodeName.get(), targetNodeName.get());
    return pageFactory.message(request, "created " + numJoblets + " joblets");
}
Also used : JobletService(io.datarouter.joblet.service.JobletService) Scanner(io.datarouter.scanner.Scanner) JobletPackage(io.datarouter.joblet.model.JobletPackage) TableSamplerTool(io.datarouter.nodewatch.util.TableSamplerTool) PrimaryKeyPercentCodecTool(io.datarouter.storage.util.PrimaryKeyPercentCodecTool) TableSample(io.datarouter.nodewatch.storage.tablesample.TableSample) OptionalString(io.datarouter.web.handler.types.optional.OptionalString) TableSampleKey(io.datarouter.nodewatch.storage.tablesample.TableSampleKey) TagCreator.h2(j2html.TagCreator.h2) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) TagCreator.br(j2html.TagCreator.br) DatarouterNodes(io.datarouter.storage.node.DatarouterNodes) Param(io.datarouter.web.handler.types.Param) Bootstrap4FormHtml(io.datarouter.web.html.j2html.bootstrap4.Bootstrap4FormHtml) JobletPriority(io.datarouter.joblet.enums.JobletPriority) PhysicalSortedStorageNode(io.datarouter.storage.node.op.raw.SortedStorage.PhysicalSortedStorageNode) TableSamplerService(io.datarouter.nodewatch.service.TableSamplerService) Mav(io.datarouter.web.handler.mav.Mav) StringTool(io.datarouter.util.string.StringTool) PrimaryKey(io.datarouter.model.key.primary.PrimaryKey) ContainerTag(j2html.tags.ContainerTag) CopyTableJoblet(io.datarouter.plugin.copytable.CopyTableJoblet) List(java.util.List) HtmlForm(io.datarouter.web.html.form.HtmlForm) Databean(io.datarouter.model.databean.Databean) BaseHandler(io.datarouter.web.handler.BaseHandler) CopyTableJobletParams(io.datarouter.plugin.copytable.CopyTableJoblet.CopyTableJobletParams) Bootstrap4PageFactory(io.datarouter.web.html.j2html.bootstrap4.Bootstrap4PageFactory) TagCreator.div(j2html.TagCreator.div) StringTool(io.datarouter.util.string.StringTool) ArrayList(java.util.ArrayList) TableSample(io.datarouter.nodewatch.storage.tablesample.TableSample) OptionalString(io.datarouter.web.handler.types.optional.OptionalString) TableSampleKey(io.datarouter.nodewatch.storage.tablesample.TableSampleKey) PhysicalSortedStorageNode(io.datarouter.storage.node.op.raw.SortedStorage.PhysicalSortedStorageNode) HtmlForm(io.datarouter.web.html.form.HtmlForm) JobletPackage(io.datarouter.joblet.model.JobletPackage) BaseHandler(io.datarouter.web.handler.BaseHandler)

Example 5 with JobletPackage

use of io.datarouter.joblet.model.JobletPackage in project datarouter by hotpads.

the class JobletTableProcessorHandler method defaultHandler.

@Handler(defaultHandler = true)
private <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> Mav defaultHandler(@Param(P_nodeName) OptionalString nodeName, @Param(P_scanBatchSize) OptionalString scanBatchSize, @Param(P_processorName) OptionalString processorName, @Param(P_submitAction) OptionalString submitAction) {
    String errorScanBatchSize = null;
    if (submitAction.isPresent()) {
        try {
            if (scanBatchSize.map(StringTool::nullIfEmpty).isPresent()) {
                Integer.valueOf(scanBatchSize.get());
            }
        } catch (Exception e) {
            errorScanBatchSize = "Please specify an integer";
        }
    }
    List<String> possibleNodes = tableSamplerService.scanCountableNodes().map(node -> node.getClientId().getName() + "." + node.getFieldInfo().getTableName()).append("").sort().list();
    List<String> possibleProcessors = Scanner.of(processorRegistry.getAll()).map(Class::getSimpleName).append("").sort().list();
    var form = new HtmlForm().withMethod("post");
    form.addSelectField().withDisplay("Node Name").withName(P_nodeName).withValues(possibleNodes);
    form.addTextField().withDisplay("Scan Batch Size").withError(errorScanBatchSize).withName(P_scanBatchSize).withPlaceholder(DEFAULT_SCAN_BATCH_SIZE + "").withValue(scanBatchSize.orElse(null));
    form.addSelectField().withDisplay("Processor Name").withName(P_processorName).withValues(possibleProcessors);
    form.addButton().withDisplay("Create Joblets").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle("Table Processor - Joblets").withContent(Html.makeContent(form)).buildMav();
    }
    @SuppressWarnings("unchecked") PhysicalSortedStorageNode<PK, D, ?> sourceNode = (PhysicalSortedStorageNode<PK, D, ?>) nodes.getNode(nodeName.get());
    String tableName = sourceNode.getFieldInfo().getTableName();
    List<TableSample> samples = tableSamplerService.scanSamplesForNode(sourceNode).list();
    TableSampleKey previousSampleKey = null;
    List<JobletPackage> jobletPackages = new ArrayList<>();
    long totalItemsProcessed = 1;
    long counter = 1;
    int actualScanBatchSize = scanBatchSize.map(StringTool::nullIfEmpty).map(Integer::valueOf).orElse(DEFAULT_SCAN_BATCH_SIZE);
    long numJoblets = 0;
    for (TableSample sample : samples) {
        PK fromKeyExclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, previousSampleKey);
        PK toKeyInclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, sample.getKey());
        var jobletPackage = createJobletPackage(tableName, nodeName.get(), fromKeyExclusive, toKeyInclusive, actualScanBatchSize, processorName.get(), sample.getNumRows(), counter, numJoblets);
        jobletPackages.add(jobletPackage);
        ++numJoblets;
        counter++;
        totalItemsProcessed++;
        previousSampleKey = sample.getKey();
    }
    // include any rows created since the last sample
    PK fromKeyExclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, previousSampleKey);
    var jobletPackage = createJobletPackage(tableName, nodeName.get(), fromKeyExclusive, // open-ended
    null, actualScanBatchSize, processorName.get(), // we have no idea about the true estNumDatabeans
    1, counter, numJoblets);
    ++numJoblets;
    jobletPackages.add(jobletPackage);
    totalItemsProcessed++;
    // jobletPackage.size() == counter == numJoblets
    counter++;
    // shuffle as optimization to spread write load. could be optional
    Scanner.of(jobletPackages).shuffle().flush(jobletService::submitJobletPackages);
    changelogRecorderService.recordChangelogForTableProcessor(getSessionInfo(), "Joblet", nodeName.get(), processorName.get());
    return pageFactory.message(request, "jobletsCreated=" + numJoblets + " totalSamplesProcessed=" + totalItemsProcessed);
}
Also used : JobletService(io.datarouter.joblet.service.JobletService) Scanner(io.datarouter.scanner.Scanner) JobletPackage(io.datarouter.joblet.model.JobletPackage) TableSamplerTool(io.datarouter.nodewatch.util.TableSamplerTool) PrimaryKeyPercentCodecTool(io.datarouter.storage.util.PrimaryKeyPercentCodecTool) TableSample(io.datarouter.nodewatch.storage.tablesample.TableSample) TableProcessorJobletParams(io.datarouter.plugin.copytable.tableprocessor.TableProcessorJoblet.TableProcessorJobletParams) OptionalString(io.datarouter.web.handler.types.optional.OptionalString) TableSampleKey(io.datarouter.nodewatch.storage.tablesample.TableSampleKey) TagCreator.h2(j2html.TagCreator.h2) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) TagCreator.br(j2html.TagCreator.br) Config(io.datarouter.storage.config.Config) DatarouterNodes(io.datarouter.storage.node.DatarouterNodes) Param(io.datarouter.web.handler.types.Param) Bootstrap4FormHtml(io.datarouter.web.html.j2html.bootstrap4.Bootstrap4FormHtml) JobletPriority(io.datarouter.joblet.enums.JobletPriority) PhysicalSortedStorageNode(io.datarouter.storage.node.op.raw.SortedStorage.PhysicalSortedStorageNode) TableSamplerService(io.datarouter.nodewatch.service.TableSamplerService) TableProcessorRegistry(io.datarouter.plugin.copytable.tableprocessor.TableProcessorRegistry) Mav(io.datarouter.web.handler.mav.Mav) StringTool(io.datarouter.util.string.StringTool) PrimaryKey(io.datarouter.model.key.primary.PrimaryKey) ContainerTag(j2html.tags.ContainerTag) List(java.util.List) HtmlForm(io.datarouter.web.html.form.HtmlForm) Databean(io.datarouter.model.databean.Databean) TableProcessorJoblet(io.datarouter.plugin.copytable.tableprocessor.TableProcessorJoblet) BaseHandler(io.datarouter.web.handler.BaseHandler) Bootstrap4PageFactory(io.datarouter.web.html.j2html.bootstrap4.Bootstrap4PageFactory) TagCreator.div(j2html.TagCreator.div) StringTool(io.datarouter.util.string.StringTool) ArrayList(java.util.ArrayList) TableSample(io.datarouter.nodewatch.storage.tablesample.TableSample) OptionalString(io.datarouter.web.handler.types.optional.OptionalString) TableSampleKey(io.datarouter.nodewatch.storage.tablesample.TableSampleKey) PhysicalSortedStorageNode(io.datarouter.storage.node.op.raw.SortedStorage.PhysicalSortedStorageNode) HtmlForm(io.datarouter.web.html.form.HtmlForm) JobletPackage(io.datarouter.joblet.model.JobletPackage) BaseHandler(io.datarouter.web.handler.BaseHandler)

Aggregations

JobletPackage (io.datarouter.joblet.model.JobletPackage)6 JobletPriority (io.datarouter.joblet.enums.JobletPriority)3 Databean (io.datarouter.model.databean.Databean)3 Scanner (io.datarouter.scanner.Scanner)3 BaseHandler (io.datarouter.web.handler.BaseHandler)3 HtmlForm (io.datarouter.web.html.form.HtmlForm)3 ArrayList (java.util.ArrayList)3 List (java.util.List)3 Inject (javax.inject.Inject)3 JobletService (io.datarouter.joblet.service.JobletService)2 JobletRequest (io.datarouter.joblet.storage.jobletrequest.JobletRequest)2 PrimaryKey (io.datarouter.model.key.primary.PrimaryKey)2 TableSamplerService (io.datarouter.nodewatch.service.TableSamplerService)2 TableSample (io.datarouter.nodewatch.storage.tablesample.TableSample)2 TableSampleKey (io.datarouter.nodewatch.storage.tablesample.TableSampleKey)2 TableSamplerTool (io.datarouter.nodewatch.util.TableSamplerTool)2 Config (io.datarouter.storage.config.Config)2 DatarouterNodes (io.datarouter.storage.node.DatarouterNodes)2 PhysicalSortedStorageNode (io.datarouter.storage.node.op.raw.SortedStorage.PhysicalSortedStorageNode)2 PrimaryKeyPercentCodecTool (io.datarouter.storage.util.PrimaryKeyPercentCodecTool)2