Examples with ShardWorker build.buildfarm.v1test.ShardWorker used on opensource projects

Example 1 with ShardWorker

use of build.buildfarm.v1test.ShardWorker in project bazel-buildfarm by bazelbuild.

the class RedisShardBackplane method removeInvalidWorkers.

private void removeInvalidWorkers(JedisCluster jedis, long testedAt, List<ShardWorker> workers) {
    if (!workers.isEmpty()) {
        for (ShardWorker worker : workers) {
            String name = worker.getEndpoint();
            String reason = format("registration expired at %d, tested at %d", worker.getExpireAt(), testedAt);
            WorkerChange workerChange = WorkerChange.newBuilder().setEffectiveAt(toTimestamp(Instant.now())).setName(name).setRemove(WorkerChange.Remove.newBuilder().setSource(source).setReason(reason).build()).build();
            try {
                String workerChangeJson = JsonFormat.printer().print(workerChange);
                removeWorkerAndPublish(jedis, name, workerChangeJson);
            } catch (InvalidProtocolBufferException e) {
                logger.log(Level.SEVERE, "error printing workerChange", e);
            }
        }
    }
}

Example 2 with ShardWorker

use of build.buildfarm.v1test.ShardWorker in project bazel-buildfarm by bazelbuild.

the class Worker method startFailsafeRegistration.

private void startFailsafeRegistration() {
    String endpoint = config.getPublicName();
    ShardWorker.Builder worker = ShardWorker.newBuilder().setEndpoint(endpoint);
    int registrationIntervalMillis = 10000;
    int registrationOffsetMillis = registrationIntervalMillis * 3;
    new Thread(new Runnable() {

        long workerRegistrationExpiresAt = 0;

        ShardWorker nextRegistration(long now) {
            return worker.setExpireAt(now + registrationOffsetMillis).build();
        }

        long nextInterval(long now) {
            return now + registrationIntervalMillis;
        }

        boolean isWorkerPausedFromNewWork() {
            try {
                File pausedFile = new File(config.getRoot() + "/.paused");
                if (pausedFile.exists() && !isPaused) {
                    isPaused = true;
                    logger.log(Level.INFO, "The current worker is paused from taking on new work!");
                    pipeline.stopMatchingOperations();
                    workerPausedMetric.inc();
                    runIndexerForWorker();
                }
            } catch (Exception e) {
                logger.log(Level.WARNING, "Could not open .paused file.", e);
            }
            return isPaused;
        }

        void registerIfExpired() {
            long now = System.currentTimeMillis();
            if (now >= workerRegistrationExpiresAt && !inGracefulShutdown && !isWorkerPausedFromNewWork()) {
                // worker must be registered to match
                addWorker(nextRegistration(now));
                // update every 10 seconds
                workerRegistrationExpiresAt = nextInterval(now);
            }
        }

        @Override
        public void run() {
            try {
                while (!server.isShutdown()) {
                    registerIfExpired();
                    SECONDS.sleep(1);
                }
            } catch (InterruptedException e) {
            // ignore
            } finally {
                try {
                    stop();
                } catch (InterruptedException ie) {
                    logger.log(SEVERE, "interrupted while stopping worker", ie);
                // ignore
                }
            }
        }
    }).start();
}

Example 3 with ShardWorker

use of build.buildfarm.v1test.ShardWorker in project bazel-buildfarm by bazelbuild.

the class WorkerProfile method fetchWorkers.

private static Set<String> fetchWorkers(JedisCluster jedis, RedisShardBackplaneConfig config, long now) {
    Set<String> workers = Sets.newConcurrentHashSet();
    for (Map.Entry<String, String> entry : jedis.hgetAll(config.getWorkersHashName()).entrySet()) {
        String json = entry.getValue();
        try {
            if (json != null) {
                ShardWorker.Builder builder = ShardWorker.newBuilder();
                JsonFormat.parser().merge(json, builder);
                ShardWorker worker = builder.build();
                if (worker.getExpireAt() > now) {
                    workers.add(worker.getEndpoint());
                }
            }
        } catch (InvalidProtocolBufferException e) {
            e.printStackTrace();
        }
    }
    return workers;
}

Example 4 with ShardWorker

use of build.buildfarm.v1test.ShardWorker in project bazel-buildfarm by bazelbuild.

the class RedisShardBackplane method fetchAndExpireWorkers.

private Set<String> fetchAndExpireWorkers(JedisCluster jedis, long now) {
    Set<String> workers = Sets.newConcurrentHashSet();
    ImmutableList.Builder<ShardWorker> invalidWorkers = ImmutableList.builder();
    for (Map.Entry<String, String> entry : jedis.hgetAll(config.getWorkersHashName()).entrySet()) {
        String json = entry.getValue();
        String name = entry.getKey();
        try {
            if (json == null) {
                invalidWorkers.add(ShardWorker.newBuilder().setEndpoint(name).build());
            } else {
                ShardWorker.Builder builder = ShardWorker.newBuilder();
                JsonFormat.parser().merge(json, builder);
                ShardWorker worker = builder.build();
                if (worker.getExpireAt() <= now) {
                    invalidWorkers.add(worker);
                } else {
                    workers.add(worker.getEndpoint());
                }
            }
        } catch (InvalidProtocolBufferException e) {
            invalidWorkers.add(ShardWorker.newBuilder().setEndpoint(name).build());
        }
    }
    removeInvalidWorkers(jedis, now, invalidWorkers.build());
    return workers;
}