-
Notifications
You must be signed in to change notification settings - Fork 120
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: replace usage of virtual threads for service output reading (#1520)
### Motivation A previous pull request replaced the old service output reading with a new, virtual thread based implementation. However, it was discovered that virtual threads are not actually a good fit for the use case, as file I/O is actually blocking on virtual threads and will block all other virtual thread operations. ### Modification Replace virtual threads for service output reading with a scaling scheduled thread pool to allow the same reading throughput regardless of the service count. ### Result The virtual thread scheduler is no longer blocked due to blocking file I/O operations due to service output reading.
- Loading branch information
Showing
4 changed files
with
209 additions
and
40 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
91 changes: 91 additions & 0 deletions
91
...ain/java/eu/cloudnetservice/node/service/defaults/log/ProcessServiceLogReadScheduler.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| /* | ||
| * Copyright 2019-2024 CloudNetService team & contributors | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package eu.cloudnetservice.node.service.defaults.log; | ||
|
|
||
| import com.google.common.util.concurrent.ThreadFactoryBuilder; | ||
| import jakarta.inject.Singleton; | ||
| import java.util.concurrent.ScheduledThreadPoolExecutor; | ||
| import java.util.concurrent.ThreadPoolExecutor; | ||
| import java.util.concurrent.TimeUnit; | ||
| import java.util.concurrent.atomic.AtomicInteger; | ||
| import lombok.NonNull; | ||
|
|
||
| @Singleton | ||
| public final class ProcessServiceLogReadScheduler { | ||
|
|
||
| private static final int LOG_READ_DELAY_MS = Integer.getInteger("cloudnet.process-log-read-delay", 25); | ||
| private static final int READ_WORKER_MAXIMUM = Integer.getInteger("cloudnet.process-log-worker-maximum", 25); | ||
| private static final int READ_ACTIONS_PER_WORKER = Integer.getInteger("cloudnet.process-log-actions-per-worker", 5); | ||
|
|
||
| private final AtomicInteger runningReaderActions; | ||
| private final ScheduledThreadPoolExecutor executor; | ||
|
|
||
| public ProcessServiceLogReadScheduler() { | ||
| var threadFactory = new ThreadFactoryBuilder() | ||
| .setDaemon(true) | ||
| .setPriority(Thread.NORM_PRIORITY) | ||
| .setNameFormat("process-log-reader-%d") | ||
| .build(); | ||
| this.executor = new ScheduledThreadPoolExecutor(1, threadFactory, new ThreadPoolExecutor.DiscardPolicy()); | ||
| this.runningReaderActions = new AtomicInteger(0); | ||
| } | ||
|
|
||
| public void schedule(@NonNull ProcessServiceLogCache logCache) { | ||
| var runningReaderActions = this.runningReaderActions.getAndIncrement(); | ||
| if (runningReaderActions != 0 && runningReaderActions % READ_ACTIONS_PER_WORKER == 0) { | ||
| var expectedWorkerCount = (runningReaderActions / READ_ACTIONS_PER_WORKER) + 1; | ||
| this.adjustWorkerCount(expectedWorkerCount); | ||
| } | ||
|
|
||
| var readTask = new ProcessServiceLogReadTask(logCache, this); | ||
| this.executor.scheduleWithFixedDelay(readTask, 0, LOG_READ_DELAY_MS, TimeUnit.MILLISECONDS); | ||
| } | ||
|
|
||
| private void notifyLogCacheReadEnd() { | ||
| var runningReaderActions = this.runningReaderActions.decrementAndGet(); | ||
| if (runningReaderActions != 0 && runningReaderActions % READ_ACTIONS_PER_WORKER == 0) { | ||
| var expectedWorkerCount = runningReaderActions / READ_ACTIONS_PER_WORKER; | ||
| this.adjustWorkerCount(expectedWorkerCount); | ||
| } | ||
| } | ||
|
|
||
| private void adjustWorkerCount(int expectedWorkerCount) { | ||
| var newCorePoolSize = Math.min(expectedWorkerCount, READ_WORKER_MAXIMUM); | ||
| if (this.executor.getCorePoolSize() != newCorePoolSize) { | ||
| this.executor.setCorePoolSize(expectedWorkerCount); | ||
| } | ||
| } | ||
|
|
||
| private record ProcessServiceLogReadTask( | ||
| @NonNull ProcessServiceLogCache logCache, | ||
| @NonNull ProcessServiceLogReadScheduler scheduler | ||
| ) implements Runnable { | ||
|
|
||
| private static final RuntimeException CANCEL_EXCEPTION = new RuntimeException("cancelled, reached stream EOF"); | ||
|
|
||
| @Override | ||
| public void run() { | ||
| // read the content from the stream, in case the stream closed notify the | ||
| // scheduler about this and stop scheduling the next by throwing an exception | ||
| var streamsStillOpen = this.logCache.readProcessOutputContent(); | ||
| if (!streamsStillOpen) { | ||
| this.scheduler.notifyLogCacheReadEnd(); | ||
| throw CANCEL_EXCEPTION; | ||
| } | ||
| } | ||
| } | ||
| } |