diff --git a/.github/workflows/build_dev_manual.yml b/.github/workflows/build_dev_manual.yml index 0d6d96efe1..1068634203 100644 --- a/.github/workflows/build_dev_manual.yml +++ b/.github/workflows/build_dev_manual.yml @@ -25,6 +25,7 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 + - uses: sbt/setup-sbt@v1.1.5 - name: Generate dev documentation website run: sh ./scripts/doc.sh buildDev - name: Commit files diff --git a/.github/workflows/publish_documentation.yaml b/.github/workflows/publish_documentation.yaml index a5e1084997..0ee56e6012 100644 --- a/.github/workflows/publish_documentation.yaml +++ b/.github/workflows/publish_documentation.yaml @@ -18,6 +18,7 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 + - uses: sbt/setup-sbt@v1.1.5 - name: setup node uses: actions/setup-node@v3 with: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 00b0ab4c20..b4705ca272 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,6 +24,7 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 + - uses: sbt/setup-sbt@v1.1.5 - name: setup node uses: actions/setup-node@v3 with: diff --git a/.github/workflows/server_build_and_test.yaml b/.github/workflows/server_build_and_test.yaml index be7a0184d7..ef535f0227 100644 --- a/.github/workflows/server_build_and_test.yaml +++ b/.github/workflows/server_build_and_test.yaml @@ -18,6 +18,8 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 + - name: install sbt + uses: sbt/setup-sbt@v1.1.5 # setup node to use yarn - uses: actions/setup-node@v2 with: diff --git a/kubernetes/helm/otoroshi/crds-with-schema.yaml b/kubernetes/helm/otoroshi/crds-with-schema.yaml index 86e089dc32..2607e6bb18 100644 --- a/kubernetes/helm/otoroshi/crds-with-schema.yaml +++ b/kubernetes/helm/otoroshi/crds-with-schema.yaml @@ -2316,7 +2316,7 @@ spec: throttlingQuota: type: "integer" format: "int64" - description: "Authorized number of calls per second globally, measured\ + description: "Authorized number of calls per window globally, measured\ \ on 10 seconds" anonymousReporting: type: "boolean" @@ -3325,7 +3325,7 @@ spec: perIpThrottlingQuota: type: "integer" format: "int64" - description: "Authorized number of calls per second globally per IP\ + description: "Authorized number of calls per window globally per IP\ \ address, measured on 10 seconds" useCircuitBreakers: type: "boolean" @@ -3501,7 +3501,7 @@ spec: throttlingQuota: type: "integer" format: "int64" - description: "Authorized number of calls per second, measured on 10\ + description: "Authorized number of calls per window, measured on 10\ \ seconds" constrainedServicesOnly: type: "boolean" diff --git a/otoroshi/app/controllers/SwaggerController.scala b/otoroshi/app/controllers/SwaggerController.scala index 09af6c82ab..ca5110f67f 100644 --- a/otoroshi/app/controllers/SwaggerController.scala +++ b/otoroshi/app/controllers/SwaggerController.scala @@ -896,7 +896,7 @@ class SwaggerController(cc: ControllerComponents, assetsBuilder: AssetsBuilder)( "clientName" -> SimpleStringType ~~> "The name of the api key, for humans ;-)", "authorizedEntities" -> SimpleArrayType ~~> "The group/service ids (prefixed by group_ or service_ on which the key is authorized", "enabled" -> SimpleBooleanType ~~> "Whether or not the key is enabled. If disabled, resources won't be available to calls using this key", - "throttlingQuota" -> SimpleLongType ~~> "Authorized number of calls per second, measured on 10 seconds", + "throttlingQuota" -> SimpleLongType ~~> "Authorized number of calls per window, measured on 10 seconds", "dailyQuota" -> SimpleLongType ~~> "Authorized number of calls per day", "monthlyQuota" -> SimpleLongType ~~> "Authorized number of calls per month", "metadata" -> SimpleObjectType ~~> "Bunch of metadata for the key" @@ -990,8 +990,8 @@ class SwaggerController(cc: ControllerComponents, assetsBuilder: AssetsBuilder)( "apiReadOnly" -> SimpleBooleanType ~~> "If enabled, Admin API won't be able to write/update/delete entities", "u2fLoginOnly" -> SimpleBooleanType ~~> "If enabled, login to backoffice through Auth0 will be disabled", "ipFiltering" -> Ref("IpFiltering"), - "throttlingQuota" -> SimpleLongType ~~> "Authorized number of calls per second globally, measured on 10 seconds", - "perIpThrottlingQuota" -> SimpleLongType ~~> "Authorized number of calls per second globally per IP address, measured on 10 seconds", + "throttlingQuota" -> SimpleLongType ~~> "Authorized number of calls per window globally", + "perIpThrottlingQuota" -> SimpleLongType ~~> "Authorized number of calls per window globally per IP address", "elasticWritesConfigs" -> ArrayOf(Ref("ElasticConfig")) ~~> "Configs. for Elastic writes", "elasticReadsConfig" -> Ref("ElasticConfig") ~~> "Config. for elastic reads", "analyticsWebhooks" -> ArrayOf(Ref("Webhook")) ~~> "Webhook that will receive all internal Otoroshi events", @@ -1176,9 +1176,9 @@ class SwaggerController(cc: ControllerComponents, assetsBuilder: AssetsBuilder)( "description" -> "Quotas state for an api key on a service group", "type" -> "object", "required" -> Json.arr( - "authorizedCallsPerSec", - "currentCallsPerSec", - "remainingCallsPerSec", + "authorizedCallsPerWindow", + "throttlingCallsPerWindow", + "remainingCallsPerWindow", "authorizedCallsPerDay", "currentCallsPerDay", "remainingCallsPerDay", @@ -1187,9 +1187,9 @@ class SwaggerController(cc: ControllerComponents, assetsBuilder: AssetsBuilder)( "remainingCallsPerMonth" ), "properties" -> Json.obj( - "authorizedCallsPerSec" -> SimpleLongType ~~> "The number of authorized calls per second", - "currentCallsPerSec" -> SimpleLongType ~~> "The current number of calls per second", - "remainingCallsPerSec" -> SimpleLongType ~~> "The remaining number of calls per second", + "authorizedCallsPerWindow" -> SimpleLongType ~~> "The number of authorized calls per window", + "throttlingCallsPerWindow" -> SimpleLongType ~~> "The current number of calls per window", + "remainingCallsPerWindow" -> SimpleLongType ~~> "The remaining number of calls per window", "authorizedCallsPerDay" -> SimpleLongType ~~> "The number of authorized calls per day", "currentCallsPerDay" -> SimpleLongType ~~> "The current number of calls per day", "remainingCallsPerDay" -> SimpleLongType ~~> "The remaining number of calls per day", diff --git a/otoroshi/app/gateway/generic.scala b/otoroshi/app/gateway/generic.scala index 6996cf7534..b730328d56 100644 --- a/otoroshi/app/gateway/generic.scala +++ b/otoroshi/app/gateway/generic.scala @@ -1279,7 +1279,7 @@ object ReverseProxyHelper { val quota = maybeQuota.getOrElse(globalConfig.perIpThrottlingQuota) val (restrictionsNotPassing, restrictionsResponse) = descriptor.restrictions.handleRestrictions(descriptor.id, descriptor.some, None, req, attrs) - if (secCalls > (quota * 10L)) { + if (secCalls > quota) { errorResult(TooManyRequests, "[IP] You performed too much requests", "errors.too.much.requests") } else { if (!isSecured && descriptor.forceHttps) { diff --git a/otoroshi/app/models/apikey.scala b/otoroshi/app/models/apikey.scala index 32b4cae91d..42a679c70e 100644 --- a/otoroshi/app/models/apikey.scala +++ b/otoroshi/app/models/apikey.scala @@ -45,15 +45,9 @@ import scala.jdk.CollectionConverters.asScalaBufferConverter import scala.util.{Failure, Success, Try} case class RemainingQuotas( - // secCalls: Long = RemainingQuotas.MaxValue, - // secCallsRemaining: Long = RemainingQuotas.MaxValue, - // dailyCalls: Long = RemainingQuotas.MaxValue, - // dailyCallsRemaining: Long = RemainingQuotas.MaxValue, - // monthlyCalls: Long = RemainingQuotas.MaxValue, - // monthlyCallsRemaining: Long = RemainingQuotas.MaxValue - authorizedCallsPerSec: Long = RemainingQuotas.MaxValue, - currentCallsPerSec: Long = RemainingQuotas.MaxValue, - remainingCallsPerSec: Long = RemainingQuotas.MaxValue, + authorizedCallsPerWindow: Long = RemainingQuotas.MaxValue, + throttlingCallsPerWindow: Long = RemainingQuotas.MaxValue, + remainingCallsPerWindow: Long = RemainingQuotas.MaxValue, authorizedCallsPerDay: Long = RemainingQuotas.MaxValue, currentCallsPerDay: Long = RemainingQuotas.MaxValue, remainingCallsPerDay: Long = RemainingQuotas.MaxValue, @@ -61,12 +55,42 @@ case class RemainingQuotas( currentCallsPerMonth: Long = RemainingQuotas.MaxValue, remainingCallsPerMonth: Long = RemainingQuotas.MaxValue ) { - def toJson: JsObject = RemainingQuotas.fmt.writes(this) + def toJson: JsObject = RemainingQuotas.fmt.writes(this).as[JsObject] } object RemainingQuotas { val MaxValue: Long = 10000000L - implicit val fmt = Json.format[RemainingQuotas] + implicit val fmt = new Format[RemainingQuotas] { + + override def reads(json: JsValue): JsResult[RemainingQuotas] = Try { + RemainingQuotas( + authorizedCallsPerWindow = json.select("authorizedCallsPerWindow").asOpt[Long].getOrElse(json.select("authorizedCallsPerSec").asLong), + throttlingCallsPerWindow = json.select("throttlingCallsPerWindow").asOpt[Long].getOrElse(json.select("currentCallsPerSec").asLong), + remainingCallsPerWindow = json.select("remainingCallsPerWindow").asOpt[Long].getOrElse(json.select("authorizedCallsPerSec").asLong), + authorizedCallsPerDay = json.select("authorizedCallsPerDay").asLong, + currentCallsPerDay = json.select("currentCallsPerDay").asLong, + remainingCallsPerDay = json.select("remainingCallsPerDay").asLong, + authorizedCallsPerMonth = json.select("authorizedCallsPerMonth").asLong, + currentCallsPerMonth = json.select("currentCallsPerMonth").asLong, + remainingCallsPerMonth = json.select("remainingCallsPerMonth").asLong, + ) + } match { + case Failure(e) => JsError(e.getMessage) + case Success(e) => JsSuccess(e) + } + + override def writes(o: RemainingQuotas): JsValue = Json.obj( + "authorizedCallsPerWindow" -> o.authorizedCallsPerWindow, + "throttlingCallsPerWindow" -> o.throttlingCallsPerWindow, + "remainingCallsPerWindow" -> o.remainingCallsPerWindow, + "authorizedCallsPerDay" -> o.authorizedCallsPerDay, + "currentCallsPerDay" -> o.currentCallsPerDay, + "remainingCallsPerDay" -> o.remainingCallsPerDay, + "authorizedCallsPerMonth" -> o.authorizedCallsPerMonth, + "currentCallsPerMonth" -> o.currentCallsPerMonth, + "remainingCallsPerMonth" -> o.remainingCallsPerMonth, + ) + } } case class ApiKeyRotation( @@ -244,7 +268,7 @@ case class ApiKey( quotas <- env.datastores.apiKeyDataStore.remainingQuotas(this) rotation <- env.datastores.apiKeyDataStore.keyRotation(this) } yield { - val within = (quotas.currentCallsPerSec <= (throttlingQuota * env.throttlingWindow)) && + val within = quotas.remainingCallsPerWindow > 0 && (quotas.currentCallsPerDay < dailyQuota) && (quotas.currentCallsPerMonth < monthlyQuota) (within, rotation, quotas) @@ -1906,9 +1930,9 @@ object ApiKeyHelper { .build Try(verifier.verify(jwt)) .filter { token => - val aud = token.getAudience.asScala.headOption.filter(v => + val aud = Option(token.getAudience).flatMap(_.asScala.headOption.filter(v => v.startsWith("http://") || v.startsWith("https://") - ) + )) if (aud.isDefined) { val currentUrl = req.theUrl val audience = aud.get diff --git a/otoroshi/app/models/config.scala b/otoroshi/app/models/config.scala index 34358bf884..3f694dbf97 100644 --- a/otoroshi/app/models/config.scala +++ b/otoroshi/app/models/config.scala @@ -1026,7 +1026,7 @@ object GlobalConfig { } trait GlobalConfigDataStore extends BasicStore[GlobalConfig] { - def incrementCallsForIpAddressWithTTL(ip: String, ttl: Int = 10)(implicit ec: ExecutionContext): Future[Long] + def incrementCallsForIpAddress(ip: String)(implicit ec: ExecutionContext): Future[Long] def quotaForIpAddress(ip: String)(implicit ec: ExecutionContext): Future[Option[Long]] def isOtoroshiEmpty()(implicit ec: ExecutionContext): Future[Boolean] def withinThrottlingQuota()(implicit ec: ExecutionContext, env: Env): Future[Boolean] diff --git a/otoroshi/app/next/plugins/context.scala b/otoroshi/app/next/plugins/context.scala index 97737679d5..dea43e0948 100644 --- a/otoroshi/app/next/plugins/context.scala +++ b/otoroshi/app/next/plugins/context.scala @@ -113,9 +113,9 @@ class ContextValidation extends NgAccessValidator { | "otoroshi.core.RequestWebsocket" : false, | "otoroshi.core.RequestCounterOut" : 0, | "otoroshi.core.RemainingQuotas" : { - | "authorizedCallsPerSec" : 10000000, - | "currentCallsPerSec" : 0, - | "remainingCallsPerSec" : 10000000, + | "authorizedCallsPerWindow" : 10000000, + | "throttlingCallsPerWindow" : 0, + | "remainingCallsPerWindow" : 10000000, | "authorizedCallsPerDay" : 10000000, | "currentCallsPerDay" : 2, | "remainingCallsPerDay" : 9999998, diff --git a/otoroshi/app/next/plugins/quotas.scala b/otoroshi/app/next/plugins/quotas.scala index 2bfb625abb..1acb4f1f0f 100644 --- a/otoroshi/app/next/plugins/quotas.scala +++ b/otoroshi/app/next/plugins/quotas.scala @@ -63,7 +63,7 @@ class GlobalPerIpAddressThrottling extends NgAccessValidator { ): Future[NgAccess] = { val globalConfig = env.datastores.globalConfigDataStore.latest() val quota = quotas.maybeQuota.getOrElse(globalConfig.perIpThrottlingQuota) - if (quotas.secCalls > (quota * 10L)) { + if (quotas.secCalls > quota) { errorResult(ctx, Results.TooManyRequests, "[IP] You performed too much requests", "errors.too.much.requests") } else { NgAccess.NgAllowed.vfuture @@ -266,7 +266,7 @@ class NgServiceQuotas extends NgAccessValidator { )(implicit ec: ExecutionContext, env: Env): Future[Boolean] = env.datastores.rawDataStore .get(throttlingKey(route.id)) - .map(_.map(_.utf8String.toLong).getOrElse(0L) <= (qconf.throttlingQuota * env.throttlingWindow)) + .map(_.map(_.utf8String.toLong).getOrElse(0L) <= qconf.throttlingQuota) private def withinDailyQuota(route: NgRoute, qconf: NgServiceQuotasConfig)(implicit ec: ExecutionContext, @@ -559,7 +559,7 @@ object NgCustomThrottling { for { secCalls <- env.datastores.rawDataStore.incrby(throttlingKey(expr, group), increment) secTtl <- env.datastores.rawDataStore.pttl(throttlingKey(expr, group)).filter(_ > -1).recoverWith { case _ => - env.datastores.rawDataStore.pexpire(throttlingKey(expr, group), ttl) // env.throttlingWindow * 1000) + env.datastores.rawDataStore.pexpire(throttlingKey(expr, group), ttl) // env.throttlingWindow * 1000 } } yield () } @@ -593,7 +593,7 @@ class NgCustomThrottling extends NgAccessValidator { )(implicit ec: ExecutionContext, env: Env): Future[Boolean] = { env.datastores.rawDataStore .get(NgCustomThrottling.throttlingKey(qconf.computeExpression(ctx, env), qconf.computeGroup(ctx, env))) - .map(_.map(_.utf8String.toLong).getOrElse(0L) <= (qconf.throttlingQuota * env.throttlingWindow)) + .map(_.map(_.utf8String.toLong).getOrElse(0L) <= qconf.throttlingQuota) } def forbidden(ctx: NgAccessContext)(implicit env: Env, ec: ExecutionContext): Future[NgAccess] = { diff --git a/otoroshi/app/next/proxy/engine.scala b/otoroshi/app/next/proxy/engine.scala index f2a85b1b9d..55a844a2d5 100644 --- a/otoroshi/app/next/proxy/engine.scala +++ b/otoroshi/app/next/proxy/engine.scala @@ -2017,8 +2017,10 @@ class ProxyEngine() extends RequestHandler { // quotasValidationFor increments calls for ip address FEither(env.datastores.globalConfigDataStore.quotasValidationFor(remoteAddress).flatMap { r => val (within, secCalls, maybeQuota) = r +// println(s"maybe quota - $maybeQuota") val quota = maybeQuota.getOrElse(globalConfig.perIpThrottlingQuota) - if (secCalls > (quota * 10L)) { + // if (secCalls > (quota * 10L)) { + if (secCalls > quota) { errorResult(Results.TooManyRequests, "[IP] You performed too much requests", "errors.too.much.requests") } else { if (!within) { diff --git a/otoroshi/app/openapi/openapi-cfg.json b/otoroshi/app/openapi/openapi-cfg.json index 9ca89f1954..04d68c2799 100644 --- a/otoroshi/app/openapi/openapi-cfg.json +++ b/otoroshi/app/openapi/openapi-cfg.json @@ -1594,7 +1594,7 @@ "otoroshi.models.ApiKey.restrictions": "Apikey restrictions settings", "otoroshi.models.ApiKey.rotation": "Apikey rotation settings", "otoroshi.models.ApiKey.tags": "Apikey tags", - "otoroshi.models.ApiKey.throttlingQuota": "Authorized number of calls per second, measured on 10 seconds", + "otoroshi.models.ApiKey.throttlingQuota": "Authorized number of calls per window", "otoroshi.models.ApiKey.validUntil": "Date until when the apikey is valid", "otoroshi.models.ApiKeyConstraints.basicAuth": "Settings to extract basic auth style apikey", "otoroshi.models.ApiKeyConstraints.clientIdAuth": "Settings to extract client_id only apikey", @@ -1798,7 +1798,7 @@ "otoroshi.models.GlobalConfig.metadata": "Entity metadata", "otoroshi.models.GlobalConfig.middleFingers": "Use middle finger emoji as a response character for endless HTTP responses", "otoroshi.models.GlobalConfig.otoroshiId": "Unique id for this otoroshi instance", - "otoroshi.models.GlobalConfig.perIpThrottlingQuota": "Authorized number of calls per second globally per IP address, measured on 10 seconds", + "otoroshi.models.GlobalConfig.perIpThrottlingQuota": "Authorized number of calls per window globally per IP address", "otoroshi.models.GlobalConfig.plugins": "global plugins settings", "otoroshi.models.GlobalConfig.proxies": "Web proxies settings", "otoroshi.models.GlobalConfig.quotasSettings": "Settings to generate alert when an apikey almost exceeded or exceeded its quotas", @@ -1808,7 +1808,7 @@ "otoroshi.models.GlobalConfig.streamEntityOnly": "HTTP will be streamed only. Doesn't work with old browsers", "otoroshi.models.GlobalConfig.tags": "Entity tags", "otoroshi.models.GlobalConfig.templates": "The otoroshi default templates for entities", - "otoroshi.models.GlobalConfig.throttlingQuota": "Authorized number of calls per second globally, measured on 10 seconds", + "otoroshi.models.GlobalConfig.throttlingQuota": "Authorized number of calls per window globally", "otoroshi.models.GlobalConfig.tlsSettings": "TLS settings", "otoroshi.models.GlobalConfig.trustXForwarded": "Use X-Forwarded-* headers for routing", "otoroshi.models.GlobalConfig.u2fLoginOnly": "If enabled, login to backoffice through Auth0 will be disabled", @@ -1957,13 +1957,13 @@ "otoroshi.models.RegionMatch.region": "Region name", "otoroshi.models.RemainingQuotas.authorizedCallsPerDay": "Number of authorized call per day", "otoroshi.models.RemainingQuotas.authorizedCallsPerMonth": "Number of authorized call per month", - "otoroshi.models.RemainingQuotas.authorizedCallsPerSec": "Number of authorized call per second", + "otoroshi.models.RemainingQuotas.authorizedCallsPerWindow": "Number of authorized call per window", "otoroshi.models.RemainingQuotas.currentCallsPerDay": "Current number of call per day", "otoroshi.models.RemainingQuotas.currentCallsPerMonth": "Current number of call per month", - "otoroshi.models.RemainingQuotas.currentCallsPerSec": "Current number of call per second", + "otoroshi.models.RemainingQuotas.throttlingCallsPerWindow": "Current number of call per window", "otoroshi.models.RemainingQuotas.remainingCallsPerDay": "Remaining number of call per day", "otoroshi.models.RemainingQuotas.remainingCallsPerMonth": "Remaining number of call per month", - "otoroshi.models.RemainingQuotas.remainingCallsPerSec": "Remaining number of call per second", + "otoroshi.models.RemainingQuotas.remainingCallsPerWindow": "Remaining number of call per window", "otoroshi.models.RestrictionPath.method": "Method of the http request", "otoroshi.models.RestrictionPath.path": "Path of the http request", "otoroshi.models.Restrictions.allowLast": "Evalute allowed paths after everything else", diff --git a/otoroshi/app/plugins/quotas.scala b/otoroshi/app/plugins/quotas.scala index 74c236c6e6..229dce6571 100644 --- a/otoroshi/app/plugins/quotas.scala +++ b/otoroshi/app/plugins/quotas.scala @@ -195,10 +195,10 @@ class ServiceQuotas extends AccessValidator { env.clusterAgent.incrementApi(descriptor.id, increment) for { _ <- env.datastores.rawDataStore.incrby(totalCallsKey(descriptor.id), increment) - secCalls <- env.datastores.rawDataStore.incrby(throttlingKey(descriptor.id), increment) secTtl <- env.datastores.rawDataStore.pttl(throttlingKey(descriptor.id)).filter(_ > -1).recoverWith { case _ => - env.datastores.rawDataStore.pexpire(throttlingKey(descriptor.id), env.throttlingWindow * 1000) - } + env.datastores.rawDataStore.pexpire(throttlingKey(descriptor.id), env.throttlingWindow * 1000) + } + secCalls <- env.datastores.rawDataStore.incrby(throttlingKey(descriptor.id), increment) dailyCalls <- env.datastores.rawDataStore.incrby(dailyQuotaKey(descriptor.id), increment) dailyTtl <- env.datastores.rawDataStore.pttl(dailyQuotaKey(descriptor.id)).filter(_ > -1).recoverWith { case _ => env.datastores.rawDataStore.pexpire(dailyQuotaKey(descriptor.id), toDayEnd.toInt) @@ -209,9 +209,9 @@ class ServiceQuotas extends AccessValidator { } } yield () // RemainingQuotas( - // authorizedCallsPerSec = qconf.throttlingQuota, - // currentCallsPerSec = (secCalls / env.throttlingWindow).toInt, - // remainingCallsPerSec = qconf.throttlingQuota - (secCalls / env.throttlingWindow).toInt, + // authorizedCallsPerWindow = qconf.throttlingQuota, + // throttlingCallsPerWindow = (secCalls / env.throttlingWindow).toInt, + // remainingCallsPerWindow = qconf.throttlingQuota - (secCalls / env.throttlingWindow).toInt, // authorizedCallsPerDay = qconf.dailyQuota, // currentCallsPerDay = dailyCalls, // remainingCallsPerDay = qconf.dailyQuota - dailyCalls, @@ -238,7 +238,7 @@ class ServiceQuotas extends AccessValidator { env.datastores.rawDataStore .get(throttlingKey(descriptor.id)) .fast - .map(_.map(_.utf8String.toLong).getOrElse(0L) <= (qconf.throttlingQuota * env.throttlingWindow)) + .map(_.map(_.utf8String.toLong).getOrElse(0L) <= qconf.throttlingQuota) private def withinDailyQuota(descriptor: ServiceDescriptor, qconf: ServiceQuotasConfig)(implicit ec: ExecutionContext, diff --git a/otoroshi/app/storage/drivers/inmemory/InMemoryDataStores.scala b/otoroshi/app/storage/drivers/inmemory/InMemoryDataStores.scala index 8de69a7984..929e23cb9b 100644 --- a/otoroshi/app/storage/drivers/inmemory/InMemoryDataStores.scala +++ b/otoroshi/app/storage/drivers/inmemory/InMemoryDataStores.scala @@ -49,14 +49,13 @@ class InMemoryDataStores( ) val materializer = Materializer(actorSystem) val _optimized = configuration.getOptionalWithFileSupport[Boolean]("app.inmemory.optimized").getOrElse(false) - val _modern = configuration.getOptionalWithFileSupport[Boolean]("app.inmemory.modern").getOrElse(false) + val _modern = configuration.getOptionalWithFileSupport[Boolean]("app.inmemory.modern").getOrElse(true) // lazy val redis = new SwappableInMemoryRedis(_optimized, env, actorSystem) - lazy val _redis = if (_modern) { + lazy val swredis = if (_modern) { new ModernSwappableInMemoryRedis(_optimized, env, actorSystem) } else { new SwappableInMemoryRedis(_optimized, env, actorSystem) } - lazy val swredis = if (env.isDev) new SwappableRedisLikeMetricsWrapper(_redis, env) else _redis def redis(): otoroshi.storage.RedisLike = swredis diff --git a/otoroshi/app/storage/drivers/inmemory/SwappableInMemoryRedis.scala b/otoroshi/app/storage/drivers/inmemory/SwappableInMemoryRedis.scala index dcc708fd22..2c95f999b3 100644 --- a/otoroshi/app/storage/drivers/inmemory/SwappableInMemoryRedis.scala +++ b/otoroshi/app/storage/drivers/inmemory/SwappableInMemoryRedis.scala @@ -12,7 +12,7 @@ import otoroshi.utils.syntax.implicits.BetterSyntax import play.api.Logger import play.api.libs.json.{JsValue, Json} -import java.util.concurrent.atomic.AtomicReference +import java.util.concurrent.atomic.{AtomicLong, AtomicReference} import java.util.concurrent.{ConcurrentHashMap, TimeUnit} import java.util.regex.Pattern import scala.collection.concurrent.TrieMap @@ -513,7 +513,15 @@ class ModernSwappableInMemoryRedis(_optimized: Boolean, env: Env, actorSystem: A def rawGet(key: String): Future[Option[Any]] = memory.get(key).future - override def get(key: String): Future[Option[ByteString]] = memory.getTyped[ByteString](key).future + override def get(key: String): Future[Option[ByteString]] = { + //memory.getTyped[ByteString](key).future + memory.get(key) match { + case Some(bs: ByteString) => bs.some.vfuture + case Some(counter: AtomicLong) => ByteString(counter.get().toString).some.vfuture + case _ => None.vfuture + } + } + override def set( key: String, @@ -546,10 +554,32 @@ class ModernSwappableInMemoryRedis(_optimized: Boolean, env: Env, actorSystem: A override def incr(key: String): Future[Long] = incrby(key, 1L) override def incrby(key: String, increment: Long): Future[Long] = { - val value: Long = memory.getTyped[ByteString](key).map(_.utf8String.toLong).getOrElse(0L) - val newValue: Long = value + increment - memory.put(key, ByteString(newValue.toString)) - newValue.future + (memory.get(key) match { + case Some(bs: ByteString) => { + val asLng = bs.utf8String.toLong + val cnt = new AtomicLong(asLng) + val fcnt = cnt.addAndGet(increment) + memory.put(key, cnt) + fcnt + } + case Some(cnt: AtomicLong) => { + val fcnt = cnt.addAndGet(increment) + memory.put(key, cnt) + fcnt + } + case _ => { + val cnt = new AtomicLong(increment) + memory.put(key, cnt) match { + case Some(bs: ByteString) => cnt.addAndGet(bs.utf8String.toLong) + case Some(c: AtomicLong) => cnt.addAndGet(c.get()) + case _ => increment + } + } + }).vfuture + // val value: Long = memory.getTyped[ByteString](key).map(_.utf8String.toLong).getOrElse(0L) + // val newValue: Long = value + increment + // memory.put(key, ByteString(newValue.toString)) + // newValue.future } override def exists(key: String): Future[Boolean] = memory.containsKey(key).future @@ -693,4 +723,4 @@ class ModernSwappableInMemoryRedis(_optimized: Boolean, env: Env, actorSystem: A } override def health()(implicit ec: ExecutionContext): Future[DataStoreHealth] = Healthy.future -} +} \ No newline at end of file diff --git a/otoroshi/app/storage/drivers/reactivepg/reactivepg.scala b/otoroshi/app/storage/drivers/reactivepg/reactivepg.scala index a1f5bdf4a0..7829e2d219 100644 --- a/otoroshi/app/storage/drivers/reactivepg/reactivepg.scala +++ b/otoroshi/app/storage/drivers/reactivepg/reactivepg.scala @@ -291,7 +291,7 @@ class ReactivePgDataStores( def setupCleanup(): Unit = { implicit val ec = reactivePgActorSystem.dispatcher - cancel.set(reactivePgActorSystem.scheduler.scheduleAtFixedRate(10.seconds, 30.second)(SchedulerHelper.runnable { + cancel.set(reactivePgActorSystem.scheduler.scheduleAtFixedRate(10.seconds, 30.seconds)(SchedulerHelper.runnable { try { client .query(s"DELETE FROM $schemaDotTable WHERE (ttl_starting_at + ttl) < NOW();") @@ -876,7 +876,12 @@ class ReactivePgRedis( |values ($$1, 'counter', $$2) |on conflict (key) |do - | update set type = 'counter', counter = $schemaDotTable.counter + $$2 returning counter; + | update set type = 'counter', + | counter = CASE + | WHEN ($schemaDotTable.ttl_starting_at + $schemaDotTable.ttl > NOW()) THEN $schemaDotTable.counter + $$2 + | ELSE $$2 + | END + | returning counter |""".stripMargin, Seq(key, increment.asInstanceOf[AnyRef]) ) { row => diff --git a/otoroshi/app/storage/stores/KvApiKeyDataStore.scala b/otoroshi/app/storage/stores/KvApiKeyDataStore.scala index dc3683119f..a06ab15d6f 100644 --- a/otoroshi/app/storage/stores/KvApiKeyDataStore.scala +++ b/otoroshi/app/storage/stores/KvApiKeyDataStore.scala @@ -77,19 +77,21 @@ class KvApiKeyDataStore(redisCli: RedisLike, _env: Env) extends ApiKeyDataStore override def remainingQuotas(apiKey: ApiKey)(implicit ec: ExecutionContext, env: Env): Future[RemainingQuotas] = for { - secCalls <- redisCli.get(throttlingKey(apiKey.clientId)).fast.map(_.map(_.utf8String.toLong).getOrElse(0L)) - dailyCalls <- redisCli.get(dailyQuotaKey(apiKey.clientId)).fast.map(_.map(_.utf8String.toLong).getOrElse(0L)) - monthlyCalls <- redisCli.get(monthlyQuotaKey(apiKey.clientId)).fast.map(_.map(_.utf8String.toLong).getOrElse(0L)) + throttlingCallsPerWindow <- redisCli.get(throttlingKey(apiKey.clientId)).fast.map(_.map(_.utf8String.toLong).getOrElse(0L)) + dailyCalls <- redisCli.get(dailyQuotaKey(apiKey.clientId)).fast.map(_.map(_.utf8String.toLong).getOrElse(0L)) + monthlyCalls <- redisCli.get(monthlyQuotaKey(apiKey.clientId)).fast.map(_.map(_.utf8String.toLong).getOrElse(0L)) } yield RemainingQuotas( - authorizedCallsPerSec = apiKey.throttlingQuota, - currentCallsPerSec = (secCalls / env.throttlingWindow).toInt, - remainingCallsPerSec = apiKey.throttlingQuota - (secCalls / env.throttlingWindow).toInt, + authorizedCallsPerWindow = apiKey.throttlingQuota, + throttlingCallsPerWindow = throttlingCallsPerWindow, + remainingCallsPerWindow = Math.max(0, apiKey.throttlingQuota - throttlingCallsPerWindow), + authorizedCallsPerDay = apiKey.dailyQuota, currentCallsPerDay = dailyCalls, - remainingCallsPerDay = apiKey.dailyQuota - dailyCalls, + remainingCallsPerDay = Math.max(0, apiKey.dailyQuota - dailyCalls), + authorizedCallsPerMonth = apiKey.monthlyQuota, currentCallsPerMonth = monthlyCalls, - remainingCallsPerMonth = apiKey.monthlyQuota - monthlyCalls + remainingCallsPerMonth = Math.max(0, apiKey.monthlyQuota - monthlyCalls) ) override def resetQuotas(apiKey: ApiKey)(implicit ec: ExecutionContext, env: Env): Future[RemainingQuotas] = { @@ -111,9 +113,9 @@ class KvApiKeyDataStore(redisCli: RedisLike, _env: Env) extends ApiKeyDataStore redisCli.expire(monthlyQuotaKey(apiKey.clientId), (toMonthEnd / 1000).toInt) } } yield RemainingQuotas( - authorizedCallsPerSec = apiKey.throttlingQuota, - currentCallsPerSec = (0L / env.throttlingWindow).toInt, - remainingCallsPerSec = apiKey.throttlingQuota - (0L / env.throttlingWindow).toInt, + authorizedCallsPerWindow = apiKey.throttlingQuota, + throttlingCallsPerWindow = 0, + remainingCallsPerWindow = 0, authorizedCallsPerDay = apiKey.dailyQuota, currentCallsPerDay = 0, remainingCallsPerDay = apiKey.dailyQuota - 0, @@ -134,29 +136,34 @@ class KvApiKeyDataStore(redisCli: RedisLike, _env: Env) extends ApiKeyDataStore env.clusterAgent.incrementApi(apiKey.clientId, increment) for { _ <- redisCli.incrby(totalCallsKey(apiKey.clientId), increment) - secCalls <- redisCli.incrby(throttlingKey(apiKey.clientId), increment) + secTtl <- redisCli.pttl(throttlingKey(apiKey.clientId)).filter(_ > -1).recoverWith { case _ => - redisCli.expire(throttlingKey(apiKey.clientId), env.throttlingWindow) - } - dailyCalls <- redisCli.incrby(dailyQuotaKey(apiKey.clientId), increment) + redisCli.expire(throttlingKey(apiKey.clientId), env.throttlingWindow) + } + secCalls <- redisCli.incrby(throttlingKey(apiKey.clientId), increment) + dailyTtl <- redisCli.pttl(dailyQuotaKey(apiKey.clientId)).filter(_ > -1).recoverWith { case _ => - redisCli.expire(dailyQuotaKey(apiKey.clientId), (toDayEnd / 1000).toInt) - } - monthlyCalls <- redisCli.incrby(monthlyQuotaKey(apiKey.clientId), increment) + redisCli.expire(dailyQuotaKey(apiKey.clientId), (toDayEnd / 1000).toInt) + } + dailyCalls <- redisCli.incrby(dailyQuotaKey(apiKey.clientId), increment) + monthlyTtl <- redisCli.pttl(monthlyQuotaKey(apiKey.clientId)).filter(_ > -1).recoverWith { case _ => - redisCli.expire(monthlyQuotaKey(apiKey.clientId), (toMonthEnd / 1000).toInt) - } - } yield RemainingQuotas( - authorizedCallsPerSec = apiKey.throttlingQuota, - currentCallsPerSec = (secCalls / env.throttlingWindow).toInt, - remainingCallsPerSec = apiKey.throttlingQuota - (secCalls / env.throttlingWindow).toInt, - authorizedCallsPerDay = apiKey.dailyQuota, - currentCallsPerDay = dailyCalls, - remainingCallsPerDay = apiKey.dailyQuota - dailyCalls, - authorizedCallsPerMonth = apiKey.monthlyQuota, - currentCallsPerMonth = monthlyCalls, - remainingCallsPerMonth = apiKey.monthlyQuota - monthlyCalls - ) + redisCli.expire(monthlyQuotaKey(apiKey.clientId), (toMonthEnd / 1000).toInt) + } + monthlyCalls <- redisCli.incrby(monthlyQuotaKey(apiKey.clientId), increment) + } yield { + RemainingQuotas( + authorizedCallsPerWindow = apiKey.throttlingQuota, + throttlingCallsPerWindow = secCalls, + remainingCallsPerWindow = (apiKey.throttlingQuota - secCalls).toInt, + authorizedCallsPerDay = apiKey.dailyQuota, + currentCallsPerDay = dailyCalls, + remainingCallsPerDay = apiKey.dailyQuota - dailyCalls, + authorizedCallsPerMonth = apiKey.monthlyQuota, + currentCallsPerMonth = monthlyCalls, + remainingCallsPerMonth = apiKey.monthlyQuota - monthlyCalls + ) + } } override def withingQuotas(apiKey: ApiKey)(implicit ec: ExecutionContext, env: Env): Future[Boolean] = @@ -170,7 +177,7 @@ class KvApiKeyDataStore(redisCli: RedisLike, _env: Env) extends ApiKeyDataStore redisCli .get(throttlingKey(apiKey.clientId)) .fast - .map(_.map(_.utf8String.toLong).getOrElse(0L) <= (apiKey.throttlingQuota * env.throttlingWindow)) + .map(_.map(_.utf8String.toLong).getOrElse(0L) <= apiKey.throttlingQuota) override def withinDailyQuota(apiKey: ApiKey)(implicit ec: ExecutionContext, env: Env): Future[Boolean] = redisCli.get(dailyQuotaKey(apiKey.clientId)).fast.map(_.map(_.utf8String.toLong).getOrElse(0L) < apiKey.dailyQuota) diff --git a/otoroshi/app/storage/stores/KvGlobalConfigDataStore.scala b/otoroshi/app/storage/stores/KvGlobalConfigDataStore.scala index 28accf525a..5fd9d37bb0 100644 --- a/otoroshi/app/storage/stores/KvGlobalConfigDataStore.scala +++ b/otoroshi/app/storage/stores/KvGlobalConfigDataStore.scala @@ -40,7 +40,7 @@ class KvGlobalConfigDataStore(redisCli: RedisLike, _env: Env) private val quotasForIpAddressCache = new UnboundedConcurrentHashMap[String, java.util.concurrent.atomic.AtomicLong]() // TODO: check growth over time - def incrementCallsForIpAddressWithTTL(ipAddress: String, ttl: Int = 10)(implicit + def incrementCallsForIpAddress(ipAddress: String)(implicit ec: ExecutionContext ): Future[Long] = { @@ -53,13 +53,19 @@ class KvGlobalConfigDataStore(redisCli: RedisLike, _env: Env) callsForIpAddressCache.get(ipAddress).set(secCalls) } redisCli.pttl(s"${_env.storageRoot}:throttling:perip:$ipAddress").filter(_ > -1).recoverWith { case _ => - redisCli.expire(s"${_env.storageRoot}:throttling:perip:$ipAddress", ttl) + callsForIpAddressCache.remove(ipAddress) + redisCli.expire(s"${_env.storageRoot}:throttling:perip:$ipAddress", _env.throttlingWindow) } map (_ => secCalls) } if (callsForIpAddressCache.containsKey(ipAddress)) { - actualCall() - FastFuture.successful(callsForIpAddressCache.get(ipAddress).get) + for { + newQuota <- actualCall() + } yield { + callsForIpAddressCache + .getOrDefault(ipAddress, new java.util.concurrent.atomic.AtomicLong(newQuota)) + .get() + } } else { actualCall() } @@ -74,6 +80,7 @@ class KvGlobalConfigDataStore(redisCli: RedisLike, _env: Env) case Success(Some(quota)) if quotasForIpAddressCache.containsKey(ipAddress) => quotasForIpAddressCache.get(ipAddress).set(quota) } + if (quotasForIpAddressCache.containsKey(ipAddress)) { actualCall() FastFuture.successful(Some(quotasForIpAddressCache.get(ipAddress).get)) @@ -93,7 +100,8 @@ class KvGlobalConfigDataStore(redisCli: RedisLike, _env: Env) //singleton().map { config => redisCli.get(throttlingKey()).map { bs => throttlingQuotasCache.set(bs.map(_.utf8String.toLong).getOrElse(0L)) - throttlingQuotasCache.get() <= (config.throttlingQuota * 10L) + // throttlingQuotasCache.get() <= (config.throttlingQuota * 10L) + throttlingQuotasCache.get() <= config.throttlingQuota } //} } @@ -108,8 +116,9 @@ class KvGlobalConfigDataStore(redisCli: RedisLike, _env: Env) from: String )(implicit ec: ExecutionContext, env: Env): Future[(Boolean, Long, Option[Long])] = { val a = withinThrottlingQuota() - val b = incrementCallsForIpAddressWithTTL(from) + val b = incrementCallsForIpAddress(from) val c = quotaForIpAddress(from) + for { within <- a secCalls <- b @@ -121,8 +130,8 @@ class KvGlobalConfigDataStore(redisCli: RedisLike, _env: Env) config: otoroshi.models.GlobalConfig )(implicit ec: ExecutionContext, env: Env): Future[Unit] = for { - secCalls <- redisCli.incrby(throttlingKey(), 1L) - _ <- redisCli.pttl(throttlingKey()).filter(_ > -1).recoverWith { case _ => redisCli.expire(throttlingKey(), 10) } + _ <- redisCli.pttl(throttlingKey()).filter(_ > -1).recoverWith { case _ => redisCli.expire(throttlingKey(), env.throttlingWindow) } + secCalls <- redisCli.incrby(throttlingKey(), 1L) fu = env.metrics.markLong(s"global.throttling-quotas", secCalls) } yield () diff --git a/otoroshi/app/utils/classgraph.scala b/otoroshi/app/utils/classgraph.scala index 5b58a7e0af..7d8db96757 100644 --- a/otoroshi/app/utils/classgraph.scala +++ b/otoroshi/app/utils/classgraph.scala @@ -3,7 +3,11 @@ package io.github.classgraph object ClassgraphUtils { def clear(result: ScanResult): Unit = { result.close() - result.classNameToClassInfo.clear() - result.classNameToClassInfo = null + try { + result.classNameToClassInfo.clear() + result.classNameToClassInfo = null + } catch { + case _: Throwable => + } } } diff --git a/otoroshi/build.sbt b/otoroshi/build.sbt index 7277a5a465..05ab04344a 100644 --- a/otoroshi/build.sbt +++ b/otoroshi/build.sbt @@ -345,7 +345,7 @@ addJava "--add-opens=java.base/sun.security.ssl=ALL-UNNAMED" addJava "-Dlog4j2.formatMsgNoLookups=true" """ -Revolver.enableDebugging(port = 5005, suspend = false) +Revolver.enableDebugging(port = Integer.parseInt(sys.props.getOrElse("otoroshi.sbt.port", "5005")), suspend = false) // run with: ~reStart reStart / mainClass := Some("play.core.server.ProdServerStart") @@ -395,8 +395,6 @@ reStart / javaOptions ++= Seq( // "-Dotoroshi.next.experimental.netty-server.native.driver=IOUring", // "-Dotoroshi.storage=ext:foo", "-Dotoroshi.storage=file" - //"-Dotoroshi.storage=postgresql", // "-Dotoroshi.storage=redis", - // "-Dotoroshi.storage=lettuce", - // "-Dotoroshi.redis.lettuce.uri=redis://localhost:6379/", +// "-Dotoroshi.redis.lettuce.uri=redis://localhost:6379/", ) diff --git a/otoroshi/conf/application.conf b/otoroshi/conf/application.conf index f51d176b10..e14e8bed8a 100644 --- a/otoroshi/conf/application.conf +++ b/otoroshi/conf/application.conf @@ -1402,7 +1402,7 @@ otoroshi { swapStrategy = "Merge" # the internal memory store strategy, can be Replace or Merge swapStrategy = ${?CLUSTER_WORKER_SWAP_STRATEGY} # the internal memory store strategy, can be Replace or Merge swapStrategy = ${?OTOROSHI_CLUSTER_WORKER_SWAP_STRATEGY} # the internal memory store strategy, can be Replace or Merge - modern = false # use a modern store implementation + modern = true # use a modern store implementation modern = ${?CLUSTER_WORKER_STORE_MODERN} modern = ${?OTOROSHI_CLUSTER_WORKER_STORE_MODERN} useWs = false diff --git a/otoroshi/conf/schemas/openapi-cfg.json b/otoroshi/conf/schemas/openapi-cfg.json index 7f2cde195b..7fed8bb264 100644 --- a/otoroshi/conf/schemas/openapi-cfg.json +++ b/otoroshi/conf/schemas/openapi-cfg.json @@ -1628,7 +1628,7 @@ "otoroshi.models.ApiKey.restrictions": "Apikey restrictions settings", "otoroshi.models.ApiKey.rotation": "Apikey rotation settings", "otoroshi.models.ApiKey.tags": "Apikey tags", - "otoroshi.models.ApiKey.throttlingQuota": "Authorized number of calls per second, measured on 10 seconds", + "otoroshi.models.ApiKey.throttlingQuota": "Authorized number of calls per window", "otoroshi.models.ApiKey.validUntil": "Date until when the apikey is valid", "otoroshi.models.ApiKeyConstraints.basicAuth": "???", "otoroshi.models.ApiKeyConstraints.clientIdAuth": "???", @@ -1832,7 +1832,7 @@ "otoroshi.models.GlobalConfig.metadata": "Entity metadata", "otoroshi.models.GlobalConfig.middleFingers": "Use middle finger emoji as a response character for endless HTTP responses", "otoroshi.models.GlobalConfig.otoroshiId": "Unique id for this otoroshi instance", - "otoroshi.models.GlobalConfig.perIpThrottlingQuota": "Authorized number of calls per second globally per IP address, measured on 10 seconds", + "otoroshi.models.GlobalConfig.perIpThrottlingQuota": "Authorized number of calls per window globally per IP address", "otoroshi.models.GlobalConfig.plugins": "global plugins settings", "otoroshi.models.GlobalConfig.proxies": "Web proxies settings", "otoroshi.models.GlobalConfig.quotasSettings": "Settings to generate alert when an apikey almost exceeded or exceeded its quotas", @@ -1842,7 +1842,7 @@ "otoroshi.models.GlobalConfig.streamEntityOnly": "HTTP will be streamed only. Doesn't work with old browsers", "otoroshi.models.GlobalConfig.tags": "Entity tags", "otoroshi.models.GlobalConfig.templates": "The otoroshi default templates for entities", - "otoroshi.models.GlobalConfig.throttlingQuota": "Authorized number of calls per second globally, measured on 10 seconds", + "otoroshi.models.GlobalConfig.throttlingQuota": "Authorized number of calls per window globally", "otoroshi.models.GlobalConfig.tlsSettings": "TLS settings", "otoroshi.models.GlobalConfig.trustXForwarded": "Use X-Forwarded-* headers for routing", "otoroshi.models.GlobalConfig.u2fLoginOnly": "If enabled, login to backoffice through Auth0 will be disabled", @@ -1990,13 +1990,13 @@ "otoroshi.models.RegionMatch.region": "???", "otoroshi.models.RemainingQuotas.authorizedCallsPerDay": "Number of authorized call per day", "otoroshi.models.RemainingQuotas.authorizedCallsPerMonth": "Number of authorized call per month", - "otoroshi.models.RemainingQuotas.authorizedCallsPerSec": "Number of authorized call per second", + "otoroshi.models.RemainingQuotas.authorizedCallsPerWindow": "Number of authorized call per window", "otoroshi.models.RemainingQuotas.currentCallsPerDay": "Current number of call per day", "otoroshi.models.RemainingQuotas.currentCallsPerMonth": "Current number of call per month", - "otoroshi.models.RemainingQuotas.currentCallsPerSec": "Current number of call per second", + "otoroshi.models.RemainingQuotas.throttlingCallsPerWindow": "Current number of call per window", "otoroshi.models.RemainingQuotas.remainingCallsPerDay": "Remaining number of call per day", "otoroshi.models.RemainingQuotas.remainingCallsPerMonth": "Remaining number of call per month", - "otoroshi.models.RemainingQuotas.remainingCallsPerSec": "Remaining number of call per second", + "otoroshi.models.RemainingQuotas.remainingCallsPerWindow": "Remaining number of call per window", "otoroshi.models.RestrictionPath.method": "???", "otoroshi.models.RestrictionPath.path": "???", "otoroshi.models.Restrictions.allowLast": "???", diff --git a/otoroshi/conf/schemas/openapi-flat.json b/otoroshi/conf/schemas/openapi-flat.json index 6bda050bb4..f54d7c4f5e 100644 --- a/otoroshi/conf/schemas/openapi-flat.json +++ b/otoroshi/conf/schemas/openapi-flat.json @@ -3945,7 +3945,7 @@ "throttlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second, measured on 10 seconds" + "description" : "Authorized number of calls per window" }, "constrainedServicesOnly" : { "type" : "boolean", @@ -13040,7 +13040,7 @@ "throttlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second, measured on 10 seconds" + "description" : "Authorized number of calls per window" }, "constrainedServicesOnly" : { "type" : "boolean", @@ -20638,7 +20638,7 @@ "throttlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second, measured on 10 seconds" + "description" : "Authorized number of calls per window" }, "constrainedServicesOnly" : { "type" : "boolean", @@ -25144,7 +25144,7 @@ "throttlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second globally, measured on 10 seconds" + "description" : "Authorized number of calls per window globally" }, "anonymousReporting" : { "type" : "boolean", @@ -26460,7 +26460,7 @@ "perIpThrottlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second globally per IP address, measured on 10 seconds" + "description" : "Authorized number of calls per window globally per IP address" }, "useCircuitBreakers" : { "type" : "boolean", @@ -29802,7 +29802,7 @@ "throttlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second globally, measured on 10 seconds" + "description" : "Authorized number of calls per window globally" }, "anonymousReporting" : { "type" : "boolean", @@ -31131,7 +31131,7 @@ "perIpThrottlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second globally per IP address, measured on 10 seconds" + "description" : "Authorized number of calls per window globally per IP address" }, "useCircuitBreakers" : { "type" : "boolean", @@ -36548,15 +36548,15 @@ "type" : "object", "description" : "Remaining quotas for an apikey", "properties" : { - "currentCallsPerSec" : { + "throttlingCallsPerWindow" : { "type" : "integer", "format" : "int64", - "description" : "Current number of call per second" + "description" : "Current number of call per window" }, - "remainingCallsPerSec" : { + "remainingCallsPerWindow" : { "type" : "integer", "format" : "int64", - "description" : "Remaining number of call per second" + "description" : "Remaining number of call per window" }, "currentCallsPerDay" : { "type" : "integer", @@ -36578,10 +36578,10 @@ "format" : "int64", "description" : "Remaining number of call per month" }, - "authorizedCallsPerSec" : { + "authorizedCallsPerWindow" : { "type" : "integer", "format" : "int64", - "description" : "Number of authorized call per second" + "description" : "Number of authorized call per window" }, "authorizedCallsPerMonth" : { "type" : "integer", @@ -42974,7 +42974,7 @@ "throttlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second, measured on 10 seconds" + "description" : "Authorized number of calls per window" }, "constrainedServicesOnly" : { "type" : "boolean", @@ -51175,7 +51175,7 @@ "throttlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second, measured on 10 seconds" + "description" : "Authorized number of calls per window" }, "constrainedServicesOnly" : { "type" : "boolean", @@ -56177,7 +56177,7 @@ "throttlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second globally, measured on 10 seconds" + "description" : "Authorized number of calls per window globally" }, "anonymousReporting" : { "type" : "boolean", @@ -57506,7 +57506,7 @@ "perIpThrottlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second globally per IP address, measured on 10 seconds" + "description" : "Authorized number of calls per window globally per IP address" }, "useCircuitBreakers" : { "type" : "boolean", @@ -61682,7 +61682,7 @@ "throttlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second, measured on 10 seconds" + "description" : "Authorized number of calls per window" }, "constrainedServicesOnly" : { "type" : "boolean", diff --git a/otoroshi/conf/schemas/openapi-forms.json b/otoroshi/conf/schemas/openapi-forms.json index 2dcb258f77..0cfbf387f3 100644 --- a/otoroshi/conf/schemas/openapi-forms.json +++ b/otoroshi/conf/schemas/openapi-forms.json @@ -35032,12 +35032,12 @@ }, "otoroshi.models.RemainingQuotas" : { "schema" : { - "currentCallsPerSec" : { - "label" : "currentCallsPerSec", + "throttlingCallsPerWindow" : { + "label" : "throttlingCallsPerWindow", "type" : "number" }, - "remainingCallsPerSec" : { - "label" : "remainingCallsPerSec", + "remainingCallsPerWindow" : { + "label" : "remainingCallsPerWindow", "type" : "number" }, "currentCallsPerDay" : { @@ -35056,8 +35056,8 @@ "label" : "remainingCallsPerMonth", "type" : "number" }, - "authorizedCallsPerSec" : { - "label" : "authorizedCallsPerSec", + "authorizedCallsPerWindow" : { + "label" : "authorizedCallsPerWindow", "type" : "number" }, "authorizedCallsPerMonth" : { @@ -35069,7 +35069,7 @@ "type" : "number" } }, - "flow" : [ "remainingCallsPerDay", "authorizedCallsPerMonth", "authorizedCallsPerSec", "remainingCallsPerMonth", "currentCallsPerMonth", "authorizedCallsPerDay", "currentCallsPerDay", "remainingCallsPerSec", "currentCallsPerSec" ] + "flow" : [ "remainingCallsPerDay", "authorizedCallsPerMonth", "authorizedCallsPerWindow", "remainingCallsPerMonth", "currentCallsPerMonth", "authorizedCallsPerDay", "currentCallsPerDay", "remainingCallsPerWindow", "throttlingCallsPerWindow" ] }, "otoroshi.next.plugins.JQRequest" : { "schema" : { diff --git a/otoroshi/conf/schemas/openapi.json b/otoroshi/conf/schemas/openapi.json index 368d62e090..b0ae8a6fb6 100644 --- a/otoroshi/conf/schemas/openapi.json +++ b/otoroshi/conf/schemas/openapi.json @@ -19135,7 +19135,7 @@ "throttlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second, measured on 10 seconds" + "description" : "Authorized number of calls per window" }, "constrainedServicesOnly" : { "type" : "boolean", @@ -20477,7 +20477,7 @@ "throttlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second globally, measured on 10 seconds" + "description" : "Authorized number of calls per window globally" }, "anonymousReporting" : { "type" : "boolean", @@ -20680,7 +20680,7 @@ "perIpThrottlingQuota" : { "type" : "integer", "format" : "int64", - "description" : "Authorized number of calls per second globally per IP address, measured on 10 seconds" + "description" : "Authorized number of calls per window globally per IP address" }, "useCircuitBreakers" : { "type" : "boolean", @@ -21173,15 +21173,15 @@ "type" : "object", "description" : "Remaining quotas for an apikey", "properties" : { - "currentCallsPerSec" : { + "throttlingCallsPerWindow" : { "type" : "integer", "format" : "int64", - "description" : "Current number of call per second" + "description" : "Current number of call per window" }, - "remainingCallsPerSec" : { + "remainingCallsPerWindow" : { "type" : "integer", "format" : "int64", - "description" : "Remaining number of call per second" + "description" : "Remaining number of call per window" }, "currentCallsPerDay" : { "type" : "integer", @@ -21203,10 +21203,10 @@ "format" : "int64", "description" : "Remaining number of call per month" }, - "authorizedCallsPerSec" : { + "authorizedCallsPerWindow" : { "type" : "integer", "format" : "int64", - "description" : "Number of authorized call per second" + "description" : "Number of authorized call per window" }, "authorizedCallsPerMonth" : { "type" : "integer", diff --git a/otoroshi/javascript/src/forms/ng_plugins/NgCustomThrottling.js b/otoroshi/javascript/src/forms/ng_plugins/NgCustomThrottling.js index 7033dcaf55..5412b4f751 100644 --- a/otoroshi/javascript/src/forms/ng_plugins/NgCustomThrottling.js +++ b/otoroshi/javascript/src/forms/ng_plugins/NgCustomThrottling.js @@ -2,7 +2,7 @@ export default { id: 'cp:otoroshi.next.plugins.NgCustomThrottling', config_schema: { throttling_quota: { - label: 'Allowed calls per seconds', + label: 'Allowed calls per window', type: 'number', }, per_route: { diff --git a/otoroshi/javascript/src/pages/DangerZonePage.js b/otoroshi/javascript/src/pages/DangerZonePage.js index 3f23f69d46..1331bc4ea6 100644 --- a/otoroshi/javascript/src/pages/DangerZonePage.js +++ b/otoroshi/javascript/src/pages/DangerZonePage.js @@ -642,16 +642,16 @@ export class DangerZonePage extends Component { type: 'number', props: { label: 'Global throttling', - placeholder: `Number of requests per seconds`, - help: `The max. number of requests allowed per seconds globally on Otoroshi`, + placeholder: `Number of requests per window`, + help: `The max. number of requests allowed per window globally on Otoroshi`, }, }, perIpThrottlingQuota: { type: 'number', props: { label: 'Throttling per IP', - placeholder: `Number of requests per seconds per IP`, - help: `The max. number of requests allowed per seconds per IP address globally on Otoroshi`, + placeholder: `Number of requests per window per IP`, + help: `The max. number of requests allowed per window per IP address globally on Otoroshi`, }, }, analyticsEventsUrl: { diff --git a/otoroshi/javascript/src/pages/GlobalEventsPage.js b/otoroshi/javascript/src/pages/GlobalEventsPage.js index 8a9cbd0473..20a2324431 100644 --- a/otoroshi/javascript/src/pages/GlobalEventsPage.js +++ b/otoroshi/javascript/src/pages/GlobalEventsPage.js @@ -195,15 +195,15 @@ export class GlobalEventsPage extends Component { { title: 'Headers Count', content: (item) => item.headers.length }, { title: 'Calls per sec', - content: (item) => safe(item.remainingQuotas, (i) => i.currentCallsPerSec), + content: (item) => safe(item.remainingQuotas, (i) => i.throttlingCallsPerWindow), }, { title: 'Auth. calls per sec', - content: (item) => safe(item.remainingQuotas, (i) => i.authorizedCallsPerSec), + content: (item) => safe(item.remainingQuotas, (i) => i.authorizedCallsPerWindow), }, { title: 'Rem. calls per sec', - content: (item) => safe(item.remainingQuotas, (i) => i.remainingCallsPerSec), + content: (item) => safe(item.remainingQuotas, (i) => i.remainingCallsPerWindow), }, { title: 'Calls per day', diff --git a/otoroshi/javascript/src/pages/ServiceApiKeysPage.js b/otoroshi/javascript/src/pages/ServiceApiKeysPage.js index 3f708c3563..fc669cd74f 100644 --- a/otoroshi/javascript/src/pages/ServiceApiKeysPage.js +++ b/otoroshi/javascript/src/pages/ServiceApiKeysPage.js @@ -456,9 +456,9 @@ class DailyRemainingQuotas extends Component { render() { const quotas = this.state.quotas || { - authorizedCallsPerSec: 0, - currentCallsPerSec: 0, - remainingCallsPerSec: 0, + authorizedCallsPerWindow: 0, + throttlingCallsPerWindow: 0, + remainingCallsPerWindow: 0, authorizedCallsPerDay: 0, currentCallsPerDay: 0, remainingCallsPerDay: 0, @@ -727,9 +727,9 @@ const ApiKeysConstants = { type: 'number', props: { label: 'Throttling quota', - placeholder: 'Authorized calls per second', + placeholder: 'Authorized calls per window', suffix: 'calls per sec.', - help: 'The authorized number of calls per second', + help: 'The authorized number of calls per window', }, }, dailyQuota: { diff --git a/otoroshi/javascript/src/pages/ServiceEventsPage.js b/otoroshi/javascript/src/pages/ServiceEventsPage.js index 28980260df..54cae38bdf 100644 --- a/otoroshi/javascript/src/pages/ServiceEventsPage.js +++ b/otoroshi/javascript/src/pages/ServiceEventsPage.js @@ -212,18 +212,18 @@ export class ServiceEventsPage extends Component { { title: 'Headers Count', content: (item) => item.headers.length, filterable: false }, { title: 'Calls per sec', - filterId: 'remainingQuotas.currentCallsPerSec', - content: (item) => safe(item.remainingQuotas, (i) => i.currentCallsPerSec), + filterId: 'remainingQuotas.throttlingCallsPerWindow', + content: (item) => safe(item.remainingQuotas, (i) => i.throttlingCallsPerWindow), }, { title: 'Auth. calls per sec', - filterId: 'remainingQuotas.authorizedCallsPerSec', - content: (item) => safe(item.remainingQuotas, (i) => i.authorizedCallsPerSec), + filterId: 'remainingQuotas.authorizedCallsPerWindow', + content: (item) => safe(item.remainingQuotas, (i) => i.authorizedCallsPerWindow), }, { title: 'Rem. calls per sec', - filterId: 'remainingQuotas.remainingCallsPerSec', - content: (item) => safe(item.remainingQuotas, (i) => i.remainingCallsPerSec), + filterId: 'remainingQuotas.remainingCallsPerWindow', + content: (item) => safe(item.remainingQuotas, (i) => i.remainingCallsPerWindow), }, { title: 'Calls per day', diff --git a/otoroshi/test/Suites.scala b/otoroshi/test/Suites.scala index 37230c889c..d7044b2d2a 100644 --- a/otoroshi/test/Suites.scala +++ b/otoroshi/test/Suites.scala @@ -183,3 +183,8 @@ class GreenScoreTest extends Suites( new GreenScoreTestSpec("GreenScore", Configurations.InMemoryConfiguration) ) + +//class ApiKeysTest +// extends Suites( +// new ApiKeysSpec("ApiKeysSpec", Configurations.InMemoryConfiguration) +// ) diff --git a/otoroshi/test/functional/ApiKeysSpec.scala b/otoroshi/test/functional/ApiKeysSpec.scala index f7e23e157c..02e6089903 100644 --- a/otoroshi/test/functional/ApiKeysSpec.scala +++ b/otoroshi/test/functional/ApiKeysSpec.scala @@ -1,17 +1,15 @@ package functional -import java.util.Base64 -import java.util.concurrent.atomic.AtomicInteger - import akka.actor.ActorSystem import com.auth0.jwt.JWT import com.auth0.jwt.algorithms.Algorithm import com.typesafe.config.ConfigFactory import otoroshi.models.{ApiKey, ServiceDescriptor, ServiceGroupIdentifier, Target} -import org.scalatest.concurrent.IntegrationPatience -import org.scalatestplus.play.PlaySpec import play.api.Configuration +import java.util.Base64 +import java.util.concurrent.atomic.AtomicInteger + class ApiKeysSpec(name: String, configurationSpec: => Configuration) extends OtoroshiSpec { lazy val serviceHost = "auth.oto.tools" diff --git a/otoroshi/test/functional/MapFilterSpec.scala b/otoroshi/test/functional/MapFilterSpec.scala index da7e46cbdc..0795ee5e5d 100644 --- a/otoroshi/test/functional/MapFilterSpec.scala +++ b/otoroshi/test/functional/MapFilterSpec.scala @@ -573,9 +573,9 @@ class MapFilterSpec extends WordSpec with MustMatchers with OptionValues { | }, | "remainingQuotas": { | "remainingCallsPerMonth": 10000000, - | "authorizedCallsPerSec": 10000000, - | "currentCallsPerSec": 10000000, - | "remainingCallsPerSec": 10000000, + | "authorizedCallsPerWindow": 10000000, + | "throttlingCallsPerWindow": 10000000, + | "remainingCallsPerWindow": 10000000, | "authorizedCallsPerDay": 10000000, | "currentCallsPerDay": 10000000, | "authorizedCallsPerMonth": 10000000, diff --git a/otoroshi/test/functional/QuotasSpec.scala b/otoroshi/test/functional/QuotasSpec.scala index fc099552bb..dfc878c231 100644 --- a/otoroshi/test/functional/QuotasSpec.scala +++ b/otoroshi/test/functional/QuotasSpec.scala @@ -96,7 +96,7 @@ class QuotasSpec(name: String, configurationSpec: => Configuration) extends Otor createOtoroshiApiKey(apiKeyLowDMonthlyQuota).futureValue } - "prevent too many calls per second" in { + "prevent too many calls per window" in { def call(auth: String) = { ws.url(s"http://127.0.0.1:$port/api") .withHttpHeaders( @@ -117,7 +117,7 @@ class QuotasSpec(name: String, configurationSpec: => Configuration) extends Otor } counter200.get() > 0 mustBe true - // counter429.get() > 0 mustBe true + counter429.get() > 0 mustBe true } "prevent too many calls per day" in { diff --git a/scripts/tools/openapi-to-graphql-schema/dist/admin-api-graphql.graphql b/scripts/tools/openapi-to-graphql-schema/dist/admin-api-graphql.graphql index de1b0e3aaf..fa8a6f878f 100644 --- a/scripts/tools/openapi-to-graphql-schema/dist/admin-api-graphql.graphql +++ b/scripts/tools/openapi-to-graphql-schema/dist/admin-api-graphql.graphql @@ -144,7 +144,7 @@ An otoroshi apikey that can allow you to access some services type ApiKey { dailyQuota: Float # Authorized number of calls per day metadata: Json # Bunch of metadata for the key - throttlingQuota: Float # Authorized number of calls per second, measured on 10 seconds + throttlingQuota: Float # Authorized number of calls per window, measured on 10 seconds constrainedServicesOnly: Boolean # This apikey can only be used on services that constrained their apikey routing allowClientIdOnly: Boolean # This apikey can be used juste with the client_id value _loc: EntityLocation # The location of the apikey @@ -524,7 +524,7 @@ The global config (dynamic) for otoroshi type GlobalConfig { geolocationSettings: Json # Settings for geolocation extraction alertsEmails: [String] # Email addresses that will receive all Otoroshi alert events - throttlingQuota: Float # Authorized number of calls per second globally, measured on 10 seconds + throttlingQuota: Float # Authorized number of calls per window globally maxWebhookSize: Float # Max number of items in webhooks maxConcurrentRequests: Float # The number of authorized request processed at the same time cleverSettings: CleverCloudSettings # Optional CleverCloud configuration @@ -557,7 +557,7 @@ type GlobalConfig { letsEncryptSettings: LetsEncryptSettings # Let's encrypt (ACME) settings snowMonkeyConfig: SnowMonkeyConfig # Snowmonky settings scripts: GlobalScripts # global plugins settings. will be deprecated soon - perIpThrottlingQuota: Float # Authorized number of calls per second globally per IP address, measured on 10 seconds + perIpThrottlingQuota: Float # Authorized number of calls per window globally per IP address useCircuitBreakers: Boolean # If enabled, services will be authorized to use circuit breakers maxHttp10ResponseSize: Float # The max size in bytes of an HTTP 1.0 response tlsSettings: TlsSettings # TLS settings @@ -784,13 +784,13 @@ type NgCustomTimeouts { Remaining quotas for an apikey """ type RemainingQuotas { - currentCallsPerSec: Float # Current number of call per second - remainingCallsPerSec: Float # Remaining number of call per second + throttlingCallsPerWindow: Float # Current number of call per window + remainingCallsPerWindow: Float # Remaining number of call per window currentCallsPerDay: Float # Current number of call per day authorizedCallsPerDay: Float # Number of authorized call per day currentCallsPerMonth: Float # Current number of call per month remainingCallsPerMonth: Float # Remaining number of call per month - authorizedCallsPerSec: Float # Number of authorized call per second + authorizedCallsPerWindow: Float # Number of authorized call per window authorizedCallsPerMonth: Float # Number of authorized call per month remainingCallsPerDay: Float # Remaining number of call per day } diff --git a/scripts/tools/openapi-to-graphql-schema/dist/queries.graphql b/scripts/tools/openapi-to-graphql-schema/dist/queries.graphql index 070d740a27..b3a100b271 100644 --- a/scripts/tools/openapi-to-graphql-schema/dist/queries.graphql +++ b/scripts/tools/openapi-to-graphql-schema/dist/queries.graphql @@ -192,13 +192,13 @@ query apikeysIdQuotas($id: String) { apikeysIdQuotas(id: $id) { authorizedCallsPerDay authorizedCallsPerMonth - authorizedCallsPerSec + authorizedCallsPerWindow currentCallsPerDay currentCallsPerMonth - currentCallsPerSec + throttlingCallsPerWindow remainingCallsPerDay remainingCallsPerMonth - remainingCallsPerSec + remainingCallsPerWindow } }