Skip to content

Commit

Permalink
Change la page de stats pour tous les utilisateurs (#1780)
Browse files Browse the repository at this point in the history
  • Loading branch information
@niladic
niladic authored Aug 27, 2023
1 parent 798efa0 commit 71f9850
Show file tree
Hide file tree
Showing 21 changed files with 50 additions and 931 deletions.
218 changes: 26 additions & 192 deletions app/controllers/ApplicationController.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import models._
import models.formModels.{AnswerFormData, ApplicationFormData, InvitationFormData}
import modules.AppConfig
import org.webjars.play.WebJarsUtil
import play.api.cache.AsyncCacheApi
import play.api.data.Forms._
import play.api.data._
import play.api.data.format.{Formats, Formatter}
Expand All @@ -31,7 +30,6 @@ import play.twirl.api.Html
import serializers.Keys
import serializers.ApiModel.{ApplicationMetadata, ApplicationMetadataResult}
import services._
import views.stats.StatsData

import java.nio.file.{Files, Path, Paths}
import java.time.{LocalDate, ZonedDateTime}
Expand All @@ -47,7 +45,6 @@ import scala.util.{Failure, Success, Try}
@Singleton
case class ApplicationController @Inject() (
applicationService: ApplicationService,
cache: AsyncCacheApi,
config: AppConfig,
eventService: EventService,
fileService: FileService,
Expand Down Expand Up @@ -569,143 +566,6 @@ case class ApplicationController @Inject() (
).withHeaders(CACHE_CONTROL -> "no-store")
}

private def statsAggregates(applications: List[Application], users: List[User]): StatsData = {
val now = Time.nowParis()
val applicationsByArea: Map[Area, List[Application]] =
applications
.groupBy(_.area)
.flatMap { case (areaId: UUID, applications: Seq[Application]) =>
Area.all
.find(area => area.id === areaId)
.map(area => (area, applications))
}

val firstDate: ZonedDateTime =
if (applications.isEmpty) now else applications.map(_.creationDate).min
val months = Time.monthsBetween(firstDate, now)
val allApplications = applicationsByArea.flatMap(_._2).toList
val allApplicationsByArea = applicationsByArea.map { case (area, applications) =>
StatsData.AreaAggregates(
area = area,
StatsData.ApplicationAggregates(
applications = applications,
months = months,
usersRelatedToApplications = users
)
)
}.toList
val data = StatsData(
all = StatsData.ApplicationAggregates(
applications = allApplications,
months = months,
usersRelatedToApplications = users
),
aggregatesByArea = allApplicationsByArea
)
data
}

private def anonymousGroupsAndUsers(
groups: List[UserGroup]
): Future[(List[User], List[Application])] =
for {
users <- userService.byGroupIdsAnonymous(groups.map(_.id))
applications <- applicationService.allForUserIds(users.map(_.id), none)
} yield (users, applications)

private def generateStats(
areaIds: List[UUID],
organisationIds: List[Organisation.Id],
groupIds: List[UUID],
creationMinDate: LocalDate,
creationMaxDate: LocalDate,
rights: Authorization.UserRights
)(implicit request: RequestWithUserData[_]): Future[Html] = {
eventService.log(
StatsShowed,
"Génère les stats pour les paramètres [Territoires '" + areaIds.mkString(",") +
"' ; Organismes '" + organisationIds.mkString(",") +
"' ; Groupes '" + groupIds.mkString(",") +
"' ; Date début '" + creationMinDate +
"' ; Date fin '" + creationMaxDate + "']"
)

val usersAndApplications: Future[(List[User], List[Application])] =
(areaIds, organisationIds, groupIds) match {
case (Nil, Nil, Nil) =>
userService.allNoNameNoEmail.zip(applicationService.all())
case (_ :: _, Nil, Nil) =>
for {
groups <- userGroupService.byAreas(areaIds)
users <- (
userService.byGroupIdsAnonymous(groups.map(_.id)),
applicationService.allForAreas(areaIds, None)
).mapN(Tuple2.apply)
} yield users
case (_ :: _, _ :: _, Nil) =>
for {
groups <- userGroupService
.byOrganisationIds(organisationIds)
.map(_.filter(_.areaIds.intersect(areaIds).nonEmpty))
users <- userService.byGroupIdsAnonymous(groups.map(_.id))
applications <- applicationService
.allForUserIds(users.map(_.id), none)
.map(_.filter(application => areaIds.contains(application.area)))
} yield (users, applications)
case (_, _ :: _, _) =>
userGroupService.byOrganisationIds(organisationIds).flatMap(anonymousGroupsAndUsers)
case (_, Nil, _) =>
userGroupService
.byIdsFuture(groupIds)
.flatMap(anonymousGroupsAndUsers)
.map { case (users, allApplications) =>
val applications = allApplications.filter { application =>
application.isWithoutInvitedGroupIdsLegacyCase ||
application.invitedGroups.intersect(groupIds.toSet).nonEmpty ||
users.exists(user => user.id === application.creatorUserId)
}
(users, applications)
}
}

// Filter creation dates
def isBeforeOrEqual(d1: LocalDate, d2: LocalDate): Boolean = !d1.isAfter(d2)

val applicationsFuture = usersAndApplications.map { case (_, applications) =>
applications.filter(application =>
isBeforeOrEqual(creationMinDate, application.creationDate.toLocalDate) &&
isBeforeOrEqual(application.creationDate.toLocalDate, creationMaxDate)
)
}

// Users whose id is in the `Application`
val relatedUsersFuture: Future[List[User]] = applicationsFuture.flatMap { applications =>
val ids: Set[UUID] = applications.flatMap { application =>
application.creatorUserId :: application.invitedUsers.keys.toList
}.toSet
if (ids.size > 1000) {
// We don't want to send a giga-query to PG
// it fails with
// IOException
// Tried to send an out-of-range integer as a 2-byte value: 33484
userService.all.map(_.filter(user => ids.contains(user.id)))
} else {
userService.byIdsFuture(ids.toList, includeDisabled = true)
}
}

// Note: `users` are Users on which we make stats (count, ...)
// `relatedUsers` are Users to help Applications stats (linked orgs, ...)
for {
users <- usersAndApplications.map { case (users, _) => users }
applications <- applicationsFuture
relatedUsers <- relatedUsersFuture
} yield views.html.stats.charts(Authorization.isAdmin(rights))(
statsAggregates(applications, relatedUsers),
users
)
}

// Handles some edge cases from browser compatibility
private val localDateMapping: Mapping[LocalDate] = {
val formatter = new Formatter[LocalDate] {
Expand Down Expand Up @@ -734,9 +594,6 @@ case class ApplicationController @Inject() (
)
)

private val statsCSP =
"connect-src 'self' https://stats.data.gouv.fr; base-uri 'none'; img-src 'self' data: stats.data.gouv.fr; form-action 'self'; frame-src 'self' *.aplus.beta.gouv.fr; style-src 'self' 'unsafe-inline' stats.data.gouv.fr; object-src 'none'; script-src 'self' 'unsafe-inline' stats.data.gouv.fr; default-src 'none'; font-src 'self'; frame-ancestors 'self'"

val statsAction = loginAction.withPublicPage(Ok(views.publicStats.page))

def stats: Action[AnyContent] =
Expand Down Expand Up @@ -786,56 +643,33 @@ case class ApplicationController @Inject() (
userGroupService.byIdsFuture(allGroupsIds).flatMap { groups =>
val groupsThatCanBeFilteredBy =
groups.filter(group => dropdownGroupIds.contains[UUID](group.id))
val charts: Future[Html] =
if (
(Authorization.isAdmin(rights) && request.getQueryString("oldstats").isEmpty) ||
request.getQueryString("newstats").nonEmpty
) {
val validQueryGroups = groups.filter(group => validQueryGroupIds.contains[UUID](group.id))
val creatorGroupIds = validQueryGroups
.filter(group =>
group.organisationId
.map(id => Organisation.organismesAidants.map(_.id).contains[Organisation.Id](id))
.getOrElse(false)
)
.map(_.id)
val invitedGroupIds =
validQueryGroupIds.filterNot(id => creatorGroupIds.contains[UUID](id))

Future.successful(
views.internalStats.charts(
views.internalStats.Filters(
startDate = creationMinDate,
endDate = creationMaxDate,
areaIds,
organisationIds,
creatorGroupIds,
invitedGroupIds,
),
config
)
val charts: Future[Html] = {
val validQueryGroups = groups.filter(group => validQueryGroupIds.contains[UUID](group.id))
val creatorGroupIds = validQueryGroups
.filter(group =>
group.organisationId
.map(id => Organisation.organismesAidants.map(_.id).contains[Organisation.Id](id))
.getOrElse(false)
)
} else {
val cacheKey =
Authorization.isAdmin(rights).toString +
".stats." +
Hash.sha256(
areaIds.toString + organisationIds.toString + validQueryGroupIds.toString +
creationMinDate.toString + creationMaxDate.toString
)
.map(_.id)
val invitedGroupIds =
validQueryGroupIds.filterNot(id => creatorGroupIds.contains[UUID](id))

Future.successful(
views.internalStats.charts(
views.internalStats.Filters(
startDate = creationMinDate,
endDate = creationMaxDate,
areaIds,
organisationIds,
creatorGroupIds,
invitedGroupIds,
),
config
)
)
}

cache
.getOrElseUpdate[Html](cacheKey, 1.hours)(
generateStats(
areaIds,
organisationIds,
validQueryGroupIds,
creationMinDate,
creationMaxDate,
rights
)
)
}
charts
.map { html =>
eventService.log(
Expand All @@ -857,7 +691,7 @@ case class ApplicationController @Inject() (
creationMinDate,
creationMaxDate
)
).withHeaders(CONTENT_SECURITY_POLICY -> statsCSP)
)
}
}
}
Expand Down
34 changes: 1 addition & 33 deletions app/helper/Time.scala
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
package helper

import cats.Order
import java.time.{Instant, LocalDate, YearMonth, ZoneId, ZonedDateTime}
import java.time.{Instant, LocalDate, ZoneId, ZonedDateTime}
import java.time.format.DateTimeFormatter
import java.util.Locale
import scala.collection.immutable.ListMap

object Time {

Expand All @@ -30,12 +29,7 @@ object Time {
def formatPatternFr(date: LocalDate, pattern: String): String =
date.format(DateTimeFormatter.ofPattern(pattern, Locale.FRANCE))

// Note that .atDay(1) will yield incorrect format value
def formatMonthYearAllLetters(month: YearMonth): String =
month.atDay(15).format(monthYearAllLettersFormatter)

val adminsFormatter = DateTimeFormatter.ofPattern("dd/MM/YY-HH:mm", Locale.FRANCE)
private val monthYearAllLettersFormatter = DateTimeFormatter.ofPattern("MMMM YYYY", Locale.FRANCE)

// Note: we use an Instant here to make clear that we will set our own TZ
def formatForAdmins(date: Instant): String =
Expand All @@ -45,32 +39,6 @@ object Time {

val dateWithHourFormatter = DateTimeFormatter.ofPattern("dd/MM/YYYY H'h'", Locale.FRANCE)

def weeksMap(fromDate: ZonedDateTime, toDate: ZonedDateTime): ListMap[String, String] = {
val keyFormatter = DateTimeFormatter.ofPattern("YYYY/ww", Locale.FRANCE)
val valueFormatter = DateTimeFormatter.ofPattern("E dd MMM YYYY", Locale.FRANCE)
val weekFieldISO = java.time.temporal.WeekFields.of(Locale.FRANCE).dayOfWeek()
def recursion(date: ZonedDateTime): ListMap[String, String] =
if (date.isBefore(fromDate)) {
ListMap()
} else {
recursion(date.minusWeeks(1)) +
(date.format(keyFormatter) -> date.format(valueFormatter))
}
val toDateFirstDayOfWeek = toDate.`with`(weekFieldISO, 1)
recursion(toDateFirstDayOfWeek)
}

def monthsBetween(fromDate: ZonedDateTime, toDate: ZonedDateTime): List[YearMonth] = {
val beginning = fromDate.withDayOfMonth(1)
def recursion(date: ZonedDateTime): Vector[YearMonth] =
if (date.isBefore(beginning)) {
Vector.empty[YearMonth]
} else {
recursion(date.minusMonths(1)) :+ YearMonth.from(date)
}
recursion(toDate).toList
}

implicit final val zonedDateTimeInstance: Order[ZonedDateTime] =
new Order[ZonedDateTime] {
override def compare(x: ZonedDateTime, y: ZonedDateTime): Int = x.compareTo(y)
Expand Down
14 changes: 0 additions & 14 deletions app/models/Application.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,6 @@ case class Application(
personalDataWiped: Boolean = false,
) extends AgeModel {

// Legacy case, can be removed once data has been cleaned up.
val isWithoutInvitedGroupIdsLegacyCase: Boolean =
invitedGroupIdsAtCreation.isEmpty

val invitedGroups: Set[UUID] =
(invitedGroupIdsAtCreation ::: answers.flatMap(_.invitedGroupIds)).toSet

Expand Down Expand Up @@ -119,9 +115,6 @@ case class Application(
def invitedUsers(users: List[User]): List[User] =
invitedUsers.keys.flatMap(userId => users.find(_.id === userId)).toList

def creatorUserQualite(users: List[User]): Option[String] =
users.find(_.id === creatorUserId).map(_.qualite)

def allUserInfos = userInfos ++ answers.flatMap(_.userInfos.getOrElse(Map()))

lazy val anonymousApplication = {
Expand Down Expand Up @@ -169,13 +162,6 @@ case class Application(
// TODO: remove
def haveUserInvitedOn(user: User) = invitedUsers.keys.toList.contains(user.id)

// Stats
lazy val estimatedClosedDate = (closedDate, closed) match {
case (Some(date), _) => Some(date)
case (_, true) => Some(answers.lastOption.map(_.creationDate).getOrElse(creationDate))
case _ => None
}

lazy val resolutionTimeInMinutes: Option[Int] = if (closed) {
val lastDate = answers.lastOption.map(_.creationDate).orElse(closedDate).getOrElse(creationDate)
Some(MINUTES.between(creationDate, lastDate).toInt)
Expand Down
28 changes: 9 additions & 19 deletions app/services/NotificationService.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,25 +84,15 @@ class NotificationService @Inject() (
// Retrieve data
val userIds = (application.invitedUsers ++ answer.invitedUsers).keys
val users = userService.byIds(userIds.toList)
val (allGroups, alreadyPresentGroupIds): (List[UserGroup], Set[UUID]) =
// This legacy case can be removed once data has been fixed
if (application.isWithoutInvitedGroupIdsLegacyCase) {
(
groupService
.byIds(users.flatMap(_.groupIds))
.filter(_.email.nonEmpty)
.filter(_.areaIds.contains(application.area)),
users.filter(user => application.invitedUsers.contains(user.id)).flatMap(_.groupIds).toSet
)
} else {
val allGroupIds = application.invitedGroups.union(answer.invitedGroupIds.toSet)
(
groupService
.byIds(allGroupIds.toList)
.filter(_.email.nonEmpty),
application.invitedGroups
)
}
val (allGroups, alreadyPresentGroupIds): (List[UserGroup], Set[UUID]) = {
val allGroupIds = application.invitedGroups.union(answer.invitedGroupIds.toSet)
(
groupService
.byIds(allGroupIds.toList)
.filter(_.email.nonEmpty),
application.invitedGroups
)
}

// Send emails to users
users
Expand Down
Loading

0 comments on commit 71f9850

Please sign in to comment.