Issue/#319 Ladder map repitition (#398)

* Add sqlalchemy with asyncio

* Add test for excluding last played maps

* If all maps were played recently, just pick a random one

* Query laddergames by looking for "ladder1v1" gamemod

* Close db connections when server shuts down

* Only return map_id

* Clean up imports

* Remove mysql specific types

* Replace db_pool with sqlalchemy engine everywhere

* Add test for empty map pool case

* Remove logging cursor

* Rename faction "nomads" to "nomad"

* Add debug for game timeout

* Clean up some formatting

* Make sure to fetch ladder results.

* Add test for querying ladder history from db.

* Set ladder maps correctly.

* Replace fetchall with async for iteration where possible.

* Refactor update_game_player_stats

* Fix stuff I missed during rebase

* Row is a dict cursor

* Fix column name

* Add typing for map tuple

requirements.txt

@@ -19,3 +19,4 @@ marisa-trie
 oauthlib
 requests_oauthlib
 mock
+SQLAlchemy

server.py

@@ -37,7 +37,6 @@ def signal_handler(signal, frame):
             if not done.done():
                 done.set_result(0)
 
-
         loop = asyncio.get_event_loop()
         done = asyncio.Future()
 
@@ -52,16 +51,20 @@ def signal_handler(signal, frame):
         signal.signal(signal.SIGTERM, signal_handler)
         signal.signal(signal.SIGINT, signal_handler)
 
-        pool_fut = asyncio.async(server.db.connect(host=DB_SERVER,
-                                                   port=int(DB_PORT),
-                                                   user=DB_LOGIN,
-                                                   password=DB_PASSWORD,
-                                                   maxsize=10,
-                                                   db=DB_NAME,
-                                                   loop=loop))
-        db_pool = loop.run_until_complete(pool_fut)
-
-        players_online = PlayerService(db_pool)
+        engine_fut = asyncio.async(
+            server.db.connect_engine(
+                host=DB_SERVER,
+                port=int(DB_PORT),
+                user=DB_LOGIN,
+                password=DB_PASSWORD,
+                maxsize=10,
+                db=DB_NAME,
+                loop=loop
+            )
+        )
+        engine = loop.run_until_complete(engine_fut)
+
+        players_online = PlayerService()
 
         api_accessor = None
         if config.USE_API:
@@ -93,6 +96,11 @@ def signal_handler(signal, frame):
 
         loop.run_until_complete(done)
         players_online.broadcast_shutdown()
+
+        # Close DB connections
+        engine.close()
+        loop.run_until_complete(engine.wait_closed())
+
         loop.close()
 
     except Exception as ex:

server/db/__init__.py

@@ -1,42 +1,32 @@
-import aiomysql
-from .logging_cursor import LoggingCursor
-from aiomysql import Pool
+from aiomysql.sa import create_engine
 
-db_pool: Pool = None
+engine = None
 
 
-def set_pool(pool: Pool):
+def set_engine(engine_):
     """
-    Set the globally used pool to the given argument
+    Set the globally used engine to the given argument
     """
-    global db_pool
-    db_pool = pool
+    global engine
+    engine = engine_
 
 
-async def connect(loop,
-                  host='localhost', port=3306, user='root', password='', db='faf_test',
-                  minsize=1, maxsize=1, cursorclass=LoggingCursor) -> Pool:
-    """
-    Initialize the database pool
-    :param loop:
-    :param host:
-    :param user:
-    :param password:
-    :param db:
-    :param minsize:
-    :param maxsize:
-    :param cursorclass:
-    :return:
-    """
-    pool = await aiomysql.create_pool(host=host,
-                                      port=port,
-                                      user=user,
-                                      password=password,
-                                      db=db,
-                                      autocommit=True,
-                                      loop=loop,
-                                      minsize=minsize,
-                                      maxsize=maxsize,
-                                      cursorclass=cursorclass)
-    set_pool(pool)
-    return pool
+async def connect_engine(
+    loop, host='localhost', port=3306, user='root', password='', db='faf_test',
+    minsize=1, maxsize=1, echo=True
+):
+    engine = await create_engine(
+        host=host,
+        port=port,
+        user=user,
+        password=password,
+        db=db,
+        autocommit=True,
+        loop=loop,
+        minsize=minsize,
+        maxsize=maxsize,
+        echo=echo
+    )
+
+    set_engine(engine)
+    return engine

server/db/models.py

@@ -0,0 +1,51 @@
+from sqlalchemy import (TIMESTAMP, Boolean, Column, Enum, Float, ForeignKey,
+                        Integer, MetaData, String, Table, Text)
+
+from ..games.game import Victory
+
+metadata = MetaData()
+
+game_featuredMods = Table(
+    'game_featuredMods', metadata,
+    Column('id',            Integer,    primary_key=True),
+    Column('gamemod',       String,     unique=True),
+    Column('descripiton',   Text,       nullable=False),
+    Column('name',          String,     nullable=False),
+    Column('publish',       Boolean,    nullable=False, server_default='f'),
+    Column('order',         Integer,    nullable=False, server_default='0'),
+    Column('git_url',       String),
+    Column('git_branch',    String),
+    Column('file_extension',String),
+    Column('allow_override',Boolean)
+)
+
+game_player_stats = Table(
+    'game_player_stats', metadata,
+    Column('id',        Integer, primary_key=True),
+    Column('gameId',    Integer, ForeignKey('game_stats.id'), nullable=False),
+    Column('playerId',  Integer, nullable=False),
+    Column('AI',        Boolean, nullable=False),
+    Column('faction',   Integer, nullable=False),
+    Column('color',     Integer, nullable=False),
+    Column('team',      Integer, nullable=False),
+    Column('place',     Integer, nullable=False),
+    Column('mean',      Float,   nullable=False),
+    Column('deviation', Float,   nullable=False),
+    Column('after_mean', Float),
+    Column('after_deviation', Float),
+    Column('score',     Integer),
+    Column('scoreTime', TIMESTAMP),
+)
+
+game_stats = Table(
+    'game_stats', metadata,
+    Column('id',        Integer,    primary_key=True),
+    Column('startTime', TIMESTAMP,  nullable=False, server_default="CURRENT_TIMESTAMP"),
+    Column('endTime',   TIMESTAMP),
+    Column('gameType',  Enum(Victory), nullable=False),
+    Column('gameMod',   Integer, ForeignKey('game_featuredMods.id'), nullable=False),
+    Column('host',      Integer,                nullable=False),
+    Column('mapId',     Integer),
+    Column('gameName',  String,                 nullable=False),
+    Column('validity',  Integer,                nullable=False),
+)

server/factions.py

@@ -7,7 +7,7 @@ class Faction(IntEnum):
     aeon = 2
     cybran = 3
     seraphim = 4
-    nomads = 5
+    nomad = 5
 
     @staticmethod
     def from_string(value):

server/game_service.py

@@ -52,9 +52,7 @@ def __init__(self, player_service, game_stats_service):
         self._update_cron = aiocron.crontab('*/10 * * * *', func=self.update_data)
 
     async def initialise_game_counter(self):
-        async with db.db_pool.get() as conn:
-            cursor = await conn.cursor()
-
+        async with db.engine.acquire() as conn:
             # InnoDB, unusually, doesn't allow insertion of values greater than the next expected
             # value into an auto_increment field. We'd like to do that, because we no longer insert
             # games into the database when they don't start, so game ids aren't contiguous (as
@@ -66,50 +64,52 @@ async def initialise_game_counter(self):
             # doing LAST_UPDATE_ID to get the id number, and then doing an UPDATE when the actual
             # data to go into the row becomes available: we now only do a single insert for each
             # game, and don't end up with 800,000 junk rows in the database.
-            await cursor.execute("SELECT MAX(id) FROM game_stats")
-            (self.game_id_counter, ) = await cursor.fetchone()
+            result = await conn.execute("SELECT MAX(id) FROM game_stats")
+            row = await result.fetchone()
+            self.game_id_counter = row[0]
 
     async def update_data(self):
         """
         Loads from the database the mostly-constant things that it doesn't make sense to query every
         time we need, but which can in principle change over time.
         """
-        async with db.db_pool.get() as conn:
-            cursor = await conn.cursor()
-
-            await cursor.execute("SELECT `id`, `gamemod`, `name`, description, publish, `order` FROM game_featuredMods")
+        async with db.engine.acquire() as conn:
+            result = await conn.execute("SELECT `id`, `gamemod`, `name`, description, publish, `order` FROM game_featuredMods")
 
-            for i in range(0, cursor.rowcount):
-                id, name, full_name, description, publish, order = await cursor.fetchone()
-                self.featured_mods[name] = FeaturedMod(id, name, full_name, description, publish, order)
+            async for row in result:
+                mod_id, name, full_name, description, publish, order = (row[i] for i in range(6))
+                self.featured_mods[name] = FeaturedMod(
+                    mod_id, name, full_name, description, publish, order)
 
-            await cursor.execute("SELECT uid FROM table_mod WHERE ranked = 1")
+            result = await conn.execute("SELECT uid FROM table_mod WHERE ranked = 1")
+            rows = await result.fetchall()
 
             # Turn resultset into a list of uids
-            rows = await cursor.fetchall()
             self.ranked_mods = set(map(lambda x: x[0], rows))
 
             # Load all ladder maps
-            await cursor.execute("SELECT ladder_map.idmap, "
-                                 "table_map.name, "
-                                 "table_map.filename "
-                                 "FROM ladder_map "
-                                 "INNER JOIN table_map ON table_map.id = ladder_map.idmap")
-            self.ladder_maps = await cursor.fetchall()
+            result = await conn.execute(
+                "SELECT ladder_map.idmap, "
+                "table_map.name, "
+                "table_map.filename "
+                "FROM ladder_map "
+                "INNER JOIN table_map ON table_map.id = ladder_map.idmap")
+            self.ladder_maps = [(row[0], row[1], row[2]) async for row in result]
 
             for mod in self.featured_mods.values():
-
                 self._logger.debug("Loading featuredMod %s", mod.name)
                 if mod.name == 'ladder1v1':
                     continue
                 self.game_mode_versions[mod.name] = {}
                 t = "updates_{}".format(mod.name)
                 tfiles = t + "_files"
-                await cursor.execute("SELECT %s.fileId, MAX(%s.version) "
-                                     "FROM %s LEFT JOIN %s ON %s.fileId = %s.id "
-                                     "GROUP BY %s.fileId" % (tfiles, tfiles, tfiles, t, tfiles, t, tfiles))
-                rows = await cursor.fetchall()
-                for fileId, version in rows:
+                result = await conn.execute(
+                    "SELECT %s.fileId, MAX(%s.version) "
+                    "FROM %s LEFT JOIN %s ON %s.fileId = %s.id "
+                    "GROUP BY %s.fileId" % (tfiles, tfiles, tfiles, t, tfiles, t, tfiles))
+
+                async for row in result:
+                    fileId, version = row[0], row[1]
                     self.game_mode_versions[mod.name][fileId] = version
             # meh
             self.game_mode_versions['ladder1v1'] = self.game_mode_versions['faf']

server/gameconnection.py

@@ -3,6 +3,7 @@
 from collections import defaultdict
 
 import server.db as db
+from sqlalchemy import text
 
 from .abc.base_game import GameConnectionState
 from .connectivity import ConnectivityState
@@ -320,12 +321,12 @@ async def handle_game_mods(self, mode, args):
         elif mode == "uids":
             uids = str(args).split()
             self.game.mods = {uid: "Unknown sim mod" for uid in uids}
-            async with db.db_pool.get() as conn:
-                cursor = await conn.cursor()
-                await cursor.execute("SELECT `uid`, `name` from `table_mod` WHERE `uid` in %s", (uids,))
-                mods = await cursor.fetchall()
-                for (uid, name) in mods:
-                    self.game.mods[uid] = name
+            async with db.engine.acquire() as conn:
+                result = await conn.execute(
+                    text("SELECT `uid`, `name` from `table_mod` WHERE `uid` in :ids"),
+                    ids=tuple(uids))
+                async for row in result:
+                    self.game.mods[row["uid"]] = row["name"]
         self._mark_dirty()
 
     async def handle_player_option(self, id_, command, value):
@@ -372,23 +373,22 @@ async def handle_operation_complete(self, army, secondary, delta):
         if not int(army) == 1:
             return
 
-        secondary, delta = int(secondary), str(delta)
-
         if self.game.validity != ValidityState.COOP_NOT_RANKED:
             return
 
-        async with db.db_pool.get() as conn:
-            cursor = await conn.cursor()
+        secondary, delta = int(secondary), str(delta)
+        async with db.engine.acquire() as conn:
             # FIXME: Resolve used map earlier than this
-            await cursor.execute("SELECT `id` FROM `coop_map` WHERE `filename` = %s",
-                                 self.game.map_file_path)
-            row = await cursor.fetchone()
+            result = await conn.execute(
+                "SELECT `id` FROM `coop_map` WHERE `filename` = %s",
+                self.game.map_file_path)
+            row = await result.fetchone()
             if not row:
                 self._logger.debug("can't find coop map: %s", self.game.map_file_path)
                 return
-            (mission,) = row
+            mission = row["id"]
 
-            await cursor.execute(
+            await conn.execute(
                 """ INSERT INTO `coop_leaderboard`
                     (`mission`, `gameuid`, `secondary`, `time`, `player_count`)
                     VALUES (%s, %s, %s, %s, %s)""",
@@ -410,10 +410,8 @@ async def handle_teamkill_report(self, gametime, victim_id, victim_name, teamkil
             :param teamkiller_name: teamkiller nickname (for debug purpose only)
         """
 
-        async with db.db_pool.get() as conn:
-            cursor = await conn.cursor()
-
-            await cursor.execute(
+        async with db.engine.acquire() as conn:
+            await conn.execute(
                 """ INSERT INTO `teamkills` (`teamkiller`, `victim`, `game_id`, `gametime`)
                     VALUES (%s, %s, %s, %s)""",
                 (teamkiller_id, victim_id, self.game.id, gametime)
@@ -446,13 +444,12 @@ async def handle_game_state(self, state):
             await self.game.launch()
 
             if len(self.game.mods.keys()) > 0:
-                async with db.db_pool.get() as conn:
-                    cursor = await conn.cursor()
+                async with db.engine.acquire() as conn:
                     uids = list(self.game.mods.keys())
-                    await cursor.execute(
+                    await conn.execute(text(
                         """ UPDATE mod_stats s JOIN mod_version v ON v.mod_id = s.mod_id
-                            SET s.times_played = s.times_played + 1 WHERE v.uid in %s""",
-                        (uids,)
+                            SET s.times_played = s.times_played + 1 WHERE v.uid in :ids"""),
+                        ids=tuple(uids)
                     )
         elif state == 'Ended':
             await self.on_connection_lost()