Merge pull request #10 from neutral-info/Develop-add_filters

Develop add filters
neutral-info · Nov 19, 2020 · e7e31bf · e7e31bf
2 parents 6f2b04e + 00133c2
commit e7e31bf
Show file tree

Hide file tree

Showing 13 changed files with 178 additions and 32 deletions.
diff --git a/api/backend/application.py b/api/backend/application.py
@@ -25,6 +25,8 @@ def __init__(self):
             "http://localhost:443",
             "http://localhost:8080",
             "http://localhost:5000",
+            "http://neutral-info.herokuapp.com",
+            "https://neutral-info.herokuapp.com",
         ]
         self.add_middleware(
             CORSMiddleware,

diff --git a/api/backend/responses.py b/api/backend/responses.py
@@ -2,7 +2,12 @@
 import typing
 
 
-def success_msg(resp_data: typing.List[typing.Union[str, typing.Dict]]) -> UJSONResponse:
+def success_msg(
+    resp_data: typing.List[typing.Union[str, typing.Dict]]
+) -> UJSONResponse:
     resp = {"msg": "success", "status": 200}
-    resp.update({"data":resp_data})
-    return UJSONResponse(resp, headers={"Access-Control-Allow-Origin": "*"},)
+    resp.update({"data": resp_data})
+    return UJSONResponse(
+        resp,
+        headers={"Access-Control-Allow-Origin": "*"},
+    )
diff --git a/api/config.py b/api/config.py
@@ -5,4 +5,3 @@
 MYSQL_DATA_PASSWORD = os.environ.get("MYSQL_DATA_PASSWORD", "test")
 MYSQL_DATA_PORT = int(os.environ.get("MYSQL_DATA_PORT", "3306"))
 MYSQL_DATA_DATABASE = os.environ.get("MYSQL_DATA_DATABASE", "JSON2")
-var_test = os.environ.get("var_test", "vat_test_default")
diff --git a/api/tools/clients.py b/api/tools/clients.py
@@ -10,7 +10,9 @@
 )
 
 
-def data_db(database: str,):
+def data_db(
+    database: str,
+):
     connect = mysql.connector.connect(
         host=MYSQL_DATA_HOST,
         port=MYSQL_DATA_PORT,

diff --git a/api/tools/data_dict.py b/api/tools/data_dict.py
@@ -1,9 +1,13 @@
 TABLE_DICT = dict(
     News="vwNews",
-    NewsPosition="NewsPosition",
+    Position="position",
+    Channel="channel",
+    Producer="producer",
 )
 
 DATABASE_DICT = dict(
     News="JSON2",
-    NewsPosition="JSON2",
+    Position="JSON2",
+    Channel="JSON2",
+    Producer="JSON2",
 )
diff --git a/api/tools/db.py b/api/tools/db.py
@@ -34,6 +34,9 @@ def get_keywords_page_sql(
     positions: str,
     volumeMin: int,
     volumeMax: int,
+    author: str,
+    channel: str,
+    producer: str,
     orderby: str,
     ordertype: str,
 ) -> str:
@@ -43,17 +46,17 @@ def get_keywords_page_sql(
         SELECT {0}
         FROM `{1}`
         WHERE 1=1
-        """.format(colname, table)
+        """.format(
+        colname, table
+    )
 
     if keywords:
         keywords_statement = []
         for k in keywords.split(","):
             k = k.strip()
             keywords_statement.append(f" `keywords` like '%{k}%' ")
 
-        keywords_statement = (
-            "AND ( " + "OR".join(keywords_statement) + " )"
-        )
+        keywords_statement = "AND ( " + "OR".join(keywords_statement) + " )"
         sql = f" {sql} {keywords_statement} "
 
     if positions:
@@ -62,9 +65,7 @@ def get_keywords_page_sql(
             p = p.strip()
             position_statement.append(f" `position` like '%{p}%'")
 
-        position_statement = (
-            "AND ( " + " AND ".join(position_statement) + " )"
-        )
+        position_statement = "AND ( " + " AND ".join(position_statement) + " )"
         sql = f" {sql} {position_statement} "
 
     if volumeMin:
@@ -75,6 +76,33 @@ def get_keywords_page_sql(
         volumeRange_statement = f"AND `volume_now` <= {volumeMax} "
         sql = f" {sql} {volumeRange_statement} "
 
+    if author:
+        author_statement = []
+        for a in author.split(","):
+            a = a.strip()
+            author_statement.append(f" `author_desc` like '%{a}%'")
+
+        author_statement = "AND ( " + " AND ".join(author_statement) + " )"
+        sql = f" {sql} {author_statement} "
+
+    if channel:
+        channel_statement = []
+        for c in channel.split(","):
+            c = c.strip()
+            channel_statement.append(f" `channel_desc` like '%{c}%'")
+
+        channel_statement = "AND ( " + " AND ".join(channel_statement) + " )"
+        sql = f" {sql} {channel_statement} "
+
+    if producer:
+        producer_statement = []
+        for p in producer.split(","):
+            p = p.strip()
+            producer_statement.append(f" `producer_desc` like '%{p}%'")
+
+        producer_statement = "AND ( " + " AND ".join(producer_statement) + " )"
+        sql = f" {sql} {producer_statement} "
+
     if colname != "COUNT(*)":
         order_limit_statement = f"""
                                 ORDER BY `{orderby}` {ordertype}
@@ -110,6 +138,9 @@ def create_pages_sql(
     positions: str,
     volumeMin: int,
     volumeMax: int,
+    author: str,
+    channel: str,
+    producer: str,
     datatype: str,
     orderby: str,
     ordertype: str,
@@ -126,6 +157,9 @@ def create_pages_sql(
         positions,
         volumeMin,
         volumeMax,
+        author,
+        channel,
+        producer,
         orderby,
         ordertype,
     )
@@ -141,6 +175,9 @@ def load_pages(
     positions: str = "",
     volumeMin: int = None,
     volumeMax: int = None,
+    author: str = "",
+    channel: str = "",
+    producer: str = "",
     datatype: str = "",
     orderby: str = "",
     ordertype: str = "",
@@ -157,6 +194,9 @@ def load_pages(
         positions,
         volumeMin,
         volumeMax,
+        author,
+        channel,
+        producer,
         datatype,
         orderby,
         ordertype,
@@ -172,3 +212,20 @@ def load_pages(
     connect.close()
 
     return data
+
+
+def load_items(
+    database: str = "",
+    table: str = "",
+):
+    sql = f"SELECT DISTINCT {table}_desc FROM {table};"
+    logger.info(f"sql cmd:{sql}")
+
+    connect = clients.get_db_client(database)
+    cursor = connect.cursor()
+    cursor.execute(sql)
+    data = get_fetch_alllist(cursor)
+    cursor.close()
+    connect.close()
+
+    return data
diff --git a/api/tools/load.py b/api/tools/load.py
@@ -1,11 +1,9 @@
 import typing
 
 from loguru import logger
-from api.tools.db import load_pages
-from api.tools.data_dict import (
-    TABLE_DICT,
-    DATABASE_DICT,
-)
+
+from api.tools.data_dict import DATABASE_DICT, TABLE_DICT
+from api.tools.db import load_items, load_pages
 
 
 def NID_pages(
@@ -16,6 +14,9 @@ def NID_pages(
     positions: str,
     volumeMin: int,
     volumeMax: int,
+    author: str,
+    channel: str,
+    producer: str,
     ordertype: str,
     orderby: str,
     datatype: str = "page",
@@ -30,6 +31,9 @@ def NID_pages(
         positions=positions,
         volumeMin=volumeMin,
         volumeMax=volumeMax,
+        author=author,
+        channel=channel,
+        producer=producer,
         datatype=datatype,
         orderby=orderby,
         ordertype=ordertype,
@@ -38,3 +42,15 @@ def NID_pages(
     logger.info(f"NID_pages parameter: {parameter}")
     data = load_pages(**parameter)
     return data
+
+
+def NID_items(dataset: str):
+
+    parameter = dict(
+        table=TABLE_DICT.get(dataset),
+        database=DATABASE_DICT.get(dataset),
+    )
+
+    logger.info(f"NID_pages parameter: {parameter}")
+    data = load_items(**parameter)
+    return data
diff --git a/api/v1/__init__.py b/api/v1/__init__.py
@@ -1,2 +1 @@
-
-from api.v1.api import api_router
+from api.v1.api import api_router
diff --git a/api/v1/endpoints/item.py b/api/v1/endpoints/item.py
@@ -1,19 +1,31 @@
+import typing
+
 from fastapi import APIRouter
 from loguru import logger
+from starlette.requests import Request
 
 from api.backend.responses import success_msg
+from api.tools import load
 from api.v1.schema.input import ItemTypeInput
+from api.v1.schema.items import ItemList
 
 router = APIRouter()
 
 
-@router.get("/item")
-async def get_data(itemtype: ItemTypeInput):
+def convert_lisdict2list(
+    listdictdata: typing.List[typing.Dict[str, typing.Union[str]]]
+) -> typing.List[str]:
     convert_data = []
-    if itemtype == "Position":
-        convert_data = ["時代力量", "國民黨", "民進黨", "親民黨", "民眾黨", "無立場"]
-    elif itemtype == "Channel":
-        convert_data = ["新聞", "社群論壇", "聊天", "部落格", "內容農場", "影音"]
+    for d in listdictdata:
+        for k in d:
+            convert_data.append(d[k])
+    return convert_data
+
+
+@router.get("/item", response_model=ItemList)
+async def get_data(request: Request, itemtype: ItemTypeInput):
+    raw_data = load.NID_items(dataset=itemtype)
+    convert_data = convert_lisdict2list(raw_data)
 
     logger.info(f"get itemtype:{itemtype} data")
     return success_msg(convert_data)
diff --git a/api/v1/endpoints/keyword.py b/api/v1/endpoints/keyword.py
@@ -15,25 +15,48 @@
 
 
 def convert_vwNews2News(
-    listdictdata: typing.List[typing.Dict[str, typing.Union[str, list, float]]], keywords: str, countsinfo: int,
+    listdictdata: typing.List[typing.Dict[str, typing.Union[str, list, float]]],
+    keywords: str,
+    countsinfo: int,
 ) -> KeywordList:
 
     group_data_News = []
     for d in listdictdata:
-        d["position"] = [{"party": p.split("*")[0], "trend": float(p.split("*")[1])} for p in d["position"].split("|")]
+        d["position"] = [
+            {"party": p.split("*")[0], "trend": float(p.split("*")[1])}
+            for p in d["position"].split("|")
+        ]
         d["keywords"] = d["keywords"].split(",")
         d["producer"] = {
             "id": d["producer_id"],
             "desc": d["producer_desc"],
             "position": [
-                {"party": p.split("*")[0], "trend": float(p.split("*")[1])} for p in d["producer_position"].split("|")
+                {"party": p.split("*")[0], "trend": float(p.split("*")[1])}
+                for p in d["producer_position"].split("|")
             ],
         }
-
         d.pop("producer_id", None)
         d.pop("producer_desc", None)
         d.pop("producer_position", None)
 
+        d["author"] = {
+            "id": d["author_id"],
+            "desc": d["author_desc"],
+            "position": [
+                {"party": p.split("*")[0], "trend": float(p.split("*")[1])}
+                for p in d["author_position"].split("|")
+            ],
+        }
+        d.pop("author_id", None)
+        d.pop("author_desc", None)
+        d.pop("author_position", None)
+
+        d["channel"] = {
+            "id": d["channel_id"],
+            "desc": d["channel_desc"],
+        }
+        d.pop("channel_id", None)
+        d.pop("channel_desc", None)
         group_data_News.append(News(**d).dict())
 
     convert_data = {}
@@ -54,12 +77,14 @@ async def get_data(
     powerMin: int = None,
     powerMax: int = None,
     positions: str = None,
+    author: str = None,
     channel: str = None,
+    producer: str = None,
     keywords: str = None,
     pageNo: int = 1,
     pageSize: int = 5,
 ):
-    # FIXME: need to add powerMin, powerMax, channel filter
+    # FIXME: need to add powerMin, powerMax
     ret = load.NID_pages(
         dataset=dataset,
         pageNo=pageNo,
@@ -68,6 +93,9 @@ async def get_data(
         positions=positions,
         volumeMin=volumeMin,
         volumeMax=volumeMax,
+        author=author,
+        channel=channel,
+        producer=producer,
         datatype="page",
         orderby=orderby,
         ordertype=ordertype,
@@ -82,6 +110,9 @@ async def get_data(
         positions=positions,
         volumeMin=volumeMin,
         volumeMax=volumeMax,
+        author=author,
+        channel=channel,
+        producer=producer,
         datatype="count",
         orderby=orderby,
         ordertype=ordertype,

diff --git a/api/v1/schema/input.py b/api/v1/schema/input.py
@@ -8,6 +8,7 @@ class DataSetInput(str, Enum):
 class ItemTypeInput(str, Enum):
     Position = "Position"
     Channel = "Channel"
+    Producer = "Producer"
 
 
 class OrderByInput(str, Enum):

diff --git a/api/v1/schema/items.py b/api/v1/schema/items.py
@@ -0,0 +1,6 @@
+import typing
+from pydantic import BaseModel
+
+
+class ItemList(BaseModel):
+    News: typing.List[str]