diff --git a/Lib/gftools/push/__init__.py b/Lib/gftools/push/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Lib/gftools/push/items.py b/Lib/gftools/push/items.py new file mode 100644 index 000000000..5edb343ec --- /dev/null +++ b/Lib/gftools/push/items.py @@ -0,0 +1,210 @@ +import logging +from abc import ABC +from dataclasses import dataclass + +from fontTools.ttLib import TTFont # type: ignore +from gftools.designers_pb2 import DesignerInfoProto +from gftools.fonts_public_pb2 import FamilyProto +from gftools.push.utils import google_path_to_repo_path +from gftools.util.google_fonts import ReadProto +from gftools.utils import font_version, download_family_from_Google_Fonts, PROD_FAMILY_DOWNLOAD +import zipfile +from bs4 import BeautifulSoup # type: ignore +from pathlib import Path +from axisregistry.axes_pb2 import AxisProto +from google.protobuf.json_format import MessageToDict # type: ignore +from typing import Optional + + +log = logging.getLogger("gftools.items") + + +def jsonify(item): + if item == None: + return item + if isinstance(item, (bool, int, float, str)): + return item + elif isinstance(item, dict): + return {k: jsonify(v) for k, v in item.items()} + elif isinstance(item, (tuple, list)): + return [jsonify(i) for i in item] + if hasattr(item, "to_json"): + return item.to_json() + return item + + +class Itemer(ABC): + def to_json(self): + return jsonify(self.__dict__) + + +@dataclass +class Family(Itemer): + name: str + version: str + + @classmethod + def from_ttfont(cls, fp: str | Path): + ttFont = TTFont(fp) + name = ttFont["name"].getBestFamilyName() + version = font_version(ttFont) + return cls(name, version) + + @classmethod + def from_fp(cls, fp: str | Path): + ttf = list(fp.glob("*.ttf"))[0] # type: ignore + return cls.from_ttfont(ttf) + + @classmethod + def from_gf_json(cls, data, dl_url: str=PROD_FAMILY_DOWNLOAD): + return cls.from_gf(data["family"], dl_url) + + @classmethod + def from_gf(cls, name: str, dl_url: str=PROD_FAMILY_DOWNLOAD): + try: + fonts = download_family_from_Google_Fonts(name, dl_url=dl_url) + ttFont = TTFont(fonts[0]) + version = font_version(ttFont) + name = ttFont["name"].getBestFamilyName() + return cls(name, version) + except zipfile.BadZipFile: + return None + + +@dataclass +class AxisFallback(Itemer): + name: str + value: float + + +@dataclass +class Axis(Itemer): + tag: str + display_name: str + min_value: float + default_value: float + max_value: float + precision: float + fallback: list[AxisFallback] + fallback_only: bool + description: str + + @classmethod + def from_gf_json(cls, axis): + return cls( + tag=axis["tag"], + display_name=axis["displayName"], + min_value=axis["min"], + default_value=axis["defaultValue"], + max_value=axis["max"], + precision=axis["precision"], + fallback=[ + AxisFallback(name=f["name"], value=f["value"]) + for f in axis["fallbacks"] + ], + fallback_only=axis["fallbackOnly"], + description=axis["description"], + ) + + @classmethod + def from_fp(cls, fp: Path): + log.info("Getting axis data") + + fp = google_path_to_repo_path(fp) + data = MessageToDict(ReadProto(AxisProto(), fp)) + + return cls( + tag=data["tag"], + display_name=data["displayName"], + min_value=data["minValue"], + default_value=data["defaultValue"], + max_value=data["maxValue"], + precision=data["precision"], + fallback=[ + AxisFallback(name=f["name"], value=f["value"]) + for f in data["fallback"] + ], + fallback_only=data["fallbackOnly"], + description=data["description"] + ) + + def to_json(self): + d = self.__dict__ + d["fallback"] = [f.__dict__ for f in self.fallback] + return d + + +@dataclass +class FamilyMeta(Itemer): + name: str + designer: list[str] + license: str + category: str + subsets: list[str] + stroke: str + classifications: list[str] + description: str + primary_script: Optional[str] = None + + @classmethod + def from_fp(cls, fp: Path): + meta_fp = fp / "METADATA.pb" + data = ReadProto(FamilyProto(), meta_fp) + description = open(fp / "DESCRIPTION.en_us.html").read() + stroke = data.category[0] if not data.stroke else data.stroke.replace(" ", "_").upper() + return cls( + name=data.name, + designer=data.designer.split(","), + license=data.license.lower(), + category=data.category[0], + subsets=sorted([s for s in data.subsets if s != "menu"]), + stroke=stroke, + classifications=[c.lower() for c in data.classifications], + description=parse_html(description), + primary_script=None if data.primary_script == "" else data.primary_script + ) + + @classmethod + def from_gf_json(cls, meta): + stroke = ( + None if meta["stroke"] == None else meta["stroke"].replace(" ", "_").upper() + ) + return cls( + name=meta["family"], + designer=[i["name"] for i in meta["designers"]], + license=meta["license"].lower(), + category=meta["category"].replace(" ", "_").upper(), + subsets=sorted(list(meta["coverage"].keys())), + stroke=stroke, + classifications=[c.lower() for c in meta["classifications"]], + description=parse_html(meta["description"]), + primary_script=None if meta["primaryScript"] == "" else meta["primaryScript"] + ) + + +def parse_html(string: str): + return BeautifulSoup(string.replace("\n", " ").replace(" ", " "), features="lxml").prettify().strip() + + +@dataclass +class Designer(Itemer): + name: str + bio: str + + @classmethod + def from_gf_json(cls, data): + return cls(data["name"], data["bio"]) + + @classmethod + def from_fp(cls, fp): + meta = ReadProto(DesignerInfoProto(), fp / "info.pb") + name = meta.designer + bio_fp = fp / "bio.html" + if not bio_fp.exists(): + return cls(name, None) + with open(bio_fp) as doc: + bio = doc.read() + return cls(name, bio) + + +Items = Axis | Designer | Family | FamilyMeta \ No newline at end of file diff --git a/Lib/gftools/push/servers.py b/Lib/gftools/push/servers.py new file mode 100644 index 000000000..7c5f04ac8 --- /dev/null +++ b/Lib/gftools/push/servers.py @@ -0,0 +1,184 @@ +import json +import logging +import os +from configparser import ConfigParser +from datetime import datetime +from functools import cache +from pathlib import Path + +import requests # type: ignore +from gftools.push.items import Axis, AxisFallback, Designer, Family, FamilyMeta, Itemer, Items +from gftools.utils import ( + PROD_FAMILY_DOWNLOAD, +) + +log = logging.getLogger("gftools.servers") + + + +# This module uses api endpoints which shouldn't be public. Ask +# Marc Foley for the .gf_push_config.ini file. Place this file in your +# home directory. Environment variables can also be used instead. +config = ConfigParser() +config.read(os.path.join(os.path.expanduser("~"), ".gf_push_config.ini")) + +SANDBOX_META_URL = os.environ.get("SANDBOX_META_URL") or config["urls"]["sandbox_meta"] +PRODUCTION_META_URL = ( + os.environ.get("PRODUCTION_META_URL") or config["urls"]["production_meta"] +) +DEV_META_URL = os.environ.get("DEV_META_URL") or config["urls"]["dev_meta"] +SANDBOX_FAMILY_DOWNLOAD = ( + os.environ.get("SANDBOX_FAMILY_DOWNLOAD") + or config["urls"]["sandbox_family_download"] +) +DEV_FAMILY_DOWNLOAD = ( + os.environ.get("DEV_FAMILY_DOWNLOAD") or config["urls"]["dev_family_download"] +) + + +@cache +def gf_server_metadata(url: str): + """Get family json data from a Google Fonts metadata url""" + # can't do requests.get("url").json() since request text starts with ")]}'" + info = requests.get(url).json() + + return {i["family"]: i for i in info["familyMetadataList"]} + + +@cache +def gf_server_family_metadata(url: str, family: str): + """Get metadata for family on a server""" + # can't do requests.get("url").json() since request text starts with ")]}'" + url = url + f"/{family.replace(' ', '%20')}" + r = requests.get(url) + if r.status_code != 200: + return None + text = r.text + info = json.loads(text[4:]) + return info + + +class GFServer(Itemer): + def __init__(self, name: str, url: str=PRODUCTION_META_URL, dl_url: str=PROD_FAMILY_DOWNLOAD): + self.name = name + self.url = url + self.dl_url = dl_url + self.families: dict[str, Family] = {} + self.designers: dict[str, Designer] = {} + self.metadata: dict[str, FamilyMeta] = {} + self.axisregistry: dict[str, Axis] = {} + + def compare_push_item(self, item: Items): + server_item = self.find_item(item) + return server_item == item + + def find_item(self, item): + if isinstance(item, Family): + server_item = self.families.get(item.name) + elif isinstance(item, Designer): + server_item = self.designers.get(item.name) + elif isinstance(item, Axis): + server_item = self.axisregistry.get(item.tag) + elif isinstance(item, FamilyMeta): + server_item = self.metadata.get(item.name) + else: + return None + return server_item + + def update_axis_registry(self, axis_data): + for axis in axis_data: + self.axisregistry[axis["tag"]] = Axis.from_gf_json(axis) + + def update_family(self, name: str): + family = Family.from_gf(name, dl_url=self.dl_url) + if family: + self.families[name] = family + return True + return False + + def update_family_designers(self, name: str): + meta = gf_server_family_metadata(self.url, name) + for designer in meta["designers"]: + self.designers[designer["name"]] = Designer.from_gf_json(designer) + + def update_metadata(self, name: str): + meta = gf_server_family_metadata(self.url, name) + self.metadata[meta["family"]] = FamilyMeta.from_gf_json(meta) + + def update(self, last_checked: str): + meta = requests.get(self.url).json() + self.update_axis_registry(meta["axisRegistry"]) + + families_data = meta["familyMetadataList"] + for family_data in families_data: + family_name = family_data["family"] + last_modified = family_data["lastModified"] + if last_modified > last_checked: + log.info(f"Updating {family_name}") + if self.update_family(family_name): + self.update_family_designers(family_name) + self.update_metadata(family_name) + + +class GFServers(Itemer): + + DEV = "dev" + SANDBOX = "sandbox" + PRODUCTION = "production" + SERVERS = (DEV, SANDBOX, PRODUCTION) + + def __init__(self): + self.last_checked = datetime.fromordinal(1).isoformat().split("T")[0] + self.dev = GFServer(GFServers.DEV, DEV_META_URL, DEV_FAMILY_DOWNLOAD) + self.sandbox = GFServer(GFServers.SANDBOX, SANDBOX_META_URL, SANDBOX_FAMILY_DOWNLOAD) + self.production = GFServer( + GFServers.PRODUCTION, PRODUCTION_META_URL, PROD_FAMILY_DOWNLOAD + ) + + def __iter__(self): + for server in GFServers.SERVERS: + yield getattr(self, server) + + def update(self): + for server in self: + server.update(self.last_checked) + self.last_checked = datetime.now().isoformat().split("T")[0] + + def compare_item(self, item: Items): + res = item.to_json() + for server in self: + res[f"In {server.name}"] = server.compare_push_item(item) + return res + + def save(self, fp: str | Path): + data = self.to_json() + json.dump(data, open(fp, "w"), indent=4) + + @classmethod + def open(cls, fp: str | Path): + data = json.load(open(fp)) + return cls.from_dict(data) + + @classmethod + def from_dict(cls, data): + inst = cls() + inst.last_checked = data["last_checked"] + for server_name in GFServers.SERVERS: + server = getattr(inst, server_name) + + for item_type, item_value in data[server_name].items(): + if item_type == "families": + server.families = {k: Family(**v) for k, v in item_value.items()} + elif item_type == "designers": + server.designers = {k: Designer(**v) for k, v in item_value.items()} + elif item_type == "metadata": + server.metadata = { + k: FamilyMeta(**v) for k, v in item_value.items() + } + elif item_type == "axisregistry": + server.axisregistry = {k: Axis(**v) for k, v in item_value.items()} + for _, v in server.axisregistry.items(): + v.fallback = [AxisFallback(**a) for a in v.fallback] + else: + setattr(server, item_type, item_value) + return inst diff --git a/Lib/gftools/push/trafficjam.py b/Lib/gftools/push/trafficjam.py new file mode 100644 index 000000000..52f114b2e --- /dev/null +++ b/Lib/gftools/push/trafficjam.py @@ -0,0 +1,498 @@ +from __future__ import annotations + +import logging +import os +from configparser import ConfigParser +from dataclasses import dataclass +from enum import Enum +from io import TextIOWrapper +from pathlib import Path +from typing import Optional, Any + +from gftools.push.items import Axis, Designer, Family, FamilyMeta +from gftools.push.utils import google_path_to_repo_path, repo_path_to_google_path + +log = logging.getLogger("gftools.servers") + +# This module uses api endpoints which shouldn't be public. Ask +# Marc Foley for the .gf_push_config.ini file. Place this file in your +# home directory. Environment variables can also be used instead. +config = ConfigParser() +config.read(os.path.join(os.path.expanduser("~"), ".gf_push_config.ini")) + +TRAFFIC_JAM_ID = ( + os.environ.get("TRAFFIC_JAM_ID") or config["board_meta"]["traffic_jam_id"] +) +STATUS_FIELD_ID = ( + os.environ.get("STATUS_FIELD_ID") or config["board_meta"]["status_field_id"] +) +LIST_FIELD_ID = os.environ.get("LIST_FIELD_ID") or config["board_meta"]["list_field_id"] +PR_GF_ID = os.environ.get("PR_GF_ID") or config["board_meta"]["pr_gf_id"] +IN_DEV_ID = os.environ.get("IN_DEV_ID") or config["board_meta"]["in_dev_id"] +IN_SANDBOX_ID = os.environ.get("IN_SANDBOX_ID") or config["board_meta"]["in_sandbox_id"] +LIVE_ID = os.environ.get("LIVE_ID") or config["board_meta"]["live_id"] +TO_SANDBOX_ID = os.environ.get("TO_SANDBOX_ID") or config["board_meta"]["to_sandbox_id"] +TO_PRODUCTION_ID = ( + os.environ.get("TO_PRODUCTION_ID") or config["board_meta"]["to_production_id"] +) +BLOCKED_ID = os.environ.get("BLOCKED_ID") or config["board_meta"]["blocked_id"] + + +class STATUS_OPTION_IDS(Enum): + PR_GF = PR_GF_ID + IN_DEV = IN_DEV_ID + IN_SANDBOX = IN_SANDBOX_ID + LIVE = LIVE_ID + + +class LIST_OPTION_IDS(Enum): + TO_SANDBOX = TO_SANDBOX_ID + TO_PRODUCTION = TO_PRODUCTION_ID + BLOCKED = BLOCKED_ID + + +class PushCategory(Enum): + NEW = "New" + UPGRADE = "Upgrade" + OTHER = "Other" + DESIGNER_PROFILE = "Designer profile" + AXIS_REGISTRY = "Axis Registry" + KNOWLEDGE = "Knowledge" + METADATA = "Metadata / Description / License" + SAMPLE_TEXTS = "Sample texts" + BLOCKED = "Blocked" + DELETED = "Deleted" + + def values(): # type: ignore[misc] + return [i.value for i in PushCategory] + + def from_string(string: str): # type: ignore[misc] + return next((i for i in PushCategory if i.value == string), None) + + +class PushStatus(Enum): + PR_GF = "PR GF" + IN_DEV = "In Dev / PR Merged" + IN_SANDBOX = "In Sandbox" + LIVE = "Live" + + def from_string(string: str): # type: ignore[misc] + return next((i for i in PushStatus if i.value == string), None) + + +class PushList(Enum): + TO_SANDBOX = "to_sandbox" + TO_PRODUCTION = "to_production" + BLOCKED = "blocked" + + def from_string(string: str): # type: ignore[misc] + return next((i for i in PushList if i.value == string), None) + + +FAMILY_FILE_SUFFIXES = frozenset( + [".ttf", ".otf", ".html", ".pb", ".txt", ".yaml", ".png"] +) + + +GOOGLE_FONTS_TRAFFIC_JAM_QUERY = """ +{ + organization(login: "google") { + projectV2(number: 74) { + id + title + items(first: 100, after: "%s") { + totalCount + edges { + cursor + } + nodes { + id + status: fieldValueByName(name: "Status") { + ... on ProjectV2ItemFieldSingleSelectValue { + name + id + } + } + list: fieldValueByName(name: "List") { + ... on ProjectV2ItemFieldSingleSelectValue { + name + id + } + } + type + content { + ... on PullRequest { + id + files(first: 100) { + nodes { + path + } + } + url + labels(first: 10) { + nodes { + name + } + } + merged + } + } + } + } + } + } +} +""" + +GOOGLE_FONTS_UPDATE_ITEM = """ +mutation { + updateProjectV2ItemFieldValue( + input: { + projectId: "%s", + itemId: "%s", + fieldId: "%s", + value: {singleSelectOptionId: "%s"}, + } + ) { + clientMutationId + } +} +""" + + +@dataclass +class PushItem: + path: Path + category: PushCategory + status: PushStatus + url: str + push_list: Optional[PushList] = None + merged: Optional[bool] = None + id_: Optional[str] = None + + def __hash__(self) -> int: + return hash(self.path) + + def __eq__(self, other): + return self.path == other.path + + def exists(self) -> bool: + from gftools.push.utils import google_path_to_repo_path + + path = google_path_to_repo_path(self.path) + return path.exists() + + def to_json(self) -> dict[str, Any]: + category = None if not self.category else self.category.value + status = None if not self.status else self.status.value + url = None if not self.url else self.url + return { + "path": str(self.path.as_posix()), + "category": category, + "status": status, + "url": url, + } + + def item(self): + if self.category in [PushCategory.NEW, PushCategory.UPGRADE]: + return Family.from_fp(self.path) + elif self.category == PushCategory.DESIGNER_PROFILE: + return Designer.from_fp(self.path) + elif self.category == PushCategory.METADATA: + return FamilyMeta.from_fp(self.path) + elif self.category == PushCategory.AXIS_REGISTRY: + return Axis.from_fp(self.path) + return None + + def set_server(self, server: STATUS_OPTION_IDS): + from gftools.gfgithub import GitHubClient + + g = GitHubClient("google", "fonts") + mutation = GOOGLE_FONTS_UPDATE_ITEM % ( + TRAFFIC_JAM_ID, + self.id_, + STATUS_FIELD_ID, + server.value, + ) + g._run_graphql(mutation, {}) + + def set_pushlist(self, listt: LIST_OPTION_IDS): + from gftools.gfgithub import GitHubClient + + g = GitHubClient("google", "fonts") + mutation = GOOGLE_FONTS_UPDATE_ITEM % ( + TRAFFIC_JAM_ID, + self.id_, + LIST_FIELD_ID, + listt.value, + ) + g._run_graphql(mutation, {}) + if listt == LIST_OPTION_IDS.TO_SANDBOX: + self.push_list = PushList.TO_SANDBOX + elif listt == LIST_OPTION_IDS.TO_PRODUCTION: + self.push_list = PushList.TO_PRODUCTION + elif listt == LIST_OPTION_IDS.BLOCKED: + self.push_list = PushList.BLOCKED + + def block(self): + self.set_pushlist(LIST_OPTION_IDS.BLOCKED) + print(f"Blocked") + + def bump_pushlist(self): + if self.push_list == None: + self.set_pushlist(LIST_OPTION_IDS.TO_SANDBOX) + elif self.push_list == PushList.TO_SANDBOX: + self.set_pushlist(LIST_OPTION_IDS.TO_PRODUCTION) + elif self.push_list == PushList.TO_PRODUCTION: + print(f"No push list beyond to_production. Keeping item in to_production") + else: + raise ValueError(f"{self.push_list} is not supported") + + +class PushItems(list): + def __add__(self, other): + from copy import deepcopy + + new = deepcopy(self) + for i in other: + new.add(i) + return new + + def __sub__(self, other): + subbed = [i for i in self if i not in other] + new = PushItems() + for i in subbed: + new.add(i) + return new + + def to_sandbox(self): + return PushItems([i for i in self if i.push_list == PushList.TO_SANDBOX]) + + def in_sandbox(self): + return PushItems([i for i in self if i.status == PushStatus.IN_SANDBOX]) + + def in_dev(self): + return PushItems([i for i in self if i.status == PushStatus.IN_DEV]) + + def to_production(self): + return PushItems([i for i in self if i.push_list == PushList.TO_PRODUCTION]) + + def live(self): + return PushItems([i for i in self if i.status == PushStatus.LIVE]) + + def add(self, item: PushItem): + # noto font projects projects often contain an article/ dir, we remove this. + # Same for legacy VF projects which may have a static/ dir. + if "article" in item.path.parts or "static" in item.path.parts: + if item.path.is_dir(): + item.path = item.path.parent + else: + item.path = item.path.parent.parent + + # for font families, we only want the dir e.g ofl/mavenpro/MavenPro[wght].ttf --> ofl/mavenpro + elif ( + any(d in item.path.parts for d in ("ofl", "ufl", "apache", "designers")) + and item.path.suffix in FAMILY_FILE_SUFFIXES + ): + item.path = item.path.parent + + # for lang and axisregistry .textproto files, we need a transformed path + elif ( + any(d in item.path.parts for d in ("lang", "axisregistry")) + and item.path.suffix == ".textproto" + ): + item.path = repo_path_to_google_path(item.path) + + # don't include any axisreg or lang file which don't end in textproto + elif ( + any(d in item.path.parts for d in ("lang", "axisregistry")) + and item.path.suffix != ".textproto" + ): + return + + # Skip if path if it's a parent dir e.g ofl/ apache/ axisregistry/ + if len(item.path.parts) == 1: + return + + # Pop any existing item which has the same path. We always want the latest + existing_idx = next( + (idx for idx, i in enumerate(self) if i.path == item.path), None + ) + if existing_idx != None: + self.pop(existing_idx) # type: ignore + + # Pop any push items which are a child of the item's path + to_pop = None + for idx, i in enumerate(self): + if str(i.path.parent) in str(i.path) or i.path == item.path: + to_pop = idx + break + if to_pop: + self.pop(to_pop) + + self.append(item) + + def missing_paths(self) -> list[Path]: + res = [] + for item in self: + if item.category == PushCategory.DELETED: + continue + path = item.path + if any(p in ("lang", "axisregistry") for p in path.parts): + path = google_path_to_repo_path(path) + if not path.exists(): + res.append(path) + return res + + def to_server_file(self, fp: str | Path): + from collections import defaultdict + + bins = defaultdict(set) + for item in self: + if item.category == PushCategory.BLOCKED: + continue + bins[item.category.value].add(item) + + res = [] + for tag in PushCategory.values(): + if tag not in bins: + continue + res.append(f"# {tag}") + for item in sorted(bins[tag], key=lambda k: k.path): + if item.exists(): + res.append(f"{item.path.as_posix()} # {item.url}") + else: + if item.url: + res.append(f"# Deleted: {item.path.as_posix()} # {item.url}") + else: + res.append(f"# Deleted: {item.path.as_posix()}") + res.append("") + if isinstance(fp, str): + doc: TextIOWrapper = open(fp, "w") + else: + doc: TextIOWrapper = fp # type: ignore[no-redef] + doc.write("\n".join(res)) + + @classmethod + def from_server_file( + cls, + fp: str | Path | TextIOWrapper, + status: Optional[PushStatus] = None, + push_list: Optional[PushList] = None, + ): + if isinstance(fp, (str, Path)): + doc = open(fp) + else: + doc = fp + results = cls() + + lines = doc.read().split("\n") + category = PushCategory.OTHER + deleted = False + for line in lines: + if not line: + continue + + if line.startswith("# Deleted"): + line = line.replace("# Deleted: ", "") + deleted = True + + if line.startswith("#"): + category = PushCategory.from_string(line[1:].strip()) + + elif "#" in line: + path, url = line.split("#") + item = PushItem( + Path(path.strip()), + category if not deleted else PushCategory.DELETED, + status, # type: ignore + url.strip(), + push_list, + ) + results.add(item) + # some paths may not contain a PR, still add them + else: + item = PushItem( + Path(line.strip()), + category if not deleted else PushCategory.DELETED, + status, # type: ignore + "", + push_list, + ) + results.add(item) + deleted = False + return results + + @classmethod + def from_traffic_jam(cls): + log.info("Getting push items from traffic jam board") + from gftools.gfgithub import GitHubClient + + g = GitHubClient("google", "fonts") + last_item = "" + data = g._run_graphql(GOOGLE_FONTS_TRAFFIC_JAM_QUERY % last_item, {}) + board_items = data["data"]["organization"]["projectV2"]["items"]["nodes"] + + # paginate through items in board + last_item = data["data"]["organization"]["projectV2"]["items"]["edges"][-1][ + "cursor" + ] + item_count = data["data"]["organization"]["projectV2"]["items"]["totalCount"] + while len(board_items) < item_count: + data = None + while not data: + try: + data = g._run_graphql(GOOGLE_FONTS_TRAFFIC_JAM_QUERY % last_item, {}) + except: + data = None + board_items += data["data"]["organization"]["projectV2"]["items"]["nodes"] + last_item = data["data"]["organization"]["projectV2"]["items"]["edges"][-1][ + "cursor" + ] + log.info(f"Getting items up to {last_item}") + # sort items by pr number + board_items.sort(key=lambda k: k["content"]["url"]) + + results = cls() + for item in board_items: + status = item.get("status", {}).get("name", None) + if status: + status = PushStatus.from_string(status) + + push_list = item.get("list", None) + if push_list: + push_list = PushList.from_string(push_list.get("name", None)) + + if "labels" not in item["content"]: + print("PR missing labels. Skipping") + continue + labels = [i["name"] for i in item["content"]["labels"]["nodes"]] + + files = [Path(i["path"]) for i in item["content"]["files"]["nodes"]] + url = item["content"]["url"] + merged = item["content"]["merged"] + id_ = item["id"] + + # get category + if "--- blocked" in labels: + cat = PushCategory.BLOCKED + elif "I Font Upgrade" in labels or "I Small Fix" in labels: + cat = PushCategory.UPGRADE + elif "I New Font" in labels: + cat = PushCategory.NEW + elif "I Description/Metadata/OFL" in labels: + cat = PushCategory.METADATA + elif "I Designer profile" in labels: + cat = PushCategory.DESIGNER_PROFILE + elif "I Knowledge" in labels: + cat = PushCategory.KNOWLEDGE + elif "I Axis Registry" in labels: + cat = PushCategory.AXIS_REGISTRY + elif "I Lang" in labels: + cat = PushCategory.SAMPLE_TEXTS + else: + cat = PushCategory.OTHER + + for f in files: + results.add(PushItem(Path(f), cat, status, url, push_list, merged, id_)) + return results diff --git a/Lib/gftools/push/utils.py b/Lib/gftools/push/utils.py new file mode 100644 index 000000000..b4f5aa15a --- /dev/null +++ b/Lib/gftools/push/utils.py @@ -0,0 +1,53 @@ +import subprocess +from pathlib import Path + +import pygit2 # type: ignore + + +def _get_google_fonts_remote(repo): + for remote in repo.remotes: + if "google/fonts.git" in remote.url: + return remote.name + raise ValueError("Cannot find remote with url https://www.github.com/google/fonts") + + +def branch_matches_google_fonts_main(path): + repo = pygit2.Repository(path) + remote_name = _get_google_fonts_remote(repo) + + # fetch latest remote data from branch main + subprocess.run(["git", "fetch", remote_name, "main"]) + + # Check local is in sync with remote + diff = repo.diff(repo.head, f"{remote_name}/main") + if diff.stats.files_changed != 0: + raise ValueError( + "Your local branch is not in sync with the google/fonts " + "main branch. Please pull or remove any commits." + ) + return True + + +# The internal Google fonts team store the axisregistry and lang directories +# in a different location. The two functions below tranform paths to +# whichever representation you need. +def repo_path_to_google_path(fp: Path): + """lang/Lib/gflanguages/data/languages/.*.textproto --> lang/languages/.*.textproto""" + # we rename lang paths due to: https://github.com/google/fonts/pull/4679 + if "gflanguages" in fp.parts: + return Path("lang") / fp.relative_to("lang/Lib/gflanguages/data") + # https://github.com/google/fonts/pull/5147 + elif "axisregistry" in fp.parts: + return Path("axisregistry") / fp.name + return fp + + +def google_path_to_repo_path(fp: Path) -> Path: + """lang/languages/.*.textproto --> lang/Lib/gflanguages/data/languages/.*.textproto""" + if "fonts" not in fp.parts: + return fp + if "lang" in fp.parts: + return Path("lang/Lib/gflanguages/data/") / fp.relative_to("lang") + elif "axisregistry" in fp.parts: + return fp.parent / "Lib" / "axisregistry" / "data" / fp.name + return fp diff --git a/Lib/gftools/scripts/add_font.py b/Lib/gftools/scripts/add_font.py index 9ffd82f17..eb03b26dc 100755 --- a/Lib/gftools/scripts/add_font.py +++ b/Lib/gftools/scripts/add_font.py @@ -169,7 +169,7 @@ def _MakeMetadata(args, is_new): subsets = ['menu'] + subsets_in_font with ttLib.TTFont(file_family_style_weights[0][0]) as ttfont: script = primary_script(ttfont) - if script is not None and script not in ("Latn", "Cyrl", "Grek",): + if script not in ("Latn", "Cyrl", "Grek",): metadata.primary_script = script metadata.license = font_license diff --git a/Lib/gftools/scripts/manage_traffic_jam.py b/Lib/gftools/scripts/manage_traffic_jam.py new file mode 100644 index 000000000..1ed761143 --- /dev/null +++ b/Lib/gftools/scripts/manage_traffic_jam.py @@ -0,0 +1,205 @@ +""" +Google Fonts Traffic Jam manager + +Set the Status items in the Google Fonts Traffic Jam board. +https://github.com/orgs/google/projects/74 + +Users will need to have Github Hub installed. +https://hub.github.com/ + +""" +import subprocess +from rich.pretty import pprint +from gftools.push.utils import branch_matches_google_fonts_main +from gftools.push.servers import GFServers, Items +from gftools.push.trafficjam import ( + PushItem, + PushItems, + PushStatus, + PushCategory, + STATUS_OPTION_IDS, +) +import os +import argparse +from pathlib import Path +import tempfile +import json +import sys +import logging +from typing import Optional +from configparser import ConfigParser + +log = logging.getLogger("gftools.manage_traffic_jam") + +# This module uses api endpoints which shouldn't be public. Ask +# Marc Foley for the .gf_push_config.ini file. Place this file in your +# home directory. Environment variables can also be used instead. +config = ConfigParser() +config.read(os.path.join(os.path.expanduser("~"), ".gf_push_config.ini")) + + +DEV_URL = os.environ.get("DEV_META_URL") or config["urls"]["dev_url"] +SANDBOX_URL = os.environ.get("SANDBOX_URL") or config["urls"]["sandbox_url"] +PRODUCTION_URL = "https://fonts.google.com" + + +try: + subprocess.run("gh", stdout=subprocess.DEVNULL).returncode == 0 +except: + raise SystemError("Github Hub is not installed. https://hub.github.com/") + + +class ItemChecker: + def __init__(self, push_items: PushItems, gf_fp: str | Path, servers: GFServers): + self.push_items = push_items + self.gf_fp = gf_fp + self.servers = servers + self.skip_pr: Optional[str] = None + + def __enter__(self): + return self + + def __exit__(self): + self.git_checkout_main() + + def user_input(self, item: PushItem): + user_input = input( + "Bump pushlist: [y/n], block: [b] skip pr: [s], inspect: [i], quit: [q]?: " + ) + + if "y" in user_input: + item.bump_pushlist() + if "b" in user_input: + item.block() + if "s" in user_input: + self.skip_pr = item.url + if "i" in user_input: + self.vim_diff(item.item()) + self.user_input(item) + if "q" in user_input: + self.__exit__() + sys.exit() + + def git_checkout_item(self, push_item: PushItem): + if not push_item.merged: + cmd = ["gh", "pr", "checkout", push_item.url.split("/")[-1], "-f"] + subprocess.call(cmd) + else: + self.git_checkout_main() + + def git_checkout_main(self): + cmd = ["git", "checkout", "main", "-f"] + subprocess.call(cmd) + + def vim_diff(self, item: Items): + items = [("local", item)] + for server in self.servers: + items.append((server.name, server.find_item(item))) + + files = [] + for server, item in items: + tmp = tempfile.NamedTemporaryFile(suffix=server, mode="w+") + if item: + json.dump(item.to_json(), tmp, indent=4) + tmp.flush() + files.append(tmp) + subprocess.call(["vimdiff"] + [f.name for f in files]) + for f in files: + f.close() + + def display_item(self, push_item: PushItem): + res = {} + item = push_item.item() + if item: + comparison = self.servers.compare_item(item) + if push_item.category in [PushCategory.UPGRADE, PushCategory.NEW]: + res.update({ + **comparison, + **push_item.__dict__, + **{ + "dev url": "{}/specimen/{}".format(DEV_URL, item.name.replace(" ", "+")), + "sandbox url": "{}/specimen/{}".format(SANDBOX_URL, item.name.replace(" ", "+")), + "prod url": "{}/specimen/{}".format(PRODUCTION_URL, item.name.replace(" ", "+")), + } + }) + else: + res.update({**comparison,**push_item.__dict__,}) + else: + res.update(push_item.__dict__) + pprint(res) + + def update_server(self, push_item: PushItem, servers: GFServers): + item = push_item.item() + if item == None: + log.warning(f"Cannot update server for {push_item}.") + return + if item == servers.production.find_item(item): + push_item.set_server(STATUS_OPTION_IDS.LIVE) + elif item == servers.sandbox.find_item(item): + push_item.set_server(STATUS_OPTION_IDS.IN_SANDBOX) + elif item == servers.dev.find_item(item): + push_item.set_server(STATUS_OPTION_IDS.IN_DEV) + + def run(self): + for push_item in self.push_items: + if any( + [ + push_item.status == PushStatus.LIVE, + not push_item.exists(), + push_item.url == self.skip_pr, + ] + ): + continue + + if push_item.category == PushCategory.OTHER: + print("no push category defined. Skipping") + continue + + self.git_checkout_item(push_item) + self.update_server(push_item, self.servers) + self.display_item(push_item) + self.user_input(push_item) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser.add_argument("fonts_repo", type=Path) + parser.add_argument( + "-f", "--filter", choices=(None, "lists", "in_dev", "in_sandbox"), default=None + ) + parser.add_argument("-s", "--server-data", default=(Path("~") / ".gf_server_data.json").expanduser()) + args = parser.parse_args(args) + + branch_matches_google_fonts_main(args.fonts_repo) + + if not args.server_data.exists(): + log.warn(f"{args.server_data} not found. Generating file. This may take a while") + servers = GFServers() + else: + servers = GFServers.open(args.server_data) + servers.update() + servers.save(args.server_data) + + os.chdir(args.fonts_repo) + + push_items = PushItems.from_traffic_jam() + if args.filter == "lists": + prod_path = args.fonts_repo / "to_production.txt" + production_file = PushItems.from_server_file(prod_path, PushStatus.IN_SANDBOX) + + sandbox_path = args.fonts_repo / "to_sandbox.txt" + sandbox_file = PushItems.from_server_file(sandbox_path, PushStatus.IN_DEV) + + urls = [i.url for i in production_file + sandbox_file] + push_items = PushItems(i for i in push_items if i.url in urls) + elif args.filter == "in_dev": + push_items = push_items.in_dev() + elif args.filter == "in_sandbox": + push_items = push_items.in_sandbox() + + with ItemChecker(push_items[::-1], args.fonts_repo, servers) as checker: + checker.run() + + +if __name__ == "__main__": + main(None) diff --git a/Lib/gftools/scripts/push_stats.py b/Lib/gftools/scripts/push_stats.py index 219990edb..be75324c3 100755 --- a/Lib/gftools/scripts/push_stats.py +++ b/Lib/gftools/scripts/push_stats.py @@ -9,7 +9,7 @@ Usage: gftools push-stats path/to/google/fonts/repo out.html """ -from gftools.push import PushItems +from gftools.push.trafficjam import PushItems from jinja2 import Environment, FileSystemLoader, select_autoescape from pkg_resources import resource_filename from datetime import datetime diff --git a/Lib/gftools/scripts/push_status.py b/Lib/gftools/scripts/push_status.py index 7f05f2a56..a8646ca83 100755 --- a/Lib/gftools/scripts/push_status.py +++ b/Lib/gftools/scripts/push_status.py @@ -31,9 +31,82 @@ """ import argparse from pathlib import Path -from gftools.push import push_report, lint_server_files +from gftools.push.trafficjam import PushItems, PushStatus +from gftools.push.servers import gf_server_metadata, PRODUCTION_META_URL, SANDBOX_META_URL +from gftools.push.items import Family import os + +PUSH_STATUS_TEMPLATE = """ +***{} Status*** +New families: +{} + +Existing families, last pushed: +{} +""" + + +def lint_server_files(fp: Path): + template = "{}: Following paths are not valid:\n{}\n\n" + footnote = ( + "lang and axisregistry dir paths need to be transformed.\n" + "See https://github.com/googlefonts/gftools/issues/603" + ) + + prod_path = fp / "to_production.txt" + production_file = PushItems.from_server_file(prod_path, PushStatus.IN_SANDBOX) + prod_missing = "\n".join(map(str, production_file.missing_paths())) + prod_msg = template.format("to_production.txt", prod_missing) + + sandbox_path = fp / "to_sandbox.txt" + sandbox_file = PushItems.from_server_file(sandbox_path, PushStatus.IN_DEV) + sandbox_missing = "\n".join(map(str, sandbox_file.missing_paths())) + sandbox_msg = template.format("to_sandbox.txt", sandbox_missing) + + if prod_missing and sandbox_missing: + raise ValueError(prod_msg + sandbox_msg + footnote) + elif prod_missing: + raise ValueError(prod_msg + footnote) + elif sandbox_missing: + raise ValueError(sandbox_msg + footnote) + else: + print("Server files have valid paths") + + +def server_push_status(fp: Path, url: str): + families = [i for i in PushItems.from_server_file(fp, None, None) if isinstance(i.item(), Family)] + family_names = [i.item().name for i in families] + + gf_meta = gf_server_metadata(url) + + new_families = [f for f in family_names if f not in gf_meta] + existing_families = [f for f in family_names if f in gf_meta] + + gf_families = sorted( + [gf_meta[f] for f in existing_families], key=lambda k: k["lastModified"] + ) + existing_families = [f"{f['family']}: {f['lastModified']}" for f in gf_families] + return new_families, existing_families + + +def server_push_report(name: str, fp: Path, server_url: str): + new_families, existing_families = server_push_status(fp, server_url) + new = "\n".join(new_families) if new_families else "N/A" + existing = "\n".join(existing_families) if existing_families else "N/A" + print(PUSH_STATUS_TEMPLATE.format(name, new, existing)) + + +def push_report(fp: Path): + prod_path = fp / "to_production.txt" + server_push_report("Production", prod_path, PRODUCTION_META_URL) + + sandbox_path = fp / "to_sandbox.txt" + server_push_report("Sandbox", sandbox_path, SANDBOX_META_URL) + + + + def main(args=None): parser = argparse.ArgumentParser() parser.add_argument("path", type=Path, help="Path to google/fonts repo") diff --git a/Lib/gftools/utils.py b/Lib/gftools/utils.py index d8059bb85..880b679d7 100644 --- a/Lib/gftools/utils.py +++ b/Lib/gftools/utils.py @@ -30,8 +30,10 @@ from PIL import Image import re from fontTools import unicodedata as ftunicodedata +from fontTools.ttLib import TTFont from ufo2ft.util import classifyGlyphs from collections import Counter +from collections import defaultdict if sys.version_info[0] == 3: from configparser import ConfigParser else: @@ -40,9 +42,12 @@ # ===================================== # HELPER FUNCTIONS -def download_family_from_Google_Fonts(family, dst=None): +PROD_FAMILY_DOWNLOAD = 'https://fonts.google.com/download?family={}' + + +def download_family_from_Google_Fonts(family, dst=None, dl_url=PROD_FAMILY_DOWNLOAD): """Download a font family from Google Fonts""" - url = 'https://fonts.google.com/download?family={}'.format( + url = dl_url.format( family.replace(' ', '%20') ) fonts_zip = ZipFile(download_file(url)) @@ -536,3 +541,24 @@ def primary_script(ttFont, ignore_latin=True): most_common = script_count.most_common(1) if most_common: return most_common[0][0] + + +def autovivification(items): + if items == None: + return None + if isinstance(items, (list, tuple)): + return [autovivification(v) for v in items] + if isinstance(items, (float, int, str, bool)): + return items + d = defaultdict(lambda: defaultdict(defaultdict)) + d.update({k: autovivification(v) for k,v in items.items()}) + return d + + +def font_version(font: TTFont): + version_id = font["name"].getName(5, 3, 1, 0x409) + if not version_id: + version = str(font["head"].fontRevision) + else: + version = version_id.toUnicode() + return version \ No newline at end of file diff --git a/tests/push/test_items.py b/tests/push/test_items.py new file mode 100644 index 000000000..9c44358f0 --- /dev/null +++ b/tests/push/test_items.py @@ -0,0 +1,128 @@ +import pytest +import os +from pathlib import Path + +from gftools.push.items import Family, FamilyMeta, Designer, Axis, AxisFallback +from pkg_resources import resource_filename +import json + + +CWD = os.path.dirname(__file__) +TEST_DIR = os.path.join(CWD, "..", "..", "data", "test", "gf_fonts") +SERVER_DIR = os.path.join(CWD, "..", "..", "data", "test", "servers") +TEST_FAMILY_DIR = Path(TEST_DIR) / "ofl" / "mavenpro" +DESIGNER_DIR = Path(TEST_DIR) / "joeprince" +AXES_DIR = Path(resource_filename("axisregistry", "data")) +FAMILY_JSON = json.load(open(os.path.join(SERVER_DIR, "family.json"))) +FONTS_JSON = json.load(open(os.path.join(SERVER_DIR, "fonts.json"))) + + + +@pytest.mark.parametrize( + "type_, fp, gf_data, res", + [ + ( + Family, + TEST_FAMILY_DIR, + next(f for f in FONTS_JSON["familyMetadataList"] if f["family"] == "Maven Pro"), + Family( + name="Maven Pro", + version="Version 2.102", + ) + ), + ( + FamilyMeta, + TEST_FAMILY_DIR, + FAMILY_JSON, + FamilyMeta( + name="Maven Pro", + designer=["Joe Prince"], + license="ofl", + category="SANS_SERIF", + subsets=['latin', 'latin-ext', 'vietnamese'], + stroke="SANS_SERIF", + classifications=[], + description= \ +""" +
++ Maven Pro is a sans-serif typeface with unique curvature and flowing rhythm. Its forms make it very distinguishable and legible when in context. It blends styles of many great typefaces and is suitable for any design medium. Maven Pro’s modern design is great for the web and fits in any environment. +
++ Updated in January 2019 with a Variable Font "Weight" axis. +
++ The Maven Pro project was initiated by Joe Price, a type designer based in the USA. To contribute, see + + github.com/googlefonts/mavenproFont + +
+ +""" + ) + ), + ( + Designer, + DESIGNER_DIR, + FAMILY_JSON["designers"][0], + Designer( + name="Joe Prince", + bio=None + ) + ), + ( + Axis, + AXES_DIR / "weight.textproto", + next(a for a in FONTS_JSON["axisRegistry"] if a["tag"] == "wght"), + Axis( + tag='wght', + display_name='Weight', + min_value=1.0, + default_value=400.0, + max_value=1000.0, + precision=0, + fallback=[ + AxisFallback( + name='Thin', + value=100.0 + ), + AxisFallback( + name='ExtraLight', + value=200.0 + ), + AxisFallback( + name='Light', + value=300.0 + ), + AxisFallback( + name='Regular', + value=400.0 + ), + AxisFallback( + name='Medium', + value=500.0 + ), + AxisFallback( + name='SemiBold', + value=600.0 + ), + AxisFallback( + name='Bold', + value=700. + ), + AxisFallback( + name='ExtraBold', + value=800.0 + ), + AxisFallback( + name='Black', + value=900.0 + ) + ], + fallback_only=False, + description='Adjust the style from lighter to bolder in typographic color, by varying stroke weights, spacing and kerning, and other aspects of the type. This typically changes overall width, and so may be used in conjunction with Width and Grade axes.') + ) + ] +) +def test_item_from_fp_and_gf_data(type_, fp, gf_data, res): + assert type_.from_fp(fp) == type_.from_gf_json(gf_data) == res diff --git a/tests/push/test_servers.py b/tests/push/test_servers.py new file mode 100644 index 000000000..cc3aa1aab --- /dev/null +++ b/tests/push/test_servers.py @@ -0,0 +1,68 @@ +import pytest +from gftools.push.servers import GFServers, GFServer +from gftools.push.items import Family, Designer, FamilyMeta + + +DATA = { + "dev": {"families": {"Abel": {"name": "Abel", "version": "1.000"}}}, + "sandbox": {"families": {"Abel": {"name": "Abel", "version": "0.999"}}}, + "production": {"families": {"Abel": {"name": "Abel", "version": "0.999"}}}, + "last_checked": "2023-01-01" +} + + +@pytest.fixture +def servers(): + return GFServers.from_dict(DATA) + + +def test_servers_open_and_save(servers): + assert servers != None + assert servers.to_json() != None + + +def test_iter(servers): + assert ["dev", "sandbox", "production"] == [s.name for s in servers] + + +@pytest.mark.parametrize( + "item, res", + [ + ( + Family("Abel", "1.000"), + { + "name": "Abel", + "version": "1.000", + "In dev": True, + "In sandbox": False, + "In production": False + } + ) + ] +) +def test_compare_items(servers, item, res): + # TODO may be worth using a dataclass instead of dict + assert servers.compare_item(item) == res + + +@pytest.fixture +def server(): + return GFServer(name="Prod") + + +@pytest.mark.parametrize( + "method, family_name, res", + [ + # Test on a family which isn't updated regularly. We should + # probably use mocks at some point + ("update_family", "Allan", Family("Allan", "Version 1.002")), + ("update_family_designers", "Allan", Designer(name='Anton Koovit', bio=None)), + ("update_metadata", "Allan", FamilyMeta(name='Allan', designer=['Anton Koovit'], license='ofl', category='DISPLAY', subsets=['latin', 'latin-ext'], stroke='SERIF', classifications=['display'], description='\n \n\n Once Allan was a sign painter in Berlin. Grey paneling work in the subway, bad materials, a city split in two. Now things have changed. His (character) palette of activities have expanded tremendously: he happily spends time traveling, experimenting in the gastronomic field, all kinds of festivities are no longer foreign to him. He comes with alternate features, and hints. A typeface suited for bigger sizes and display use. Truly a type that you like to see!\n
\n \n')) + ] +) +def test_update_server(server, method, family_name, res): + assert server.find_item(res) == None + funcc = getattr(server, method) + funcc(family_name) + assert server.find_item(res) == res + diff --git a/tests/push/test_trafficjam.py b/tests/push/test_trafficjam.py new file mode 100644 index 000000000..f1866f04e --- /dev/null +++ b/tests/push/test_trafficjam.py @@ -0,0 +1,697 @@ +import pytest +import operator +from gftools.push.trafficjam import PushItem, PushItems, PushCategory, PushStatus, PushList +from pathlib import Path +import os + + +CWD = os.path.dirname(__file__) +TEST_DIR = os.path.join(CWD, "..", "..", "data", "test", "gf_fonts") +TEST_FAMILY_DIR = Path(os.path.join(TEST_DIR, "ofl", "mavenpro")) + + +@pytest.mark.parametrize( + "item1,item2,expected", + [ + ( + PushItem( + Path("ofl/mavenpro"), PushCategory.UPGRADE, PushStatus.IN_DEV, "45" + ), + PushItem( + Path("ofl/mavenpro"), PushCategory.UPGRADE, PushStatus.IN_DEV, "45" + ), + True, + ), + ( + PushItem( + Path("ofl/mavenpro"), PushCategory.UPGRADE, PushStatus.IN_DEV, "45" + ), + PushItem( + Path("ofl/mavenpro"), PushCategory.UPGRADE, PushStatus.IN_DEV, "46" + ), + True, + ), + ( + PushItem( + Path("ofl/mavenpro"), PushCategory.UPGRADE, PushStatus.IN_DEV, "45" + ), + PushItem( + Path("ofl/mavenpro2"), PushCategory.UPGRADE, PushStatus.IN_DEV, "45" + ), + False, + ), + ( + PushItem( + Path("ofl/mavenpro"), PushCategory.UPGRADE, PushStatus.IN_DEV, "45" + ), + PushItem( + Path("ofl/mavenpro"), PushCategory.UPGRADE, PushStatus.IN_SANDBOX, "45" + ), + True, + ), + ], +) +def test_push_item_eq(item1, item2, expected): + assert (item1 == item2) == expected + + +@pytest.mark.parametrize( + "items, expected_size", + [ + ( + [ + PushItem("1", "1", "1", "1"), + PushItem("1", "1", "1", "1"), + PushItem("1", "1", "1", "1"), + ], + 1, + ), + ( + [ + PushItem("1", "1", "1", "1"), + PushItem("1", "1", "1", "1"), + PushItem("2", "1", "1", "1"), + ], + 2, + ), + ], +) +def test_push_item_set(items, expected_size): + new_items = PushItems(set(items)) + assert len(new_items) == expected_size + + +@pytest.mark.parametrize( + """operator,item1,item2,expected""", + [ + # add items together + ( + operator.add, + PushItems( + [ + PushItem( + Path("ofl/mavenpro"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + ] + ), + PushItems( + [ + PushItem( + Path("ofl/mavenpro"), + PushCategory.NEW, + PushStatus.IN_SANDBOX, + "1", + ), + PushItem( + Path("ofl/amatic"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + ] + ), + PushItems( + [ + PushItem( + Path("ofl/mavenpro"), + PushCategory.NEW, + PushStatus.IN_SANDBOX, + "1", + ), + PushItem( + Path("ofl/amatic"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + ] + ), + ), + # sub items + ( + operator.sub, + PushItems( + [ + PushItem( + Path("ofl/mavenpro"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + PushItem( + Path("ofl/amatic"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + ] + ), + PushItems( + [ + PushItem( + Path("ofl/mavenpro"), + PushCategory.NEW, + PushStatus.IN_SANDBOX, + "1", + ), + ] + ), + PushItems( + [ + PushItem( + Path("ofl/amatic"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + ] + ), + ), + ], +) +def test_push_items_operators(operator, item1, item2, expected): + assert operator(item1, item2) == expected + + +@pytest.mark.parametrize( + "items, expected", + [ + # fonts filenames get removed + ( + [ + PushItem( + Path("ofl/mavenpro/MavenPro[wght].ttf"), + "update", + PushStatus.IN_DEV, + "1", + ) + ], + PushItems( + [PushItem(Path("ofl/mavenpro"), "update", PushStatus.IN_DEV, "1")] + ), + ), + # font family + ( + [ + PushItem( + Path("ofl/mavenpro/MavenPro[wght].ttf"), + "update", + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path("ofl/mavenpro/MavenPro-Italic[wght].ttf"), + "update", + PushStatus.IN_DEV, + "1", + ), + ], + PushItems( + [PushItem(Path("ofl/mavenpro"), "update", PushStatus.IN_DEV, "1")] + ), + ), + # axisregistry + ( + [ + PushItem( + Path("axisregistry/Lib/axisregistry/data/bounce.textproto"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path("axisregistry/Lib/axisregistry/data/morph.textproto"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + ], + PushItems( + [ + PushItem( + Path("axisregistry/bounce.textproto"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path("axisregistry/morph.textproto"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + ] + ), + ), + # lang + ( + [ + PushItem( + Path("lang/Lib/gflanguages/data/languages/aa_Latn.textproto"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + ], + PushItems( + [ + PushItem( + Path("lang/languages/aa_Latn.textproto"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + ] + ), + ), + # child + ( + [ + PushItem( + Path("ofl/mavenpro"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + PushItem(Path("ofl"), PushCategory.NEW, PushStatus.IN_DEV, "1"), + ], + PushItems( + [ + PushItem( + Path("ofl/mavenpro"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + ] + ), + ), + # parent + ( + [ + PushItem(Path("ofl"), PushCategory.NEW, PushStatus.IN_DEV, "1"), + PushItem( + Path("ofl/mavenpro"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + ], + PushItems( + [ + PushItem( + Path("ofl/mavenpro"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + ] + ), + ), + # parent dir + ( + [ + PushItem(Path("ofl"), PushCategory.NEW, PushStatus.IN_DEV, "1"), + PushItem(Path("apache"), PushCategory.NEW, PushStatus.IN_DEV, "1"), + PushItem( + Path("lang/authors.txt"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + ], + PushItems(), + ), + # noto article + ( + [ + PushItem( + Path("ofl/notosans/article/index.html"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + ], + PushItems( + [ + PushItem( + Path("ofl/notosans"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + ] + ), + ), + # noto full + ( + [ + PushItem( + Path("ofl/notosans/article"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path("ofl/notosans/NotoSans[wght].ttf"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path("ofl/notosans/DESCRIPTION.en_us.html"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path("ofl/notosans/upstream.yaml"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path("ofl/notosans/OFL.txt"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path("ofl/notosans/DESCRIPTION.en_us.html"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + ], + PushItems( + [ + PushItem( + Path("ofl/notosans"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + ] + ), + ), + # multi notosans + ( + [ + PushItem( + Path("ofl/notosanspsalterpahlavi/METADATA.pb"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path("ofl/notosans/METADATA.pb"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + ], + PushItems( + [ + PushItem( + Path("ofl/notosanspsalterpahlavi"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path("ofl/notosans"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + ] + ), + ), + # multi notosans 2 + ( + [ + PushItem( + Path("ofl/notosans/METADATA.pb"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path("ofl/notosanspsalterpahlavi/METADATA.pb"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + ], + PushItems( + [ + PushItem( + Path("ofl/notosans"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + PushItem( + Path("ofl/notosanspsalterpahlavi"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + ] + ), + ), + # designer + ( + [ + PushItem( + Path("catalog/designers/colophonfoundry/bio.html"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path("catalog/designers/colophonfoundry/colophonfoundry.png"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path("catalog/designers/colophonfoundry/info.pb"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + ], + PushItems( + [ + PushItem( + Path("catalog/designers/colophonfoundry"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + ] + ), + ), + # ensure latest push item is used + ( + [ + PushItem( + Path("ofl/mavenpro"), PushCategory.NEW, PushStatus.IN_DEV, "1" + ), + PushItem( + Path("ofl/mavenpro"), PushCategory.UPGRADE, PushStatus.IN_DEV, "2" + ), + ], + PushItems( + [ + PushItem( + Path("ofl/mavenpro"), + PushCategory.UPGRADE, + PushStatus.IN_DEV, + "2", + ), + ] + ), + ), + # Ensure Knowledge articles are unique + ( + [ + PushItem( + Path("cc-by-sa/knowledge/glossary/terms/xopq_axis/content.md"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path( + "cc-by-sa/knowledge/modules/using_type/lessons/the_complications_of_typographic_size/content.md" + ), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + ], + PushItems( + [ + PushItem( + Path("cc-by-sa/knowledge/glossary/terms/xopq_axis/content.md"), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + PushItem( + Path( + "cc-by-sa/knowledge/modules/using_type/lessons/the_complications_of_typographic_size/content.md" + ), + PushCategory.NEW, + PushStatus.IN_DEV, + "1", + ), + ] + ), + ), + ], +) +def test_push_items_add(items, expected): + res = PushItems() + for item in items: + res.add(item) + for got, want in zip(res, expected): + assert got.path == want.path + assert got.status == want.status + assert got.category == want.category + assert got.url == want.url + + +# TODO reactivate this. Doesn't work on GHA +# def test_push_items_from_traffic_jam(): +# items = PushItems.from_traffic_jam() +# # traffic board shouldn't be empty +# assert ( +# len(items) != 0 +# ), "board is empty! check https://github.com/orgs/google/projects/74" + + +@pytest.mark.parametrize( + "string, expected_size", + [ + ("ofl/noto # 2", 1), + ("# New\nofl/noto # 2\nofl/foobar # 3\n\n# Upgrade\nofl/mavenPro # 4", 3), + ("# New\nofl/noto\n# Deleted: lang/languages/wsg_Gong.textproto # 5", 2), + ], +) +def test_push_items_from_server_file(string, expected_size): + from io import StringIO + + data = StringIO() + data.write(string) + data.seek(0) + items = PushItems.from_server_file(data, PushStatus.IN_DEV, PushList.TO_SANDBOX) + assert len(items) == expected_size + + +@pytest.mark.parametrize( + "items,create_dirs,expected", + [ + # standard items + ( + PushItems( + [ + PushItem( + Path("ofl/mavenpro"), + PushCategory.UPGRADE, + PushStatus.IN_DEV, + "45", + ), + PushItem( + Path("ofl/amatic"), PushCategory.NEW, PushStatus.IN_DEV, "46" + ), + ] + ), + (True, True), + "# New\nofl/amatic # 46\n\n# Upgrade\nofl/mavenpro # 45\n", + ), + # deleted item + ( + PushItems( + [ + PushItem( + Path("ofl/mavenpro"), + PushCategory.UPGRADE, + PushStatus.IN_DEV, + "45", + ), + PushItem( + Path("ofl/amatic"), PushCategory.NEW, PushStatus.IN_DEV, "46" + ), + PushItem( + Path("ofl/opensans"), + PushCategory.UPGRADE, + PushStatus.IN_DEV, + "47", + ), + ] + ), + (True, False, True), + "# New\n# Deleted: ofl/amatic # 46\n\n# Upgrade\nofl/mavenpro # 45\nofl/opensans # 47\n", + ), + # duplicate items + ( + PushItems( + [ + PushItem( + Path("ofl/mavenpro"), + PushCategory.UPGRADE, + PushStatus.IN_DEV, + "45", + ), + PushItem( + Path("ofl/mavenpro"), + PushCategory.UPGRADE, + PushStatus.IN_DEV, + "45", + ), + PushItem( + Path("ofl/mavenpro"), + PushCategory.UPGRADE, + PushStatus.IN_DEV, + "45", + ), + ] + ), + (True, True), + f"# Upgrade\nofl/mavenpro # 45\n", + ), + ], +) +def test_push_items_to_server_file(items, create_dirs, expected): + from io import StringIO + import tempfile + + # We need to mock the item paths because if they don't exist, the server_file + # should mention that they're deleted + with tempfile.TemporaryDirectory() as tmp_dir: + cwd = os.getcwd() + os.chdir(tmp_dir) + for item, create_dir in zip(items, create_dirs): + new_path = tmp_dir / item.path + if create_dir: + os.makedirs(new_path, exist_ok=True) + + out = StringIO() + items.to_server_file(out) + out.seek(0) + os.chdir(cwd) + assert out.read() == expected + + +@pytest.mark.parametrize( + "path, expected", + [ + (TEST_FAMILY_DIR, []), + (Path("/foo/bar"), [Path("/foo/bar")]), + ], +) +def test_push_items_missing_paths(path, expected): + items = PushItems([PushItem(path, "a", PushStatus.IN_DEV, "a")]) + assert items.missing_paths() == expected + + +@pytest.mark.parametrize( + """item,expected""", + [ + ( + PushItem( + Path("ofl/mavenpro"), + PushCategory.UPGRADE, + PushStatus.IN_DEV, + "45", + ), + { + "path": "ofl/mavenpro", + "category": "Upgrade", + "status": "In Dev / PR Merged", + "url": "45", + }, + ), + ( + PushItem( + Path("ofl/mavenpro"), + None, + None, + "45", + ), + {"path": "ofl/mavenpro", "category": None, "status": None, "url": "45"}, + ), + ( + PushItem( + Path("ofl/mavenpro"), + None, + None, + None, + ), + {"path": "ofl/mavenpro", "category": None, "status": None, "url": None}, + ), + ], +) +def test_push_items_to_json(item, expected): + assert item.to_json() == expected