From aeec058a118e5b56e742635b80c904b3f789da5c Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Thu, 6 Jan 2022 22:22:22 +0800 Subject: [PATCH 01/33] fix(bukuserver.views): ftp on valid scheme --- bukuserver/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bukuserver/views.py b/bukuserver/views.py index 736aca4e..8dda4fff 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -81,7 +81,7 @@ def _list_entry( self, context: Any, model: Namespace, name: str) -> Markup: parsed_url = urlparse(model.url) netloc, scheme = parsed_url.netloc, parsed_url.scheme - is_scheme_valid = scheme in ('http', 'https') + is_scheme_valid = scheme in ('http', 'https', 'ftp') tag_text = [] tag_tmpl = '{0}' for tag in model.tags.split(','): From e4ba8a74fb5f7f2133932610e307f137a68239f1 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Mon, 10 Jan 2022 19:45:53 +0800 Subject: [PATCH 02/33] fix(BookmarkModelView): update - use f string - don't use string as result variablee - black formatted --- bukuserver/views.py | 140 +++++++++++++++++++++++--------------------- 1 file changed, 73 insertions(+), 67 deletions(-) diff --git a/bukuserver/views.py b/bukuserver/views.py index 8dda4fff..d30de4d6 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -6,6 +6,7 @@ from urllib.parse import urlparse import itertools import logging +import functools from flask import current_app, flash, redirect, request, url_for from flask_admin.babel import gettext @@ -77,109 +78,114 @@ def _apply_filters(self, models, filters): def _create_ajax_loader(self, name, options): pass - def _list_entry( - self, context: Any, model: Namespace, name: str) -> Markup: + def _list_entry(self, context: Any, model: Namespace, name: str) -> Markup: + LOG.debug(f"context: {context}, name: {name}") parsed_url = urlparse(model.url) netloc, scheme = parsed_url.netloc, parsed_url.scheme - is_scheme_valid = scheme in ('http', 'https', 'ftp') + is_scheme_valid = scheme in ("http", "https", "ftp") tag_text = [] - tag_tmpl = '{0}' - for tag in model.tags.split(','): - if tag: - tag_text.append(tag_tmpl.format(tag, url_for( - 'bookmark.index_view', flt2_tags_contain=tag))) + br_tag = '
' + get_index_view_url = functools.partial(url_for, 'bookmark.index_view') + for tag in filter(None, model.tags.split(",")): + tag_text.append(f'{tag}') + tag_text_markup = "".join(tag_text) if not netloc: - return Markup("""\ - {0.title}
{2}
{1}{0.description} - """.format( - model, ''.join(tag_text), Markup.escape(model.url) - )) - res = '' - if not current_app.config.get('BUKUSERVER_DISABLE_FAVICON', False): - netloc_tmpl = ' ' - res = netloc_tmpl.format( - 'http://www.google.com/s2/favicons?domain=', netloc) - title = model.title if model.title else '<EMPTY TITLE>' - open_in_new_tab = current_app.config.get('BUKUSERVER_OPEN_IN_NEW_TAB', False) - if is_scheme_valid and open_in_new_tab: - res += '{1}'.format(model, title) - elif is_scheme_valid and not open_in_new_tab: - res += '{1}'.format(model, title) - else: - res += title - if self.url_render_mode == 'netloc': - res += ' ({0})'.format( - netloc, - url_for('bookmark.index_view', flt2_url_netloc_match=netloc) + escaped_url = Markup.escape(model.url) + return Markup( + f"""{model.title}{br_tag}{escaped_url}{br_tag}{tag_text_markup}{model.description}""" ) - res += '
' - if not is_scheme_valid: - res += model.url - elif self.url_render_mode is None or self.url_render_mode == 'full': - res += '{0.url}'.format(model) - res += '
' - if self.url_render_mode != 'netloc': - res += tag_tmpl.format( - 'netloc:{}'.format(netloc), - url_for('bookmark.index_view', flt2_url_netloc_match=netloc) + res = [] + if not current_app.config.get("BUKUSERVER_DISABLE_FAVICON", False): + res.append( + f'' ) - res += ''.join(tag_text) + title = model.title if model.title else "<EMPTY TITLE>" + open_in_new_tab = current_app.config.get("BUKUSERVER_OPEN_IN_NEW_TAB", False) + url_for_index_view_netloc = get_index_view_url(flt2_url_netloc_match=netloc) + if is_scheme_valid and not open_in_new_tab: + target = 'target="_blank"' if open_in_new_tab else "" + res.append(f'{title}') + else: + res.append(title) + if self.url_render_mode == "netloc": + res.append( f'({netloc})' ) + res.append(br_tag) + if not is_scheme_valid: + res.extend((model.url, br_tag)) + elif self.url_render_mode is None or self.url_render_mode == "full": + res.extend((f'{model.url}', br_tag)) + if self.url_render_mode != "netloc": + res.append( f'netloc:{netloc}') + if tag_text_markup: + res.append("".join(tag_text)) description = model.description if description: - res += '
' - res += description.replace('\n', '
') - return Markup(res) + res.extend((br_tag, description.replace("\n", br_tag))) + return Markup("".join(res)) can_set_page_size = True can_view_details = True - column_filters = ['buku', 'id', 'url', 'title', 'tags'] - column_formatters = {'Entry': _list_entry,} - column_list = ['Entry'] + column_filters = ["buku", "id", "url", "title", "tags"] + column_formatters = { + "Entry": _list_entry, + } + column_list = ["Entry"] create_modal = True - create_modal_template = 'bukuserver/bookmark_create_modal.html' - create_template = 'bukuserver/bookmark_create.html' + create_modal_template = "bukuserver/bookmark_create_modal.html" + create_template = "bukuserver/bookmark_create.html" details_modal = True edit_modal = True - edit_modal_template = 'bukuserver/bookmark_edit_modal.html' - edit_template = 'bukuserver/bookmark_edit.html' + edit_modal_template = "bukuserver/bookmark_edit_modal.html" + edit_template = "bukuserver/bookmark_edit.html" named_filter_urls = True def __init__(self, *args, **kwargs): self.bukudb = args[0] - custom_model = CustomBukuDbModel(args[0], 'bookmark') - args = [custom_model, ] + list(args[1:]) - self.page_size = kwargs.pop('page_size', DEFAULT_PER_PAGE) - self.url_render_mode = kwargs.pop('url_render_mode', DEFAULT_URL_RENDER_MODE) + custom_model = CustomBukuDbModel(args[0], "bookmark") + args = [ + custom_model, + ] + list(args[1:]) + self.page_size = kwargs.pop("page_size", DEFAULT_PER_PAGE) + self.url_render_mode = kwargs.pop("url_render_mode", DEFAULT_URL_RENDER_MODE) super().__init__(*args, **kwargs) def create_form(self, obj=None): form = super().create_form(obj) args = request.args - if 'url' in args.keys() and not args.get("url").startswith('/bookmark/'): + if "url" in args.keys() and not args.get("url").startswith("/bookmark/"): form.url.data = args.get("url") - if 'title' in args.keys(): + if "title" in args.keys(): form.title.data = args.get("title") - if 'description' in args.keys(): + if "description" in args.keys(): form.description.data = args.get("description") return form def create_model(self, form): try: - model = SimpleNamespace(id=None, url=None, title=None, tags=None, description=None) + model = SimpleNamespace( + id=None, url=None, title=None, tags=None, description=None + ) form.populate_obj(model) - vars(model).pop('id') + vars(model).pop("id") self._on_model_change(form, model, True) tags_in = model.tags - if not tags_in.startswith(','): - tags_in = ',{}'.format(tags_in) - if not tags_in.endswith(','): - tags_in = '{},'.format(tags_in) + if not tags_in.startswith(","): + tags_in = ",{}".format(tags_in) + if not tags_in.endswith(","): + tags_in = "{},".format(tags_in) self.model.bukudb.add_rec( - url=model.url, title_in=model.title, tags_in=tags_in, desc=model.description) + url=model.url, + title_in=model.title, + tags_in=tags_in, + desc=model.description, + ) except Exception as ex: if not self.handle_view_exception(ex): - flash(gettext('Failed to create record. %(error)s', error=str(ex)), 'error') - LOG.exception('Failed to create record.') + flash( + gettext("Failed to create record. %(error)s", error=str(ex)), + "error", + ) + LOG.exception("Failed to create record.") return False else: self.after_model_change(form, model, True) From 1d42014935eec95dc278a4f9d5cf10e24ace8e90 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Tue, 11 Jan 2022 18:45:27 +0800 Subject: [PATCH 03/33] refactor(bukuserver.views): black format and pylint --- bukuserver/views.py | 384 ++++++++++++++++++++++++++++---------------- 1 file changed, 243 insertions(+), 141 deletions(-) diff --git a/bukuserver/views.py b/bukuserver/views.py index d30de4d6..fc456771 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -26,37 +26,41 @@ STATISTIC_DATA = None -DEFAULT_URL_RENDER_MODE = 'full' +DEFAULT_URL_RENDER_MODE = "full" DEFAULT_PER_PAGE = 10 LOG = logging.getLogger("bukuserver.views") class CustomAdminIndexView(AdminIndexView): - - @expose('/') + @expose("/") def index(self): - return self.render('bukuserver/home.html', form=forms.HomeForm()) - - @expose('/', methods=['POST',]) + return self.render("bukuserver/home.html", form=forms.HomeForm()) + + @expose( + "/", + methods=[ + "POST", + ], + ) def search(self): "redirect to bookmark search" form = forms.HomeForm() bbm_filter = bs_filters.BookmarkBukuFilter( - all_keywords=False, deep=form.deep.data, regex=form.regex.data) + all_keywords=False, deep=form.deep.data, regex=form.regex.data + ) op_text = bbm_filter.operation() values_combi = sorted(itertools.product([True, False], repeat=3)) for idx, (all_keywords, deep, regex) in enumerate(values_combi): if deep == form.deep.data and regex == form.regex.data and not all_keywords: choosen_idx = idx - url_op_text = op_text.replace(', ', '_').replace(' ', ' ').replace(' ', '_') - key = ''.join(['flt', str(choosen_idx), '_buku_', url_op_text]) + url_op_text = op_text.replace(", ", "_").replace(" ", " ").replace(" ", "_") + key = "".join(["flt", str(choosen_idx), "_buku_", url_op_text]) kwargs = {key: form.keyword.data} - url = url_for('bookmark.index_view', **kwargs) + url = url_for("bookmark.index_view", **kwargs) return redirect(url) class CustomBukuDbModel: # pylint: disable=too-few-public-methods - def __init__(self, bukudb_inst, name): self.bukudb = bukudb_inst self.name = name @@ -67,7 +71,6 @@ def __name__(self): class BookmarkModelView(BaseModelView): - def _apply_filters(self, models, filters): for idx, flt_name, value in filters: flt = self._filters[idx] @@ -79,15 +82,17 @@ def _create_ajax_loader(self, name, options): pass def _list_entry(self, context: Any, model: Namespace, name: str) -> Markup: - LOG.debug(f"context: {context}, name: {name}") + LOG.debug("context: %s, name: %s", context, name) parsed_url = urlparse(model.url) netloc, scheme = parsed_url.netloc, parsed_url.scheme is_scheme_valid = scheme in ("http", "https", "ftp") tag_text = [] - br_tag = '
' - get_index_view_url = functools.partial(url_for, 'bookmark.index_view') + br_tag = "
" + get_index_view_url = functools.partial(url_for, "bookmark.index_view") for tag in filter(None, model.tags.split(",")): - tag_text.append(f'{tag}') + tag_text.append( + f'{tag}' + ) tag_text_markup = "".join(tag_text) if not netloc: escaped_url = Markup.escape(model.url) @@ -108,14 +113,16 @@ def _list_entry(self, context: Any, model: Namespace, name: str) -> Markup: else: res.append(title) if self.url_render_mode == "netloc": - res.append( f'({netloc})' ) + res.append(f'({netloc})') res.append(br_tag) if not is_scheme_valid: res.extend((model.url, br_tag)) elif self.url_render_mode is None or self.url_render_mode == "full": res.extend((f'{model.url}', br_tag)) if self.url_render_mode != "netloc": - res.append( f'netloc:{netloc}') + res.append( + f'netloc:{netloc}' + ) if tag_text_markup: res.append("".join(tag_text)) description = model.description @@ -197,8 +204,11 @@ def delete_model(self, model): res = self.bukudb.delete_rec(model.id) except Exception as ex: if not self.handle_view_exception(ex): - flash(gettext('Failed to delete record. %(error)s', error=str(ex)), 'error') - LOG.exception('Failed to delete record.') + flash( + gettext("Failed to delete record. %(error)s", error=str(ex)), + "error", + ) + LOG.exception("Failed to delete record.") return False else: self.after_model_delete(model) @@ -206,23 +216,26 @@ def delete_model(self, model): def get_list(self, page, sort_field, sort_desc, search, filters, page_size=None): bukudb = self.bukudb - contain_buku_search = any(x[1] == 'buku' for x in filters) + contain_buku_search = any(x[1] == "buku" for x in filters) if contain_buku_search: mode_id = [x[0] for x in filters] if len(list(set(mode_id))) > 1: - flash(gettext('Invalid search mode combination'), 'error') + flash(gettext("Invalid search mode combination"), "error") return 0, [] keywords = [x[2] for x in filters] for idx, flt_name, value in filters: - if flt_name == 'buku': + if flt_name == "buku": flt = self._filters[idx] bookmarks = bukudb.searchdb( - keywords, all_keywords=flt.all_keywords, deep=flt.deep, regex=flt.regex) + keywords, all_keywords=flt.all_keywords, deep=flt.deep, regex=flt.regex + ) else: bookmarks = bukudb.get_rec_all() bookmarks = self._apply_filters(bookmarks, filters) if sort_field: - key_idx = [x.value for x in BookmarkField if x.name.lower() == sort_field][0] + key_idx = [x.value for x in BookmarkField if x.name.lower() == sort_field][ + 0 + ] bookmarks = sorted(bookmarks, key=lambda x: x[key_idx], reverse=sort_desc) count = len(bookmarks) if page_size and bookmarks: @@ -232,13 +245,15 @@ def get_list(self, page, sort_field, sort_desc, search, filters, page_size=None) bookmarks = [] data = [] for bookmark in bookmarks: - bm_sns = SimpleNamespace(id=None, url=None, title=None, tags=None, description=None) + bm_sns = SimpleNamespace( + id=None, url=None, title=None, tags=None, description=None + ) for field in list(BookmarkField): if field == BookmarkField.TAGS: value = bookmark[field.value] - if value.startswith(','): + if value.startswith(","): value = value[1:] - if value.endswith(','): + if value.endswith(","): value = value[:-1] setattr(bm_sns, field.name.lower(), value) else: @@ -248,13 +263,15 @@ def get_list(self, page, sort_field, sort_desc, search, filters, page_size=None) def get_one(self, id): bookmark = self.model.bukudb.get_rec_by_id(id) - bm_sns = SimpleNamespace(id=None, url=None, title=None, tags=None, description=None) + bm_sns = SimpleNamespace( + id=None, url=None, title=None, tags=None, description=None + ) for field in list(BookmarkField): - if field == BookmarkField.TAGS and bookmark[field.value].startswith(','): + if field == BookmarkField.TAGS and bookmark[field.value].startswith(","): value = bookmark[field.value] - if value.startswith(','): + if value.startswith(","): value = value[1:] - if value.endswith(','): + if value.endswith(","): value = value[:-1] setattr(bm_sns, field.name.lower(), value) else: @@ -271,66 +288,102 @@ def scaffold_list_form(self, widget=None, validators=None): pass def scaffold_sortable_columns(self): - return {x:x for x in self.scaffold_list_columns()} + return {x: x for x in self.scaffold_list_columns()} def scaffold_filters(self, name): res = [] - if name == 'buku': + if name == "buku": values_combi = sorted(itertools.product([True, False], repeat=3)) for all_keywords, deep, regex in values_combi: res.append( - bs_filters.BookmarkBukuFilter(all_keywords=all_keywords, deep=deep, regex=regex) + bs_filters.BookmarkBukuFilter( + all_keywords=all_keywords, deep=deep, regex=regex + ) ) elif name == BookmarkField.ID.name.lower(): - res.extend([ - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.EQUAL), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.NOT_EQUAL), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.IN_LIST), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.NOT_IN_LIST), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.GREATER), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.SMALLER), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.TOP_X), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.BOTTOM_X), - ]) + res.extend( + [ + bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.EQUAL), + bs_filters.BookmarkBaseFilter( + name, filter_type=FilterType.NOT_EQUAL + ), + bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.IN_LIST), + bs_filters.BookmarkBaseFilter( + name, filter_type=FilterType.NOT_IN_LIST + ), + bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.GREATER), + bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.SMALLER), + bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.TOP_X), + bs_filters.BookmarkBaseFilter( + name, filter_type=FilterType.BOTTOM_X + ), + ] + ) elif name == BookmarkField.URL.name.lower(): + def netloc_match_func(query, value, index): return filter(lambda x: urlparse(x[index]).netloc == value, query) - res.extend([ - bs_filters.BookmarkBaseFilter(name, 'netloc match', netloc_match_func), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.EQUAL), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.NOT_EQUAL), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.IN_LIST), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.NOT_IN_LIST), - ]) + res.extend( + [ + bs_filters.BookmarkBaseFilter( + name, "netloc match", netloc_match_func + ), + bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.EQUAL), + bs_filters.BookmarkBaseFilter( + name, filter_type=FilterType.NOT_EQUAL + ), + bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.IN_LIST), + bs_filters.BookmarkBaseFilter( + name, filter_type=FilterType.NOT_IN_LIST + ), + ] + ) elif name == BookmarkField.TITLE.name.lower(): - res.extend([ - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.EQUAL), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.NOT_EQUAL), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.IN_LIST), - bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.NOT_IN_LIST), - ]) + res.extend( + [ + bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.EQUAL), + bs_filters.BookmarkBaseFilter( + name, filter_type=FilterType.NOT_EQUAL + ), + bs_filters.BookmarkBaseFilter(name, filter_type=FilterType.IN_LIST), + bs_filters.BookmarkBaseFilter( + name, filter_type=FilterType.NOT_IN_LIST + ), + ] + ) elif name == BookmarkField.TAGS.name.lower(): + def tags_contain_func(query, value, index): for item in query: - for tag in item[index].split(','): + for tag in item[index].split(","): if tag and tag == value: yield item def tags_not_contain_func(query, value, index): for item in query: - for tag in item[index].split(','): + for tag in item[index].split(","): if tag and tag != value: yield item - res.extend([ - bs_filters.BookmarkBaseFilter(name, 'contain', tags_contain_func), - bs_filters.BookmarkBaseFilter(name, 'not contain', tags_not_contain_func), - bs_filters.BookmarkTagNumberEqualFilter(name, 'number equal'), - bs_filters.BookmarkTagNumberNotEqualFilter(name, 'number not equal'), - bs_filters.BookmarkTagNumberGreaterFilter(name, 'number greater than'), - bs_filters.BookmarkTagNumberSmallerFilter(name, 'number smaller than'), - ]) + res.extend( + [ + bs_filters.BookmarkBaseFilter(name, "contain", tags_contain_func), + bs_filters.BookmarkBaseFilter( + name, "not contain", tags_not_contain_func + ), + bs_filters.BookmarkTagNumberEqualFilter(name, "number equal"), + bs_filters.BookmarkTagNumberNotEqualFilter( + name, "number not equal" + ), + bs_filters.BookmarkTagNumberGreaterFilter( + name, "number greater than" + ), + bs_filters.BookmarkTagNumberSmallerFilter( + name, "number smaller than" + ), + ] + ) elif name in self.scaffold_list_columns(): pass else: @@ -349,17 +402,24 @@ def update_model(self, form, model): self._on_model_change(form, model, False) self.bukudb.delete_tag_at_index(model.id, original_tags) tags_in = model.tags - if not tags_in.startswith(','): - tags_in = ',{}'.format(tags_in) - if not tags_in.endswith(','): - tags_in = '{},'.format(tags_in) + if not tags_in.startswith(","): + tags_in = ",{}".format(tags_in) + if not tags_in.endswith(","): + tags_in = "{},".format(tags_in) res = self.bukudb.update_rec( - model.id, url=model.url, title_in=model.title, tags_in=tags_in, - desc=model.description) + model.id, + url=model.url, + title_in=model.title, + tags_in=tags_in, + desc=model.description, + ) except Exception as ex: if not self.handle_view_exception(ex): - flash(gettext('Failed to update record. %(error)s', error=str(ex)), 'error') - LOG.exception('Failed to update record.') + flash( + gettext("Failed to update record. %(error)s", error=str(ex)), + "error", + ) + LOG.exception("Failed to update record.") return False else: self.after_model_change(form, model, False) @@ -367,7 +427,6 @@ def update_model(self, form, model): class TagModelView(BaseModelView): - def _create_ajax_loader(self, name, options): pass @@ -381,31 +440,39 @@ def _apply_filters(self, models, filters): def _name_formatter(self, context, model, name): data = getattr(model, name) if not data: - return Markup('{}'.format( - url_for('bookmark.index_view', flt2_tags_number_equal=0), - '<EMPTY TAG>' - )) - return Markup('{}'.format( - url_for('bookmark.index_view', flt1_tags_contain=data), data - )) + return Markup( + '{}'.format( + url_for("bookmark.index_view", flt2_tags_number_equal=0), + "<EMPTY TAG>", + ) + ) + return Markup( + '{}'.format( + url_for("bookmark.index_view", flt1_tags_contain=data), data + ) + ) can_create = False can_set_page_size = True - column_filters = ['name', 'usage_count'] - column_formatters = {'name': _name_formatter,} + column_filters = ["name", "usage_count"] + column_formatters = { + "name": _name_formatter, + } def __init__(self, *args, **kwargs): self.bukudb = args[0] - custom_model = CustomBukuDbModel(args[0], 'tag') - args = [custom_model, ] + list(args[1:]) - self.page_size = kwargs.pop('page_size', DEFAULT_PER_PAGE) + custom_model = CustomBukuDbModel(args[0], "tag") + args = [ + custom_model, + ] + list(args[1:]) + self.page_size = kwargs.pop("page_size", DEFAULT_PER_PAGE) super().__init__(*args, **kwargs) def scaffold_list_columns(self): - return ['name', 'usage_count'] + return ["name", "usage_count"] def scaffold_sortable_columns(self): - return {x:x for x in self.scaffold_list_columns()} + return {x: x for x in self.scaffold_list_columns()} def scaffold_form(self): class CustomForm(FlaskForm): # pylint: disable=too-few-public-methods @@ -417,21 +484,27 @@ def scaffold_list_form(self, widget=None, validators=None): pass def get_list( - self, - page: int, - sort_field: str, - sort_desc: bool, - search: Optional[Any], - filters: List[Tuple[int, str, str]], - page_size: int = None) -> Tuple[int, List[SimpleNamespace]]: + self, + page: int, + sort_field: str, + sort_desc: bool, + search: Optional[Any], + filters: List[Tuple[int, str, str]], + page_size: int = None, + ) -> Tuple[int, List[SimpleNamespace]]: bukudb = self.bukudb tags = bukudb.get_tag_all()[1] tags = sorted(tags.items()) tags = self._apply_filters(tags, filters) - sort_field_dict = {'usage_count': 1, 'name': 0} + sort_field_dict = {"usage_count": 1, "name": 0} if sort_field in sort_field_dict: - tags = list(sorted( - tags, key=lambda x: x[sort_field_dict[sort_field]], reverse=sort_desc)) + tags = list( + sorted( + tags, + key=lambda x: x[sort_field_dict[sort_field]], + reverse=sort_desc, + ) + ) count = len(tags) if page_size and tags: tags = list(chunks(tags, page_size))[page] @@ -459,21 +532,27 @@ def top_most_common_func(query, value, index): most_common_item = [x[0] for x in most_common] return filter(lambda x: x[index] in most_common_item, query) - res.extend([ - bs_filters.TagBaseFilter(name, filter_type=FilterType.EQUAL), - bs_filters.TagBaseFilter(name, filter_type=FilterType.NOT_EQUAL), - bs_filters.TagBaseFilter(name, filter_type=FilterType.IN_LIST), - bs_filters.TagBaseFilter(name, filter_type=FilterType.NOT_IN_LIST), - ]) - if name == 'usage_count': - res.extend([ - bs_filters.TagBaseFilter(name, filter_type=FilterType.GREATER), - bs_filters.TagBaseFilter(name, filter_type=FilterType.SMALLER), - bs_filters.TagBaseFilter(name, filter_type=FilterType.TOP_X), - bs_filters.TagBaseFilter(name, filter_type=FilterType.BOTTOM_X), - bs_filters.TagBaseFilter(name, 'top most common', top_most_common_func), - ]) - elif name == 'name': + res.extend( + [ + bs_filters.TagBaseFilter(name, filter_type=FilterType.EQUAL), + bs_filters.TagBaseFilter(name, filter_type=FilterType.NOT_EQUAL), + bs_filters.TagBaseFilter(name, filter_type=FilterType.IN_LIST), + bs_filters.TagBaseFilter(name, filter_type=FilterType.NOT_IN_LIST), + ] + ) + if name == "usage_count": + res.extend( + [ + bs_filters.TagBaseFilter(name, filter_type=FilterType.GREATER), + bs_filters.TagBaseFilter(name, filter_type=FilterType.SMALLER), + bs_filters.TagBaseFilter(name, filter_type=FilterType.TOP_X), + bs_filters.TagBaseFilter(name, filter_type=FilterType.BOTTOM_X), + bs_filters.TagBaseFilter( + name, "top most common", top_most_common_func + ), + ] + ) + elif name == "name": pass else: return super().scaffold_filters(name) @@ -486,8 +565,11 @@ def delete_model(self, model): res = self.bukudb.delete_tag_at_index(0, model.name, chatty=False) except Exception as ex: if not self.handle_view_exception(ex): - flash(gettext('Failed to delete record. %(error)s', error=str(ex)), 'error') - LOG.exception('Failed to delete record.') + flash( + gettext("Failed to delete record. %(error)s", error=str(ex)), + "error", + ) + LOG.exception("Failed to delete record.") return False else: self.after_model_delete(model) @@ -502,8 +584,11 @@ def update_model(self, form, model): res = self.bukudb.replace_tag(original_name, [model.name]) except Exception as ex: if not self.handle_view_exception(ex): - flash(gettext('Failed to update record. %(error)s', error=str(ex)), 'error') - LOG.exception('Failed to update record.') + flash( + gettext("Failed to update record. %(error)s", error=str(ex)), + "error", + ) + LOG.exception("Failed to update record.") return False else: self.after_model_change(form, model, False) @@ -514,46 +599,55 @@ def create_model(self, form): class StatisticView(BaseView): # pylint: disable=too-few-public-methods - def __init__(self, *args, **kwargs): self.bukudb = args[0] args = list(args[1:]) super().__init__(*args, **kwargs) - @expose('/', methods=('GET', 'POST')) + @expose("/", methods=("GET", "POST")) def index(self): bukudb = self.bukudb global STATISTIC_DATA statistic_data = STATISTIC_DATA - if not statistic_data or request.method == 'POST': + if not statistic_data or request.method == "POST": all_bookmarks = bukudb.get_rec_all() netloc = [urlparse(x[1]).netloc for x in all_bookmarks] tag_set = [x[3] for x in all_bookmarks] tag_items = [] for tags in tag_set: - tag_items.extend([x.strip() for x in tags.split(',') if x.strip()]) + tag_items.extend([x.strip() for x in tags.split(",") if x.strip()]) tag_counter = Counter(tag_items) title_items = [x[2] for x in all_bookmarks] title_counter = Counter(title_items) statistic_datetime = arrow.now() STATISTIC_DATA = { - 'datetime': statistic_datetime, - 'netloc': netloc, - 'tag_counter': tag_counter, - 'title_counter': title_counter, + "datetime": statistic_datetime, + "netloc": netloc, + "tag_counter": tag_counter, + "title_counter": title_counter, } else: - netloc = statistic_data['netloc'] - statistic_datetime = statistic_data['datetime'] - tag_counter = statistic_data['tag_counter'] - title_counter = statistic_data['title_counter'] + netloc = statistic_data["netloc"] + statistic_datetime = statistic_data["datetime"] + tag_counter = statistic_data["tag_counter"] + title_counter = statistic_data["title_counter"] netloc_counter = Counter(netloc) unique_netloc_len = len(set(netloc)) colors = [ - "#F7464A", "#46BFBD", "#FDB45C", "#FEDCBA", - "#ABCDEF", "#DDDDDD", "#ABCABC", "#4169E1", - "#C71585", "#FF4500", "#FEDCBA", "#46BFBD"] + "#F7464A", + "#46BFBD", + "#FDB45C", + "#FEDCBA", + "#ABCDEF", + "#DDDDDD", + "#ABCABC", + "#4169E1", + "#C71585", + "#FF4500", + "#FEDCBA", + "#46BFBD", + ] show_netloc_table = False if unique_netloc_len > len(colors): max_netloc_item = len(colors) @@ -564,7 +658,9 @@ def index(self): max_netloc_item = unique_netloc_len most_common_netlocs = netloc_counter.most_common(max_netloc_item) most_common_netlocs = [ - [val[0], val[1], netloc_colors[idx]] for idx, val in enumerate(most_common_netlocs)] + [val[0], val[1], netloc_colors[idx]] + for idx, val in enumerate(most_common_netlocs) + ] unique_tag_len = len(tag_counter) show_tag_rank_table = False @@ -577,7 +673,9 @@ def index(self): max_tag_item = unique_tag_len most_common_tags = tag_counter.most_common(max_tag_item) most_common_tags = [ - [val[0], val[1], tag_colors[idx]] for idx, val in enumerate(most_common_tags)] + [val[0], val[1], tag_colors[idx]] + for idx, val in enumerate(most_common_tags) + ] unique_title_len = len(title_counter) show_title_rank_table = False @@ -590,10 +688,12 @@ def index(self): max_title_item = unique_title_len most_common_titles = title_counter.most_common(max_title_item) most_common_titles = [ - [val[0], val[1], title_colors[idx]] for idx, val in enumerate(most_common_titles)] + [val[0], val[1], title_colors[idx]] + for idx, val in enumerate(most_common_titles) + ] return self.render( - 'bukuserver/statistic.html', + "bukuserver/statistic.html", most_common_netlocs=most_common_netlocs, netloc_counter=netloc_counter, show_netloc_table=show_netloc_table, @@ -604,10 +704,12 @@ def index(self): title_counter=title_counter, show_title_rank_table=show_title_rank_table, datetime=statistic_datetime, - datetime_text=statistic_datetime.humanize(arrow.now(), granularity='second'), + datetime_text=statistic_datetime.humanize( + arrow.now(), granularity="second" + ), ) def chunks(arr, n): n = max(1, n) - return (arr[i:i+n] for i in range(0, len(arr), n)) + return (arr[i : i + n] for i in range(0, len(arr), n)) From 3a2c2ee3a7186b87844f721bdc12b344d0fa8552 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Tue, 11 Jan 2022 19:42:00 +0800 Subject: [PATCH 04/33] refactor(bukuserver.views): pyright --- bukuserver/views.py | 55 ++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/bukuserver/views.py b/bukuserver/views.py index fc456771..9ec2f92a 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -13,16 +13,20 @@ from flask_admin.base import AdminIndexView, BaseView, expose from flask_admin.model import BaseModelView from flask_wtf import FlaskForm -from jinja2 import Markup +from jinja2 import Markup # type: ignore import arrow import wtforms +# `from jinja2 import Markup` raise following warning if not ignored: +# "Markup" is not exported from module "jinja2" +# Import from "jinja2.utils" instead + try: from . import forms, filters as bs_filters from .filters import BookmarkField, FilterType except ImportError: - from bukuserver import forms, filters as bs_filters - from bukuserver.filters import BookmarkField, FilterType + from bukuserver import forms, filters as bs_filters # type: ignore + from bukuserver.filters import BookmarkField, FilterType # type: ignore STATISTIC_DATA = None @@ -50,12 +54,15 @@ def search(self): ) op_text = bbm_filter.operation() values_combi = sorted(itertools.product([True, False], repeat=3)) + choosen_idx = None for idx, (all_keywords, deep, regex) in enumerate(values_combi): if deep == form.deep.data and regex == form.regex.data and not all_keywords: choosen_idx = idx url_op_text = op_text.replace(", ", "_").replace(" ", " ").replace(" ", "_") - key = "".join(["flt", str(choosen_idx), "_buku_", url_op_text]) - kwargs = {key: form.keyword.data} + kwargs = {} + if choosen_idx: + key = "".join(["flt", str(choosen_idx), "_buku_", url_op_text]) + kwargs = {key: form.keyword.data} url = url_for("bookmark.index_view", **kwargs) return redirect(url) @@ -72,10 +79,11 @@ def __name__(self): class BookmarkModelView(BaseModelView): def _apply_filters(self, models, filters): - for idx, flt_name, value in filters: - flt = self._filters[idx] - clean_value = flt.clean(value) - models = list(flt.apply(models, clean_value)) + for idx, _, value in filters: + if self._filters: + flt = self._filters[idx] + clean_value = flt.clean(value) + models = list(flt.apply(models, clean_value)) return models def _create_ajax_loader(self, name, options): @@ -159,8 +167,9 @@ def __init__(self, *args, **kwargs): def create_form(self, obj=None): form = super().create_form(obj) args = request.args - if "url" in args.keys() and not args.get("url").startswith("/bookmark/"): - form.url.data = args.get("url") + args_url = args.get("url") + if args_url and not args_url.startswith("/bookmark/"): + form.url.data = args_url if "title" in args.keys(): form.title.data = args.get("title") if "description" in args.keys(): @@ -214,7 +223,7 @@ def delete_model(self, model): self.after_model_delete(model) return res - def get_list(self, page, sort_field, sort_desc, search, filters, page_size=None): + def get_list(self, page, sort_field, sort_desc, _, filters, page_size=None): bukudb = self.bukudb contain_buku_search = any(x[1] == "buku" for x in filters) if contain_buku_search: @@ -223,12 +232,16 @@ def get_list(self, page, sort_field, sort_desc, search, filters, page_size=None) flash(gettext("Invalid search mode combination"), "error") return 0, [] keywords = [x[2] for x in filters] + flt = None for idx, flt_name, value in filters: - if flt_name == "buku": + if flt_name == "buku" and self._filters: flt = self._filters[idx] - bookmarks = bukudb.searchdb( - keywords, all_keywords=flt.all_keywords, deep=flt.deep, regex=flt.regex + kwargs = ( + dict(all_keywords=flt.all_keywords, deep=flt.deep, regex=flt.regex) + if flt + else {} ) + bookmarks = bukudb.searchdb(keywords, **kwargs) else: bookmarks = bukudb.get_rec_all() bookmarks = self._apply_filters(bookmarks, filters) @@ -431,13 +444,14 @@ def _create_ajax_loader(self, name, options): pass def _apply_filters(self, models, filters): - for idx, flt_name, value in filters: - flt = self._filters[idx] - clean_value = flt.clean(value) - models = list(flt.apply(models, clean_value)) + for idx, _, value in filters: + if self._filters: + flt = self._filters[idx] + clean_value = flt.clean(value) + models = list(flt.apply(models, clean_value)) return models - def _name_formatter(self, context, model, name): + def _name_formatter(self, _, model, name): data = getattr(model, name) if not data: return Markup( @@ -492,6 +506,7 @@ def get_list( filters: List[Tuple[int, str, str]], page_size: int = None, ) -> Tuple[int, List[SimpleNamespace]]: + logging.debug("search: %s", search) bukudb = self.bukudb tags = bukudb.get_tag_all()[1] tags = sorted(tags.items()) From bb1d6d3811baf7b370e4a92a3db64f8345b7bf6e Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Wed, 12 Jan 2022 18:06:23 +0800 Subject: [PATCH 05/33] build(pre-commit): new hooks --- .pre-commit-config.yaml | 48 ++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 47842bd2..a7b210b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,20 +1,34 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 - hooks: - # - id: trailing-whitespace - # - id: end-of-file-fixer - # - id: check-yaml - - id: check-added-large-files -- repo: https://github.com/PyCQA/pylint/ - rev: '2.6' - hooks: - - id: pylint - name: pylint - entry: pylint - language: system - types: [python] - args: ['--rcfile', 'tests/.pylintrc'] - # exclude: tests/functional/|tests/input|tests/extensions/data|tests/regrtest_data/|tests/data/|doc/ +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.4.0 + hooks: + # - id: trailing-whitespace + # - id: end-of-file-fixer + # - id: check-yaml + - id: check-added-large-files +- repo: https://github.com/PyCQA/pylint/ + rev: '2.6' + hooks: + - id: pylint + name: pylint + entry: pylint + language: system + types: [python] + args: [--rcfile, tests/.pylintrc] +- repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + name: isort (python) +- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks + rev: v2.2.0 + hooks: + - id: pretty-format-toml + args: [--autofix] + - id: pretty-format-yaml + args: [--autofix] + exclude: .copier-answers.yml + # based on + # https://gitlab.com/smop/pre-commit-hooks/-/blob/master/.pre-commit-hooks.yaml From 44be0e951ae0bc07081ea4059eaf6c03809365ff Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Wed, 12 Jan 2022 18:08:01 +0800 Subject: [PATCH 06/33] refactor(bukuserver.views): isort --- bukuserver/views.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/bukuserver/views.py b/bukuserver/views.py index 9ec2f92a..74b3f130 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -1,31 +1,29 @@ """views module.""" +import functools +import itertools +import logging from argparse import Namespace from collections import Counter from types import SimpleNamespace from typing import Any, List, Optional, Tuple from urllib.parse import urlparse -import itertools -import logging -import functools +import arrow +import wtforms from flask import current_app, flash, redirect, request, url_for from flask_admin.babel import gettext from flask_admin.base import AdminIndexView, BaseView, expose from flask_admin.model import BaseModelView from flask_wtf import FlaskForm -from jinja2 import Markup # type: ignore -import arrow -import wtforms - -# `from jinja2 import Markup` raise following warning if not ignored: -# "Markup" is not exported from module "jinja2" -# Import from "jinja2.utils" instead +from markupsafe import Markup try: - from . import forms, filters as bs_filters + from . import filters as bs_filters + from . import forms from .filters import BookmarkField, FilterType except ImportError: - from bukuserver import forms, filters as bs_filters # type: ignore + from bukuserver import filters as bs_filters # type: ignore + from bukuserver import forms from bukuserver.filters import BookmarkField, FilterType # type: ignore From 9db41b7b0339ed4c3dd23ad48448d4716676bf14 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Wed, 12 Jan 2022 18:08:55 +0800 Subject: [PATCH 07/33] test(BookmarkModelView): simpler --- tests/test_views.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/tests/test_views.py b/tests/test_views.py index 62b6aaeb..a42ff5bd 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1,3 +1,4 @@ +import logging from argparse import Namespace from types import SimpleNamespace @@ -21,6 +22,7 @@ def client(tmp_path): @pytest.mark.parametrize('disable_favicon', [False, True]) def test_bookmark_model_view(tmp_path, client, disable_favicon): + logging.debug('client: %s', client) test_db = tmp_path / 'test.db' bukudb = BukuDb(dbfile=test_db.as_posix()) inst = BookmarkModelView(bukudb) @@ -28,21 +30,7 @@ def test_bookmark_model_view(tmp_path, client, disable_favicon): description='randomdesc', id=1, tags='tags1', title='Example Domain', url='http://example.com') current_app.config['BUKUSERVER_DISABLE_FAVICON'] = disable_favicon - img_html = '' - if not disable_favicon: - img_html = \ - ' ' - res = inst._list_entry(None, model, 'Entry') - exp_res = \ - ( - 'Example Domain
' - 'http://example.com
' - 'netloc:example.com' - 'tags1' - '
randomdesc') - exp_res = ''.join([img_html, exp_res]) - assert str(res) == exp_res + assert inst._list_entry(None, model, 'Entry') @pytest.fixture From 71a1274e13878531253a85f25d32609984adf3d6 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Thu, 13 Jan 2022 21:06:05 +0800 Subject: [PATCH 08/33] test(buku): update - isort format - deduplicate parse_tags test case - test no args parse_tags --- tests/test_buku.py | 52 ++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/tests/test_buku.py b/tests/test_buku.py index aaa4644f..bd18d42a 100644 --- a/tests/test_buku.py +++ b/tests/test_buku.py @@ -1,17 +1,17 @@ """test module.""" -from itertools import product -from unittest import mock -from urllib.parse import urlparse import json import logging import os import signal import sys import unittest +from itertools import product +from unittest import mock +from urllib.parse import urlparse import pytest -from buku import is_int, parse_tags, prep_tag_search +from buku import DELIM, is_int, parse_tags, prep_tag_search only_python_3_5 = pytest.mark.skipif( sys.version_info < (3, 5), reason="requires Python 3.5 or later") @@ -77,21 +77,28 @@ def test_get_PoolManager(m_myproxy): @pytest.mark.parametrize( 'keywords, exp_res', [ - (None, None), - ([], None), + ('', DELIM), + (',', DELIM), + ('tag1, tag2', ',t a g 1,t a g 2,'), + (['tag1', 'tag2', 'tag3'], ',tag1 tag2 tag3,' ), + (['tag1', 'tag2'], ',tag1 tag2,' ), (['tag1', 'tag2'], ',tag1 tag2,'), + (['tag1'], ',tag1,' ), + (['tag1,tag2', 'tag3'], ',tag1,tag2 tag3,' ), (['tag1,tag2', 'tag3'], ',tag1,tag2 tag3,'), + (['tag1,tag2', 'tag3,tag4'], ',tag1,tag2 tag3,tag4,' ), + (['tag1,tag2'], ',tag1,tag2,' ), ] ) def test_parse_tags(keywords, exp_res): """test func.""" import buku - if keywords is None: - pass - elif not keywords: - exp_res = buku.DELIM - res = buku.parse_tags(keywords) - assert res == exp_res + assert buku.parse_tags(keywords) == exp_res + + +def test_parse_tags_no_args(): + import buku + assert buku.parse_tags() == DELIM @pytest.mark.parametrize( @@ -472,6 +479,10 @@ class TestHelpers(unittest.TestCase): # @unittest.skip('skipping') def test_parse_tags(self): + """test parse tags with unittest. + + Add new test case on `test_parse_tags` instead here. + """ # call with None parsed = parse_tags(None) self.assertIsNone(parsed) @@ -559,8 +570,9 @@ def test_sigint_handler(capsys): ) def test_network_handler_with_url(url, exp_res): """test func.""" - import buku import urllib3 + + import buku buku.urllib3 = urllib3 buku.myproxy = None res = buku.network_handler(url) @@ -668,8 +680,9 @@ def test_import_org(tmpdir, newtag, exp_res): ) def test_import_html(html_text, exp_res): """test method.""" - from buku import import_html from bs4 import BeautifulSoup + + from buku import import_html html_soup = BeautifulSoup(html_text, 'html.parser') res = list(import_html(html_soup, False, None)) for item, exp_item in zip(res, exp_res): @@ -677,8 +690,9 @@ def test_import_html(html_text, exp_res): def test_import_html_and_add_parent(): - from buku import import_html from bs4 import BeautifulSoup + + from buku import import_html html_text = """

1s

""" @@ -689,8 +703,9 @@ def test_import_html_and_add_parent(): def test_import_html_and_new_tag(): - from buku import import_html from bs4 import BeautifulSoup + + from buku import import_html html_text = """
GitHub
comment for the bookmark here""" exp_res = ( @@ -724,8 +739,9 @@ def test_copy_to_clipboard(platform, params): mock.patch('buku.Popen', return_value=m_popen_retval) as m_popen, \ mock.patch('buku.shutil.which', return_value=True): m_sys.platform = platform - from buku import copy_to_clipboard import subprocess + + from buku import copy_to_clipboard copy_to_clipboard(content) if platform_recognized: m_popen.assert_called_once_with( @@ -753,8 +769,8 @@ def test_copy_to_clipboard(platform, params): ['random', None], ]) def test_convert_bookmark_set(export_type, exp_res, monkeypatch): - from buku import convert_bookmark_set import buku + from buku import convert_bookmark_set bms = [ (1, 'htttp://example.com', '', ',', '', 0), (1, 'htttp://example.org', None, ',', '', 0), From 16b88dbe47bff52b23aa19925edfb8c1171992b6 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Thu, 13 Jan 2022 21:40:37 +0800 Subject: [PATCH 09/33] test(parse_tags): merge unittest --- tests/test_buku.py | 237 +++++++++++++++++++++++---------------------- 1 file changed, 121 insertions(+), 116 deletions(-) diff --git a/tests/test_buku.py b/tests/test_buku.py index bd18d42a..4e32d25c 100644 --- a/tests/test_buku.py +++ b/tests/test_buku.py @@ -11,174 +11,209 @@ import pytest -from buku import DELIM, is_int, parse_tags, prep_tag_search +from buku import DELIM, is_int, prep_tag_search only_python_3_5 = pytest.mark.skipif( - sys.version_info < (3, 5), reason="requires Python 3.5 or later") + sys.version_info < (3, 5), reason="requires Python 3.5 or later" +) @pytest.mark.parametrize( - 'url, exp_res', + "url, exp_res", [ - ['http://example.com', False], - ['ftp://ftp.somedomain.org', False], - ['http://examplecom.', True], - ['http://.example.com', True], - ['http://example.com.', True], - ['about:newtab', True], - ['chrome://version/', True], - ] + ["http://example.com", False], + ["ftp://ftp.somedomain.org", False], + ["http://examplecom.", True], + ["http://.example.com", True], + ["http://example.com.", True], + ["about:newtab", True], + ["chrome://version/", True], + ], ) def test_is_bad_url(url, exp_res): """test func.""" import buku + res = buku.is_bad_url(url) assert res == exp_res @pytest.mark.parametrize( - 'url, exp_res', + "url, exp_res", [ - ('http://example.com/file.pdf', True), - ('http://example.com/file.txt', True), - ('http://example.com/file.jpg', False), - ] + ("http://example.com/file.pdf", True), + ("http://example.com/file.txt", True), + ("http://example.com/file.jpg", False), + ], ) def test_is_ignored_mime(url, exp_res): """test func.""" import buku + assert exp_res == buku.is_ignored_mime(url) def test_gen_headers(): """test func.""" import buku + exp_myheaders = { - 'Accept-Encoding': 'gzip,deflate', - 'User-Agent': buku.USER_AGENT, - 'Accept': '*/*', - 'Cookie': '', - 'DNT': '1' + "Accept-Encoding": "gzip,deflate", + "User-Agent": buku.USER_AGENT, + "Accept": "*/*", + "Cookie": "", + "DNT": "1", } buku.gen_headers() assert buku.MYPROXY is None assert buku.MYHEADERS == exp_myheaders -@pytest.mark.parametrize('m_myproxy', [None, mock.Mock()]) +@pytest.mark.parametrize("m_myproxy", [None, mock.Mock()]) def test_get_PoolManager(m_myproxy): """test func.""" - with mock.patch('buku.urllib3'): + with mock.patch("buku.urllib3"): import buku + buku.myproxy = m_myproxy assert buku.get_PoolManager() @pytest.mark.parametrize( - 'keywords, exp_res', + "keywords, exp_res", [ - ('', DELIM), - (',', DELIM), - ('tag1, tag2', ',t a g 1,t a g 2,'), - (['tag1', 'tag2', 'tag3'], ',tag1 tag2 tag3,' ), - (['tag1', 'tag2'], ',tag1 tag2,' ), - (['tag1', 'tag2'], ',tag1 tag2,'), - (['tag1'], ',tag1,' ), - (['tag1,tag2', 'tag3'], ',tag1,tag2 tag3,' ), - (['tag1,tag2', 'tag3'], ',tag1,tag2 tag3,'), - (['tag1,tag2', 'tag3,tag4'], ',tag1,tag2 tag3,tag4,' ), - (['tag1,tag2'], ',tag1,tag2,' ), - ] + ("", DELIM), + (",", DELIM), + ("tag1, tag2", ",t a g 1,t a g 2,"), + ([" a tag , , , ,\t,\n,\r,\x0b,\x0c"], ",a tag,"), # whitespaces + ([",,,,,"], ","), # empty tags + (["\"tag\",'tag',tag"], ",\"tag\",'tag',tag,"), # escaping quotes + (["tag,tag, tag, tag,tag , tag "], ",tag,"), # duplicates, excessive spaces + (["tag1", "tag2", "tag3"], ",tag1 tag2 tag3,"), + (["tag1", "tag2"], ",tag1 tag2,"), + (["tag1"], ",tag1,"), + (["tag1,tag2", "tag3"], ",tag1,tag2 tag3,"), + (["tag1,tag2", "tag3,tag4"], ",tag1,tag2 tag3,tag4,"), + (["tag1,tag2"], ",tag1,tag2,"), + (["z_tag,a_tag,n_tag"], ",a_tag,n_tag,z_tag,"), # sorting tags + ([' '], ","), + ([''], ","), + ([','], ","), + ([], ","), # call with empty list + ([None], ","), + (None, None), # call with None + # combo + ( + [',,z_tag, a tag ,\t,,, ,n_tag ,n_tag, a_tag, \na tag ,\r, "a_tag"'], + ',"a_tag",a tag,a_tag,n_tag,z_tag,', + ), + ], ) def test_parse_tags(keywords, exp_res): """test func.""" import buku - assert buku.parse_tags(keywords) == exp_res + + if keywords is None: + assert buku.parse_tags(keywords) is None + else: + assert buku.parse_tags(keywords) == exp_res def test_parse_tags_no_args(): import buku + assert buku.parse_tags() == DELIM @pytest.mark.parametrize( - 'records, field_filter, exp_res', + "records, field_filter, exp_res", [ [ - [(1, 'http://url1.com', 'title1', ',tag1,'), - (2, 'http://url2.com', 'title2', ',tag1,tag2,')], + [ + (1, "http://url1.com", "title1", ",tag1,"), + (2, "http://url2.com", "title2", ",tag1,tag2,"), + ], 1, - ['1\thttp://url1.com', '2\thttp://url2.com'] + ["1\thttp://url1.com", "2\thttp://url2.com"], ], [ - [(1, 'http://url1.com', 'title1', ',tag1,'), - (2, 'http://url2.com', 'title2', ',tag1,tag2,')], + [ + (1, "http://url1.com", "title1", ",tag1,"), + (2, "http://url2.com", "title2", ",tag1,tag2,"), + ], 2, - ['1\thttp://url1.com\ttag1', '2\thttp://url2.com\ttag1,tag2'] + ["1\thttp://url1.com\ttag1", "2\thttp://url2.com\ttag1,tag2"], ], [ - [(1, 'http://url1.com', 'title1', ',tag1,'), - (2, 'http://url2.com', 'title2', ',tag1,tag2,')], + [ + (1, "http://url1.com", "title1", ",tag1,"), + (2, "http://url2.com", "title2", ",tag1,tag2,"), + ], 3, - ['1\ttitle1', '2\ttitle2'] + ["1\ttitle1", "2\ttitle2"], ], [ - [(1, 'http://url1.com', 'title1', ',tag1,'), - (2, 'http://url2.com', 'title2', ',tag1,tag2,')], + [ + (1, "http://url1.com", "title1", ",tag1,"), + (2, "http://url2.com", "title2", ",tag1,tag2,"), + ], 4, - ['1\thttp://url1.com\ttitle1\ttag1', '2\thttp://url2.com\ttitle2\ttag1,tag2'] + [ + "1\thttp://url1.com\ttitle1\ttag1", + "2\thttp://url2.com\ttitle2\ttag1,tag2", + ], ], [ - [(1, 'http://url1.com', 'title1', ',tag1,'), - (2, 'http://url2.com', 'title2', ',tag1,tag2,')], + [ + (1, "http://url1.com", "title1", ",tag1,"), + (2, "http://url2.com", "title2", ",tag1,tag2,"), + ], 10, - ['http://url1.com', 'http://url2.com'] + ["http://url1.com", "http://url2.com"], ], [ - [(1, 'http://url1.com', 'title1', ',tag1,'), - (2, 'http://url2.com', 'title2', ',tag1,tag2,')], + [ + (1, "http://url1.com", "title1", ",tag1,"), + (2, "http://url2.com", "title2", ",tag1,tag2,"), + ], 20, - ['http://url1.com\ttag1', 'http://url2.com\ttag1,tag2'] + ["http://url1.com\ttag1", "http://url2.com\ttag1,tag2"], ], [ - [(1, 'http://url1.com', 'title1', ',tag1,'), - (2, 'http://url2.com', 'title2', ',tag1,tag2,')], + [ + (1, "http://url1.com", "title1", ",tag1,"), + (2, "http://url2.com", "title2", ",tag1,tag2,"), + ], 30, - ['title1', 'title2'] + ["title1", "title2"], ], [ - [(1, 'http://url1.com', 'title1', ',tag1,'), - (2, 'http://url2.com', 'title2', ',tag1,tag2,')], + [ + (1, "http://url1.com", "title1", ",tag1,"), + (2, "http://url2.com", "title2", ",tag1,tag2,"), + ], 40, - ['http://url1.com\ttitle1\ttag1', 'http://url2.com\ttitle2\ttag1,tag2'] - ] - ] + ["http://url1.com\ttitle1\ttag1", "http://url2.com\ttitle2\ttag1,tag2"], + ], + ], ) def test_print_rec_with_filter(records, field_filter, exp_res): """test func.""" - with mock.patch('buku.print', create=True) as m_print: + with mock.patch("buku.print", create=True) as m_print: import buku + buku.print_rec_with_filter(records, field_filter) for res in exp_res: m_print.assert_any_call(res) @pytest.mark.parametrize( - 'taglist, exp_res', + "taglist, exp_res", [ - [ - 'tag1, tag2+3', - ([',tag1,', ',tag2+3,'], 'OR', None) - ], - [ - 'tag1 + tag2-3 + tag4', - ([',tag1,', ',tag2-3,', ',tag4,'], 'AND', None) - ], - [ - 'tag1, tag2-3 - tag4, tag5', - ([',tag1,', ',tag2-3,'], 'OR', ',tag4,|,tag5,') - ] - ] + ["tag1, tag2+3", ([",tag1,", ",tag2+3,"], "OR", None)], + ["tag1 + tag2-3 + tag4", ([",tag1,", ",tag2-3,", ",tag4,"], "AND", None)], + ["tag1, tag2-3 - tag4, tag5", ([",tag1,", ",tag2-3,"], "OR", ",tag4,|,tag5,")], + ], ) def test_prep_tag_search(taglist, exp_res): """test prep_tag_search helper function""" @@ -477,47 +512,17 @@ def test_piped_input(argv, pipeargs, isatty): class TestHelpers(unittest.TestCase): - # @unittest.skip('skipping') - def test_parse_tags(self): - """test parse tags with unittest. - - Add new test case on `test_parse_tags` instead here. - """ - # call with None - parsed = parse_tags(None) - self.assertIsNone(parsed) - # call with empty list - parsed = parse_tags([]) - self.assertEqual(parsed, ",") - # empty tags - parsed = parse_tags([",,,,,"]) - self.assertEqual(parsed, ",") - # sorting tags - parsed = parse_tags(["z_tag,a_tag,n_tag"]) - self.assertEqual(parsed, ",a_tag,n_tag,z_tag,") - # whitespaces - parsed = parse_tags([" a tag , , , ,\t,\n,\r,\x0b,\x0c"]) - self.assertEqual(parsed, ",a tag,") - # duplicates, excessive spaces - parsed = parse_tags(["tag,tag, tag, tag,tag , tag "]) - self.assertEqual(parsed, ",tag,") - # escaping quotes - parsed = parse_tags(["\"tag\",\'tag\',tag"]) - self.assertEqual(parsed, ",\"tag\",\'tag\',tag,") - # combo - parsed = parse_tags([",,z_tag, a tag ,\t,,, ,n_tag ,n_tag, a_tag, \na tag ,\r, \"a_tag\""]) - self.assertEqual(parsed, ",\"a_tag\",a tag,a_tag,n_tag,z_tag,") - # @unittest.skip('skipping') def test_is_int(self): - self.assertTrue(is_int('0')) - self.assertTrue(is_int('1')) - self.assertTrue(is_int('-1')) - self.assertFalse(is_int('')) - self.assertFalse(is_int('one')) + self.assertTrue(is_int("0")) + self.assertTrue(is_int("1")) + self.assertTrue(is_int("-1")) + self.assertFalse(is_int("")) + self.assertFalse(is_int("one")) + # This test fails because we use os._exit() now -@unittest.skip('skipping') +@unittest.skip("skipping") def test_sigint_handler(capsys): try: # sending SIGINT to self From 4380537616bf4d5f02b8feee531a61e63bba41e9 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Thu, 13 Jan 2022 21:47:43 +0800 Subject: [PATCH 10/33] fix(bukuserver.views): use buku parse_tags --- bukuserver/views.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/bukuserver/views.py b/bukuserver/views.py index 74b3f130..92d2645a 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -17,6 +17,8 @@ from flask_wtf import FlaskForm from markupsafe import Markup +import buku + try: from . import filters as bs_filters from . import forms @@ -153,7 +155,7 @@ def _list_entry(self, context: Any, model: Namespace, name: str) -> Markup: named_filter_urls = True def __init__(self, *args, **kwargs): - self.bukudb = args[0] + self.bukudb: buku.BukuDb = args[0] custom_model = CustomBukuDbModel(args[0], "bookmark") args = [ custom_model, @@ -182,15 +184,10 @@ def create_model(self, form): form.populate_obj(model) vars(model).pop("id") self._on_model_change(form, model, True) - tags_in = model.tags - if not tags_in.startswith(","): - tags_in = ",{}".format(tags_in) - if not tags_in.endswith(","): - tags_in = "{},".format(tags_in) self.model.bukudb.add_rec( url=model.url, title_in=model.title, - tags_in=tags_in, + tags_in=buku.parse_tags([model.tags]), desc=model.description, ) except Exception as ex: @@ -405,23 +402,16 @@ def scaffold_form(self): cls = forms.BookmarkForm return cls - def update_model(self, form, model): + def update_model(self, form: forms.BookmarkForm, model: Namespace): res = False try: - original_tags = model.tags form.populate_obj(model) self._on_model_change(form, model, False) - self.bukudb.delete_tag_at_index(model.id, original_tags) - tags_in = model.tags - if not tags_in.startswith(","): - tags_in = ",{}".format(tags_in) - if not tags_in.endswith(","): - tags_in = "{},".format(tags_in) res = self.bukudb.update_rec( model.id, url=model.url, title_in=model.title, - tags_in=tags_in, + tags_in=buku.parse_tags([model.tags]), desc=model.description, ) except Exception as ex: From b0ca0f58d551fc3819207fd1cdd98f0add9c6097 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Thu, 13 Jan 2022 21:52:43 +0800 Subject: [PATCH 11/33] refactor(bukuserver.views): unify msg between log and flash --- bukuserver/views.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/bukuserver/views.py b/bukuserver/views.py index 92d2645a..a5a466e5 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -192,11 +192,12 @@ def create_model(self, form): ) except Exception as ex: if not self.handle_view_exception(ex): + msg = "Failed to create record." flash( - gettext("Failed to create record. %(error)s", error=str(ex)), + gettext("%(msg)s %(error)s", msg=msg, error=str(ex)), "error", ) - LOG.exception("Failed to create record.") + LOG.exception(msg) return False else: self.after_model_change(form, model, True) @@ -208,11 +209,12 @@ def delete_model(self, model): res = self.bukudb.delete_rec(model.id) except Exception as ex: if not self.handle_view_exception(ex): + msg = "Failed to delete record." flash( - gettext("Failed to delete record. %(error)s", error=str(ex)), + gettext("%(msg)s %(error)s", msg=msg, error=str(ex)), "error", ) - LOG.exception("Failed to delete record.") + LOG.exception(msg) return False else: self.after_model_delete(model) @@ -416,11 +418,12 @@ def update_model(self, form: forms.BookmarkForm, model: Namespace): ) except Exception as ex: if not self.handle_view_exception(ex): + msg = "Failed to update record." flash( - gettext("Failed to update record. %(error)s", error=str(ex)), + gettext("%(msg)s %(error)s", msg=msg, error=str(ex)), "error", ) - LOG.exception("Failed to update record.") + LOG.exception(msg) return False else: self.after_model_change(form, model, False) @@ -568,11 +571,12 @@ def delete_model(self, model): res = self.bukudb.delete_tag_at_index(0, model.name, chatty=False) except Exception as ex: if not self.handle_view_exception(ex): + msg = "Failed to delete record." flash( - gettext("Failed to delete record. %(error)s", error=str(ex)), + gettext("%(msg)s %(error)s", msg=msg, error=str(ex)), "error", ) - LOG.exception("Failed to delete record.") + LOG.exception(msg) return False else: self.after_model_delete(model) @@ -587,11 +591,12 @@ def update_model(self, form, model): res = self.bukudb.replace_tag(original_name, [model.name]) except Exception as ex: if not self.handle_view_exception(ex): + msg = "Failed to update record." flash( - gettext("Failed to update record. %(error)s", error=str(ex)), + gettext("%(msg)s %(error)s", msg=msg, error=str(ex)), "error", ) - LOG.exception("Failed to update record.") + LOG.exception(msg) return False else: self.after_model_change(form, model, False) From 7eb9e00f4e2b2ef6189dcd6312f1fd62d2e237c0 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Thu, 13 Jan 2022 22:03:47 +0800 Subject: [PATCH 12/33] build(pre-commit): autoupdate --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a7b210b6..d6c7d1a4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,14 +2,14 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v4.1.0 hooks: # - id: trailing-whitespace # - id: end-of-file-fixer # - id: check-yaml - id: check-added-large-files - repo: https://github.com/PyCQA/pylint/ - rev: '2.6' + rev: v2.12.2 hooks: - id: pylint name: pylint From dd775952717a76256e1bd8ee1544d55e4b2ba511 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Thu, 13 Jan 2022 22:22:14 +0800 Subject: [PATCH 13/33] test(flake8): config --- .circleci/config.yml | 2 +- tests/.flake8 | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 tests/.flake8 diff --git a/.circleci/config.yml b/.circleci/config.yml index e07a2df3..35526bea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ test-template: &test-template command: | pip install -e .[tests] pip install -r requirements.txt - python3 -m flake8 + python3 -m flake8 --config tests/.flake8 echo buku | xargs pylint --rcfile tests/.pylintrc find . -iname "*.py" ! -path "./api/*" | xargs pylint --rcfile tests/.pylintrc python3 -m pytest ./tests/test_*.py --cov buku -vv --durations=0 -c ./tests/pytest.ini diff --git a/tests/.flake8 b/tests/.flake8 new file mode 100644 index 00000000..8ac52343 --- /dev/null +++ b/tests/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length=139 +extend-ignore = E203 From be0a9c84f6b756c8e5b1f6fdf4961efa03dcfe1b Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Thu, 13 Jan 2022 23:34:14 +0800 Subject: [PATCH 14/33] Revert "test(flake8): config" This reverts commit dd775952717a76256e1bd8ee1544d55e4b2ba511. --- .circleci/config.yml | 2 +- tests/.flake8 | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 tests/.flake8 diff --git a/.circleci/config.yml b/.circleci/config.yml index 35526bea..e07a2df3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ test-template: &test-template command: | pip install -e .[tests] pip install -r requirements.txt - python3 -m flake8 --config tests/.flake8 + python3 -m flake8 echo buku | xargs pylint --rcfile tests/.pylintrc find . -iname "*.py" ! -path "./api/*" | xargs pylint --rcfile tests/.pylintrc python3 -m pytest ./tests/test_*.py --cov buku -vv --durations=0 -c ./tests/pytest.ini diff --git a/tests/.flake8 b/tests/.flake8 deleted file mode 100644 index 8ac52343..00000000 --- a/tests/.flake8 +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -max-line-length=139 -extend-ignore = E203 From 4bcd21fd4a5b82508a24f1237ed6a4b9a68d3d92 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Thu, 13 Jan 2022 23:36:41 +0800 Subject: [PATCH 15/33] test(flake8): config with tox.ini --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index d838e2a8..6708968e 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,8 @@ ignore = W292, # W504 line break after binary operator W504, + # E203 whitespace before : + E203, [testenv] usedevelop = true From 08fc444190e4365cbf9bba5588886782d8f10308 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Fri, 14 Jan 2022 09:03:20 +0800 Subject: [PATCH 16/33] fix(bukuserver.__main__): import server --- bukuserver/__main__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bukuserver/__main__.py b/bukuserver/__main__.py index 61360bba..1981062e 100644 --- a/bukuserver/__main__.py +++ b/bukuserver/__main__.py @@ -1,4 +1,7 @@ -from . import server +try: + from . import server +except ImportError: + from bukuserver import server if __name__ == '__main__': From ea0c5cd476cc8210b42661a0b75907c334285057 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Fri, 14 Jan 2022 09:25:17 +0800 Subject: [PATCH 17/33] build(pre-commit): hook darker --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d6c7d1a4..bc596c29 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,6 +22,7 @@ repos: hooks: - id: isort name: isort (python) + args: [--profile, black, --filter-files] - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks rev: v2.2.0 hooks: @@ -32,3 +33,7 @@ repos: exclude: .copier-answers.yml # based on # https://gitlab.com/smop/pre-commit-hooks/-/blob/master/.pre-commit-hooks.yaml +- repo: https://github.com/akaihola/darker + rev: 1.3.2 + hooks: + - id: darker From ab14932ed65be81298e408a5d4f0bd934287fde2 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Fri, 14 Jan 2022 10:17:39 +0800 Subject: [PATCH 18/33] fix(ApiTagView): big tag list --- bukuserver/server.py | 91 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 16 deletions(-) diff --git a/bukuserver/server.py b/bukuserver/server.py index c800d871..2d3dc850 100644 --- a/bukuserver/server.py +++ b/bukuserver/server.py @@ -1,28 +1,31 @@ #!/usr/bin/env python # pylint: disable=wrong-import-order, ungrouped-imports """Server module.""" +import os +import sys +import typing as T from typing import Any, Dict, Union # NOQA; type: ignore from unittest import mock from urllib.parse import urlparse -import os -import sys -from buku import BukuDb, __version__, network_handler from flask.cli import FlaskGroup from flask.views import MethodView from flask_admin import Admin -from flask_api import exceptions, FlaskAPI, status +from flask_api import FlaskAPI, exceptions, status from flask_bootstrap import Bootstrap from flask_paginate import Pagination, get_page_parameter, get_per_page_parameter + +import buku +from buku import BukuDb, __version__, network_handler + try: from flask_reverse_proxy_fix.middleware import ReverseProxyPrefixFix except ImportError: ReverseProxyPrefixFix = None -from markupsafe import Markup import click import flask -from flask import ( # type: ignore - __version__ as flask_version, +from flask import __version__ as flask_version # type: ignore +from flask import ( abort, current_app, flash, @@ -32,11 +35,12 @@ request, url_for, ) +from markupsafe import Markup try: - from . import response, forms, views + from . import forms, response, views except ImportError: - from bukuserver import response, forms, views + from bukuserver import forms, response, views STATISTIC_DATA = None @@ -301,19 +305,74 @@ def shell_context(): return app +def search_tag( + db: BukuDb, stag: T.Optional[str] = None, limit: T.Optional[int] = None +) -> T.Tuple[T.List[str], T.Dict[str, int]]: + """search tag. + + db: + buku db instance + stag: + search tag + limit: + positive integer limit + + Returns + ------- + tuple + list of unique tags sorted alphabetically and dictionary of tag and its usage count + + Raises + ------ + ValueError + if limit is not positive + """ + if limit is not None and limit < 1: + raise ValueError("limit must be positive") + tags = [] + unique_tags = [] + dic = {} + query_args = None + base_query = "SELECT DISTINCT tags , COUNT(tags) FROM bookmarks GROUP BY tags" + if stag and limit is None: + query_args = f"{base_query} WHERE tags LIKE ?", (stag,) + elif stag and limit: + query_args = f"{base_query} WHERE tags LIKE ? LIMIT ?", (stag, limit) + elif not stag and limit: + query_args = f"{base_query} LIMIT ?", (limit,) + else: + raise ValueError(f"unknown condition: search tag: {stag}, limit: {limit}") + + for row in db.cur.execute(*query_args): + tagset = row[0].strip(buku.DELIM).split(buku.DELIM) + for tag in tagset: + if tag not in tags: + dic[tag] = row[1] + tags += (tag,) + else: + dic[tag] += row[1] + + if not tags: + return tags, dic + + if tags[0] == "": + unique_tags = sorted(tags[1:]) + else: + unique_tags = sorted(tags) + + return unique_tags, dic + + class ApiTagView(MethodView): - def get(self, tag: Union[str, None]): + def get(self, tag: T.Optional[str]): bukudb = get_bukudb() if tag is None: - tags = bukudb.get_tag_all() - result = {'tags': tags[0]} - return result - tags = bukudb.get_tag_all() + return {"tags": search_tag(db=bukudb, limit=5)[0]} + tags = search_tag(db=bukudb, stag=tag) if tag not in tags[1]: raise exceptions.NotFound() - res = dict(name=tag, usage_count=tags[1][tag]) - return res + return dict(name=tag, usage_count=tags[1][tag]) def put(self, tag: str): bukudb = get_bukudb() From bf7b223b57d5a470351236288e8e83dedf98aa85 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Fri, 14 Jan 2022 10:47:12 +0800 Subject: [PATCH 19/33] refactor(test_buku): revert unrelated code --- tests/test_buku.py | 153 +++++++++++++++++++++------------------------ 1 file changed, 71 insertions(+), 82 deletions(-) diff --git a/tests/test_buku.py b/tests/test_buku.py index 4e32d25c..e6c4bb48 100644 --- a/tests/test_buku.py +++ b/tests/test_buku.py @@ -14,21 +14,20 @@ from buku import DELIM, is_int, prep_tag_search only_python_3_5 = pytest.mark.skipif( - sys.version_info < (3, 5), reason="requires Python 3.5 or later" -) + sys.version_info < (3, 5), reason="requires Python 3.5 or later") @pytest.mark.parametrize( - "url, exp_res", + 'url, exp_res', [ - ["http://example.com", False], - ["ftp://ftp.somedomain.org", False], - ["http://examplecom.", True], - ["http://.example.com", True], - ["http://example.com.", True], - ["about:newtab", True], - ["chrome://version/", True], - ], + ['http://example.com', False], + ['ftp://ftp.somedomain.org', False], + ['http://examplecom.', True], + ['http://.example.com', True], + ['http://example.com.', True], + ['about:newtab', True], + ['chrome://version/', True], + ] ) def test_is_bad_url(url, exp_res): """test func.""" @@ -39,12 +38,12 @@ def test_is_bad_url(url, exp_res): @pytest.mark.parametrize( - "url, exp_res", + 'url, exp_res', [ - ("http://example.com/file.pdf", True), - ("http://example.com/file.txt", True), - ("http://example.com/file.jpg", False), - ], + ('http://example.com/file.pdf', True), + ('http://example.com/file.txt', True), + ('http://example.com/file.jpg', False), + ] ) def test_is_ignored_mime(url, exp_res): """test func.""" @@ -58,21 +57,21 @@ def test_gen_headers(): import buku exp_myheaders = { - "Accept-Encoding": "gzip,deflate", - "User-Agent": buku.USER_AGENT, - "Accept": "*/*", - "Cookie": "", - "DNT": "1", + 'Accept-Encoding': 'gzip,deflate', + 'User-Agent': buku.USER_AGENT, + 'Accept': '*/*', + 'Cookie': '', + 'DNT': '1' } buku.gen_headers() assert buku.MYPROXY is None assert buku.MYHEADERS == exp_myheaders -@pytest.mark.parametrize("m_myproxy", [None, mock.Mock()]) +@pytest.mark.parametrize('m_myproxy', [None, mock.Mock()]) def test_get_PoolManager(m_myproxy): """test func.""" - with mock.patch("buku.urllib3"): + with mock.patch('buku.urllib3'): import buku buku.myproxy = m_myproxy @@ -126,80 +125,61 @@ def test_parse_tags_no_args(): @pytest.mark.parametrize( - "records, field_filter, exp_res", + 'records, field_filter, exp_res', [ [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], + [(1, 'http://url1.com', 'title1', ',tag1,'), + (2, 'http://url2.com', 'title2', ',tag1,tag2,')], 1, - ["1\thttp://url1.com", "2\thttp://url2.com"], + ['1\thttp://url1.com', '2\thttp://url2.com'] ], [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], + [(1, 'http://url1.com', 'title1', ',tag1,'), + (2, 'http://url2.com', 'title2', ',tag1,tag2,')], 2, - ["1\thttp://url1.com\ttag1", "2\thttp://url2.com\ttag1,tag2"], + ['1\thttp://url1.com\ttag1', '2\thttp://url2.com\ttag1,tag2'] ], [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], + [(1, 'http://url1.com', 'title1', ',tag1,'), + (2, 'http://url2.com', 'title2', ',tag1,tag2,')], 3, - ["1\ttitle1", "2\ttitle2"], + ['1\ttitle1', '2\ttitle2'] ], [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], + [(1, 'http://url1.com', 'title1', ',tag1,'), + (2, 'http://url2.com', 'title2', ',tag1,tag2,')], 4, - [ - "1\thttp://url1.com\ttitle1\ttag1", - "2\thttp://url2.com\ttitle2\ttag1,tag2", - ], + ['1\thttp://url1.com\ttitle1\ttag1', '2\thttp://url2.com\ttitle2\ttag1,tag2'] ], [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], + [(1, 'http://url1.com', 'title1', ',tag1,'), + (2, 'http://url2.com', 'title2', ',tag1,tag2,')], 10, - ["http://url1.com", "http://url2.com"], + ['http://url1.com', 'http://url2.com'] ], [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], + [(1, 'http://url1.com', 'title1', ',tag1,'), + (2, 'http://url2.com', 'title2', ',tag1,tag2,')], 20, - ["http://url1.com\ttag1", "http://url2.com\ttag1,tag2"], + ['http://url1.com\ttag1', 'http://url2.com\ttag1,tag2'] ], [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], + [(1, 'http://url1.com', 'title1', ',tag1,'), + (2, 'http://url2.com', 'title2', ',tag1,tag2,')], 30, - ["title1", "title2"], + ['title1', 'title2'] ], [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], + [(1, 'http://url1.com', 'title1', ',tag1,'), + (2, 'http://url2.com', 'title2', ',tag1,tag2,')], 40, - ["http://url1.com\ttitle1\ttag1", "http://url2.com\ttitle2\ttag1,tag2"], - ], - ], + ['http://url1.com\ttitle1\ttag1', 'http://url2.com\ttitle2\ttag1,tag2'] + ] + ] ) def test_print_rec_with_filter(records, field_filter, exp_res): """test func.""" - with mock.patch("buku.print", create=True) as m_print: + with mock.patch('buku.print', create=True) as m_print: import buku buku.print_rec_with_filter(records, field_filter) @@ -208,12 +188,21 @@ def test_print_rec_with_filter(records, field_filter, exp_res): @pytest.mark.parametrize( - "taglist, exp_res", + 'taglist, exp_res', [ - ["tag1, tag2+3", ([",tag1,", ",tag2+3,"], "OR", None)], - ["tag1 + tag2-3 + tag4", ([",tag1,", ",tag2-3,", ",tag4,"], "AND", None)], - ["tag1, tag2-3 - tag4, tag5", ([",tag1,", ",tag2-3,"], "OR", ",tag4,|,tag5,")], - ], + [ + 'tag1, tag2+3', + ([',tag1,', ',tag2+3,'], 'OR', None) + ], + [ + 'tag1 + tag2-3 + tag4', + ([',tag1,', ',tag2-3,', ',tag4,'], 'AND', None) + ], + [ + 'tag1, tag2-3 - tag4, tag5', + ([',tag1,', ',tag2-3,'], 'OR', ',tag4,|,tag5,') + ] + ] ) def test_prep_tag_search(taglist, exp_res): """test prep_tag_search helper function""" @@ -512,17 +501,17 @@ def test_piped_input(argv, pipeargs, isatty): class TestHelpers(unittest.TestCase): + # @unittest.skip('skipping') # @unittest.skip('skipping') def test_is_int(self): - self.assertTrue(is_int("0")) - self.assertTrue(is_int("1")) - self.assertTrue(is_int("-1")) - self.assertFalse(is_int("")) - self.assertFalse(is_int("one")) - + self.assertTrue(is_int('0')) + self.assertTrue(is_int('1')) + self.assertTrue(is_int('-1')) + self.assertFalse(is_int('')) + self.assertFalse(is_int('one')) # This test fails because we use os._exit() now -@unittest.skip("skipping") +@unittest.skip('skipping') def test_sigint_handler(capsys): try: # sending SIGINT to self From be08fb972e5327e07e87b6132be603658a0f7dce Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Fri, 14 Jan 2022 11:11:35 +0800 Subject: [PATCH 20/33] build(pre-commit): hook pylint use hook config from example https://pylint.pycqa.org/en/latest/user_guide/pre-commit-integration.html?highlight=pre%20commit --- .pre-commit-config.yaml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bc596c29..17b271a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,15 +8,6 @@ repos: # - id: end-of-file-fixer # - id: check-yaml - id: check-added-large-files -- repo: https://github.com/PyCQA/pylint/ - rev: v2.12.2 - hooks: - - id: pylint - name: pylint - entry: pylint - language: system - types: [python] - args: [--rcfile, tests/.pylintrc] - repo: https://github.com/pycqa/isort rev: 5.10.1 hooks: @@ -37,3 +28,15 @@ repos: rev: 1.3.2 hooks: - id: darker +- repo: local + hooks: + - id: pylint + name: pylint + entry: pylint + language: system + types: [python] + args: [-rn, -sn, --rcfile=tests/.pylintrc] + # "-rn", # Only display messages + # "-sn", # Don't display the score + # based on + # https://pylint.pycqa.org/en/latest/user_guide/pre-commit-integration.html From b655d6292a2bf1eef775b05a9247148508ad350b Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Fri, 14 Jan 2022 11:36:58 +0800 Subject: [PATCH 21/33] refactor(search_tag): type hint and simpler variable --- bukuserver/server.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/bukuserver/server.py b/bukuserver/server.py index 2d3dc850..7722fabe 100644 --- a/bukuserver/server.py +++ b/bukuserver/server.py @@ -329,9 +329,8 @@ def search_tag( """ if limit is not None and limit < 1: raise ValueError("limit must be positive") - tags = [] - unique_tags = [] - dic = {} + tags: T.Set[str] = set() + dic: T.Dict[str, int] = {} query_args = None base_query = "SELECT DISTINCT tags , COUNT(tags) FROM bookmarks GROUP BY tags" if stag and limit is None: @@ -343,24 +342,19 @@ def search_tag( else: raise ValueError(f"unknown condition: search tag: {stag}, limit: {limit}") + row: T.Tuple[str, int] for row in db.cur.execute(*query_args): tagset = row[0].strip(buku.DELIM).split(buku.DELIM) for tag in tagset: + if not tag: + continue + tags.add(tag) if tag not in tags: dic[tag] = row[1] - tags += (tag,) else: dic[tag] += row[1] - if not tags: - return tags, dic - - if tags[0] == "": - unique_tags = sorted(tags[1:]) - else: - unique_tags = sorted(tags) - - return unique_tags, dic + return list(sorted(tags)), dic class ApiTagView(MethodView): From 4ae5c08ea7a87af71820079882170977ceb4f8bd Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Fri, 14 Jan 2022 11:54:47 +0800 Subject: [PATCH 22/33] fix(pylint): too-few-public-methods BookmarkletView --- bukuserver/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bukuserver/server.py b/bukuserver/server.py index 7722fabe..bacb42e4 100644 --- a/bukuserver/server.py +++ b/bukuserver/server.py @@ -607,7 +607,7 @@ def delete(self): return res -class BookmarkletView(MethodView): +class BookmarkletView(MethodView): # pylint: disable=too-few-public-methods def get(self): url = request.args.get('url') title = request.args.get('title') From 880c93c3afc7ad8c0f4ea7cb1082d727d0ad49c0 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Fri, 14 Jan 2022 11:55:34 +0800 Subject: [PATCH 23/33] fix(search_tag): no key on dic --- bukuserver/server.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bukuserver/server.py b/bukuserver/server.py index bacb42e4..c89e61d1 100644 --- a/bukuserver/server.py +++ b/bukuserver/server.py @@ -344,12 +344,11 @@ def search_tag( row: T.Tuple[str, int] for row in db.cur.execute(*query_args): - tagset = row[0].strip(buku.DELIM).split(buku.DELIM) - for tag in tagset: + for tag in row[0].strip(buku.DELIM).split(buku.DELIM): if not tag: continue tags.add(tag) - if tag not in tags: + if tag not in dic: dic[tag] = row[1] else: dic[tag] += row[1] From 9802ccc6713e20d80940be401ddad419cc85ed17 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Fri, 14 Jan 2022 13:20:48 +0800 Subject: [PATCH 24/33] fix(search_tag): simpler query --- bukuserver/server.py | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/bukuserver/server.py b/bukuserver/server.py index c89e61d1..9cf9bef9 100644 --- a/bukuserver/server.py +++ b/bukuserver/server.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # pylint: disable=wrong-import-order, ungrouped-imports """Server module.""" +import collections import os import sys import typing as T @@ -330,30 +331,19 @@ def search_tag( if limit is not None and limit < 1: raise ValueError("limit must be positive") tags: T.Set[str] = set() - dic: T.Dict[str, int] = {} - query_args = None - base_query = "SELECT DISTINCT tags , COUNT(tags) FROM bookmarks GROUP BY tags" - if stag and limit is None: - query_args = f"{base_query} WHERE tags LIKE ?", (stag,) - elif stag and limit: - query_args = f"{base_query} WHERE tags LIKE ? LIMIT ?", (stag, limit) - elif not stag and limit: - query_args = f"{base_query} LIMIT ?", (limit,) - else: - raise ValueError(f"unknown condition: search tag: {stag}, limit: {limit}") - + counter = collections.Counter() + query_list = ["SELECT DISTINCT tags , COUNT(tags) FROM bookmarks"] + if stag: + query_list.append("where tags LIKE :search_tag") + query_list.append("GROUP BY tags") row: T.Tuple[str, int] - for row in db.cur.execute(*query_args): + for row in db.cur.execute(" ".join(query_list), {"search_tag": f"%{stag}%"}): for tag in row[0].strip(buku.DELIM).split(buku.DELIM): if not tag: continue tags.add(tag) - if tag not in dic: - dic[tag] = row[1] - else: - dic[tag] += row[1] - - return list(sorted(tags)), dic + counter[tag] += row[1] + return list(sorted(tags)), dict(counter.most_common(limit)) class ApiTagView(MethodView): From d9e9d20ad3e57b0c8dd3fc423b9b21e143ba68ca Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Fri, 14 Jan 2022 13:48:08 +0800 Subject: [PATCH 25/33] feat(BookmarkModelView): handle magnet --- bukuserver/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bukuserver/views.py b/bukuserver/views.py index a5a466e5..7d58bf39 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -93,7 +93,7 @@ def _list_entry(self, context: Any, model: Namespace, name: str) -> Markup: LOG.debug("context: %s, name: %s", context, name) parsed_url = urlparse(model.url) netloc, scheme = parsed_url.netloc, parsed_url.scheme - is_scheme_valid = scheme in ("http", "https", "ftp") + is_scheme_valid = scheme in ("http", "https", "ftp", "magnet") tag_text = [] br_tag = "
" get_index_view_url = functools.partial(url_for, "bookmark.index_view") @@ -102,7 +102,7 @@ def _list_entry(self, context: Any, model: Namespace, name: str) -> Markup: f'{tag}' ) tag_text_markup = "".join(tag_text) - if not netloc: + if not netloc and not is_scheme_valid: escaped_url = Markup.escape(model.url) return Markup( f"""{model.title}{br_tag}{escaped_url}{br_tag}{tag_text_markup}{model.description}""" From a42c12529996f27b9540de4cdcd46cfd97a29a66 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Fri, 14 Jan 2022 13:48:51 +0800 Subject: [PATCH 26/33] fix(BookmarkForm): remove url validators --- bukuserver/forms.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bukuserver/forms.py b/bukuserver/forms.py index 51a197f4..9d8dd3c5 100644 --- a/bukuserver/forms.py +++ b/bukuserver/forms.py @@ -1,7 +1,7 @@ """Forms module.""" # pylint: disable=too-few-public-methods, missing-docstring -from flask_wtf import FlaskForm import wtforms +from flask_wtf import FlaskForm class SearchBookmarksForm(FlaskForm): @@ -16,8 +16,7 @@ class HomeForm(SearchBookmarksForm): class BookmarkForm(FlaskForm): - url = wtforms.StringField( - validators=[wtforms.validators.DataRequired(), wtforms.validators.URL(require_tld=False)]) + url = wtforms.StringField(validators=[wtforms.validators.DataRequired()]) title = wtforms.StringField() tags = wtforms.StringField() description = wtforms.TextAreaField() From 8f318f1426305ede9247dc3d6a7e9ce9367b3c20 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Fri, 14 Jan 2022 13:53:51 +0800 Subject: [PATCH 27/33] fix(BookmarkModelView): don't check scheme manually --- bukuserver/views.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bukuserver/views.py b/bukuserver/views.py index 7d58bf39..4a99d8f3 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -92,8 +92,7 @@ def _create_ajax_loader(self, name, options): def _list_entry(self, context: Any, model: Namespace, name: str) -> Markup: LOG.debug("context: %s, name: %s", context, name) parsed_url = urlparse(model.url) - netloc, scheme = parsed_url.netloc, parsed_url.scheme - is_scheme_valid = scheme in ("http", "https", "ftp", "magnet") + netloc = parsed_url.netloc tag_text = [] br_tag = "
" get_index_view_url = functools.partial(url_for, "bookmark.index_view") @@ -102,7 +101,7 @@ def _list_entry(self, context: Any, model: Namespace, name: str) -> Markup: f'{tag}' ) tag_text_markup = "".join(tag_text) - if not netloc and not is_scheme_valid: + if not netloc and not parsed_url.scheme: escaped_url = Markup.escape(model.url) return Markup( f"""{model.title}{br_tag}{escaped_url}{br_tag}{tag_text_markup}{model.description}""" @@ -115,7 +114,7 @@ def _list_entry(self, context: Any, model: Namespace, name: str) -> Markup: title = model.title if model.title else "<EMPTY TITLE>" open_in_new_tab = current_app.config.get("BUKUSERVER_OPEN_IN_NEW_TAB", False) url_for_index_view_netloc = get_index_view_url(flt2_url_netloc_match=netloc) - if is_scheme_valid and not open_in_new_tab: + if parsed_url.scheme and not open_in_new_tab: target = 'target="_blank"' if open_in_new_tab else "" res.append(f'{title}') else: @@ -123,7 +122,7 @@ def _list_entry(self, context: Any, model: Namespace, name: str) -> Markup: if self.url_render_mode == "netloc": res.append(f'({netloc})') res.append(br_tag) - if not is_scheme_valid: + if not parsed_url.scheme: res.extend((model.url, br_tag)) elif self.url_render_mode is None or self.url_render_mode == "full": res.extend((f'{model.url}', br_tag)) From f1d11b46bb8dc76e522cf42fae3fa742fe26e73f Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Fri, 14 Jan 2022 14:28:10 +0800 Subject: [PATCH 28/33] fix(BookmarkModelView): dont get favicons if no netloc --- bukuserver/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bukuserver/views.py b/bukuserver/views.py index 4a99d8f3..c0a9e55e 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -107,7 +107,7 @@ def _list_entry(self, context: Any, model: Namespace, name: str) -> Markup: f"""{model.title}{br_tag}{escaped_url}{br_tag}{tag_text_markup}{model.description}""" ) res = [] - if not current_app.config.get("BUKUSERVER_DISABLE_FAVICON", False): + if not current_app.config.get("BUKUSERVER_DISABLE_FAVICON", False) and netloc: res.append( f'' ) From af794a77de65ebec9137d5b12f9cac6a707c3b35 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Tue, 18 Jan 2022 07:55:44 +0800 Subject: [PATCH 29/33] build(bukuserver): use fixed version of flask-reverse-proxy-fix --- bukuserver/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bukuserver/requirements.txt b/bukuserver/requirements.txt index b13e5c86..08377813 100644 --- a/bukuserver/requirements.txt +++ b/bukuserver/requirements.txt @@ -8,7 +8,7 @@ Flask-Admin>=1.5.1 Flask-API>=0.6.9 Flask-Bootstrap>=3.3.7.1 flask-paginate>=0.5.1 -flask-reverse-proxy-fix>=0.2.1 +https://github.com/rachmadaniHaryono/flask-reverse-proxy-fix/archive/refs/tags/v0.2.2rc1.zip Flask-WTF>=0.14.2 Flask>=1.0.2,<2.0 idna>=2.5 From f9eb421f0f338ee6d902f4104ff9b133230efde6 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Tue, 18 Jan 2022 08:03:00 +0800 Subject: [PATCH 30/33] build(setup): add fork of flask-reverse-proxy-fix --- setup.py | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/setup.py b/setup.py index 0a1baa70..5def3611 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import re import shutil -from setuptools import setup, find_packages +from setuptools import find_packages, setup if os.path.isfile('buku'): shutil.copyfile('buku', 'buku.py') @@ -34,29 +34,29 @@ server_require = [ - 'appdirs>=1.4.3', - 'arrow>=0.12.1', - 'beautifulsoup4>=4.5.3', - 'cffi>=1.9.1', - 'click>=6.7', - 'Flask-Admin>=1.5.1', - 'Flask-API>=0.6.9', - 'Flask-Bootstrap>=3.3.7.1', - 'flask-paginate>=0.5.1', - 'flask-reverse-proxy-fix>=0.2.1', - 'Flask-WTF>=0.14.2', - 'Flask>=1.0.2,<2.0', - 'idna>=2.5', - 'itsdangerous>=0.24', - 'Jinja2>=2.10.1', - 'MarkupSafe>=1.0', - 'packaging>=16.8', - 'pyasn1>=0.2.3', - 'pycparser>=2.17', - 'requests>=2.21.0', - 'six>=1.10.0', - 'urllib3>=1.25.2', - 'Werkzeug>=0.11.15', + "appdirs>=1.4.3", + "arrow>=0.12.1", + "beautifulsoup4>=4.5.3", + "cffi>=1.9.1", + "click>=6.7", + "Flask-Admin>=1.5.1", + "Flask-API>=0.6.9", + "Flask-Bootstrap>=3.3.7.1", + "flask-paginate>=0.5.1", + "https://github.com/rachmadaniHaryono/flask-reverse-proxy-fix/archive/refs/tags/v0.2.2rc1.zip", + "Flask-WTF>=0.14.2", + "Flask>=1.0.2,<2.0", + "idna>=2.5", + "itsdangerous>=0.24", + "Jinja2>=2.10.1", + "MarkupSafe>=1.0", + "packaging>=16.8", + "pyasn1>=0.2.3", + "pycparser>=2.17", + "requests>=2.21.0", + "six>=1.10.0", + "urllib3>=1.25.2", + "Werkzeug>=0.11.15", ] setup( From db698e8d34e33c8f4864811577a6c1f5064916bc Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Tue, 18 Jan 2022 08:21:01 +0800 Subject: [PATCH 31/33] build: flask-reverse-proxy-fix based on pep508 --- bukuserver/requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bukuserver/requirements.txt b/bukuserver/requirements.txt index 08377813..62e43310 100644 --- a/bukuserver/requirements.txt +++ b/bukuserver/requirements.txt @@ -8,7 +8,7 @@ Flask-Admin>=1.5.1 Flask-API>=0.6.9 Flask-Bootstrap>=3.3.7.1 flask-paginate>=0.5.1 -https://github.com/rachmadaniHaryono/flask-reverse-proxy-fix/archive/refs/tags/v0.2.2rc1.zip +flask-reverse-proxy-fix @ https://github.com/rachmadaniHaryono/flask-reverse-proxy-fix/archive/refs/tags/v0.2.2rc1.zip Flask-WTF>=0.14.2 Flask>=1.0.2,<2.0 idna>=2.5 diff --git a/setup.py b/setup.py index 5def3611..274cc1b7 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ "Flask-API>=0.6.9", "Flask-Bootstrap>=3.3.7.1", "flask-paginate>=0.5.1", - "https://github.com/rachmadaniHaryono/flask-reverse-proxy-fix/archive/refs/tags/v0.2.2rc1.zip", + "flask-reverse-proxy-fix @ https://github.com/rachmadaniHaryono/flask-reverse-proxy-fix/archive/refs/tags/v0.2.2rc1.zip", "Flask-WTF>=0.14.2", "Flask>=1.0.2,<2.0", "idna>=2.5", From 715d0c9b2d3bb177f320d1bbe03705e2065c127c Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Sat, 22 Jan 2022 06:44:06 +0800 Subject: [PATCH 32/33] fix(BookmarkModelView): use buku method when creating bookmark --- bukuserver/views.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bukuserver/views.py b/bukuserver/views.py index c0a9e55e..be480a51 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -183,12 +183,15 @@ def create_model(self, form): form.populate_obj(model) vars(model).pop("id") self._on_model_change(form, model, True) - self.model.bukudb.add_rec( - url=model.url, - title_in=model.title, - tags_in=buku.parse_tags([model.tags]), - desc=model.description, - ) + if not model.url.strip(): + raise ValueError(f"url invalid: {model.url}") + kwargs = {"url": model.url} + if model.tags.strip(): + kwargs["tags_in"] = buku.parse_tags([model.tags]) + for key, item in (("title_in", model.title), ("desc", model.description)): + if item.strip(): + kwargs[key] = item + self.model.bukudb.add_rec(**kwargs) except Exception as ex: if not self.handle_view_exception(ex): msg = "Failed to create record." From ea6fdd8b2415cb3fcd51aed409562fe92d3f7962 Mon Sep 17 00:00:00 2001 From: Rachmadani Haryono Date: Sat, 22 Jan 2022 06:50:57 +0800 Subject: [PATCH 33/33] fix(BookmarkModelView): dont render tag with netloc when no netloc --- bukuserver/views.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bukuserver/views.py b/bukuserver/views.py index be480a51..30864cac 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -113,20 +113,22 @@ def _list_entry(self, context: Any, model: Namespace, name: str) -> Markup: ) title = model.title if model.title else "<EMPTY TITLE>" open_in_new_tab = current_app.config.get("BUKUSERVER_OPEN_IN_NEW_TAB", False) - url_for_index_view_netloc = get_index_view_url(flt2_url_netloc_match=netloc) + url_for_index_view_netloc = None + if netloc: + url_for_index_view_netloc = get_index_view_url(flt2_url_netloc_match=netloc) if parsed_url.scheme and not open_in_new_tab: target = 'target="_blank"' if open_in_new_tab else "" res.append(f'{title}') else: res.append(title) - if self.url_render_mode == "netloc": + if self.url_render_mode == "netloc" and url_for_index_view_netloc: res.append(f'({netloc})') res.append(br_tag) if not parsed_url.scheme: res.extend((model.url, br_tag)) elif self.url_render_mode is None or self.url_render_mode == "full": res.extend((f'{model.url}', br_tag)) - if self.url_render_mode != "netloc": + if self.url_render_mode != "netloc" and url_for_index_view_netloc: res.append( f'netloc:{netloc}' )