chore(deps): bump sphinx from 6.2.1 to 7.0.0

.flake8

@@ -10,7 +10,9 @@ exclude =
     # This contains our built documentation
     build,
     # This contains builds of flake8 that we don't want to check
-    dist
+    dist,
+    # This contains various user scripts
+    scripts
 per-file-ignores =
     tests/*.py:E501,D100,D101,D102,D103,D104,D107,
     __init__.py:F401,F403,

.github/ISSUE_TEMPLATE/bug_report.md

@@ -21,7 +21,7 @@ A clear and concise description of what you expected to happen.
 **Device (please complete the following information):**
 
 - OS [e.g. debian, macOS]:
-- Censys Python Version [e.g. 2.1.0]: <!-- $ censys --version -->
+- Censys Python Version [e.g. 2.2.1]: <!-- $ censys --version -->
 - Shell [e.g. bash, zsh]: <!-- $ echo $SHELL -->
 
 **Additional context**

.gitignore

@@ -83,9 +83,15 @@ dmypy.json
 # Config
 *.cfg
 
+# Scripts
+scripts/*.py
+
 # Output
 censys-query-output.*
 censys-view-*
 *.json
-*.csv
+*.txt
 *.xml
+
+# Autocomplete
+!censys/cli/data/*_autocomplete.json

.pre-commit-config.yaml

@@ -34,16 +34,30 @@ repos:
         pass_filenames: false
         language: system
         types: [python]
+      - id: system
+        name: pytest
+        entry: poetry run pytest --cov --cov-fail-under=100
+        pass_filenames: false
+        language: system
+        types: [python]
       - id: system
         name: flake8
         entry: poetry run flake8 .
         pass_filenames: false
         language: system
         types: [python]
+        require_serial: true
+        args: [--darglint-ignore-regex, .*] # Skip docstring checks
       - id: system
         name: mypy
         entry: poetry run mypy censys
         pass_filenames: false
         language: system
         types: [python]
         require_serial: true
+      - id: system
+        name: update autocomplete
+        entry: bash scripts/update_autocomplete.sh
+        pass_filenames: false
+        language: system
+        types: []

README.md

@@ -7,7 +7,7 @@
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-organge.svg?logo=git&logoColor=organge)](http://makeapullrequest.com)
 [![License](https://img.shields.io/github/license/censys/censys-python?logo=apache)](https://github.com/censys/censys-python/blob/main/LICENSE)
 
-An easy-to-use and lightweight API wrapper for Censys APIs ([censys.io](https://censys.io/)). Python 3.7.2+ is currently supported.
+An easy-to-use and lightweight API wrapper for Censys APIs ([censys.io](https://censys.io/)). Python 3.7.2+ is currently supported. This library has been tested on Python 3.7 and 3.x (Currently version 3.11).
 
 > **Notice:** The Censys Search v1 endpoints are deprecated as of Nov. 30, 2021. Please begin using v2 endpoints to query hosts and certificates and check out our [support center](https://support.censys.io/hc/en-us/sections/360013076551-Censys-Search-2-0) for resources.
 
@@ -19,7 +19,9 @@ An easy-to-use and lightweight API wrapper for Censys APIs ([censys.io](https://
 - [Manage assets, events, and seeds in Censys ASM](https://censys-python.readthedocs.io/en/stable/usage-asm.html)
 - [Command-line interface](https://censys-python.readthedocs.io/en/stable/cli.html)
 
+<!-- markdownlint-disable MD033 -->
 <a href="https://asciinema.org/a/500416" target="_blank"><img src="https://asciinema.org/a/500416.svg" width="600"/></a>
+<!-- markdownlint-enable MD033 -->
 
 ## Getting Started
 
@@ -35,6 +37,22 @@ To upgraded using `pip`.
 pip install --upgrade censys
 ```
 
+Or, you can install the library from source using `poetry`.
+
+```sh
+git clone https://github.com/censys/censys-python.git
+cd censys-python/
+poetry install
+```
+
+Optionally, you can enable tab completion for the CLI by adding this line to your `~/.bashrc`, `~/.zshrc`, or equivalent.
+
+> Please note that autocomplete is supported for field names in the `search` command.
+
+```sh
+eval "$(register-python-argcomplete censys)"
+```
+
 To configure your search credentials run `censys config` or set both `CENSYS_API_ID` and `CENSYS_API_SECRET` environment variables.
 
 ```sh

censys/cli/__init__.py

@@ -1,7 +1,10 @@
 #!/usr/bin/env python3
+# PYTHON_ARGCOMPLETE_OK
 """Interact with the Censys Search API through the command line."""
 import sys
 
+import argcomplete
+
 from .args import get_parser
 from censys.common.version import __version__
 
@@ -10,6 +13,9 @@ def main():
     """Main cli function."""
     parser = get_parser()
 
+    # Adds autocomplete
+    argcomplete.autocomplete(parser)
+
     # Executes by subcommand
     args = parser.parse_args()
 

censys/cli/commands/search.py

@@ -1,7 +1,9 @@
 """Censys search CLI."""
 import argparse
+import json
 import sys
 import webbrowser
+from pathlib import Path
 from typing import Any, Dict, List
 from urllib.parse import urlencode
 
@@ -12,6 +14,43 @@
 
 Results = List[dict]
 
+DATA_DIR = Path(__file__).parent.parent / "data"
+HOSTS_AUTOCOMPLETE = DATA_DIR / "hosts_autocomplete.json"
+CERTIFICATES_AUTOCOMPLETE = DATA_DIR / "certificates_autocomplete.json"
+
+
+def fields_completer(
+    prefix: str, parsed_args: argparse.Namespace, **kwargs
+) -> List[str]:
+    """Fields completer.
+
+    Args:
+        prefix (str): Prefix to complete.
+        parsed_args (Namespace): Argparse Namespace.
+        **kwargs: Keyword arguments.
+
+    Returns:
+        List[str]: List of fields.
+    """
+    index_type = parsed_args.index_type
+    autocomplete_json = {}
+    if index_type == "hosts":
+        with HOSTS_AUTOCOMPLETE.open() as autocomplete_file:
+            autocomplete_json = json.load(autocomplete_file)
+    elif index_type == "certificates":
+        with CERTIFICATES_AUTOCOMPLETE.open() as autocomplete_file:
+            autocomplete_json = json.load(autocomplete_file)
+    else:
+        return []
+
+    autocomplete_data = autocomplete_json.get("data", [])
+    fields = [field["value"] for field in autocomplete_data]
+
+    if not prefix:
+        # Returns first 20 fields if no prefix is provided
+        return fields[:20]
+    return fields
+
 
 def cli_search(args: argparse.Namespace):
     """Search subcommand.
@@ -109,7 +148,7 @@ def include(parent_parser: argparse._SubParsersAction, parents: dict):
         "query",
         type=str,
         help="a string written in Censys Search syntax",
-    )
+    ).completer = fields_completer
 
     index_metavar = "|".join(V2_INDEXES)
     index_default = "hosts"
@@ -127,8 +166,8 @@ def include(parent_parser: argparse._SubParsersAction, parents: dict):
         "--format",
         type=str,
         default="screen",
-        choices=["screen", "json", "csv"],
-        metavar="screen|json|csv",
+        choices=["screen", "json"],
+        metavar="screen|json",
         help=argparse.SUPPRESS,
     )
     search_parser.add_argument(
@@ -189,6 +228,6 @@ def include(parent_parser: argparse._SubParsersAction, parents: dict):
         type=str,
         nargs="+",
         help="additional fields to return in the matched certificates",
-    )
+    ).completer = fields_completer
 
     search_parser.set_defaults(func=cli_search)

censys/cli/commands/subdomains.py

@@ -39,7 +39,9 @@ def cli_subdomains(args: argparse.Namespace):  # pragma: no cover
         certificate_query = f"names: {args.domain}"
 
         with err_console.status(f"Querying {args.domain} subdomains"):
-            query = client.search(certificate_query, fields=["names"], pages=args.pages)
+            query = client.search(
+                certificate_query, per_page=100, pages=args.pages
+            )  # 100 is the max per page
 
             # Flatten the result, and remove duplicates
             for hits in query: