Remove server functionality from media-parser

This commit removes all server-related functionalities from the media-parser package. As a result, the package now solely exists for parsing media from social networks. This creates a cleaner codebase and focuses on the core functionality of the package. Signed-off-by: Jag_k <[email protected]>
jag-k · May 8, 2024 · 404b6b5 · 404b6b5
1 parent f4b058c
commit 404b6b5
Show file tree

Hide file tree

Showing 36 changed files with 1,448 additions and 2,698 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,6 @@
 .idea/
+.vscode/
+.fleet/
 __pycache__/
 *.py[cod]
 dist/
@@ -7,3 +9,9 @@ __pypackages__/
 
 config/
 !**/.gitkeep
+
+*.local
+*.local.*
+
+*.private
+*.private.*
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,12 +1,10 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
 repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  # Ruff version.
-  rev: 'v0.4.2'
+  rev: 'v0.4.3'
   hooks:
     - id: ruff
       args: [ --fix, --exit-non-zero-on-fix ]
+    - id: ruff-format
 
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v4.6.0
@@ -16,16 +14,12 @@ repos:
   - id: check-yaml
   - id: check-added-large-files
 
-- repo: https://github.com/psf/black
-  rev: "23.10.0"
-  hooks:
-  - id: black
 
 - repo: https://github.com/python-poetry/poetry
   rev: '1.8.0'
   hooks:
-    - id: poetry-check
     - id: poetry-lock
+    - id: poetry-check
     - id: poetry-export
       args: [ "--with", "docs", "--without-hashes", "-o", "docs/requirements.txt" ]
 

diff --git a/docs/.readthedocs.yaml b/docs/.readthedocs.yaml
@@ -5,19 +5,19 @@
 # Required
 version: 2
 
-# Set the OS, Python version and other tools you might need
+# Set the OS, Python version, and other tools you might need
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "3.12"
 
 # Build documentation in the "docs/" directory with Sphinx
 sphinx:
   configuration: docs/conf.py
   builder: html
   fail_on_warning: true
 
-# Optionally build your docs in additional formats such as PDF and ePub
+# Optionally, build your docs in additional formats such as PDF and ePub
 formats:
   - pdf
   - epub
@@ -28,3 +28,4 @@ formats:
 python:
   install:
     - requirements: docs/requirements.txt
+      method: pip
diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css
@@ -0,0 +1,5 @@
+/* Make `required` gray and unselectable */
+span.sig-name.descname > span.pre:nth-child(2) {
+    color: gray;
+    user-select: none;
+}
diff --git a/docs/conf.py b/docs/conf.py
@@ -6,14 +6,11 @@
 from pathlib import Path
 from tomllib import load
 
-import yaml
 from sphinx.application import Sphinx
 
 PACKAGE_NAME = "media_parser"
 SPHINX_SOURCE_PATH = Path(__file__).resolve().parent
 BASE_PATH = SPHINX_SOURCE_PATH.parent
-SPEC_PATH = SPHINX_SOURCE_PATH / "specs"
-SPEC_PATH.mkdir(exist_ok=True, parents=True)
 
 with (BASE_PATH / "pyproject.toml").open("rb") as f:
     poetry = load(f)["tool"]["poetry"]
@@ -39,9 +36,6 @@
     "autodoc2",
     "sphinx_copybutton",
     "sphinxext.opengraph",
-    # "sphinxcontrib.openapi",
-    # "sphinxcontrib.redoc",
-    # "swagger_plugin_for_sphinx",
     "sphinx_inline_tabs",
     "sphinx.ext.duration",
     "sphinx.ext.coverage",
@@ -61,6 +55,9 @@
 html_logo = (SPHINX_SOURCE_PATH / "_static" / "logo.png").as_posix()
 html_static_path = [(SPHINX_SOURCE_PATH / "_static").as_posix()]
 html_search_options = {"type": "default"}
+html_css_files = [
+    "css/custom.css",
+]
 html_theme_options = {
     "navigation_with_keys": True,
     "footer_icons": [
@@ -125,10 +122,6 @@
         "path": f"{AUTODOC2_RELATIVE_PATH}/models",
         "module": f"{PACKAGE_NAME}.models",
     },
-    {
-        "path": f"{AUTODOC2_RELATIVE_PATH}/client.py",
-        "module": f"{PACKAGE_NAME}.client",
-    },
 ]
 autodoc2_render_plugin = "myst"
 
@@ -145,23 +138,18 @@
 {%- endfor %}
 ```
 
-[^1]: Created with [sphinx-autodoc2](https://github.com/chrisjsewell/sphinx-autodoc2)"""
+[^1]: Created with [sphinx-autodoc2](https://github.com/chrisjsewell/sphinx-autodoc2)
+"""
 
 # https://github.com/sphinx-extensions2/sphinx-autodoc2/pull/29
 autodoc2_index_filename = "index.md"
 
 
 def setup(app: Sphinx):
-    from main import app as _app
     from pygments.lexers.data import JsonLexer
 
     app.add_lexer("json5", JsonLexer)
     app.connect("builder-inited", parser_config)
-    api = _app.openapi()
-    with (SPEC_PATH / "openapi.json").open("w", encoding="utf-8") as f:
-        json.dump(api, f, ensure_ascii=False, indent=2)
-    with (SPEC_PATH / "openapi.yml").open("w", encoding="utf-8") as f:
-        yaml.dump(api, f, allow_unicode=True)
 
 
 def parser_config(_: Sphinx):
@@ -203,14 +191,12 @@ def parser_config(_: Sphinx):
             text.append(f"``````{{object}} {name}")
 
             if name in required:
-                text += [
-                    f"**This field is required to enable `{parser_type}` parser!**",
-                    "",
-                ]
+                text[-1] += " (required)"
+            type_ = value.get("type", value.get("anyOf", [{}])[0].get("type", None))
             text += [
                 value.get("description", ""),
                 "",
-                f"**type**: `{type_to_string(value['type'])}`{{l=python}}",
+                f"**type**: `{type_to_string(type_)}`{{l=python}}",
                 "",
             ]
             ph = type("placeholder", (), {})
@@ -234,40 +220,11 @@ def parser_config(_: Sphinx):
 
 
 def type_to_string(type_: str) -> str:
-    if type_ == "array":
-        return "list"
-    if type_ == "integer":
-        return "int"
-    if type_ == "number":
-        return "float"
-    if type_ == "boolean":
-        return "bool"
-    if type_ == "object":
-        return "dict"
-    if type_ == "string":
-        return "str"
-    return type_
-
-
-swagger = [
-    {
-        "name": "Service API",
-        "page": "openapi",
-        "id": "my-page",
-        "options": {
-            "url": "specs/openapi.yml",
-        },
-    }
-]
-
-redoc_uri = "https://cdn.jsdelivr.net/npm/redoc@next/bundles/redoc.standalone.js"
-
-redoc = [
-    {
-        # "name": "Media API",
-        "page": "usage/api",
-        "spec": "specs/openapi.json",
-        "embed": True,
-        "opts": {"hide-hostname": True, "hide-loading": True},
-    }
-]
+    return {
+        "array": "list",
+        "integer": "int",
+        "number": "float",
+        "boolean": "bool",
+        "object": "dict",
+        "string": "str",
+    }.get(type_, type_)
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -1,82 +1,71 @@
-aiohttp==3.8.6 ; python_version >= "3.11" and python_version < "4.0"
-aiosignal==1.3.1 ; python_version >= "3.11" and python_version < "4.0"
-alabaster==0.7.13 ; python_version >= "3.11" and python_version < "4.0"
-anyio==3.7.1 ; python_version >= "3.11" and python_version < "4.0"
-astroid==2.15.8 ; python_version >= "3.11" and python_version < "4.0"
-asttokens==2.4.0 ; python_version >= "3.11" and python_version < "4.0"
-async-lru==2.0.4 ; python_version >= "3.11" and python_version < "4.0"
-async-timeout==4.0.3 ; python_version >= "3.11" and python_version < "4.0"
-attrs==23.1.0 ; python_version >= "3.11" and python_version < "4.0"
-babel==2.13.0 ; python_version >= "3.11" and python_version < "4.0"
-beautifulsoup4==4.12.2 ; python_version >= "3.11" and python_version < "4.0"
-certifi==2023.7.22 ; python_version >= "3.11" and python_version < "4.0"
-charset-normalizer==3.3.0 ; python_version >= "3.11" and python_version < "4.0"
-click==8.1.7 ; python_version >= "3.11" and python_version < "4.0"
-colorama==0.4.6 ; python_version >= "3.11" and python_version < "4.0" and (sys_platform == "win32" or platform_system == "Windows")
-contextvars==2.4 ; python_version >= "3.11" and python_version < "4.0"
-contourpy==1.1.1 ; python_version >= "3.11" and python_version < "4.0"
-cycler==0.12.1 ; python_version >= "3.11" and python_version < "4.0"
-dnspython==2.4.2 ; python_version >= "3.11" and python_version < "4.0"
-docutils==0.20.1 ; python_version >= "3.11" and python_version < "4.0"
-executing==2.0.0 ; python_version >= "3.11" and python_version < "4.0"
-fastapi==0.104.0 ; python_version >= "3.11" and python_version < "4.0"
-fonttools==4.43.1 ; python_version >= "3.11" and python_version < "4.0"
-frozenlist==1.4.0 ; python_version >= "3.11" and python_version < "4.0"
-furo==2023.9.10 ; python_version >= "3.11" and python_version < "4.0"
-h11==0.14.0 ; python_version >= "3.11" and python_version < "4.0"
-httpcore==0.18.0 ; python_version >= "3.11" and python_version < "4.0"
-httpx==0.25.0 ; python_version >= "3.11" and python_version < "4.0"
-idna==3.4 ; python_version >= "3.11" and python_version < "4.0"
-imagesize==1.4.1 ; python_version >= "3.11" and python_version < "4.0"
-immutables==0.20 ; python_version >= "3.11" and python_version < "4.0"
-jinja2==3.1.2 ; python_version >= "3.11" and python_version < "4.0"
-jsonref==1.1.0 ; python_version >= "3.11" and python_version < "4.0"
-kiwisolver==1.4.5 ; python_version >= "3.11" and python_version < "4.0"
-lazy-object-proxy==1.9.0 ; python_version >= "3.11" and python_version < "4.0"
-markdown-it-py==3.0.0 ; python_version >= "3.11" and python_version < "4.0"
-markupsafe==2.1.3 ; python_version >= "3.11" and python_version < "4.0"
-matplotlib==3.8.0 ; python_version >= "3.11" and python_version < "4.0"
-mdit-py-plugins==0.4.0 ; python_version >= "3.11" and python_version < "4.0"
-mdurl==0.1.2 ; python_version >= "3.11" and python_version < "4.0"
-motor==3.3.1 ; python_version >= "3.11" and python_version < "4.0"
-multidict==6.0.4 ; python_version >= "3.11" and python_version < "4.0"
-myst-parser==2.0.0 ; python_version >= "3.11" and python_version < "4.0"
-numpy==1.25.2 ; python_version >= "3.11" and python_version < "4.0"
-packaging==23.2 ; python_version >= "3.11" and python_version < "4.0"
-pillow==10.1.0 ; python_version >= "3.11" and python_version < "4.0"
-pure-eval==0.2.2 ; python_version >= "3.11" and python_version < "4.0"
-pydantic==1.10.13 ; python_version >= "3.11" and python_version < "4.0"
-pydantic[dotenv]==1.10.13 ; python_version >= "3.11" and python_version < "4.0"
-pygments==2.16.1 ; python_version >= "3.11" and python_version < "4.0"
-pymongo==4.5.0 ; python_version >= "3.11" and python_version < "4.0"
-pyparsing==3.1.1 ; python_version >= "3.11" and python_version < "4.0"
-python-dateutil==2.8.2 ; python_version >= "3.11" and python_version < "4.0"
-python-dotenv==1.0.0 ; python_version >= "3.11" and python_version < "4.0"
-pytube==15.0.0 ; python_version >= "3.11" and python_version < "4.0"
-pyyaml==6.0.1 ; python_version >= "3.11" and python_version < "4.0"
-requests==2.31.0 ; python_version >= "3.11" and python_version < "4.0"
-sentry-sdk[fastapi,httpx,pure-eval,pymongo]==1.32.0 ; python_version >= "3.11" and python_version < "4.0"
-setuptools-scm==8.0.4 ; python_version >= "3.11" and python_version < "4.0"
-setuptools==68.2.2 ; python_version >= "3.11" and python_version < "4.0"
-six==1.16.0 ; python_version >= "3.11" and python_version < "4.0"
-sniffio==1.3.0 ; python_version >= "3.11" and python_version < "4.0"
-snowballstemmer==2.2.0 ; python_version >= "3.11" and python_version < "4.0"
-soupsieve==2.5 ; python_version >= "3.11" and python_version < "4.0"
-sphinx-autodoc2 @ git+https://github.com/jag-k/sphinx-autodoc2.git@f84739a7639230c8b8f77c4098ffe4b95b67a62d ; python_version >= "3.11" and python_version < "4.0"
-sphinx-basic-ng==1.0.0b2 ; python_version >= "3.11" and python_version < "4.0"
-sphinx-copybutton==0.5.2 ; python_version >= "3.11" and python_version < "4.0"
-sphinx-inline-tabs==2023.4.21 ; python_version >= "3.11" and python_version < "4.0"
-sphinx==7.1.1 ; python_version >= "3.11" and python_version < "4.0"
-sphinxcontrib-applehelp==1.0.7 ; python_version >= "3.11" and python_version < "4.0"
-sphinxcontrib-devhelp==1.0.5 ; python_version >= "3.11" and python_version < "4.0"
-sphinxcontrib-htmlhelp==2.0.4 ; python_version >= "3.11" and python_version < "4.0"
-sphinxcontrib-jsmath==1.0.1 ; python_version >= "3.11" and python_version < "4.0"
-sphinxcontrib-qthelp==1.0.6 ; python_version >= "3.11" and python_version < "4.0"
-sphinxcontrib-serializinghtml==1.1.9 ; python_version >= "3.11" and python_version < "4.0"
-sphinxext-opengraph==0.8.2 ; python_version >= "3.11" and python_version < "4.0"
-starlette==0.27.0 ; python_version >= "3.11" and python_version < "4.0"
-typing-extensions==4.8.0 ; python_version >= "3.11" and python_version < "4.0"
-urllib3==2.0.7 ; python_version >= "3.11" and python_version < "4.0"
-uvicorn==0.23.2 ; python_version >= "3.11" and python_version < "4.0"
-wrapt==1.15.0 ; python_version >= "3.11" and python_version < "4.0"
-yarl==1.9.2 ; python_version >= "3.11" and python_version < "4.0"
+aiohttp==3.9.5 ; python_version >= "3.12" and python_version < "4.0"
+aiosignal==1.3.1 ; python_version >= "3.12" and python_version < "4.0"
+alabaster==0.7.16 ; python_version >= "3.12" and python_version < "4.0"
+annotated-types==0.6.0 ; python_version >= "3.12" and python_version < "4.0"
+anyio==4.3.0 ; python_version >= "3.12" and python_version < "4.0"
+astroid==3.2.0 ; python_version >= "3.12" and python_version < "4.0"
+async-lru==2.0.4 ; python_version >= "3.12" and python_version < "4.0"
+attrs==23.2.0 ; python_version >= "3.12" and python_version < "4.0"
+babel==2.15.0 ; python_version >= "3.12" and python_version < "4.0"
+beautifulsoup4==4.12.3 ; python_version >= "3.12" and python_version < "4.0"
+certifi==2024.2.2 ; python_version >= "3.12" and python_version < "4.0"
+charset-normalizer==3.3.2 ; python_version >= "3.12" and python_version < "4.0"
+colorama==0.4.6 ; python_version >= "3.12" and python_version < "4.0" and sys_platform == "win32"
+contextvars==2.4 ; python_version >= "3.12" and python_version < "4.0"
+contourpy==1.2.1 ; python_version >= "3.12" and python_version < "4.0"
+cycler==0.12.1 ; python_version >= "3.12" and python_version < "4.0"
+dnspython==2.6.1 ; python_version >= "3.12" and python_version < "4.0"
+docutils==0.20.1 ; python_version >= "3.12" and python_version < "4.0"
+fonttools==4.51.0 ; python_version >= "3.12" and python_version < "4.0"
+frozenlist==1.4.1 ; python_version >= "3.12" and python_version < "4.0"
+furo==2023.9.10 ; python_version >= "3.12" and python_version < "4.0"
+h11==0.14.0 ; python_version >= "3.12" and python_version < "4.0"
+httpcore==1.0.5 ; python_version >= "3.12" and python_version < "4.0"
+httpx==0.27.0 ; python_version >= "3.12" and python_version < "4.0"
+idna==3.7 ; python_version >= "3.12" and python_version < "4.0"
+imagesize==1.4.1 ; python_version >= "3.12" and python_version < "4.0"
+immutables==0.20 ; python_version >= "3.12" and python_version < "4.0"
+jinja2==3.1.4 ; python_version >= "3.12" and python_version < "4.0"
+jsonref==1.1.0 ; python_version >= "3.12" and python_version < "4.0"
+kiwisolver==1.4.5 ; python_version >= "3.12" and python_version < "4.0"
+markdown-it-py==3.0.0 ; python_version >= "3.12" and python_version < "4.0"
+markupsafe==2.1.5 ; python_version >= "3.12" and python_version < "4.0"
+matplotlib==3.8.4 ; python_version >= "3.12" and python_version < "4.0"
+mdit-py-plugins==0.4.0 ; python_version >= "3.12" and python_version < "4.0"
+mdurl==0.1.2 ; python_version >= "3.12" and python_version < "4.0"
+motor==3.4.0 ; python_version >= "3.12" and python_version < "4.0"
+multidict==6.0.5 ; python_version >= "3.12" and python_version < "4.0"
+myst-parser==2.0.0 ; python_version >= "3.12" and python_version < "4.0"
+numpy==1.26.4 ; python_version >= "3.12" and python_version < "4.0"
+packaging==24.0 ; python_version >= "3.12" and python_version < "4.0"
+pillow==10.3.0 ; python_version >= "3.12" and python_version < "4.0"
+pydantic-core==2.18.2 ; python_version >= "3.12" and python_version < "4.0"
+pydantic-settings==2.2.1 ; python_version >= "3.12" and python_version < "4.0"
+pydantic==2.7.1 ; python_version >= "3.12" and python_version < "4.0"
+pygments==2.18.0 ; python_version >= "3.12" and python_version < "4.0"
+pymongo==4.7.2 ; python_version >= "3.12" and python_version < "4.0"
+pyparsing==3.1.2 ; python_version >= "3.12" and python_version < "4.0"
+python-dateutil==2.9.0.post0 ; python_version >= "3.12" and python_version < "4.0"
+python-dotenv==1.0.1 ; python_version >= "3.12" and python_version < "4.0"
+pytube==15.0.0 ; python_version >= "3.12" and python_version < "4.0"
+pyyaml==6.0.1 ; python_version >= "3.12" and python_version < "4.0"
+requests==2.31.0 ; python_version >= "3.12" and python_version < "4.0"
+six==1.16.0 ; python_version >= "3.12" and python_version < "4.0"
+sniffio==1.3.1 ; python_version >= "3.12" and python_version < "4.0"
+snowballstemmer==2.2.0 ; python_version >= "3.12" and python_version < "4.0"
+soupsieve==2.5 ; python_version >= "3.12" and python_version < "4.0"
+sphinx-autodoc2 @ git+https://github.com/jag-k/sphinx-autodoc2.git@d326f4e2a76aa0253ccadd3749a090b9bba77c66 ; python_version >= "3.12" and python_version < "4.0"
+sphinx-basic-ng==1.0.0b2 ; python_version >= "3.12" and python_version < "4.0"
+sphinx-copybutton==0.5.2 ; python_version >= "3.12" and python_version < "4.0"
+sphinx-inline-tabs==2023.4.21 ; python_version >= "3.12" and python_version < "4.0"
+sphinx==7.1.1 ; python_version >= "3.12" and python_version < "4.0"
+sphinxcontrib-applehelp==1.0.8 ; python_version >= "3.12" and python_version < "4.0"
+sphinxcontrib-devhelp==1.0.6 ; python_version >= "3.12" and python_version < "4.0"
+sphinxcontrib-htmlhelp==2.0.5 ; python_version >= "3.12" and python_version < "4.0"
+sphinxcontrib-jsmath==1.0.1 ; python_version >= "3.12" and python_version < "4.0"
+sphinxcontrib-qthelp==1.0.7 ; python_version >= "3.12" and python_version < "4.0"
+sphinxcontrib-serializinghtml==1.1.10 ; python_version >= "3.12" and python_version < "4.0"
+sphinxext-opengraph==0.8.2 ; python_version >= "3.12" and python_version < "4.0"
+typing-extensions==4.11.0 ; python_version >= "3.12" and python_version < "4.0"
+urllib3==2.2.1 ; python_version >= "3.12" and python_version < "4.0"
+yarl==1.9.4 ; python_version >= "3.12" and python_version < "4.0"
diff --git a/docs/usage/index.md b/docs/usage/index.md
@@ -1,11 +1,9 @@
 # Usage
 
-API documentation available on `/docs` endpoint.
-
 ## Content
 ```{toctree}
 :maxdepth: 2
 
-client
-server
+library
+../apidocs/parsers-config
 ```