Support 202406 (#93)

fixes #67
fixes #72

Co-authored-by: 20C <[email protected]>

CHANGELOG.md

@@ -4,8 +4,10 @@
 ## Unreleased
 ### Added
 - Client.update_all() method to update all objects in the database - this existed in v1 and has now been readded
+- Allow option for logging to co-exist with existing loggers (#67)
 ### Fixed
 - fixes automatic solving of unique constraint errors (#85)
+- fixes -O , --output-format commands not supporting all output formats supported (#72)
 
 
 ## 2.0.0

CHANGELOG.yaml

@@ -1,8 +1,10 @@
 Unreleased:
   added:
   - Client.update_all() method to update all objects in the database - this existed in v1 and has now been readded
+  - Allow option for logging to co-exist with existing loggers (#67)
   fixed:
   - fixes automatic solving of unique constraint errors (#85)
+  - fixes -O , --output-format commands not supporting all output formats supported (#72)
   changed: []
   deprecated: []
   removed: []

docs/cli.md

@@ -46,7 +46,7 @@ By default, this produces a shallow output (depth = 0). Set the depth with `--de
 
 You may also change the output format to anything munge supports, so to get json, it would be:
 
-    peeringdb -O json get net1
+    peeringdb get -O json net1
 
 ## whois `<obj><id>`
 Fetches a specific object and outputs to stdout, supports everything that `get` does, as well as:

docs/config.md

@@ -28,4 +28,9 @@ This document describes the environment variables used to configure the PeeringD
 
 ### General ORM Configuration
 
-- **PDB_ORM_BACKEND**: The backend to use for the ORM. Default is `django_peeringdb`.
+- **PDB_ORM_BACKEND**: The backend to use for the ORM. Default is `django_peeringdb`.
+
+### Logging Configuration
+
+- **ALLOW_OTHER_LOGGERS**: To allow other loggers to be used (1 for true, 0 for false). Default is `0`.
+- **LOG_LEVEL**: The python logging level to use. Default is `INFO`

src/peeringdb/__init__.py

@@ -4,15 +4,18 @@
 
 import logging
 import sys
+from distutils.util import strtobool
 from importlib import import_module
 
 import pkg_resources
 
+from peeringdb.util import get_log_level
+
 __version__ = pkg_resources.require("peeringdb")[0].version
 _log_level = logging.INFO
 
 
-def _config_logs(lvl=None, name=None):
+def _config_logs(level=None, name=None, allow_other_loggers=False):
     """
     Set up or change logging configuration.
 
@@ -24,13 +27,23 @@ def _config_logs(lvl=None, name=None):
     # maybe better for log files
     # FORMAT='[%(levelname)s]:%(message)s',
 
-    # Reset handlers
-    for h in list(logging.root.handlers):
-        logging.root.removeHandler(h)
+    if isinstance(level, str):
+        level = get_log_level(level)
 
     global _log_level
-    if lvl:
-        _log_level = lvl
+    if level:
+        _log_level = level
+
+    if not isinstance(allow_other_loggers, bool):
+        try:
+            allow_other_loggers = strtobool(str(allow_other_loggers))
+        except Exception:
+            allow_other_loggers = False
+
+    if not allow_other_loggers:
+        # Reset handlers
+        for h in list(logging.root.handlers):
+            logging.root.removeHandler(h)
 
     logging.basicConfig(level=_log_level, format=logging_format, stream=sys.stdout)
     _log = logging.getLogger(__name__)

src/peeringdb/_update.py

@@ -6,7 +6,7 @@
 from datetime import datetime
 from typing import List, Union
 
-from peeringdb import _config_logs, get_backend
+from peeringdb import get_backend
 from peeringdb._sync import extract_relations, set_many_relations, set_single_relations
 from peeringdb.fetch import Fetcher
 from peeringdb.private import private_data_has_been_fetched
@@ -24,7 +24,6 @@ class Updater:
     """
 
     def __init__(self, fetcher: Fetcher):
-        _config_logs()
         self._log = logging.getLogger(__name__)
         self.resources = {}
         self.backend = get_backend()

src/peeringdb/cli.py

@@ -42,8 +42,6 @@ def check_load_config(config_dir):
 
 
 def main(args=sys.argv):
-    peeringdb._config_logs()
-
     parser = ArgumentParser()
     parser.add_argument(
         "--version", action="version", version="%(prog)s " + peeringdb.__version__

src/peeringdb/client.py

@@ -3,6 +3,7 @@
 import munge.util
 
 from peeringdb import (
+    _config_logs,
     backend_initialized,
     config,
     get_backend,
@@ -43,6 +44,8 @@ def __init__(self, cfg=None, **kwargs):
         if cfg is None:
             cfg = config.load_config()
         self.config = cfg
+        log_config = cfg["log"]
+        _config_logs(**log_config)
         orm_config = cfg["orm"]
         orm_name = orm_config["backend"]
         if not backend_initialized():

src/peeringdb/commands.py

@@ -1,3 +1,4 @@
+import io
 import logging
 import os
 import subprocess
@@ -9,7 +10,7 @@
 from peeringdb import config as cfg
 from peeringdb import resource, util
 from peeringdb.client import Client
-from peeringdb.output._yaml import dump
+from peeringdb.output._dict import dump_python_dict
 from peeringdb.whois import WhoisFormat
 
 
@@ -99,8 +100,12 @@ def handle(config, poids, output_format, depth, remote, **_):
                 else:
                     print(f"Not found: {tag}-{pk}", file=sys.stderr)
                     return 1
-
-            dump(obj, depth, sys.stdout)
+            try:
+                codec = munge.get_codec(output_format)()
+                codec.dump(dump_python_dict(obj, depth), sys.stdout)
+            except TypeError as e:
+                print(f"Output format not supported: {output_format}", file=sys.stderr)
+                return 1
 
 
 class Whois:
@@ -234,9 +239,10 @@ def handle(config, verbose, quiet, init, since, **kwargs):
 
         loglvl = 1 + (verbose or 0) - (quiet or 0)
         if loglvl > 1:
-            peeringdb._config_logs(logging.DEBUG)
-        if loglvl < 1:
+            config.update({"log": {"level": logging.DEBUG}})
+        elif loglvl < 1:
             peeringdb._config_logs(logging.WARNING)
+            config.update({"log": {"level": logging.WARNING}})
 
         client = Client(config, **kwargs)
         # todo verify table schema

src/peeringdb/config.py

@@ -93,8 +93,15 @@ class OrmDbSchema(_schema.Schema):
         migrate = _schema.Bool("migrate", default=True)
         database = OrmDbSchema()
 
+    class LogSchema(_schema.Schema):
+        allow_other_loggers = _schema.Int(
+            "allow_other_loggers", default=os.environ.get("ALLOW_OTHER_LOGGERS", 0)
+        )
+        level = _schema.Str("log_level", default=os.environ.get("LOG_LEVEL", "INFO"))
+
     sync = SyncSchema()
     orm = OrmSchema()
+    log = LogSchema()
 
 
 CLIENT_SCHEMA = ClientSchema()

src/peeringdb/output/_yaml.py → src/peeringdb/output/_dict.py

@@ -1,15 +1,16 @@
-import yaml
+import json
+from datetime import datetime
+from decimal import Decimal
+from ipaddress import IPv4Address, IPv6Address
+
+import django_countries.fields
 
 from peeringdb import get_backend
 from peeringdb.util import group_fields
 
-# from peeringdb.debug import try_or_debug
-
 
-# Wrap orm object into node for graph traversal
-class YamlWrap:
+class DictWrap:
     def __init__(self, o, depth):
-        self.tag = "tag:yaml.org,2002:map"
         self.object = o
         self.depth = depth
         backend = get_backend()
@@ -18,7 +19,7 @@ def __init__(self, o, depth):
     @staticmethod
     def _resolve_one(name, value, depth):
         if depth > 0:
-            return YamlWrap(value, depth - 1)
+            return DictWrap(value, depth - 1).to_dict()
         elif value is None:
             return None
         else:
@@ -27,17 +28,25 @@ def _resolve_one(name, value, depth):
     @staticmethod
     def _resolve_many(name, value, depth):
         if depth > 1:
-            return [YamlWrap(o, depth - 1) for o in value.all()]
+            return [DictWrap(o, depth - 1).to_dict() for o in value.all()]
         elif depth == 1:
             return list(value.values_list("id", flat=True))
 
     def resolve(self, group, name, value):
         if group == "scalars":
+            if isinstance(value, datetime):
+                return value.isoformat()
+            elif isinstance(value, django_countries.fields.Country):
+                return str(value)
+            elif isinstance(value, Decimal):
+                return float(value)
+            elif isinstance(value, (IPv4Address, IPv6Address)):
+                return str(value)
             return value
         elif group == "single_refs":
-            return YamlWrap._resolve_one(name, value, self.depth)
+            return DictWrap._resolve_one(name, value, self.depth)
         elif group == "many_refs":
-            return YamlWrap._resolve_many(name, value, self.depth)
+            return DictWrap._resolve_many(name, value, self.depth)
         else:
             raise ValueError(group)
 
@@ -47,30 +56,18 @@ def field_values(self):
                 continue
             for name in self.fields[group]:
                 value = self.resolve(group, name, getattr(self.object, name))
+                if value is None:
+                    value = "None"
                 yield name, value
 
-
-def represent_wrapped(dumper, wrap):
-    _dict = {}
-    for name, value in wrap.field_values():
-        _dict[name] = value
-    alist = [(k, _dict[k]) for k in sorted(_dict)]
-    return dumper.represent_mapping(wrap.tag, alist)
-
-
-def default_representer(dumper, data):
-    return dumper.represent_str(str(data))
-
-
-def _init():
-    dumper = yaml.SafeDumper
-    for cls in get_backend().CUSTOM_FIELDS:
-        dumper.add_representer(cls, default_representer)
-    dumper.add_representer(YamlWrap, represent_wrapped)
+    def to_dict(self):
+        data = {}
+        for name, value in self.field_values():
+            data[name] = value
+        return data
 
 
-def dump(obj, depth, file):
-    _init()
+def dump_python_dict(obj, depth):
     if get_backend().is_concrete(type(obj)):
-        obj = YamlWrap(obj, depth)
-    yaml.safe_dump(obj, stream=file, default_flow_style=False)
+        obj = DictWrap(obj, depth).to_dict()
+    return obj

src/peeringdb/util.py

@@ -102,3 +102,18 @@ def client_load(client, path):
             des = serializers.deserialize("json", fin)
             for obj in des:
                 obj.save()
+
+
+def get_log_level(level_str):
+    """
+    Convert a string log level to its corresponding logging module level.
+    """
+    levels = {
+        "CRITICAL": logging.CRITICAL,
+        "ERROR": logging.ERROR,
+        "WARNING": logging.WARNING,
+        "INFO": logging.INFO,
+        "DEBUG": logging.DEBUG,
+        "NOTSET": logging.NOTSET,
+    }
+    return levels.get(level_str.strip().upper())

tests/helper/__init__.py

@@ -32,6 +32,7 @@
         # caching will be tested specifically, so keep blank
         "cache_url": "",
     },
+    "log": {"allow_other_loggers": 0, "level": "INFO"},
 }
 
 CONFIG_CACHING = {
@@ -59,6 +60,7 @@
         # caching will be tested specifically, so keep blank
         "cache_url": "cache://localhost",
     },
+    "log": {"allow_other_loggers": 0, "level": "INFO"},
 }
 
 _DATA_PATH = None