diff --git a/confuse/core.py b/confuse/core.py index 6c6c4b0..37cdbcf 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -19,7 +19,7 @@ import errno import os -import yaml +import ruamel.yaml as yaml from collections import OrderedDict from . import util @@ -527,7 +527,10 @@ def _add_user_source(self): exists. """ filename = self.user_config_path() - self.add(YamlSource(filename, loader=self.loader, optional=True)) + source = YamlSource(filename, loader=self.loader, optional=True) + # Save value to keep comments + self.value = source.load() + self.add(source) def _add_default_source(self): """Add the package's default configuration settings. This looks @@ -537,8 +540,11 @@ def _add_default_source(self): if self.modname: if self._package_path: filename = os.path.join(self._package_path, DEFAULT_FILENAME) - self.add(YamlSource(filename, loader=self.loader, - optional=True, default=True)) + source = YamlSource(filename, loader=self.loader, + optional=True, default=True) + # Save value to keep comments + self.value = source.load() + self.add(source) def read(self, user=True, defaults=True): """Find and read the files for this configuration and set them @@ -647,24 +653,13 @@ def dump(self, full=True, redact=False): temp_root.redactions = self.redactions out_dict = temp_root.flatten(redact=redact) - yaml_out = yaml.dump(out_dict, Dumper=yaml_util.Dumper, + # Update configuration value + self.value.update(out_dict) + + return yaml.dump(self.value, Dumper=yaml_util.Dumper, default_flow_style=None, indent=4, width=1000) - - # Restore comments to the YAML text. - default_source = None - for source in self.sources: - if source.default: - default_source = source - break - if default_source and default_source.filename: - with open(default_source.filename, 'rb') as fp: - default_data = fp.read() - yaml_out = yaml_util.restore_yaml_comments( - yaml_out, default_data.decode('utf-8')) - - return yaml_out - + def reload(self): """Reload all sources from the file system. diff --git a/confuse/exceptions.py b/confuse/exceptions.py index 782260e..40f3692 100644 --- a/confuse/exceptions.py +++ b/confuse/exceptions.py @@ -1,6 +1,6 @@ from __future__ import division, absolute_import, print_function -import yaml +import ruamel.yaml as yaml __all__ = [ 'ConfigError', 'NotFoundError', 'ConfigValueError', 'ConfigTypeError', diff --git a/confuse/sources.py b/confuse/sources.py index 2b0f53b..52b9b20 100644 --- a/confuse/sources.py +++ b/confuse/sources.py @@ -73,7 +73,6 @@ def __init__(self, filename=None, default=False, base_for_paths=False, super(YamlSource, self).__init__({}, filename, default, base_for_paths) self.loader = loader self.optional = optional - self.load() def load(self): """Load YAML data from the source's filename. @@ -84,6 +83,8 @@ def load(self): value = yaml_util.load_yaml(self.filename, loader=self.loader) or {} self.update(value) + # Return value for round tripping + return value class EnvSource(ConfigSource): diff --git a/confuse/yaml_util.py b/confuse/yaml_util.py index 2cb4b52..da7bf47 100644 --- a/confuse/yaml_util.py +++ b/confuse/yaml_util.py @@ -1,14 +1,14 @@ from __future__ import division, absolute_import, print_function from collections import OrderedDict -import yaml +import ruamel.yaml as yaml from .exceptions import ConfigReadError from .util import BASESTRING # YAML loading. -class Loader(yaml.SafeLoader): +class Loader(yaml.RoundTripLoader): """A customized YAML loader. This loader deviates from the official YAML spec in a few convenient ways: @@ -28,30 +28,30 @@ def construct_yaml_map(self, node): value = self.construct_mapping(node) data.update(value) - def construct_mapping(self, node, deep=False): - if isinstance(node, yaml.MappingNode): - self.flatten_mapping(node) - else: - raise yaml.constructor.ConstructorError( - None, None, - u'expected a mapping node, but found %s' % node.id, - node.start_mark - ) - - mapping = OrderedDict() - for key_node, value_node in node.value: - key = self.construct_object(key_node, deep=deep) - try: - hash(key) - except TypeError as exc: - raise yaml.constructor.ConstructorError( - u'while constructing a mapping', - node.start_mark, 'found unacceptable key (%s)' % exc, - key_node.start_mark - ) - value = self.construct_object(value_node, deep=deep) - mapping[key] = value - return mapping + # def construct_mapping(self, node, deep=False): + # if isinstance(node, yaml.MappingNode): + # self.flatten_mapping(node) + # else: + # raise yaml.constructor.ConstructorError( + # None, None, + # u'expected a mapping node, but found %s' % node.id, + # node.start_mark + # ) + + # mapping = OrderedDict() + # for key_node, value_node in node.value: + # key = self.construct_object(key_node, deep=deep) + # try: + # hash(key) + # except TypeError as exc: + # raise yaml.constructor.ConstructorError( + # u'while constructing a mapping', + # node.start_mark, 'found unacceptable key (%s)' % exc, + # key_node.start_mark + # ) + # value = self.construct_object(value_node, deep=deep) + # mapping[key] = value + # return mapping # Allow bare strings to begin with %. Directives are still detected. def check_plain(self): @@ -64,12 +64,13 @@ def add_constructors(loader): and maps. Call this method on a custom Loader class to make it behave like Confuse's own Loader """ - loader.add_constructor('tag:yaml.org,2002:str', - Loader._construct_unicode) - loader.add_constructor('tag:yaml.org,2002:map', - Loader.construct_yaml_map) - loader.add_constructor('tag:yaml.org,2002:omap', - Loader.construct_yaml_map) + # Disable this for now + # loader.add_constructor('tag:yaml.org,2002:str', + # Loader._construct_unicode) + # loader.add_constructor('tag:yaml.org,2002:map', + # Loader.construct_yaml_map) + # loader.add_constructor('tag:yaml.org,2002:omap', + # Loader.construct_yaml_map) Loader.add_constructors(Loader) @@ -133,35 +134,35 @@ def parse_as_scalar(value, loader=Loader): # YAML dumping. -class Dumper(yaml.SafeDumper): +class Dumper(yaml.RoundTripDumper): """A PyYAML Dumper that represents OrderedDicts as ordinary mappings (in order, of course). """ # From http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py - def represent_mapping(self, tag, mapping, flow_style=None): - value = [] - node = yaml.MappingNode(tag, value, flow_style=flow_style) - if self.alias_key is not None: - self.represented_objects[self.alias_key] = node - best_style = False - if hasattr(mapping, 'items'): - mapping = list(mapping.items()) - for item_key, item_value in mapping: - node_key = self.represent_data(item_key) - node_value = self.represent_data(item_value) - if not (isinstance(node_key, yaml.ScalarNode) - and not node_key.style): - best_style = False - if not (isinstance(node_value, yaml.ScalarNode) - and not node_value.style): - best_style = False - value.append((node_key, node_value)) - if flow_style is None: - if self.default_flow_style is not None: - node.flow_style = self.default_flow_style - else: - node.flow_style = best_style - return node + # def represent_mapping(self, tag, mapping, flow_style=None): + # value = [] + # node = yaml.MappingNode(tag, value, flow_style=flow_style) + # if self.alias_key is not None: + # self.represented_objects[self.alias_key] = node + # best_style = False + # if hasattr(mapping, 'items'): + # mapping = list(mapping.items()) + # for item_key, item_value in mapping: + # node_key = self.represent_data(item_key) + # node_value = self.represent_data(item_value) + # if not (isinstance(node_key, yaml.ScalarNode) + # and not node_key.style): + # best_style = False + # if not (isinstance(node_value, yaml.ScalarNode) + # and not node_value.style): + # best_style = False + # value.append((node_key, node_value)) + # if flow_style is None: + # if self.default_flow_style is not None: + # node.flow_style = self.default_flow_style + # else: + # node.flow_style = best_style + # return node def represent_list(self, data): """If a list has less than 4 items, represent it in inline style @@ -190,39 +191,8 @@ def represent_none(self, data): return self.represent_scalar('tag:yaml.org,2002:null', '') -Dumper.add_representer(OrderedDict, Dumper.represent_dict) +# This code doesn't work yet with round tripping +# Dumper.add_representer(OrderedDict, Dumper.represent_dict) Dumper.add_representer(bool, Dumper.represent_bool) Dumper.add_representer(type(None), Dumper.represent_none) Dumper.add_representer(list, Dumper.represent_list) - - -def restore_yaml_comments(data, default_data): - """Scan default_data for comments (we include empty lines in our - definition of comments) and place them before the same keys in data. - Only works with comments that are on one or more own lines, i.e. - not next to a yaml mapping. - """ - comment_map = dict() - default_lines = iter(default_data.splitlines()) - for line in default_lines: - if not line: - comment = "\n" - elif line.startswith("#"): - comment = "{0}\n".format(line) - else: - continue - while True: - line = next(default_lines) - if line and not line.startswith("#"): - break - comment += "{0}\n".format(line) - key = line.split(':')[0].strip() - comment_map[key] = comment - out_lines = iter(data.splitlines()) - out_data = "" - for line in out_lines: - key = line.split(':')[0].strip() - if key in comment_map: - out_data += comment_map[key] - out_data += "{0}\n".format(line) - return out_data diff --git a/pyproject.toml b/pyproject.toml index 2abd5f0..fca86b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ author = "Adrian Sampson" author-email = "adrian@radbox.org" home-page = "https://github.com/beetbox/confuse" requires = [ - "pyyaml" + "ruamel.yaml" ] description-file = "README.rst" requires-python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4" diff --git a/requirements.txt b/requirements.txt index dbfc709..4c0f633 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -PyYAML \ No newline at end of file +ruamel.yaml \ No newline at end of file diff --git a/test/test_yaml.py b/test/test_yaml.py index da8ab36..49c167b 100644 --- a/test/test_yaml.py +++ b/test/test_yaml.py @@ -1,7 +1,7 @@ from __future__ import division, absolute_import, print_function import confuse -import yaml +import ruamel.yaml as yaml import unittest from . import TempDir diff --git a/tox.ini b/tox.ini index 0906920..ea4d3f8 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ deps = coverage nose nose-show-skipped - pyyaml + ruamel.yaml pathlib