From 61d39d748994018c39078ebe990bd927280a3e19 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 15 Jul 2020 13:08:43 +0200 Subject: [PATCH 01/80] Remove flake support This support was half-baked and needs rethinking. --- nix/eval-machine-info.nix | 20 ++------------ nixops/backends/__init__.py | 3 -- nixops/deployment.py | 41 --------------------------- nixops/script_defs.py | 55 +++++++++---------------------------- 4 files changed, 16 insertions(+), 103 deletions(-) diff --git a/nix/eval-machine-info.nix b/nix/eval-machine-info.nix index 8acdcff99..066814f53 100644 --- a/nix/eval-machine-info.nix +++ b/nix/eval-machine-info.nix @@ -1,6 +1,5 @@ { system ? builtins.currentSystem , networkExprs -, flakeUri ? null , checkConfigurationOptions ? true , uuid , deploymentName @@ -29,26 +28,13 @@ let operator = { key }: map exprToKey ((getNetworkFromExpr key).require or []); }; in - map ({ key }: getNetworkFromExpr key) networkExprClosure - ++ optional (flakeUri != null) - ((call (builtins.getFlake flakeUri).outputs.nixopsConfigurations.default) // { _file = "<${flakeUri}>"; }); + map ({ key }: getNetworkFromExpr key) networkExprClosure; network = zipAttrs networks; - evalConfig = - if flakeUri != null - then - if network ? nixpkgs - then (builtins.head (network.nixpkgs)).lib.nixosSystem - else throw "NixOps network must have a 'nixpkgs' attribute" - else import (pkgs.path + "/nixos/lib/eval-config.nix"); + evalConfig = import (pkgs.path + "/nixos/lib/eval-config.nix"); - pkgs = if flakeUri != null - then - if network ? nixpkgs - then (builtins.head network.nixpkgs).legacyPackages.${system} - else throw "NixOps network must have a 'nixpkgs' attribute" - else (builtins.head (network.network)).nixpkgs or (import { inherit system; }); + pkgs = (builtins.head (network.network)).nixpkgs or (import { inherit system; }); inherit (pkgs) lib; diff --git a/nixops/backends/__init__.py b/nixops/backends/__init__.py index c716cded2..1d12c6c69 100644 --- a/nixops/backends/__init__.py +++ b/nixops/backends/__init__.py @@ -116,9 +116,6 @@ class MachineState( cur_toplevel: Optional[str] = nixops.util.attr_property("toplevel", None) new_toplevel: Optional[str] - # Immutable flake URI from which this machine was built. - cur_flake_uri: Optional[str] = nixops.util.attr_property("curFlakeUri", None) - # Time (in Unix epoch) the instance was started, if known. start_time: Optional[int] = nixops.util.attr_property("startTime", None, int) diff --git a/nixops/deployment.py b/nixops/deployment.py index 6c85922c4..18f84c337 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -76,8 +76,6 @@ class Deployment: name: Optional[str] = nixops.util.attr_property("name", None) nix_exprs = nixops.util.attr_property("nixExprs", [], "json") nix_path = nixops.util.attr_property("nixPath", [], "json") - flake_uri = nixops.util.attr_property("flakeUri", None) - cur_flake_uri = nixops.util.attr_property("curFlakeUri", None) args = nixops.util.attr_property("args", {}, "json") description = nixops.util.attr_property("description", default_description) configs_path = nixops.util.attr_property("configsPath", None) @@ -128,8 +126,6 @@ def __init__( self.definitions: Optional[Definitions] = None - self._cur_flake_uri: Optional[str] = None - @property def tempdir(self) -> nixops.util.SelfDeletingDir: if not self._tempdir: @@ -138,20 +134,6 @@ def tempdir(self) -> nixops.util.SelfDeletingDir: ) return self._tempdir - def _get_cur_flake_uri(self): - assert self.flake_uri is not None - if self._cur_flake_uri is None: - out = json.loads( - subprocess.check_output( - ["nix", "flake", "metadata", "--json", "--", self.flake_uri], - stderr=self.logger.log_file, - ) - ) - self._cur_flake_uri = out["url"].replace( - "ref=HEAD&rev=0000000000000000000000000000000000000000&", "" - ) # FIXME - return self._cur_flake_uri - @property def machines(self) -> Dict[str, nixops.backends.GenericMachineState]: return _filter_machines(self.resources) @@ -458,18 +440,6 @@ def _eval_flags(self, exprs: List[str]) -> List[str]: ] ) - if self.flake_uri is not None: - flags.extend( - [ - # "--pure-eval", # FIXME - "--argstr", - "flakeUri", - self._get_cur_flake_uri(), - "--allowed-uris", - self.expr_path, - ] - ) - return flags def set_arg(self, name: str, value: str) -> None: @@ -812,8 +782,6 @@ def build_configs( # Set the NixOS version suffix, if we're building from Git. # That way ‘nixos-version’ will show something useful on the # target machines. - # - # TODO: Implement flake compatible version nixos_path = str(self.evaluate_config("nixpkgs")) get_version_script = nixos_path + "/modules/installer/tools/get-version-suffix" if os.path.exists(nixos_path + "/.git") and os.path.exists(get_version_script): @@ -1011,10 +979,6 @@ def set_profile(): if dry_activate: return None - self.cur_flake_uri = ( - self._get_cur_flake_uri() if self.flake_uri is not None else None - ) - if res != 0 and res != 100: raise Exception( "unable to activate new configuration (exit code {})".format( @@ -1041,9 +1005,6 @@ def set_profile(): # configuration. m.cur_configs_path = configs_path m.cur_toplevel = m.new_toplevel - m.cur_flake_uri = ( - self._get_cur_flake_uri() if self.flake_uri is not None else None - ) except Exception: # This thread shouldn't throw an exception because @@ -1555,8 +1516,6 @@ def _rollback( max_concurrent_activate=max_concurrent_activate, ) - self.cur_flake_uri = None - def rollback(self, **kwargs: Any) -> None: with self._get_deployment_lock(): self._rollback(**kwargs) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index cd7329e85..6ef422b92 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -151,34 +151,16 @@ def modify_deployment(args, depl: nixops.deployment.Deployment): nix_exprs = args.nix_exprs templates = args.templates or [] - if args.flake is None: - for i in templates: - nix_exprs.append("".format(i)) - if len(nix_exprs) == 0: - raise Exception( - "you must specify the path to a Nix expression and/or use ‘-t’" - ) - depl.nix_exprs = [os.path.abspath(x) if x[0:1] != "<" else x for x in nix_exprs] - depl.nix_path = [ - nixops.util.abs_nix_path(x) for x in sum(args.nix_path or [], []) - ] - else: - if nix_exprs: - raise Exception( - "you cannot specify a Nix expression in conjunction with '--flake'" - ) - if args.nix_path: - raise Exception( - "you cannot specify a Nix search path ('-I') in conjunction with '--flake'" - ) - if len(templates) != 0: - raise Exception( - "you cannot specify a template ('-t') in conjunction with '--flake'" - ) - # FIXME: should absolutize args.flake if it's a local path. - depl.flake_uri = args.flake - depl.nix_exprs = [] - depl.nix_path = [] + for i in templates: + nix_exprs.append("".format(i)) + if len(nix_exprs) == 0: + raise Exception( + "you must specify the path to a Nix expression and/or use ‘-t’" + ) + depl.nix_exprs = [os.path.abspath(x) if x[0:1] != "<" else x for x in nix_exprs] + depl.nix_path = [ + nixops.util.abs_nix_path(x) for x in sum(args.nix_path or [], []) + ] def op_create(args): @@ -344,14 +326,9 @@ def name_to_key(name: str) -> Tuple[str, str, List[object]]: print("Network UUID:", depl.uuid) print("Network description:", depl.description) - if depl.flake_uri is None: - print("Nix expressions:", " ".join(depl.nix_exprs)) - if depl.nix_path != []: - print("Nix path:", " ".join(["-I " + x for x in depl.nix_path])) - else: - print("Flake URI:", depl.flake_uri) - if depl.cur_flake_uri is not None: - print("Deployed flake URI:", depl.cur_flake_uri) + print("Nix expressions:", " ".join(depl.nix_exprs)) + if depl.nix_path != []: + print("Nix path:", " ".join(["-I " + x for x in depl.nix_path])) if depl.rollback_enabled: print("Nix profile:", depl.get_profile()) @@ -1139,12 +1116,6 @@ def add_common_modify_options(subparser: ArgumentParser): metavar="TEMPLATE", help="name of template to be used", ) - subparser.add_argument( - "--flake", - dest="flake", - metavar="FLAKE_URI", - help="URI of the flake that defines the network", - ) def add_common_deployment_options(subparser: ArgumentParser): From 6a8289bf09aefe5013a82e1eb0b61c4b3392b89d Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 12 Mar 2020 01:41:33 +0000 Subject: [PATCH 02/80] Add deploy without creating a state file --- nixops/evaluation.py | 44 +++++++++++++++++++++++++++++++++++++++++++ nixops/script_defs.py | 12 ++++++++++-- 2 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 nixops/evaluation.py diff --git a/nixops/evaluation.py b/nixops/evaluation.py new file mode 100644 index 000000000..b94c3026f --- /dev/null +++ b/nixops/evaluation.py @@ -0,0 +1,44 @@ +from dataclasses import dataclass +import subprocess +import typing +import json + + +@dataclass +class NetworkEval: + + description: str = "Unnamed NixOps network" + enableRollback: bool = False + enableState: bool = True + + +def _eval_attr( + attr, nix_exprs: typing.List[str] +) -> typing.Dict[typing.Any, typing.Any]: + p = subprocess.run( + [ + "nix-instantiate", + "--eval-only", + "--json", + "--strict", + # Arg + "--arg", + "checkConfigurationOptions", + "false", + # Attr + "-A", + attr, + ] + + nix_exprs, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if p.returncode != 0: + raise RuntimeError(p.stderr.decode()) + + return json.loads(p.stdout) + + +def eval_network(nix_exprs: typing.List[str]) -> NetworkEval: + result = _eval_attr("network", nix_exprs) + return NetworkEval(**result) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 6ef422b92..d4414e352 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -26,6 +26,7 @@ from nixops.plugins.manager import PluginManager from nixops.plugins import get_plugin_manager +from nixops.evaluation import eval_network PluginManager.load() @@ -168,8 +169,15 @@ def op_create(args): depl = sf.create_deployment() sys.stderr.write("created deployment ‘{0}’\n".format(depl.uuid)) modify_deployment(args, depl) - if args.name or args.deployment: - set_name(depl, args.name or args.deployment) + + # When deployment is created without state "name" does not exist + name = args.deployment + if "name" in args: + name = args.name or args.deployment + + if name: + set_name(depl, name) + sys.stdout.write(depl.uuid + "\n") From e9426b0fddf46df925a2f6fa2cdb1dd7128bc20c Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sun, 22 Mar 2020 13:23:42 -0400 Subject: [PATCH 03/80] add_common_{modify,deployment}_options: add types --- nixops/script_defs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index d4414e352..20681c01c 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -1109,7 +1109,7 @@ def add_subparser( return subparser -def add_common_modify_options(subparser: ArgumentParser): +def add_common_modify_options(subparser: ArgumentParser) -> None: subparser.add_argument( "nix_exprs", nargs="*", @@ -1126,7 +1126,7 @@ def add_common_modify_options(subparser: ArgumentParser): ) -def add_common_deployment_options(subparser: ArgumentParser): +def add_common_deployment_options(subparser: ArgumentParser) -> None: subparser.add_argument( "--include", nargs="+", From 2292ee36cef7aae072de44ec6c54f054e77d7009 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sun, 22 Mar 2020 22:20:00 -0400 Subject: [PATCH 04/80] Begin to describe an API for storage backends --- nixops/plugins/__init__.py | 7 ++++++ nixops/plugins/hookspecs.py | 1 - nixops/plugins/manager.py | 19 +++++++++++++- nixops/script_defs.py | 1 - nixops/storage/__init__.py | 49 +++++++++++++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 nixops/storage/__init__.py diff --git a/nixops/plugins/__init__.py b/nixops/plugins/__init__.py index 768a1ad0e..200dd6886 100644 --- a/nixops/plugins/__init__.py +++ b/nixops/plugins/__init__.py @@ -3,6 +3,7 @@ from nixops.backends import MachineState from typing import List, Dict, Optional, Union, Tuple +from nixops.storage import BackendRegistration from functools import lru_cache from typing import Generator import pluggy @@ -93,3 +94,9 @@ def docs(self) -> List[Tuple[str, str]]: :return a list of tuples (plugin_name, doc_path) """ return [] + + def storage_backends(self) -> BackendRegistration: + """ Extend the core nixops cli parser + :return a set of plugin parser extensions + """ + return {} diff --git a/nixops/plugins/hookspecs.py b/nixops/plugins/hookspecs.py index a48622d3d..bb9fde7c2 100644 --- a/nixops/plugins/hookspecs.py +++ b/nixops/plugins/hookspecs.py @@ -11,4 +11,3 @@ def plugin() -> Plugin: """ Register a plugin base class """ - pass diff --git a/nixops/plugins/manager.py b/nixops/plugins/manager.py index 19c9bf144..1a0b217b8 100644 --- a/nixops/plugins/manager.py +++ b/nixops/plugins/manager.py @@ -1,11 +1,14 @@ from __future__ import annotations from nixops.backends import MachineState -from typing import List, Dict, Generator, Tuple, Any, Set +from typing import List, Dict, Generator, Tuple, Any, Set, Type import importlib +from nixops.storage import StorageBackend from . import get_plugins, MachineHooks, DeploymentHooks +import nixops.ansi import nixops +import sys NixosConfigurationType = List[Dict[Tuple[str, ...], Any]] @@ -76,3 +79,17 @@ def parser(parser, subparsers): def docs() -> Generator[Tuple[str, str], None, None]: for plugin in get_plugins(): yield from plugin.docs() + + @staticmethod + def storage_backends(): + storage_backends: Dict[str, Type[StorageBackend]] = {} + for plugin in get_plugins(): + for name, backend in plugin.storage_backends().items(): + if name not in storage_backends: + storage_backends[name] = backend + else: + sys.stderr.write( + nixops.ansi.ansi_warn( + f"Two plugins tried to provide the '{name}' storage backend." + ) + ) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 20681c01c..1bd1d28d5 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -26,7 +26,6 @@ from nixops.plugins.manager import PluginManager from nixops.plugins import get_plugin_manager -from nixops.evaluation import eval_network PluginManager.load() diff --git a/nixops/storage/__init__.py b/nixops/storage/__init__.py new file mode 100644 index 000000000..143e331c0 --- /dev/null +++ b/nixops/storage/__init__.py @@ -0,0 +1,49 @@ +from __future__ import annotations +from typing import Dict, Any, Type, TYPE_CHECKING +from typing_extensions import Protocol, TypedDict + +if TYPE_CHECKING: + import nixops.statefile + + +class ArgumentDescription(TypedDict): + optional: bool + required: bool + default: Any + description: str + + +StorageArgDescriptions = Dict[str, ArgumentDescription] +StorageArgValues = Dict[str, Any] + + +class StorageBackend(Protocol): + @staticmethod + def arguments() -> StorageArgDescriptions: + raise NotImplementedError + + def __init__(self, args: StorageArgValues) -> None: + raise NotImplementedError + + # fetchToFile: acquire a lock and download the state file to + # the local disk. Note: no arguments will be passed over kwargs. + # Making it part of the type definition allows adding new + # arguments later. + def fetchToFile(self, path: str, **kwargs) -> None: + raise NotImplementedError + + # onOpen: receive the StateFile object for last-minute, backend + # specific changes to the state file. + # Note: no arguments will be passed over kwargs. Making it part + # of the type definition allows adding new arguments later. + def onOpen(self, sf: nixops.statefile.StateFile, **kwargs) -> None: + pass + + # uploadFromFile: upload the new state file and release any locks + # Note: no arguments will be passed over kwargs. Making it part of + # the type definition allows adding new arguments later. + def uploadFromFile(self, path: str, **kwargs) -> None: + raise NotImplementedError + + +BackendRegistration = Dict[str, Type[StorageBackend]] From 29388ae179edf4477bd2485cee2288eeca0ba4d2 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 23 Mar 2020 19:43:58 -0400 Subject: [PATCH 05/80] script_defs: accept a network file, not a state file --- nixops/script_defs.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 1bd1d28d5..bd5207e84 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -1023,12 +1023,11 @@ def add_subparser( ) -> ArgumentParser: subparser = subparsers.add_parser(name, help=help) subparser.add_argument( - "--state", - "-s", - dest="state_file", + "--network", + dest="network_file", metavar="FILE", - default=nixops.statefile.get_default_state_file(), - help="path to state file", + default=f"{os.getcwd()}/network.nix", + help="path to a network.nix", ) subparser.add_argument( "--deployment", From ba716a49cf5f8a57bf40655645d108467034f6cf Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 23 Mar 2020 17:47:31 -0400 Subject: [PATCH 06/80] evaluation: parse out storage provider data --- nixops/evaluation.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index b94c3026f..e817e8c7e 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -5,11 +5,16 @@ @dataclass -class NetworkEval: +class GenericStorageConfig: + provider: str + configuration: typing.Dict[typing.Any, typing.Any] + +@dataclass +class NetworkEval: + storage: GenericStorageConfig description: str = "Unnamed NixOps network" enableRollback: bool = False - enableState: bool = True def _eval_attr( @@ -41,4 +46,21 @@ def _eval_attr( def eval_network(nix_exprs: typing.List[str]) -> NetworkEval: result = _eval_attr("network", nix_exprs) + + storage = result.get("storage") + if storage is None: + raise Exception("Missing property: network.storage must be configured.") + if len(storage.keys()) > 1: + raise Exception( + "Invalid property: network.storage can only have one defined storage backend." + ) + + key = list(storage.keys()).pop() + if key is None: + raise Exception( + "Missing property: network.storage has no defined storage backend." + ) + + result["storage"] = GenericStorageConfig(provider=key, configuration=storage[key]) + return NetworkEval(**result) From 3ee765dc613f364ddb3c43a03df6f6af46270552 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 24 Mar 2020 14:11:45 -0400 Subject: [PATCH 07/80] deployment: HACK: make the network.nix always part of the nix exprs --- nixops/script_defs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index bd5207e84..aacff1f2a 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -35,6 +35,7 @@ def deployment(args: Namespace) -> Generator[nixops.deployment.Deployment, None, None]: with network_state(args) as sf: depl = open_deployment(sf, args) + depl.nix_exprs = [os.path.abspath(args.network_file)] yield depl From 5515e92d805c9f30ba22233de7758508061d321e Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 24 Mar 2020 17:37:29 -0400 Subject: [PATCH 08/80] network_state: use a storage backend --- nixops/script_defs.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index aacff1f2a..f7861b516 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -19,6 +19,7 @@ import logging import logging.handlers import json +from tempfile import TemporaryDirectory import pipes from typing import Tuple, List, Optional, Union, Generator import nixops.ansi @@ -41,11 +42,34 @@ def deployment(args: Namespace) -> Generator[nixops.deployment.Deployment, None, @contextlib.contextmanager def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None, None]: - state = nixops.statefile.StateFile(args.state_file) - try: - yield state - finally: - state.close() + network_file: str = args.network_file + network = eval_network([network_file]) + storage_class: Optional[Type[StorageBackend]] = storage_backends.get( + network.storage.provider + ) + if storage_class is None: + sys.stderr.write( + nixops.util.ansi_warn( + f"The network requires the '{network.storage.provider}' state provider, " + "but no plugin provides it.\n" + ) + ) + raise Exception("Missing storage provider plugin.") + + storage: StorageBackend = storage_class(network.storage.configuration) + + with TemporaryDirectory("nixops") as statedir: + statefile = statedir + "/state.nixops" + storage.fetchToFile(statefile) + + state = nixops.statefile.StateFile(statefile) + try: + storage.onOpen(state) + + yield state + finally: + state.close() + storage.uploadFromFile(statefile) def op_list_plugins(args): From 952beb7966459585d314d7ca3bf6b8c24c7111db Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 24 Mar 2020 17:37:48 -0400 Subject: [PATCH 09/80] LegacyBackend: init --- nixops/plugin.py | 14 ++++++++++ nixops/plugins/manager.py | 3 +-- nixops/script_defs.py | 7 ++--- nixops/storage/__init__.py | 3 +++ nixops/storage/legacy.py | 55 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 nixops/plugin.py create mode 100644 nixops/storage/legacy.py diff --git a/nixops/plugin.py b/nixops/plugin.py new file mode 100644 index 000000000..517a931bd --- /dev/null +++ b/nixops/plugin.py @@ -0,0 +1,14 @@ +from nixops.storage import StorageBackend, BackendRegistration +from nixops.storage.legacy import LegacyBackend +import nixops.plugins + + +class InternalPlugin(nixops.plugins.Plugin): + + def storage_backends(self) -> BackendRegistration: + return {"legacy": LegacyBackend} + + +@nixops.plugins.hookimpl +def plugin(): + return InternalPlugin() diff --git a/nixops/plugins/manager.py b/nixops/plugins/manager.py index 1a0b217b8..35a47639a 100644 --- a/nixops/plugins/manager.py +++ b/nixops/plugins/manager.py @@ -4,7 +4,7 @@ from typing import List, Dict, Generator, Tuple, Any, Set, Type import importlib -from nixops.storage import StorageBackend +from nixops.storage import StorageBackend, storage_backends from . import get_plugins, MachineHooks, DeploymentHooks import nixops.ansi import nixops @@ -82,7 +82,6 @@ def docs() -> Generator[Tuple[str, str], None, None]: @staticmethod def storage_backends(): - storage_backends: Dict[str, Type[StorageBackend]] = {} for plugin in get_plugins(): for name, backend in plugin.storage_backends().items(): if name not in storage_backends: diff --git a/nixops/script_defs.py b/nixops/script_defs.py index f7861b516..a008f3d30 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -21,12 +21,14 @@ import json from tempfile import TemporaryDirectory import pipes -from typing import Tuple, List, Optional, Union, Generator +from typing import Tuple, List, Optional, Union, Generator, Type import nixops.ansi from nixops.plugins.manager import PluginManager from nixops.plugins import get_plugin_manager +from nixops.evaluation import eval_network +from nixops.storage import StorageBackend, storage_backends PluginManager.load() @@ -49,7 +51,7 @@ def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None ) if storage_class is None: sys.stderr.write( - nixops.util.ansi_warn( + nixops.ansi.ansi_warn( f"The network requires the '{network.storage.provider}' state provider, " "but no plugin provides it.\n" ) @@ -61,7 +63,6 @@ def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None with TemporaryDirectory("nixops") as statedir: statefile = statedir + "/state.nixops" storage.fetchToFile(statefile) - state = nixops.statefile.StateFile(statefile) try: storage.onOpen(state) diff --git a/nixops/storage/__init__.py b/nixops/storage/__init__.py index 143e331c0..b1e102d20 100644 --- a/nixops/storage/__init__.py +++ b/nixops/storage/__init__.py @@ -47,3 +47,6 @@ def uploadFromFile(self, path: str, **kwargs) -> None: BackendRegistration = Dict[str, Type[StorageBackend]] + + +storage_backends: Dict[str, Type[StorageBackend]] = {} diff --git a/nixops/storage/legacy.py b/nixops/storage/legacy.py new file mode 100644 index 000000000..b0ecac696 --- /dev/null +++ b/nixops/storage/legacy.py @@ -0,0 +1,55 @@ +from nixops.storage import StorageArgDescriptions, StorageArgValues +import nixops.statefile +import sys +import os +import os.path + + +class LegacyBackend: + @staticmethod + def arguments() -> StorageArgDescriptions: + raise NotImplementedError + + def __init__(self, args: StorageArgValues) -> None: + pass + + # fetchToFile: acquire a lock and download the state file to + # the local disk. Note: no arguments will be passed over kwargs. + # Making it part of the type definition allows adding new + # arguments later. + def fetchToFile(self, path: str, **kwargs) -> None: + os.symlink(os.path.abspath(self.state_location()), path) + + def onOpen(self, sf: nixops.statefile.StateFile, **kwargs) -> None: + pass + + def state_location(self) -> str: + env_override = os.environ.get("NIXOPS_STATE", os.environ.get("CHARON_STATE")) + if env_override is not None: + return env_override + + home_dir = os.environ.get("HOME", "") + charon_dir = f"{home_dir}/.charon" + nixops_dir = f"{home_dir}/.nixops" + + if not os.path.exists(nixops_dir): + if os.path.exists(charon_dir): + sys.stderr.write( + "renaming ‘{0}’ to ‘{1}’...\n".format(charon_dir, nixops_dir) + ) + os.rename(charon_dir, nixops_dir) + if os.path.exists(nixops_dir + "/deployments.charon"): + os.rename( + nixops_dir + "/deployments.charon", + nixops_dir + "/deployments.nixops", + ) + else: + os.makedirs(nixops_dir, 0o700) + + return nixops_dir + "/deployments.nixops" + + # uploadFromFile: upload the new state file and release any locks + # Note: no arguments will be passed over kwargs. Making it part of + # the type definition allows adding new arguments later. + def uploadFromFile(self, path: str, **kwargs) -> None: + pass From 81717db132fcdedf2e0349358842a2be94a1ad82 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 23 Mar 2020 22:13:18 -0400 Subject: [PATCH 10/80] MemoryBackend: init --- nixops/plugin.py | 6 +++--- nixops/plugins/manager.py | 10 ++++++---- nixops/storage/memory.py | 27 +++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 nixops/storage/memory.py diff --git a/nixops/plugin.py b/nixops/plugin.py index 517a931bd..0bcbdb8c8 100644 --- a/nixops/plugin.py +++ b/nixops/plugin.py @@ -1,12 +1,12 @@ -from nixops.storage import StorageBackend, BackendRegistration +from nixops.storage import BackendRegistration from nixops.storage.legacy import LegacyBackend +from nixops.storage.memory import MemoryBackend import nixops.plugins class InternalPlugin(nixops.plugins.Plugin): - def storage_backends(self) -> BackendRegistration: - return {"legacy": LegacyBackend} + return {"legacy": LegacyBackend, "memory": MemoryBackend} @nixops.plugins.hookimpl diff --git a/nixops/plugins/manager.py b/nixops/plugins/manager.py index 35a47639a..b930e316c 100644 --- a/nixops/plugins/manager.py +++ b/nixops/plugins/manager.py @@ -1,10 +1,10 @@ from __future__ import annotations from nixops.backends import MachineState -from typing import List, Dict, Generator, Tuple, Any, Set, Type +from typing import List, Dict, Generator, Tuple, Any, Set import importlib -from nixops.storage import StorageBackend, storage_backends +from nixops.storage import storage_backends from . import get_plugins, MachineHooks, DeploymentHooks import nixops.ansi import nixops @@ -54,8 +54,8 @@ def machine_hooks() -> Generator[MachineHooks, None, None]: continue yield machine_hooks - @staticmethod - def load(): + @classmethod + def load(cls): seen: Set[str] = set() for plugin in get_plugins(): for mod in plugin.load(): @@ -63,6 +63,8 @@ def load(): importlib.import_module(mod) seen.add(mod) + cls.storage_backends() + @staticmethod def nixexprs() -> List[str]: nixexprs: List[str] = [] diff --git a/nixops/storage/memory.py b/nixops/storage/memory.py new file mode 100644 index 000000000..2656cf12c --- /dev/null +++ b/nixops/storage/memory.py @@ -0,0 +1,27 @@ +import nixops.statefile +from nixops.storage import StorageArgDescriptions, StorageArgValues + + +class MemoryBackend: + @staticmethod + def arguments() -> StorageArgDescriptions: + raise NotImplementedError + + def __init__(self, args: StorageArgValues) -> None: + pass + + # fetchToFile: acquire a lock and download the state file to + # the local disk. Note: no arguments will be passed over kwargs. + # Making it part of the type definition allows adding new + # arguments later. + def fetchToFile(self, path: str, **kwargs) -> None: + pass + + def onOpen(self, sf: nixops.statefile.StateFile, **kwargs) -> None: + sf.create_deployment() + + # uploadFromFile: upload the new state file and release any locks + # Note: no arguments will be passed over kwargs. Making it part of + # the type definition allows adding new arguments later. + def uploadFromFile(self, path: str, **kwargs) -> None: + pass From bb64023b2b083c60357276b47451578aa939ad0f Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 23 Apr 2020 15:08:59 -0400 Subject: [PATCH 11/80] fixup d6b34b2251c9b30da3377b970cb617c73ef208ab: storage API: use Mapping over Dict --- nixops/storage/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixops/storage/__init__.py b/nixops/storage/__init__.py index b1e102d20..93bad42ae 100644 --- a/nixops/storage/__init__.py +++ b/nixops/storage/__init__.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import Dict, Any, Type, TYPE_CHECKING +from typing import Mapping, Dict, Any, Type, TYPE_CHECKING from typing_extensions import Protocol, TypedDict if TYPE_CHECKING: @@ -14,7 +14,7 @@ class ArgumentDescription(TypedDict): StorageArgDescriptions = Dict[str, ArgumentDescription] -StorageArgValues = Dict[str, Any] +StorageArgValues = Mapping[str, Any] class StorageBackend(Protocol): From 41f658a6705eb2879f702a9af7e7d8896c1391fa Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 23 Apr 2020 15:10:51 -0400 Subject: [PATCH 12/80] fixup d6b34b2251c9b30da3377b970cb617c73ef208ab: storage API: drop the arguments junk --- nixops/storage/__init__.py | 12 ------------ nixops/storage/legacy.py | 6 +----- nixops/storage/memory.py | 6 +----- 3 files changed, 2 insertions(+), 22 deletions(-) diff --git a/nixops/storage/__init__.py b/nixops/storage/__init__.py index 93bad42ae..7905ed51a 100644 --- a/nixops/storage/__init__.py +++ b/nixops/storage/__init__.py @@ -6,22 +6,10 @@ import nixops.statefile -class ArgumentDescription(TypedDict): - optional: bool - required: bool - default: Any - description: str - - -StorageArgDescriptions = Dict[str, ArgumentDescription] StorageArgValues = Mapping[str, Any] class StorageBackend(Protocol): - @staticmethod - def arguments() -> StorageArgDescriptions: - raise NotImplementedError - def __init__(self, args: StorageArgValues) -> None: raise NotImplementedError diff --git a/nixops/storage/legacy.py b/nixops/storage/legacy.py index b0ecac696..ea761f6f8 100644 --- a/nixops/storage/legacy.py +++ b/nixops/storage/legacy.py @@ -1,4 +1,4 @@ -from nixops.storage import StorageArgDescriptions, StorageArgValues +from nixops.storage import StorageArgValues import nixops.statefile import sys import os @@ -6,10 +6,6 @@ class LegacyBackend: - @staticmethod - def arguments() -> StorageArgDescriptions: - raise NotImplementedError - def __init__(self, args: StorageArgValues) -> None: pass diff --git a/nixops/storage/memory.py b/nixops/storage/memory.py index 2656cf12c..34117119d 100644 --- a/nixops/storage/memory.py +++ b/nixops/storage/memory.py @@ -1,12 +1,8 @@ import nixops.statefile -from nixops.storage import StorageArgDescriptions, StorageArgValues +from nixops.storage import StorageArgValues class MemoryBackend: - @staticmethod - def arguments() -> StorageArgDescriptions: - raise NotImplementedError - def __init__(self, args: StorageArgValues) -> None: pass From 8a59a5831aee297cc7fbbb34065f4e1e979b8217 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 28 Apr 2020 09:58:29 -0400 Subject: [PATCH 13/80] eval network: use ImmutableValidatedObject for good errors Parse the result of evaluating the network's `network` argument as a typed mapping. Extend this validation to the Storage params. --- nixops/evaluation.py | 38 ++++++++++++++++++++++---------------- nixops/script_defs.py | 4 +++- nixops/storage/__init__.py | 14 ++++++++++---- nixops/storage/legacy.py | 13 +++++++++++-- nixops/storage/memory.py | 13 +++++++++++-- 5 files changed, 57 insertions(+), 25 deletions(-) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index e817e8c7e..f7b781b0f 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -1,22 +1,27 @@ -from dataclasses import dataclass import subprocess import typing +from typing import Optional, Mapping, Any import json +from nixops.util import ImmutableValidatedObject -@dataclass -class GenericStorageConfig: +class GenericStorageConfig(ImmutableValidatedObject): provider: str - configuration: typing.Dict[typing.Any, typing.Any] + configuration: typing.Mapping[typing.Any, typing.Any] -@dataclass -class NetworkEval: +class NetworkEval(ImmutableValidatedObject): storage: GenericStorageConfig description: str = "Unnamed NixOps network" enableRollback: bool = False +class RawNetworkEval(ImmutableValidatedObject): + storage: Mapping[str, Any] + description: Optional[str] + enableRollback: Optional[bool] + + def _eval_attr( attr, nix_exprs: typing.List[str] ) -> typing.Dict[typing.Any, typing.Any]: @@ -45,22 +50,23 @@ def _eval_attr( def eval_network(nix_exprs: typing.List[str]) -> NetworkEval: - result = _eval_attr("network", nix_exprs) + raw_eval = RawNetworkEval(**_eval_attr("network", nix_exprs)) - storage = result.get("storage") - if storage is None: - raise Exception("Missing property: network.storage must be configured.") - if len(storage.keys()) > 1: + if len(raw_eval.storage) > 1: raise Exception( "Invalid property: network.storage can only have one defined storage backend." ) - key = list(storage.keys()).pop() - if key is None: + try: + key = list(raw_eval.storage.keys()).pop() + value = raw_eval.storage[key] + except KeyError: raise Exception( "Missing property: network.storage has no defined storage backend." ) - result["storage"] = GenericStorageConfig(provider=key, configuration=storage[key]) - - return NetworkEval(**result) + return NetworkEval( + enableRollback=raw_eval.enableRollback or False, + description=raw_eval.description or "Unnamed NixOps network", + storage={"provider": key, "configuration": value}, + ) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index a008f3d30..71608793a 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -58,7 +58,9 @@ def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None ) raise Exception("Missing storage provider plugin.") - storage: StorageBackend = storage_class(network.storage.configuration) + storage_class_options = storage_class.options() + foo = storage_class_options(**network.storage.configuration) + storage: StorageBackend = storage_class(foo) with TemporaryDirectory("nixops") as statedir: statefile = statedir + "/state.nixops" diff --git a/nixops/storage/__init__.py b/nixops/storage/__init__.py index 7905ed51a..6d6498699 100644 --- a/nixops/storage/__init__.py +++ b/nixops/storage/__init__.py @@ -1,16 +1,22 @@ from __future__ import annotations -from typing import Mapping, Dict, Any, Type, TYPE_CHECKING -from typing_extensions import Protocol, TypedDict +from typing import Mapping, Dict, Any, Type, TypeVar, TYPE_CHECKING, Protocol, Callable + +# from typing_extensions import Protocol, TypedDict if TYPE_CHECKING: import nixops.statefile +T = TypeVar("T", covariant=True) StorageArgValues = Mapping[str, Any] -class StorageBackend(Protocol): - def __init__(self, args: StorageArgValues) -> None: +class StorageBackend(Protocol[T]): + @staticmethod + def options() -> Callable[..., T]: + pass + + def __init__(self, args: T) -> None: raise NotImplementedError # fetchToFile: acquire a lock and download the state file to diff --git a/nixops/storage/legacy.py b/nixops/storage/legacy.py index ea761f6f8..9d00db621 100644 --- a/nixops/storage/legacy.py +++ b/nixops/storage/legacy.py @@ -1,12 +1,21 @@ -from nixops.storage import StorageArgValues import nixops.statefile import sys import os import os.path +from nixops.util import ImmutableValidatedObject +from typing import Type + + +class LegacyBackendOptions(ImmutableValidatedObject): + pass class LegacyBackend: - def __init__(self, args: StorageArgValues) -> None: + @staticmethod + def options() -> Type[LegacyBackendOptions]: + return LegacyBackendOptions + + def __init__(self, args: LegacyBackendOptions) -> None: pass # fetchToFile: acquire a lock and download the state file to diff --git a/nixops/storage/memory.py b/nixops/storage/memory.py index 34117119d..0dd19416a 100644 --- a/nixops/storage/memory.py +++ b/nixops/storage/memory.py @@ -1,9 +1,18 @@ import nixops.statefile -from nixops.storage import StorageArgValues +from nixops.util import ImmutableValidatedObject +from typing import Type + + +class MemoryBackendOptions(ImmutableValidatedObject): + pass class MemoryBackend: - def __init__(self, args: StorageArgValues) -> None: + @staticmethod + def options() -> Type[MemoryBackendOptions]: + return MemoryBackendOptions + + def __init__(self, args: MemoryBackendOptions) -> None: pass # fetchToFile: acquire a lock and download the state file to From 97c4a412ec66b5c4dc6f6be464cc3555bbb6fe5d Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 28 Apr 2020 09:50:42 -0400 Subject: [PATCH 14/80] nixops.exceptions: add NixError for Nix-related problems --- nixops/__main__.py | 6 ++++++ nixops/exceptions.py | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 nixops/exceptions.py diff --git a/nixops/__main__.py b/nixops/__main__.py index 905a8ac34..19e3bceff 100755 --- a/nixops/__main__.py +++ b/nixops/__main__.py @@ -705,6 +705,8 @@ def main() -> None: args = parser.parse_args() setup_logging(args) + from nixops.exceptions import NixError + try: nixops.deployment.DEBUG = args.debug args.op(args) @@ -719,6 +721,10 @@ def main() -> None: if args.debug or args.show_trace or str(e) == "": e.print_all_backtraces() sys.exit(1) + except NixError as e: + sys.stderr.write(str(e)) + sys.stderr.flush() + sys.exit(1) if __name__ == "__main__": diff --git a/nixops/exceptions.py b/nixops/exceptions.py new file mode 100644 index 000000000..cce58cd3d --- /dev/null +++ b/nixops/exceptions.py @@ -0,0 +1,2 @@ +class NixError(Exception): + pass From b3c0fa53f845fc62ce191a16e17eb2e9bf83c22d Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 28 Apr 2020 09:51:04 -0400 Subject: [PATCH 15/80] eval_network, eval_attrs: handle missing 'network' --- nixops/evaluation.py | 62 ++++++++++++++++++++++++++++++++++++------- nixops/script_defs.py | 2 +- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index f7b781b0f..c60d8729a 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -3,6 +3,11 @@ from typing import Optional, Mapping, Any import json from nixops.util import ImmutableValidatedObject +from nixops.exceptions import NixError + + +class MalformedNetworkError(NixError): + pass class GenericStorageConfig(ImmutableValidatedObject): @@ -22,9 +27,12 @@ class RawNetworkEval(ImmutableValidatedObject): enableRollback: Optional[bool] -def _eval_attr( - attr, nix_exprs: typing.List[str] -) -> typing.Dict[typing.Any, typing.Any]: +class EvalResult(ImmutableValidatedObject): + exists: bool + value: Any + + +def _eval_attr(attr, nix_expr: str) -> EvalResult: p = subprocess.run( [ "nix-instantiate", @@ -36,21 +44,57 @@ def _eval_attr( "checkConfigurationOptions", "false", # Attr - "-A", + "--argstr", + "attr", attr, - ] - + nix_exprs, + "--arg", + "nix_expr", + nix_expr, + "--expr", + """ + { nix_expr, attr }: + let + ret = (import nix_expr); + in { + exists = ret ? "${attr}"; + value = ret."${attr}" or null; + } + """, + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) if p.returncode != 0: raise RuntimeError(p.stderr.decode()) - return json.loads(p.stdout) + return EvalResult(**json.loads(p.stdout)) + +def eval_network(nix_expr: str) -> NetworkEval: + result = _eval_attr("network", nix_expr) + if not result.exists: + raise MalformedNetworkError( + """ +TODO: improve this error to be less specific about conversion. link to +docs? + + +WARNING: NixOps 1.0 -> 2.0 conversion step required + +NixOps 2.0 added support for multiple storage backends. + +Upgrade steps: +1. Open %s +2. Add: + network.storage.legacy = { + databasefile = "~/.nixops/deployments.nixops" + } +3. Rerun +""" + % nix_expr + ) -def eval_network(nix_exprs: typing.List[str]) -> NetworkEval: - raw_eval = RawNetworkEval(**_eval_attr("network", nix_exprs)) + raw_eval = RawNetworkEval(**result.value) if len(raw_eval.storage) > 1: raise Exception( diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 71608793a..ac86da277 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -45,7 +45,7 @@ def deployment(args: Namespace) -> Generator[nixops.deployment.Deployment, None, @contextlib.contextmanager def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None, None]: network_file: str = args.network_file - network = eval_network([network_file]) + network = eval_network(network_file) storage_class: Optional[Type[StorageBackend]] = storage_backends.get( network.storage.provider ) From bfa6ffcf463a150452282852b81eeaf8c023b443 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 28 Apr 2020 10:06:42 -0400 Subject: [PATCH 16/80] eval_network: check the network property is a dictionary --- nixops/evaluation.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index c60d8729a..cc5831bc7 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -94,6 +94,24 @@ def eval_network(nix_expr: str) -> NetworkEval: % nix_expr ) + if not isinstance(result.value, dict): + raise MalformedNetworkError( + """ +TODO: improve this error + +The network.nix has a `network` attribute set, but it is of the wrong +type. A valid network attribute looks like this: + + { + network = { + storage = { + /* storage driver details */ + }; + }; + } +""" + ) + raw_eval = RawNetworkEval(**result.value) if len(raw_eval.storage) > 1: From c2335f2206834f09683507f03040665e92dcfca0 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 28 Apr 2020 10:13:41 -0400 Subject: [PATCH 17/80] add a test for when the network.nix has no network attr --- tests/storage/unspecified.nix | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/storage/unspecified.nix diff --git a/tests/storage/unspecified.nix b/tests/storage/unspecified.nix new file mode 100644 index 000000000..8e9225669 --- /dev/null +++ b/tests/storage/unspecified.nix @@ -0,0 +1,5 @@ +/* +Expect a reasonable error message when the `network` attribute is missing +*/ +{ +} From 9a59a3e55dfcee5ce452a8f2482beb136b4b1c6d Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 28 Apr 2020 10:14:17 -0400 Subject: [PATCH 18/80] add a test for when the network.nix's network is of the wrong type --- tests/storage/wrong-type.nix | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/storage/wrong-type.nix diff --git a/tests/storage/wrong-type.nix b/tests/storage/wrong-type.nix new file mode 100644 index 000000000..662fecbd9 --- /dev/null +++ b/tests/storage/wrong-type.nix @@ -0,0 +1,7 @@ +/* +Expect a reasonable error message when the `network` attribute +has a value of the wrong type +*/ +{ + network = "meh"; +} From f45fcf9ced72eea02b4117c132cf1cf7bb25e01a Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 28 Apr 2020 10:14:33 -0400 Subject: [PATCH 19/80] Add a test for when the network's value is an empty attrset --- tests/storage/empty.nix | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tests/storage/empty.nix diff --git a/tests/storage/empty.nix b/tests/storage/empty.nix new file mode 100644 index 000000000..a206eb205 --- /dev/null +++ b/tests/storage/empty.nix @@ -0,0 +1,16 @@ +/* +Expect a reasonable error message when the `network` attribute +has an empty attributeset for network. + +2020-04-28 User gets a not terrible message: + + TypeError: type of storage must be collections.abc.Mapping; + got NoneType instead + +and we're going to punt on this error handling from here, since we +have now reached a point where ImmutableValidatedObject handles +errors. +*/ +{ + network = {}; +} From 1b4571424bdf33f913d4a43ae535a964ae8b1938 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 28 Apr 2020 10:17:31 -0400 Subject: [PATCH 20/80] util: ImmutableValidatedObject: don't set _frozen(?) --- nixops/util.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nixops/util.py b/nixops/util.py index 178203e62..43c453c37 100644 --- a/nixops/util.py +++ b/nixops/util.py @@ -171,6 +171,9 @@ def _transform_value(key: Any, value: Any) -> Any: return value for key in set(list(anno.keys()) + list(kwargs.keys())): + if key == "_frozen": + continue + # If a default value: # class SomeSubClass(ImmutableValidatedObject): # x: int = 1 From 2ff99b5d67e04f20e108f7bc5fd2187a3a2db98b Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 29 Apr 2020 11:15:20 -0400 Subject: [PATCH 21/80] storage backends: strengthen typecheck, making T invariant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before, a single storage backend could use a different T at every call site. Not anymore, the T's must be aligned. Co-authored-by: Adam Höse --- nixops/storage/__init__.py | 20 +++++++++++++++++--- nixops/storage/legacy.py | 9 ++++++--- nixops/storage/memory.py | 9 ++++++--- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/nixops/storage/__init__.py b/nixops/storage/__init__.py index 6d6498699..2866787b1 100644 --- a/nixops/storage/__init__.py +++ b/nixops/storage/__init__.py @@ -1,17 +1,31 @@ from __future__ import annotations -from typing import Mapping, Dict, Any, Type, TypeVar, TYPE_CHECKING, Protocol, Callable +from typing import Mapping, Dict, Any, Type, TypeVar, TYPE_CHECKING, Callable -# from typing_extensions import Protocol, TypedDict +from typing_extensions import Protocol if TYPE_CHECKING: import nixops.statefile -T = TypeVar("T", covariant=True) +T = TypeVar("T") StorageArgValues = Mapping[str, Any] class StorageBackend(Protocol[T]): + # Hack: Make T a mypy invariant. According to PEP-0544, a + # Protocol[T] whose T is only used in function arguments and + # returns is "de-facto covariant". + # + # However, a Protocol[T] which requires an attribute of type T is + # invariant, "since it has a mutable attribute". + # + # I don't really get it, to be honest. That said, since it is + # defined by the type, please set it ... even though mypy doesn't + # force you to. What even. + # + # See: https://www.python.org/dev/peps/pep-0544/#generic-protocols + __options: Type[T] + @staticmethod def options() -> Callable[..., T]: pass diff --git a/nixops/storage/legacy.py b/nixops/storage/legacy.py index 9d00db621..e1c522487 100644 --- a/nixops/storage/legacy.py +++ b/nixops/storage/legacy.py @@ -1,18 +1,21 @@ +from nixops.storage import StorageBackend import nixops.statefile import sys import os import os.path from nixops.util import ImmutableValidatedObject -from typing import Type +from typing import Callable class LegacyBackendOptions(ImmutableValidatedObject): pass -class LegacyBackend: +class LegacyBackend(StorageBackend[LegacyBackendOptions]): + __options = LegacyBackendOptions + @staticmethod - def options() -> Type[LegacyBackendOptions]: + def options() -> Callable[..., LegacyBackendOptions]: return LegacyBackendOptions def __init__(self, args: LegacyBackendOptions) -> None: diff --git a/nixops/storage/memory.py b/nixops/storage/memory.py index 0dd19416a..e9736b84c 100644 --- a/nixops/storage/memory.py +++ b/nixops/storage/memory.py @@ -1,15 +1,18 @@ import nixops.statefile +from nixops.storage import StorageBackend from nixops.util import ImmutableValidatedObject -from typing import Type +from typing import Callable class MemoryBackendOptions(ImmutableValidatedObject): pass -class MemoryBackend: +class MemoryBackend(StorageBackend[MemoryBackendOptions]): + __options = MemoryBackendOptions + @staticmethod - def options() -> Type[MemoryBackendOptions]: + def options() -> Callable[..., MemoryBackendOptions]: return MemoryBackendOptions def __init__(self, args: MemoryBackendOptions) -> None: From 1e8aa36b14ec40f522dd1d096e1e7fc0bde951a7 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 29 Apr 2020 11:23:21 -0400 Subject: [PATCH 22/80] storage backends: simplify options() type: just handle kwargs, return options type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The return type of Callable[..., T] and Type[T] were mixed up and confusing mypy. Co-authored-by: Adam Höse --- nixops/script_defs.py | 5 ++--- nixops/storage/__init__.py | 2 +- nixops/storage/legacy.py | 4 ++-- nixops/storage/memory.py | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index ac86da277..e25d16480 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -58,9 +58,8 @@ def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None ) raise Exception("Missing storage provider plugin.") - storage_class_options = storage_class.options() - foo = storage_class_options(**network.storage.configuration) - storage: StorageBackend = storage_class(foo) + storage_class_options = storage_class.options(**network.storage.configuration) + storage: StorageBackend = storage_class(storage_class_options) with TemporaryDirectory("nixops") as statedir: statefile = statedir + "/state.nixops" diff --git a/nixops/storage/__init__.py b/nixops/storage/__init__.py index 2866787b1..c88164509 100644 --- a/nixops/storage/__init__.py +++ b/nixops/storage/__init__.py @@ -27,7 +27,7 @@ class StorageBackend(Protocol[T]): __options: Type[T] @staticmethod - def options() -> Callable[..., T]: + def options(**kwargs) -> T: pass def __init__(self, args: T) -> None: diff --git a/nixops/storage/legacy.py b/nixops/storage/legacy.py index e1c522487..5aa250298 100644 --- a/nixops/storage/legacy.py +++ b/nixops/storage/legacy.py @@ -15,8 +15,8 @@ class LegacyBackend(StorageBackend[LegacyBackendOptions]): __options = LegacyBackendOptions @staticmethod - def options() -> Callable[..., LegacyBackendOptions]: - return LegacyBackendOptions + def options(**kwargs) -> LegacyBackendOptions: + return LegacyBackendOptions(**kwargs) def __init__(self, args: LegacyBackendOptions) -> None: pass diff --git a/nixops/storage/memory.py b/nixops/storage/memory.py index e9736b84c..41892f93b 100644 --- a/nixops/storage/memory.py +++ b/nixops/storage/memory.py @@ -12,8 +12,8 @@ class MemoryBackend(StorageBackend[MemoryBackendOptions]): __options = MemoryBackendOptions @staticmethod - def options() -> Callable[..., MemoryBackendOptions]: - return MemoryBackendOptions + def options(**kwargs) -> MemoryBackendOptions: + return MemoryBackendOptions(**kwargs) def __init__(self, args: MemoryBackendOptions) -> None: pass From d85ba2c6a2a1354c192ed214e24461bc41eaf71a Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 29 Apr 2020 13:48:00 -0400 Subject: [PATCH 23/80] locks: init protocol with a noop --- nixops/locks/__init__.py | 40 ++++++++++++++++++++++++++++++++++++++++ nixops/locks/noop.py | 23 +++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 nixops/locks/__init__.py create mode 100644 nixops/locks/noop.py diff --git a/nixops/locks/__init__.py b/nixops/locks/__init__.py new file mode 100644 index 000000000..c3ba726c5 --- /dev/null +++ b/nixops/locks/__init__.py @@ -0,0 +1,40 @@ +from typing import TypeVar, Type +from typing_extensions import Protocol + + +LockOptions = TypeVar("LockOptions") + + +class LockDriver(Protocol[LockOptions]): + # Hack: Make T a mypy invariant. According to PEP-0544, a + # Protocol[T] whose T is only used in function arguments and + # returns is "de-facto covariant". + # + # However, a Protocol[T] which requires an attribute of type T is + # invariant, "since it has a mutable attribute". + # + # I don't really get it, to be honest. That said, since it is + # defined by the type, please set it ... even though mypy doesn't + # force you to. What even. + # + # See: https://www.python.org/dev/peps/pep-0544/#generic-protocols + __options: Type[LockOptions] + + @staticmethod + def options(**kwargs) -> LockOptions: + pass + + def __init__(self, args: LockOptions) -> None: + raise NotImplementedError + + # lock: acquire a lock. + # Note: no arguments will be passed over kwargs. Making it part of + # the type definition allows adding new arguments later. + def lock(self, **kwargs) -> None: + raise NotImplementedError + + # unlock: release the lock. + # Note: no arguments will be passed over kwargs. Making it part of + # the type definition allows adding new arguments later. + def unlock(self, **kwargs) -> None: + raise NotImplementedError diff --git a/nixops/locks/noop.py b/nixops/locks/noop.py new file mode 100644 index 000000000..71632d7df --- /dev/null +++ b/nixops/locks/noop.py @@ -0,0 +1,23 @@ +from nixops.util import ImmutableValidatedObject +from . import LockDriver + + +class NoopLockOptions(ImmutableValidatedObject): + pass + + +class NoopLock(LockDriver[NoopLockOptions]): + __options = NoopLockOptions + + @staticmethod + def options(**kwargs) -> NoopLockOptions: + return NoopLockOptions(**kwargs) + + def __init__(self, args: NoopLockOptions) -> None: + pass + + def unlock(self, **_kwargs) -> None: + pass + + def lock(self, **_kwargs) -> None: + pass From 76ff3a3329526d492469fb435e611e24a7250c03 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 29 Apr 2020 14:22:27 -0400 Subject: [PATCH 24/80] evaluation.nix: handle Lock configuration too --- nixops/evaluation.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index cc5831bc7..17437563c 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -15,14 +15,21 @@ class GenericStorageConfig(ImmutableValidatedObject): configuration: typing.Mapping[typing.Any, typing.Any] +class GenericLockConfig(ImmutableValidatedObject): + provider: str + configuration: typing.Mapping[typing.Any, typing.Any] + + class NetworkEval(ImmutableValidatedObject): storage: GenericStorageConfig + lock: GenericLockConfig description: str = "Unnamed NixOps network" enableRollback: bool = False class RawNetworkEval(ImmutableValidatedObject): storage: Mapping[str, Any] + lock: Optional[Mapping[str, Any]] description: Optional[str] enableRollback: Optional[bool] @@ -127,8 +134,28 @@ def eval_network(nix_expr: str) -> NetworkEval: "Missing property: network.storage has no defined storage backend." ) + lock: Mapping[str, Any] = raw_eval.lock or {} + if len(lock) > 1: + raise MalformedNetworkError( + "Invalid property: network.lock can only have one defined lock backend." + ) + + lock_config: Optional[Mapping[str, Any]] + try: + lock_key = list(lock.keys()).pop() + lock_config = { + "provider": lock_key, + "configuration": lock[lock_key], + } + except IndexError: + lock_config = { + "provider": "noop", + "configuration": {}, + } + return NetworkEval( enableRollback=raw_eval.enableRollback or False, description=raw_eval.description or "Unnamed NixOps network", storage={"provider": key, "configuration": value}, + lock=lock_config, ) From 7202a10882c02183454c873bb3202dad7cf6438c Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 29 Apr 2020 14:25:37 -0400 Subject: [PATCH 25/80] Use the network's lock when accessing its state --- nixops/locks/__init__.py | 5 ++++- nixops/plugin.py | 10 ++++++---- nixops/plugins/__init__.py | 6 +++++- nixops/plugins/manager.py | 15 +++++++++++++++ nixops/script_defs.py | 23 ++++++++++++++++++++++- nixops/storage/__init__.py | 2 +- nixops/storage/legacy.py | 1 - nixops/storage/memory.py | 1 - 8 files changed, 53 insertions(+), 10 deletions(-) diff --git a/nixops/locks/__init__.py b/nixops/locks/__init__.py index c3ba726c5..4786a6fdd 100644 --- a/nixops/locks/__init__.py +++ b/nixops/locks/__init__.py @@ -1,4 +1,4 @@ -from typing import TypeVar, Type +from typing import TypeVar, Type, Dict from typing_extensions import Protocol @@ -38,3 +38,6 @@ def lock(self, **kwargs) -> None: # the type definition allows adding new arguments later. def unlock(self, **kwargs) -> None: raise NotImplementedError + + +lock_drivers: Dict[str, Type[LockDriver]] = {} diff --git a/nixops/plugin.py b/nixops/plugin.py index 0bcbdb8c8..bebbefa84 100644 --- a/nixops/plugin.py +++ b/nixops/plugin.py @@ -1,14 +1,16 @@ from nixops.storage import BackendRegistration + from nixops.storage.legacy import LegacyBackend from nixops.storage.memory import MemoryBackend import nixops.plugins +from nixops.locks import LockDriver +from nixops.locks.noop import NoopLock +from typing import Dict, Type class InternalPlugin(nixops.plugins.Plugin): def storage_backends(self) -> BackendRegistration: return {"legacy": LegacyBackend, "memory": MemoryBackend} - -@nixops.plugins.hookimpl -def plugin(): - return InternalPlugin() + def lock_drivers(self) -> Dict[str, Type[LockDriver]]: + return {"noop": NoopLock} diff --git a/nixops/plugins/__init__.py b/nixops/plugins/__init__.py index 200dd6886..04b1e957b 100644 --- a/nixops/plugins/__init__.py +++ b/nixops/plugins/__init__.py @@ -1,9 +1,10 @@ from __future__ import annotations from nixops.backends import MachineState -from typing import List, Dict, Optional, Union, Tuple +from typing import List, Dict, Optional, Union, Tuple, Type from nixops.storage import BackendRegistration +from nixops.locks import LockDriver from functools import lru_cache from typing import Generator import pluggy @@ -95,6 +96,9 @@ def docs(self) -> List[Tuple[str, str]]: """ return [] + def lock_drivers(self) -> Dict[str, Type[LockDriver]]: + return {} + def storage_backends(self) -> BackendRegistration: """ Extend the core nixops cli parser :return a set of plugin parser extensions diff --git a/nixops/plugins/manager.py b/nixops/plugins/manager.py index b930e316c..ebf1cb97e 100644 --- a/nixops/plugins/manager.py +++ b/nixops/plugins/manager.py @@ -5,6 +5,7 @@ import importlib from nixops.storage import storage_backends +from nixops.locks import lock_drivers from . import get_plugins, MachineHooks, DeploymentHooks import nixops.ansi import nixops @@ -64,6 +65,7 @@ def load(cls): seen.add(mod) cls.storage_backends() + cls.lock_drivers() @staticmethod def nixexprs() -> List[str]: @@ -94,3 +96,16 @@ def storage_backends(): f"Two plugins tried to provide the '{name}' storage backend." ) ) + + @staticmethod + def lock_drivers(): + for plugin in get_plugins(): + for name, driver in plugin.lock_drivers().items(): + if name not in lock_drivers: + lock_drivers[name] = driver + else: + sys.stderr.write( + nixops.ansi.ansi_warn( + f"Two plugins tried to provide the '{name}' lock driver." + ) + ) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index e25d16480..8a1774926 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -2,6 +2,8 @@ from nixops.nix_expr import py2nix from nixops.parallel import run_tasks +from nixops.storage import StorageBackend +from nixops.locks import LockDriver import contextlib import nixops.statefile @@ -28,7 +30,8 @@ from nixops.plugins import get_plugin_manager from nixops.evaluation import eval_network -from nixops.storage import StorageBackend, storage_backends +from nixops.storage import storage_backends +from nixops.locks import lock_drivers PluginManager.load() @@ -58,11 +61,28 @@ def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None ) raise Exception("Missing storage provider plugin.") + lock: LockDriver + lock_class: Type[LockDriver] + try: + lock_class = lock_drivers[network.lock.provider] + except KeyError: + sys.stderr.write( + nixops.ansi.ansi_warn( + f"The network requires the '{network.lock.provider}' lock driver, " + "but no plugin provides it.\n" + ) + ) + raise Exception("Missing lock driver plugin.") + else: + lock_class_options = lock_class.options(**network.lock.configuration) + lock = lock_class(lock_class_options) + storage_class_options = storage_class.options(**network.storage.configuration) storage: StorageBackend = storage_class(storage_class_options) with TemporaryDirectory("nixops") as statedir: statefile = statedir + "/state.nixops" + lock.lock() storage.fetchToFile(statefile) state = nixops.statefile.StateFile(statefile) try: @@ -72,6 +92,7 @@ def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None finally: state.close() storage.uploadFromFile(statefile) + lock.unlock() def op_list_plugins(args): diff --git a/nixops/storage/__init__.py b/nixops/storage/__init__.py index c88164509..bfb5acf48 100644 --- a/nixops/storage/__init__.py +++ b/nixops/storage/__init__.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import Mapping, Dict, Any, Type, TypeVar, TYPE_CHECKING, Callable +from typing import Mapping, Dict, Any, Type, TypeVar, TYPE_CHECKING from typing_extensions import Protocol diff --git a/nixops/storage/legacy.py b/nixops/storage/legacy.py index 5aa250298..fdfa5f44d 100644 --- a/nixops/storage/legacy.py +++ b/nixops/storage/legacy.py @@ -4,7 +4,6 @@ import os import os.path from nixops.util import ImmutableValidatedObject -from typing import Callable class LegacyBackendOptions(ImmutableValidatedObject): diff --git a/nixops/storage/memory.py b/nixops/storage/memory.py index 41892f93b..7acc23ad6 100644 --- a/nixops/storage/memory.py +++ b/nixops/storage/memory.py @@ -1,7 +1,6 @@ import nixops.statefile from nixops.storage import StorageBackend from nixops.util import ImmutableValidatedObject -from typing import Callable class MemoryBackendOptions(ImmutableValidatedObject): From ad87a56851e94caa082db85179a9f8f02a98fcc0 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 15 Jul 2020 17:15:16 +0200 Subject: [PATCH 26/80] Squash with 869d2765a --- nixops/locks/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/locks/__init__.py b/nixops/locks/__init__.py index 4786a6fdd..7bd440c3c 100644 --- a/nixops/locks/__init__.py +++ b/nixops/locks/__init__.py @@ -1,4 +1,4 @@ -from typing import TypeVar, Type, Dict +from typing import TypeVar, Type from typing_extensions import Protocol From 98ea4cc7e4042a3211effb708a407c740ba999fd Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 15 Jul 2020 17:18:21 +0200 Subject: [PATCH 27/80] Squash with 869d2765a --- nixops/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/storage/__init__.py b/nixops/storage/__init__.py index bfb5acf48..0d48ff6ff 100644 --- a/nixops/storage/__init__.py +++ b/nixops/storage/__init__.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import Mapping, Dict, Any, Type, TypeVar, TYPE_CHECKING +from typing import Mapping, Any, Type, TypeVar, TYPE_CHECKING from typing_extensions import Protocol From 310badad1821c982b8b1ce8b1bed66a4343258e2 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 29 Apr 2020 14:26:45 -0400 Subject: [PATCH 28/80] storage: fixup handling of edge casesin network parsing --- nixops/evaluation.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index 17437563c..aa94e490d 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -28,7 +28,7 @@ class NetworkEval(ImmutableValidatedObject): class RawNetworkEval(ImmutableValidatedObject): - storage: Mapping[str, Any] + storage: Optional[Mapping[str, Any]] lock: Optional[Mapping[str, Any]] description: Optional[str] enableRollback: Optional[bool] @@ -82,8 +82,9 @@ def eval_network(nix_expr: str) -> NetworkEval: if not result.exists: raise MalformedNetworkError( """ -TODO: improve this error to be less specific about conversion. link to -docs? +TODO: improve this error to be less specific about conversion, and less +about storage backends, and more about the construction of a network +attribute value. link to docs about storage drivers and lock drivers. WARNING: NixOps 1.0 -> 2.0 conversion step required @@ -104,7 +105,9 @@ def eval_network(nix_expr: str) -> NetworkEval: if not isinstance(result.value, dict): raise MalformedNetworkError( """ -TODO: improve this error +TODO: improve this error to be less specific about conversion, and less +about storage backends, and more about the construction of a network +attribute value. link to docs about storage drivers and lock drivers. The network.nix has a `network` attribute set, but it is of the wrong type. A valid network attribute looks like this: @@ -121,16 +124,18 @@ def eval_network(nix_expr: str) -> NetworkEval: raw_eval = RawNetworkEval(**result.value) - if len(raw_eval.storage) > 1: - raise Exception( + storage: Mapping[str, Any] = raw_eval.storage or {} + if len(storage) > 1: + raise MalformedNetworkError( "Invalid property: network.storage can only have one defined storage backend." ) - + storage_config: Optional[Mapping[str, Any]] try: - key = list(raw_eval.storage.keys()).pop() - value = raw_eval.storage[key] - except KeyError: - raise Exception( + storage_key = list(storage.keys()).pop() + storage_value = storage[storage_key] + storage_config = {"provider": storage_key, "configuration": storage_value} + except IndexError: + raise MalformedNetworkError( "Missing property: network.storage has no defined storage backend." ) @@ -156,6 +161,6 @@ def eval_network(nix_expr: str) -> NetworkEval: return NetworkEval( enableRollback=raw_eval.enableRollback or False, description=raw_eval.description or "Unnamed NixOps network", - storage={"provider": key, "configuration": value}, + storage=storage_config, lock=lock_config, ) From 524a2d5e20ba87a442a8db6a016892b0715c7a04 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 29 Apr 2020 14:28:09 -0400 Subject: [PATCH 29/80] ImmutableValidatedObject: support validating Unions with nested ImmutableValidatedObject MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adam Höse --- nixops/util.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nixops/util.py b/nixops/util.py index 43c453c37..b78a1e790 100644 --- a/nixops/util.py +++ b/nixops/util.py @@ -152,6 +152,13 @@ def _transform_value(key: Any, value: Any) -> Any: if inspect.isclass(ann) and issubclass(ann, ImmutableValidatedObject): value = ann(**value) + elif hasattr(ann, "__origin__") and ann.__origin__ == Union: + for arg in ann.__args__: + if inspect.isclass(arg) and issubclass( + arg, ImmutableValidatedObject + ): + value = arg(**value) + break # Support Sequence[ImmutableValidatedObject] if isinstance(value, tuple) and not isinstance(ann, str): From 1304da43601fed7d61d9665394fad29f9dce6197 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 29 Apr 2020 14:35:15 -0400 Subject: [PATCH 30/80] Add and improve slome storage test expressions --- tests/storage/legacy.nix | 3 +++ tests/storage/missing-plugin.nix | 3 +++ tests/storage/multiple.nix | 14 ++++++++++++++ tests/storage/network-wrong-type.nix | 7 +++++++ tests/storage/s3.nix | 4 ++++ tests/storage/withlock.nix | 12 ++++++++++++ tests/storage/wrong-type.nix | 4 ++-- 7 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 tests/storage/legacy.nix create mode 100644 tests/storage/missing-plugin.nix create mode 100644 tests/storage/multiple.nix create mode 100644 tests/storage/network-wrong-type.nix create mode 100644 tests/storage/s3.nix create mode 100644 tests/storage/withlock.nix diff --git a/tests/storage/legacy.nix b/tests/storage/legacy.nix new file mode 100644 index 000000000..c9f571a10 --- /dev/null +++ b/tests/storage/legacy.nix @@ -0,0 +1,3 @@ +{ + network.storage.legacy = {}; +} diff --git a/tests/storage/missing-plugin.nix b/tests/storage/missing-plugin.nix new file mode 100644 index 000000000..9e1b22cf1 --- /dev/null +++ b/tests/storage/missing-plugin.nix @@ -0,0 +1,3 @@ +{ + network.storage."þis-storage-backend-doesn't-exist" = {}; +} diff --git a/tests/storage/multiple.nix b/tests/storage/multiple.nix new file mode 100644 index 000000000..3c39a8e77 --- /dev/null +++ b/tests/storage/multiple.nix @@ -0,0 +1,14 @@ +/* +Users should get a reasonable error messages if they accidentally +specify multiple storage backends +*/ +{ + network = rec { + storage.s3 = { + bucket = "hi"; + whatever = "there"; + }; + + storage.legacy = {}; + }; +} diff --git a/tests/storage/network-wrong-type.nix b/tests/storage/network-wrong-type.nix new file mode 100644 index 000000000..662fecbd9 --- /dev/null +++ b/tests/storage/network-wrong-type.nix @@ -0,0 +1,7 @@ +/* +Expect a reasonable error message when the `network` attribute +has a value of the wrong type +*/ +{ + network = "meh"; +} diff --git a/tests/storage/s3.nix b/tests/storage/s3.nix new file mode 100644 index 000000000..244d836ad --- /dev/null +++ b/tests/storage/s3.nix @@ -0,0 +1,4 @@ +{ + network.storage.s3 = {}; + /* network.lock.s3 = {}; */ +} diff --git a/tests/storage/withlock.nix b/tests/storage/withlock.nix new file mode 100644 index 000000000..d5d87605c --- /dev/null +++ b/tests/storage/withlock.nix @@ -0,0 +1,12 @@ +{ + network = rec { + storage.s3 = { + bucket = "hi"; + whatever = "there"; + }; + + locking.s3 = storage.s3 // { + dynamodb_table = "blah"; + }; + }; +} diff --git a/tests/storage/wrong-type.nix b/tests/storage/wrong-type.nix index 662fecbd9..a07b4f5f1 100644 --- a/tests/storage/wrong-type.nix +++ b/tests/storage/wrong-type.nix @@ -1,7 +1,7 @@ /* -Expect a reasonable error message when the `network` attribute +Expect a reasonable error message when the `network.storage` attribute has a value of the wrong type */ { - network = "meh"; + network.storage = "meh"; } From f3f6d4db41ca2edec00fd4c32fec677f342f3ece Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 29 Apr 2020 14:35:25 -0400 Subject: [PATCH 31/80] Add some test expressions for locks --- tests/locks/missing-plugin.nix | 4 ++++ tests/locks/multiple.nix | 11 +++++++++++ tests/locks/noop.nix | 4 ++++ tests/locks/s3.nix | 4 ++++ tests/locks/withlock.nix | 12 ++++++++++++ tests/locks/wrong-type.nix | 8 ++++++++ 6 files changed, 43 insertions(+) create mode 100644 tests/locks/missing-plugin.nix create mode 100644 tests/locks/multiple.nix create mode 100644 tests/locks/noop.nix create mode 100644 tests/locks/s3.nix create mode 100644 tests/locks/withlock.nix create mode 100644 tests/locks/wrong-type.nix diff --git a/tests/locks/missing-plugin.nix b/tests/locks/missing-plugin.nix new file mode 100644 index 000000000..4cf0b76d9 --- /dev/null +++ b/tests/locks/missing-plugin.nix @@ -0,0 +1,4 @@ +{ + network.storage.memory = {}; + network.lock."þis-l©k-backend-doesn't-exist" = {}; +} diff --git a/tests/locks/multiple.nix b/tests/locks/multiple.nix new file mode 100644 index 000000000..d3bd1da84 --- /dev/null +++ b/tests/locks/multiple.nix @@ -0,0 +1,11 @@ +/* +Users should get a reasonable error messages if they accidentally +specify multiple lock backends +*/ +{ + network = { + storage.memory = {}; + lock.legacy = {}; + lock.noop = {}; + }; +} diff --git a/tests/locks/noop.nix b/tests/locks/noop.nix new file mode 100644 index 000000000..7752b7712 --- /dev/null +++ b/tests/locks/noop.nix @@ -0,0 +1,4 @@ +{ + network.storage.memory = {}; + network.lock.noop = {}; +} diff --git a/tests/locks/s3.nix b/tests/locks/s3.nix new file mode 100644 index 000000000..9f52dca83 --- /dev/null +++ b/tests/locks/s3.nix @@ -0,0 +1,4 @@ +{ + network.storage.memory = {}; + network.lock.s3 = {}; +} diff --git a/tests/locks/withlock.nix b/tests/locks/withlock.nix new file mode 100644 index 000000000..92a10c67f --- /dev/null +++ b/tests/locks/withlock.nix @@ -0,0 +1,12 @@ +{ + network = rec { + storage.s3 = { + bucket = "hi"; + whatever = "there"; + }; + + lock.s3 = storage.s3 // { + dynamodb_table = "blah"; + }; + }; +} diff --git a/tests/locks/wrong-type.nix b/tests/locks/wrong-type.nix new file mode 100644 index 000000000..5a9cf578c --- /dev/null +++ b/tests/locks/wrong-type.nix @@ -0,0 +1,8 @@ +/* +Expect a reasonable error message when the `network.lock` attribute +has a value of the wrong type +*/ +{ + network.storage.memory = {}; + network.lock = "meh"; +} From 0f1d68726fa1a0241401b46c72bdb21bb4f6cff5 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 29 Apr 2020 15:14:23 -0400 Subject: [PATCH 32/80] fixup d2f845a9f5c47d2e1f5bcbdf30583c4f5d655bb5 with no locking indications --- nixops/storage/__init__.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/nixops/storage/__init__.py b/nixops/storage/__init__.py index 0d48ff6ff..955c69861 100644 --- a/nixops/storage/__init__.py +++ b/nixops/storage/__init__.py @@ -33,21 +33,20 @@ def options(**kwargs) -> T: def __init__(self, args: T) -> None: raise NotImplementedError - # fetchToFile: acquire a lock and download the state file to - # the local disk. Note: no arguments will be passed over kwargs. - # Making it part of the type definition allows adding new - # arguments later. + # fetchToFile: download the state file to the local disk. + # Note: no arguments will be passed over kwargs. Making it part of + # the type definition allows adding new arguments later. def fetchToFile(self, path: str, **kwargs) -> None: raise NotImplementedError # onOpen: receive the StateFile object for last-minute, backend # specific changes to the state file. - # Note: no arguments will be passed over kwargs. Making it part - # of the type definition allows adding new arguments later. + # Note: no arguments will be passed over kwargs. Making it part of + # the type definition allows adding new arguments later. def onOpen(self, sf: nixops.statefile.StateFile, **kwargs) -> None: pass - # uploadFromFile: upload the new state file and release any locks + # uploadFromFile: upload the new version of the state file # Note: no arguments will be passed over kwargs. Making it part of # the type definition allows adding new arguments later. def uploadFromFile(self, path: str, **kwargs) -> None: From a2e25f15538e4544af5d809e79dbf98d057a1771 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 29 Apr 2020 15:15:37 -0400 Subject: [PATCH 33/80] fixup f52369f6b4c1cb94c7cba244ef737e814537ba3b^C --- nixops/storage/memory.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nixops/storage/memory.py b/nixops/storage/memory.py index 7acc23ad6..d1a5a7100 100644 --- a/nixops/storage/memory.py +++ b/nixops/storage/memory.py @@ -17,17 +17,20 @@ def options(**kwargs) -> MemoryBackendOptions: def __init__(self, args: MemoryBackendOptions) -> None: pass - # fetchToFile: acquire a lock and download the state file to - # the local disk. Note: no arguments will be passed over kwargs. - # Making it part of the type definition allows adding new - # arguments later. + # fetchToFile: download the state file to the local disk. + # Note: no arguments will be passed over kwargs. Making it part of + # the type definition allows adding new arguments later. def fetchToFile(self, path: str, **kwargs) -> None: pass + # onOpen: receive the StateFile object for last-minute, backend + # specific changes to the state file. + # Note: no arguments will be passed over kwargs. Making it part of + # the type definition allows adding new arguments later. def onOpen(self, sf: nixops.statefile.StateFile, **kwargs) -> None: sf.create_deployment() - # uploadFromFile: upload the new state file and release any locks + # uploadFromFile: upload the new version of the state file # Note: no arguments will be passed over kwargs. Making it part of # the type definition allows adding new arguments later. def uploadFromFile(self, path: str, **kwargs) -> None: From 87b64d8d5ea4a0eadb6476e3031aae9657a9716e Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 15 Jul 2020 15:35:54 +0200 Subject: [PATCH 34/80] Split argument parser into it's own file Just to make things more manageable --- nixops/__main__.py | 667 +-------------------------------------------- nixops/args.py | 658 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 662 insertions(+), 663 deletions(-) create mode 100644 nixops/args.py diff --git a/nixops/__main__.py b/nixops/__main__.py index 19e3bceff..66622fdf6 100755 --- a/nixops/__main__.py +++ b/nixops/__main__.py @@ -1,6 +1,7 @@ #! /usr/bin/env python3 # -*- coding: utf-8 -*- import sys +import os def setup_debugger() -> None: @@ -28,671 +29,11 @@ def hook(_type: Type[BaseException], value: BaseException, tb: TracebackType): setup_debugger() -from argparse import ArgumentParser, _SubParsersAction, SUPPRESS, REMAINDER -import os from nixops.parallel import MultipleExceptions -from nixops.script_defs import ( - add_subparser, - op_list_deployments, - op_create, - add_common_modify_options, - op_modify, - op_clone, - op_delete, - op_info, - op_check, - op_set_args, - op_deploy, - add_common_deployment_options, - op_send_keys, - op_destroy, - op_delete_resources, - op_stop, - op_start, - op_reboot, - op_show_arguments, - op_show_physical, - op_ssh, - op_ssh_for_each, - op_scp, - op_mount, - op_rename, - op_backup, - op_backup_status, - op_remove_backup, - op_clean_backups, - op_restore, - op_show_option, - op_list_generations, - op_rollback, - op_delete_generation, - op_show_console_output, - op_dump_nix_paths, - op_export, - op_import, - op_edit, - op_copy_closure, - op_list_plugins, - parser_plugin_hooks, - setup_logging, - error, -) -import sys +from nixops.script_defs import setup_logging +from nixops.script_defs import error +from nixops.args import parser import nixops -import nixops.ansi - -# Set up the parser. -parser = ArgumentParser(description="NixOS cloud deployment tool", prog="nixops") -parser.add_argument("--version", action="version", version="NixOps @version@") -parser.add_argument( - "--pdb", action="store_true", help="Invoke pdb on unhandled exception" -) - -subparsers: _SubParsersAction = parser.add_subparsers( - help="sub-command help", metavar="operation", required=True -) - -subparser = add_subparser(subparsers, "list", help="list all known deployments") -subparser.set_defaults(op=op_list_deployments) - -subparser = add_subparser(subparsers, "create", help="create a new deployment") -subparser.set_defaults(op=op_create) -subparser.add_argument( - "--name", "-n", dest="name", metavar="NAME", help=SUPPRESS -) # obsolete, use -d instead -add_common_modify_options(subparser) - -subparser = add_subparser(subparsers, "modify", help="modify an existing deployment") -subparser.set_defaults(op=op_modify) -subparser.add_argument( - "--name", "-n", dest="name", metavar="NAME", help="new symbolic name of deployment" -) -add_common_modify_options(subparser) - -subparser = add_subparser(subparsers, "clone", help="clone an existing deployment") -subparser.set_defaults(op=op_clone) -subparser.add_argument( - "--name", - "-n", - dest="name", - metavar="NAME", - help="symbolic name of the cloned deployment", -) - -subparser = add_subparser(subparsers, "delete", help="delete a deployment") -subparser.add_argument( - "--force", action="store_true", help="force deletion even if resources still exist" -) -subparser.add_argument("--all", action="store_true", help="delete all deployments") -subparser.set_defaults(op=op_delete) - -subparser = add_subparser(subparsers, "info", help="show the state of the deployment") -subparser.set_defaults(op=op_info) -subparser.add_argument("--all", action="store_true", help="show all deployments") -subparser.add_argument( - "--plain", action="store_true", help="do not pretty-print the output" -) -subparser.add_argument( - "--no-eval", - action="store_true", - help="do not evaluate the deployment specification", -) - -subparser = add_subparser( - subparsers, - "check", - help="check the state of the machines in the network" - " (note that this might alter the internal nixops state to consolidate with the real state of the resource)", -) -subparser.set_defaults(op=op_check) -subparser.add_argument("--all", action="store_true", help="check all deployments") -subparser.add_argument( - "--include", - nargs="+", - metavar="MACHINE-NAME", - help="check only the specified machines", -) -subparser.add_argument( - "--exclude", - nargs="+", - metavar="MACHINE-NAME", - help="check all except the specified machines", -) - -subparser = add_subparser( - subparsers, - "set-args", - help="persistently set arguments to the deployment specification", -) -subparser.set_defaults(op=op_set_args) -subparser.add_argument( - "--arg", - nargs=2, - action="append", - dest="args", - metavar=("NAME", "VALUE"), - help="pass a Nix expression value", -) -subparser.add_argument( - "--argstr", - nargs=2, - action="append", - dest="argstrs", - metavar=("NAME", "VALUE"), - help="pass a string value", -) -subparser.add_argument( - "--unset", - nargs=1, - action="append", - dest="unset", - metavar="NAME", - help="unset previously set argument", -) - - -subparser = add_subparser(subparsers, "deploy", help="deploy the network configuration") -subparser.set_defaults(op=op_deploy) -subparser.add_argument( - "--kill-obsolete", "-k", action="store_true", help="kill obsolete virtual machines" -) -subparser.add_argument( - "--dry-run", action="store_true", help="evaluate and print what would be built" -) -subparser.add_argument( - "--dry-activate", - action="store_true", - help="show what will be activated on the machines in the network", -) -subparser.add_argument( - "--test", - action="store_true", - help="build and activate the new configuration; do not enable it in the bootloader. Rebooting the system will roll back automatically.", -) -subparser.add_argument( - "--boot", - action="store_true", - help="build the new configuration and enable it in the bootloader; do not activate it. Upon reboot, the system will use the new configuration.", -) -subparser.add_argument( - "--repair", action="store_true", help="use --repair when calling nix-build (slow)" -) -subparser.add_argument( - "--evaluate-only", action="store_true", help="only call nix-instantiate and exit" -) -subparser.add_argument( - "--plan-only", - action="store_true", - help="show the diff between the configuration and the state and exit", -) -subparser.add_argument( - "--build-only", - action="store_true", - help="build only; do not perform deployment actions", -) -subparser.add_argument( - "--create-only", action="store_true", help="exit after creating missing machines" -) -subparser.add_argument( - "--copy-only", action="store_true", help="exit after copying closures" -) -subparser.add_argument( - "--allow-recreate", - action="store_true", - help="recreate resources machines that have disappeared", -) -subparser.add_argument( - "--always-activate", - action="store_true", - help="activate unchanged configurations as well", -) -add_common_deployment_options(subparser) - -subparser = add_subparser(subparsers, "send-keys", help="send encryption keys") -subparser.set_defaults(op=op_send_keys) -subparser.add_argument( - "--include", - nargs="+", - metavar="MACHINE-NAME", - help="send keys to only the specified machines", -) -subparser.add_argument( - "--exclude", - nargs="+", - metavar="MACHINE-NAME", - help="send keys to all except the specified machines", -) - -subparser = add_subparser( - subparsers, "destroy", help="destroy all resources in the specified deployment" -) -subparser.set_defaults(op=op_destroy) -subparser.add_argument( - "--include", - nargs="+", - metavar="MACHINE-NAME", - help="destroy only the specified machines", -) -subparser.add_argument( - "--exclude", - nargs="+", - metavar="MACHINE-NAME", - help="destroy all except the specified machines", -) -subparser.add_argument( - "--wipe", action="store_true", help="securely wipe data on the machines" -) -subparser.add_argument("--all", action="store_true", help="destroy all deployments") - -subparser = add_subparser( - subparsers, - "delete-resources", - help="deletes the resource from the local NixOps state file.", -) -subparser.set_defaults(op=op_delete_resources) -subparser.add_argument( - "--include", - nargs="+", - metavar="RESOURCE-NAME", - help="delete only the specified resources", -) -subparser.add_argument( - "--exclude", - nargs="+", - metavar="RESOURCE-NAME", - help="delete all resources except the specified resources", -) - -subparser = add_subparser( - subparsers, "stop", help="stop all virtual machines in the network" -) -subparser.set_defaults(op=op_stop) -subparser.add_argument( - "--include", - nargs="+", - metavar="MACHINE-NAME", - help="stop only the specified machines", -) -subparser.add_argument( - "--exclude", - nargs="+", - metavar="MACHINE-NAME", - help="stop all except the specified machines", -) - -subparser = add_subparser( - subparsers, "start", help="start all virtual machines in the network" -) -subparser.set_defaults(op=op_start) -subparser.add_argument( - "--include", - nargs="+", - metavar="MACHINE-NAME", - help="start only the specified machines", -) -subparser.add_argument( - "--exclude", - nargs="+", - metavar="MACHINE-NAME", - help="start all except the specified machines", -) - -subparser = add_subparser( - subparsers, "reboot", help="reboot all virtual machines in the network" -) -subparser.set_defaults(op=op_reboot) -subparser.add_argument( - "--include", - nargs="+", - metavar="MACHINE-NAME", - help="reboot only the specified machines", -) -subparser.add_argument( - "--exclude", - nargs="+", - metavar="MACHINE-NAME", - help="reboot all except the specified machines", -) -subparser.add_argument( - "--no-wait", action="store_true", help="do not wait until the machines are up again" -) -subparser.add_argument( - "--rescue", - action="store_true", - help="reboot machines into the rescue system" " (if available)", -) -subparser.add_argument( - "--hard", - action="store_true", - help="send a hard reset (power switch) to the machines" " (if available)", -) - -subparser = add_subparser( - subparsers, "show-arguments", help="print the arguments to the network expressions" -) -subparser.set_defaults(op=op_show_arguments) - -subparser = add_subparser( - subparsers, "show-physical", help="print the physical network expression" -) -subparser.add_argument( - "--backup", - dest="backupid", - default=None, - help="print physical network expression for given backup id", -) -subparser.set_defaults(op=op_show_physical) - -subparser = add_subparser( - subparsers, "ssh", help="login on the specified machine via SSH" -) -subparser.set_defaults(op=op_ssh) -subparser.add_argument("machine", metavar="MACHINE", help="identifier of the machine") -subparser.add_argument( - "args", metavar="SSH_ARGS", nargs=REMAINDER, help="SSH flags and/or command", -) - -subparser = add_subparser( - subparsers, "ssh-for-each", help="execute a command on each machine via SSH" -) -subparser.set_defaults(op=op_ssh_for_each) -subparser.add_argument( - "args", metavar="ARG", nargs="*", help="additional arguments to SSH" -) -subparser.add_argument("--parallel", "-p", action="store_true", help="run in parallel") -subparser.add_argument( - "--include", - nargs="+", - metavar="MACHINE-NAME", - help="run command only on the specified machines", -) -subparser.add_argument( - "--exclude", - nargs="+", - metavar="MACHINE-NAME", - help="run command on all except the specified machines", -) -subparser.add_argument( - "--all", action="store_true", help="run ssh-for-each for all deployments" -) - -subparser = add_subparser( - subparsers, "scp", help="copy files to or from the specified machine via scp" -) -subparser.set_defaults(op=op_scp) -subparser.add_argument( - "--from", - dest="scp_from", - action="store_true", - help="copy a file from specified machine", -) -subparser.add_argument( - "--to", dest="scp_to", action="store_true", help="copy a file to specified machine" -) -subparser.add_argument("machine", metavar="MACHINE", help="identifier of the machine") -subparser.add_argument("source", metavar="SOURCE", help="source file location") -subparser.add_argument("destination", metavar="DEST", help="destination file location") - -subparser = add_subparser( - subparsers, - "mount", - help="mount a directory from the specified machine into the local filesystem", -) -subparser.set_defaults(op=op_mount) -subparser.add_argument( - "machine", - metavar="MACHINE[:PATH]", - help="identifier of the machine, optionally followed by a path", -) -subparser.add_argument("destination", metavar="PATH", help="local path") -subparser.add_argument( - "--sshfs-option", - "-o", - action="append", - metavar="OPTIONS", - help="mount options passed to sshfs", -) - -subparser = add_subparser(subparsers, "rename", help="rename machine in network") -subparser.set_defaults(op=op_rename) -subparser.add_argument( - "current_name", metavar="FROM", help="current identifier of the machine" -) -subparser.add_argument("new_name", metavar="TO", help="new identifier of the machine") - -subparser = add_subparser( - subparsers, - "backup", - help="make snapshots of persistent disks in network (currently EC2-only)", -) -subparser.set_defaults(op=op_backup) -subparser.add_argument( - "--include", - nargs="+", - metavar="MACHINE-NAME", - help="perform backup actions on the specified machines only", -) -subparser.add_argument( - "--exclude", - nargs="+", - metavar="MACHINE-NAME", - help="do not perform backup actions on the specified machines", -) -subparser.add_argument( - "--freeze", - dest="freeze_fs", - action="store_true", - default=False, - help="freeze filesystems for non-root filesystems that support this (e.g. xfs)", -) -subparser.add_argument( - "--force", - dest="force", - action="store_true", - default=False, - help="start new backup even if previous is still running", -) -subparser.add_argument( - "--devices", - nargs="+", - metavar="DEVICE-NAME", - help="only backup the specified devices", -) - -subparser = add_subparser(subparsers, "backup-status", help="get status of backups") -subparser.set_defaults(op=op_backup_status) -subparser.add_argument( - "backupid", default=None, nargs="?", help="use specified backup in stead of latest" -) -subparser.add_argument( - "--include", - nargs="+", - metavar="MACHINE-NAME", - help="perform backup actions on the specified machines only", -) -subparser.add_argument( - "--exclude", - nargs="+", - metavar="MACHINE-NAME", - help="do not perform backup actions on the specified machines", -) -subparser.add_argument( - "--wait", - dest="wait", - action="store_true", - default=False, - help="wait until backup is finished", -) -subparser.add_argument( - "--latest", - dest="latest", - action="store_true", - default=False, - help="show status of latest backup only", -) - -subparser = add_subparser(subparsers, "remove-backup", help="remove a given backup") -subparser.set_defaults(op=op_remove_backup) -subparser.add_argument("backupid", metavar="BACKUP-ID", help="backup ID to remove") -subparser.add_argument( - "--keep-physical", - dest="keep_physical", - default=False, - action="store_true", - help="do not remove the physical backups, only remove backups from nixops state", -) - -subparser = add_subparser(subparsers, "clean-backups", help="remove old backups") -subparser.set_defaults(op=op_clean_backups) -subparser.add_argument( - "--keep", dest="keep", type=int, help="number of backups to keep around" -) -subparser.add_argument( - "--keep-days", - metavar="N", - dest="keep_days", - type=int, - help="keep backups newer than N days", -) -subparser.add_argument( - "--keep-physical", - dest="keep_physical", - default=False, - action="store_true", - help="do not remove the physical backups, only remove backups from nixops state", -) - -subparser = add_subparser( - subparsers, - "restore", - help="restore machines based on snapshots of persistent disks in network (currently EC2-only)", -) -subparser.set_defaults(op=op_restore) -subparser.add_argument( - "--backup-id", default=None, help="use specified backup in stead of latest" -) -subparser.add_argument( - "--include", - nargs="+", - metavar="MACHINE-NAME", - help="perform backup actions on the specified machines only", -) -subparser.add_argument( - "--exclude", - nargs="+", - metavar="MACHINE-NAME", - help="do not perform backup actions on the specified machines", -) -subparser.add_argument( - "--devices", - nargs="+", - metavar="DEVICE-NAME", - help="only restore the specified devices", -) - -subparser = add_subparser( - subparsers, "show-option", help="print the value of a configuration option" -) -subparser.set_defaults(op=op_show_option) -subparser.add_argument("machine", metavar="MACHINE", help="identifier of the machine") -subparser.add_argument("option", metavar="OPTION", help="option name") -subparser.add_argument( - "--xml", action="store_true", help="print the option value in XML format" -) -subparser.add_argument( - "--json", action="store_true", help="print the option value in JSON format" -) -subparser.add_argument( - "--include-physical", - action="store_true", - help="include the physical specification in the evaluation", -) - -subparser = add_subparser( - subparsers, - "list-generations", - help="list previous configurations to which you can roll back", -) -subparser.set_defaults(op=op_list_generations) - -subparser = add_subparser( - subparsers, "rollback", help="roll back to a previous configuration" -) -subparser.set_defaults(op=op_rollback) -subparser.add_argument( - "generation", - type=int, - metavar="GENERATION", - help="number of the desired configuration (see ‘nixops list-generations’)", -) -add_common_deployment_options(subparser) - -subparser = add_subparser( - subparsers, "delete-generation", help="remove a previous configuration" -) -subparser.set_defaults(op=op_delete_generation) -subparser.add_argument( - "generation", - type=int, - metavar="GENERATION", - help="number of the desired configuration (see ‘nixops list-generations’)", -) -add_common_deployment_options(subparser) - -subparser = add_subparser( - subparsers, - "show-console-output", - help="print the machine's console output on stdout", -) -subparser.set_defaults(op=op_show_console_output) -subparser.add_argument("machine", metavar="MACHINE", help="identifier of the machine") -add_common_deployment_options(subparser) - -subparser = add_subparser( - subparsers, "dump-nix-paths", help="dump Nix paths referenced in deployments" -) -subparser.add_argument( - "--all", action="store_true", help="dump Nix paths for all deployments" -) -subparser.set_defaults(op=op_dump_nix_paths) -add_common_deployment_options(subparser) - -subparser = add_subparser(subparsers, "export", help="export the state of a deployment") -subparser.add_argument("--all", action="store_true", help="export all deployments") -subparser.set_defaults(op=op_export) - -subparser = add_subparser( - subparsers, "import", help="import deployments into the state file" -) -subparser.add_argument( - "--include-keys", - action="store_true", - help="import public SSH hosts keys to .ssh/known_hosts", -) -subparser.set_defaults(op=op_import) - -subparser = add_subparser( - subparsers, "edit", help="open the deployment specification in $EDITOR" -) -subparser.set_defaults(op=op_edit) - -subparser = add_subparser( - subparsers, "copy-closure", help="copy closure to a target machine" -) -subparser.add_argument("machine", help="identifier of the machine") -subparser.add_argument("storepath", help="store path of the closure to be copied") -subparser.set_defaults(op=op_copy_closure) - -subparser = subparsers.add_parser( - "list-plugins", help="list the available nixops plugins" -) -subparser.set_defaults(op=op_list_plugins) -subparser.add_argument( - "--verbose", "-v", action="store_true", help="Provide extra plugin information" -) -subparser.add_argument("--debug", action="store_true", help="enable debug output") - -parser_plugin_hooks(parser, subparsers) def main() -> None: diff --git a/nixops/args.py b/nixops/args.py new file mode 100644 index 000000000..40ffe01e5 --- /dev/null +++ b/nixops/args.py @@ -0,0 +1,658 @@ +from argparse import ArgumentParser, _SubParsersAction, SUPPRESS, REMAINDER +from nixops.script_defs import ( + add_subparser, + op_list_deployments, + op_create, + add_common_modify_options, + op_modify, + op_clone, + op_delete, + op_info, + op_check, + op_set_args, + op_deploy, + add_common_deployment_options, + op_send_keys, + op_destroy, + op_delete_resources, + op_stop, + op_start, + op_reboot, + op_show_arguments, + op_show_physical, + op_ssh, + op_ssh_for_each, + op_scp, + op_mount, + op_rename, + op_backup, + op_backup_status, + op_remove_backup, + op_clean_backups, + op_restore, + op_show_option, + op_list_generations, + op_rollback, + op_delete_generation, + op_show_console_output, + op_dump_nix_paths, + op_export, + op_import, + op_edit, + op_copy_closure, + op_list_plugins, + parser_plugin_hooks, +) + +# Set up the parser. +parser = ArgumentParser(description="NixOS cloud deployment tool", prog="nixops") +parser.add_argument("--version", action="version", version="NixOps @version@") +parser.add_argument( + "--pdb", action="store_true", help="Invoke pdb on unhandled exception" +) + +subparsers: _SubParsersAction = parser.add_subparsers( + help="sub-command help", metavar="operation", required=True +) + +subparser = add_subparser(subparsers, "list", help="list all known deployments") +subparser.set_defaults(op=op_list_deployments) + +subparser = add_subparser(subparsers, "create", help="create a new deployment") +subparser.set_defaults(op=op_create) +subparser.add_argument( + "--name", "-n", dest="name", metavar="NAME", help=SUPPRESS +) # obsolete, use -d instead +add_common_modify_options(subparser) + +subparser = add_subparser(subparsers, "modify", help="modify an existing deployment") +subparser.set_defaults(op=op_modify) +subparser.add_argument( + "--name", "-n", dest="name", metavar="NAME", help="new symbolic name of deployment" +) +add_common_modify_options(subparser) + +subparser = add_subparser(subparsers, "clone", help="clone an existing deployment") +subparser.set_defaults(op=op_clone) +subparser.add_argument( + "--name", + "-n", + dest="name", + metavar="NAME", + help="symbolic name of the cloned deployment", +) + +subparser = add_subparser(subparsers, "delete", help="delete a deployment") +subparser.add_argument( + "--force", action="store_true", help="force deletion even if resources still exist" +) +subparser.add_argument("--all", action="store_true", help="delete all deployments") +subparser.set_defaults(op=op_delete) + +subparser = add_subparser(subparsers, "info", help="show the state of the deployment") +subparser.set_defaults(op=op_info) +subparser.add_argument("--all", action="store_true", help="show all deployments") +subparser.add_argument( + "--plain", action="store_true", help="do not pretty-print the output" +) +subparser.add_argument( + "--no-eval", + action="store_true", + help="do not evaluate the deployment specification", +) + +subparser = add_subparser( + subparsers, + "check", + help="check the state of the machines in the network" + " (note that this might alter the internal nixops state to consolidate with the real state of the resource)", +) +subparser.set_defaults(op=op_check) +subparser.add_argument("--all", action="store_true", help="check all deployments") +subparser.add_argument( + "--include", + nargs="+", + metavar="MACHINE-NAME", + help="check only the specified machines", +) +subparser.add_argument( + "--exclude", + nargs="+", + metavar="MACHINE-NAME", + help="check all except the specified machines", +) + +subparser = add_subparser( + subparsers, + "set-args", + help="persistently set arguments to the deployment specification", +) +subparser.set_defaults(op=op_set_args) +subparser.add_argument( + "--arg", + nargs=2, + action="append", + dest="args", + metavar=("NAME", "VALUE"), + help="pass a Nix expression value", +) +subparser.add_argument( + "--argstr", + nargs=2, + action="append", + dest="argstrs", + metavar=("NAME", "VALUE"), + help="pass a string value", +) +subparser.add_argument( + "--unset", + nargs=1, + action="append", + dest="unset", + metavar="NAME", + help="unset previously set argument", +) + + +subparser = add_subparser(subparsers, "deploy", help="deploy the network configuration") +subparser.set_defaults(op=op_deploy) +subparser.add_argument( + "--kill-obsolete", "-k", action="store_true", help="kill obsolete virtual machines" +) +subparser.add_argument( + "--dry-run", action="store_true", help="evaluate and print what would be built" +) +subparser.add_argument( + "--dry-activate", + action="store_true", + help="show what will be activated on the machines in the network", +) +subparser.add_argument( + "--test", + action="store_true", + help="build and activate the new configuration; do not enable it in the bootloader. Rebooting the system will roll back automatically.", +) +subparser.add_argument( + "--boot", + action="store_true", + help="build the new configuration and enable it in the bootloader; do not activate it. Upon reboot, the system will use the new configuration.", +) +subparser.add_argument( + "--repair", action="store_true", help="use --repair when calling nix-build (slow)" +) +subparser.add_argument( + "--evaluate-only", action="store_true", help="only call nix-instantiate and exit" +) +subparser.add_argument( + "--plan-only", + action="store_true", + help="show the diff between the configuration and the state and exit", +) +subparser.add_argument( + "--build-only", + action="store_true", + help="build only; do not perform deployment actions", +) +subparser.add_argument( + "--create-only", action="store_true", help="exit after creating missing machines" +) +subparser.add_argument( + "--copy-only", action="store_true", help="exit after copying closures" +) +subparser.add_argument( + "--allow-recreate", + action="store_true", + help="recreate resources machines that have disappeared", +) +subparser.add_argument( + "--always-activate", + action="store_true", + help="activate unchanged configurations as well", +) +add_common_deployment_options(subparser) + +subparser = add_subparser(subparsers, "send-keys", help="send encryption keys") +subparser.set_defaults(op=op_send_keys) +subparser.add_argument( + "--include", + nargs="+", + metavar="MACHINE-NAME", + help="send keys to only the specified machines", +) +subparser.add_argument( + "--exclude", + nargs="+", + metavar="MACHINE-NAME", + help="send keys to all except the specified machines", +) + +subparser = add_subparser( + subparsers, "destroy", help="destroy all resources in the specified deployment" +) +subparser.set_defaults(op=op_destroy) +subparser.add_argument( + "--include", + nargs="+", + metavar="MACHINE-NAME", + help="destroy only the specified machines", +) +subparser.add_argument( + "--exclude", + nargs="+", + metavar="MACHINE-NAME", + help="destroy all except the specified machines", +) +subparser.add_argument( + "--wipe", action="store_true", help="securely wipe data on the machines" +) +subparser.add_argument("--all", action="store_true", help="destroy all deployments") + +subparser = add_subparser( + subparsers, + "delete-resources", + help="deletes the resource from the local NixOps state file.", +) +subparser.set_defaults(op=op_delete_resources) +subparser.add_argument( + "--include", + nargs="+", + metavar="RESOURCE-NAME", + help="delete only the specified resources", +) +subparser.add_argument( + "--exclude", + nargs="+", + metavar="RESOURCE-NAME", + help="delete all resources except the specified resources", +) + +subparser = add_subparser( + subparsers, "stop", help="stop all virtual machines in the network" +) +subparser.set_defaults(op=op_stop) +subparser.add_argument( + "--include", + nargs="+", + metavar="MACHINE-NAME", + help="stop only the specified machines", +) +subparser.add_argument( + "--exclude", + nargs="+", + metavar="MACHINE-NAME", + help="stop all except the specified machines", +) + +subparser = add_subparser( + subparsers, "start", help="start all virtual machines in the network" +) +subparser.set_defaults(op=op_start) +subparser.add_argument( + "--include", + nargs="+", + metavar="MACHINE-NAME", + help="start only the specified machines", +) +subparser.add_argument( + "--exclude", + nargs="+", + metavar="MACHINE-NAME", + help="start all except the specified machines", +) + +subparser = add_subparser( + subparsers, "reboot", help="reboot all virtual machines in the network" +) +subparser.set_defaults(op=op_reboot) +subparser.add_argument( + "--include", + nargs="+", + metavar="MACHINE-NAME", + help="reboot only the specified machines", +) +subparser.add_argument( + "--exclude", + nargs="+", + metavar="MACHINE-NAME", + help="reboot all except the specified machines", +) +subparser.add_argument( + "--no-wait", action="store_true", help="do not wait until the machines are up again" +) +subparser.add_argument( + "--rescue", + action="store_true", + help="reboot machines into the rescue system" " (if available)", +) +subparser.add_argument( + "--hard", + action="store_true", + help="send a hard reset (power switch) to the machines" " (if available)", +) + +subparser = add_subparser( + subparsers, "show-arguments", help="print the arguments to the network expressions" +) +subparser.set_defaults(op=op_show_arguments) + +subparser = add_subparser( + subparsers, "show-physical", help="print the physical network expression" +) +subparser.add_argument( + "--backup", + dest="backupid", + default=None, + help="print physical network expression for given backup id", +) +subparser.set_defaults(op=op_show_physical) + +subparser = add_subparser( + subparsers, "ssh", help="login on the specified machine via SSH" +) +subparser.set_defaults(op=op_ssh) +subparser.add_argument("machine", metavar="MACHINE", help="identifier of the machine") +subparser.add_argument( + "args", metavar="SSH_ARGS", nargs=REMAINDER, help="SSH flags and/or command", +) + +subparser = add_subparser( + subparsers, "ssh-for-each", help="execute a command on each machine via SSH" +) +subparser.set_defaults(op=op_ssh_for_each) +subparser.add_argument( + "args", metavar="ARG", nargs="*", help="additional arguments to SSH" +) +subparser.add_argument("--parallel", "-p", action="store_true", help="run in parallel") +subparser.add_argument( + "--include", + nargs="+", + metavar="MACHINE-NAME", + help="run command only on the specified machines", +) +subparser.add_argument( + "--exclude", + nargs="+", + metavar="MACHINE-NAME", + help="run command on all except the specified machines", +) +subparser.add_argument( + "--all", action="store_true", help="run ssh-for-each for all deployments" +) + +subparser = add_subparser( + subparsers, "scp", help="copy files to or from the specified machine via scp" +) +subparser.set_defaults(op=op_scp) +subparser.add_argument( + "--from", + dest="scp_from", + action="store_true", + help="copy a file from specified machine", +) +subparser.add_argument( + "--to", dest="scp_to", action="store_true", help="copy a file to specified machine" +) +subparser.add_argument("machine", metavar="MACHINE", help="identifier of the machine") +subparser.add_argument("source", metavar="SOURCE", help="source file location") +subparser.add_argument("destination", metavar="DEST", help="destination file location") + +subparser = add_subparser( + subparsers, + "mount", + help="mount a directory from the specified machine into the local filesystem", +) +subparser.set_defaults(op=op_mount) +subparser.add_argument( + "machine", + metavar="MACHINE[:PATH]", + help="identifier of the machine, optionally followed by a path", +) +subparser.add_argument("destination", metavar="PATH", help="local path") +subparser.add_argument( + "--sshfs-option", + "-o", + action="append", + metavar="OPTIONS", + help="mount options passed to sshfs", +) + +subparser = add_subparser(subparsers, "rename", help="rename machine in network") +subparser.set_defaults(op=op_rename) +subparser.add_argument( + "current_name", metavar="FROM", help="current identifier of the machine" +) +subparser.add_argument("new_name", metavar="TO", help="new identifier of the machine") + +subparser = add_subparser( + subparsers, + "backup", + help="make snapshots of persistent disks in network (currently EC2-only)", +) +subparser.set_defaults(op=op_backup) +subparser.add_argument( + "--include", + nargs="+", + metavar="MACHINE-NAME", + help="perform backup actions on the specified machines only", +) +subparser.add_argument( + "--exclude", + nargs="+", + metavar="MACHINE-NAME", + help="do not perform backup actions on the specified machines", +) +subparser.add_argument( + "--freeze", + dest="freeze_fs", + action="store_true", + default=False, + help="freeze filesystems for non-root filesystems that support this (e.g. xfs)", +) +subparser.add_argument( + "--force", + dest="force", + action="store_true", + default=False, + help="start new backup even if previous is still running", +) +subparser.add_argument( + "--devices", + nargs="+", + metavar="DEVICE-NAME", + help="only backup the specified devices", +) + +subparser = add_subparser(subparsers, "backup-status", help="get status of backups") +subparser.set_defaults(op=op_backup_status) +subparser.add_argument( + "backupid", default=None, nargs="?", help="use specified backup in stead of latest" +) +subparser.add_argument( + "--include", + nargs="+", + metavar="MACHINE-NAME", + help="perform backup actions on the specified machines only", +) +subparser.add_argument( + "--exclude", + nargs="+", + metavar="MACHINE-NAME", + help="do not perform backup actions on the specified machines", +) +subparser.add_argument( + "--wait", + dest="wait", + action="store_true", + default=False, + help="wait until backup is finished", +) +subparser.add_argument( + "--latest", + dest="latest", + action="store_true", + default=False, + help="show status of latest backup only", +) + +subparser = add_subparser(subparsers, "remove-backup", help="remove a given backup") +subparser.set_defaults(op=op_remove_backup) +subparser.add_argument("backupid", metavar="BACKUP-ID", help="backup ID to remove") +subparser.add_argument( + "--keep-physical", + dest="keep_physical", + default=False, + action="store_true", + help="do not remove the physical backups, only remove backups from nixops state", +) + +subparser = add_subparser(subparsers, "clean-backups", help="remove old backups") +subparser.set_defaults(op=op_clean_backups) +subparser.add_argument( + "--keep", dest="keep", type=int, help="number of backups to keep around" +) +subparser.add_argument( + "--keep-days", + metavar="N", + dest="keep_days", + type=int, + help="keep backups newer than N days", +) +subparser.add_argument( + "--keep-physical", + dest="keep_physical", + default=False, + action="store_true", + help="do not remove the physical backups, only remove backups from nixops state", +) + +subparser = add_subparser( + subparsers, + "restore", + help="restore machines based on snapshots of persistent disks in network (currently EC2-only)", +) +subparser.set_defaults(op=op_restore) +subparser.add_argument( + "--backup-id", default=None, help="use specified backup in stead of latest" +) +subparser.add_argument( + "--include", + nargs="+", + metavar="MACHINE-NAME", + help="perform backup actions on the specified machines only", +) +subparser.add_argument( + "--exclude", + nargs="+", + metavar="MACHINE-NAME", + help="do not perform backup actions on the specified machines", +) +subparser.add_argument( + "--devices", + nargs="+", + metavar="DEVICE-NAME", + help="only restore the specified devices", +) + +subparser = add_subparser( + subparsers, "show-option", help="print the value of a configuration option" +) +subparser.set_defaults(op=op_show_option) +subparser.add_argument("machine", metavar="MACHINE", help="identifier of the machine") +subparser.add_argument("option", metavar="OPTION", help="option name") +subparser.add_argument( + "--xml", action="store_true", help="print the option value in XML format" +) +subparser.add_argument( + "--json", action="store_true", help="print the option value in JSON format" +) +subparser.add_argument( + "--include-physical", + action="store_true", + help="include the physical specification in the evaluation", +) + +subparser = add_subparser( + subparsers, + "list-generations", + help="list previous configurations to which you can roll back", +) +subparser.set_defaults(op=op_list_generations) + +subparser = add_subparser( + subparsers, "rollback", help="roll back to a previous configuration" +) +subparser.set_defaults(op=op_rollback) +subparser.add_argument( + "generation", + type=int, + metavar="GENERATION", + help="number of the desired configuration (see ‘nixops list-generations’)", +) +add_common_deployment_options(subparser) + +subparser = add_subparser( + subparsers, "delete-generation", help="remove a previous configuration" +) +subparser.set_defaults(op=op_delete_generation) +subparser.add_argument( + "generation", + type=int, + metavar="GENERATION", + help="number of the desired configuration (see ‘nixops list-generations’)", +) +add_common_deployment_options(subparser) + +subparser = add_subparser( + subparsers, + "show-console-output", + help="print the machine's console output on stdout", +) +subparser.set_defaults(op=op_show_console_output) +subparser.add_argument("machine", metavar="MACHINE", help="identifier of the machine") +add_common_deployment_options(subparser) + +subparser = add_subparser( + subparsers, "dump-nix-paths", help="dump Nix paths referenced in deployments" +) +subparser.add_argument( + "--all", action="store_true", help="dump Nix paths for all deployments" +) +subparser.set_defaults(op=op_dump_nix_paths) +add_common_deployment_options(subparser) + +subparser = add_subparser(subparsers, "export", help="export the state of a deployment") +subparser.add_argument("--all", action="store_true", help="export all deployments") +subparser.set_defaults(op=op_export) + +subparser = add_subparser( + subparsers, "import", help="import deployments into the state file" +) +subparser.add_argument( + "--include-keys", + action="store_true", + help="import public SSH hosts keys to .ssh/known_hosts", +) +subparser.set_defaults(op=op_import) + +subparser = add_subparser( + subparsers, "edit", help="open the deployment specification in $EDITOR" +) +subparser.set_defaults(op=op_edit) + +subparser = add_subparser( + subparsers, "copy-closure", help="copy closure to a target machine" +) +subparser.add_argument("machine", help="identifier of the machine") +subparser.add_argument("storepath", help="store path of the closure to be copied") +subparser.set_defaults(op=op_copy_closure) + +subparser = subparsers.add_parser( + "list-plugins", help="list the available nixops plugins" +) +subparser.set_defaults(op=op_list_plugins) +subparser.add_argument( + "--verbose", "-v", action="store_true", help="Provide extra plugin information" +) +subparser.add_argument("--debug", action="store_true", help="enable debug output") + +parser_plugin_hooks(parser, subparsers) From a389d3938eae571910b5006c30dfb7436535d787 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 15 Jul 2020 15:36:29 +0200 Subject: [PATCH 35/80] Fix internal plugin --- nixops/plugin.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nixops/plugin.py b/nixops/plugin.py index bebbefa84..79a78c3f5 100644 --- a/nixops/plugin.py +++ b/nixops/plugin.py @@ -14,3 +14,8 @@ def storage_backends(self) -> BackendRegistration: def lock_drivers(self) -> Dict[str, Type[LockDriver]]: return {"noop": NoopLock} + + +@nixops.plugins.hookimpl +def plugin(): + return InternalPlugin() From e3857528e84e8dbe0e1f8e2e3b838da11fb67e9f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 15 Jul 2020 15:37:26 +0200 Subject: [PATCH 36/80] Memory storage: Run modify_deployment() when creating internal state Otherwise `nixops info` & friends will think the deployment is not created and no machines show up. --- nixops/storage/memory.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nixops/storage/memory.py b/nixops/storage/memory.py index d1a5a7100..d2fce3288 100644 --- a/nixops/storage/memory.py +++ b/nixops/storage/memory.py @@ -28,7 +28,13 @@ def fetchToFile(self, path: str, **kwargs) -> None: # Note: no arguments will be passed over kwargs. Making it part of # the type definition allows adding new arguments later. def onOpen(self, sf: nixops.statefile.StateFile, **kwargs) -> None: - sf.create_deployment() + from nixops.script_defs import modify_deployment + from nixops.args import parser + + depl = sf.create_deployment() + args = parser.parse_args() + + modify_deployment(args, depl) # uploadFromFile: upload the new version of the state file # Note: no arguments will be passed over kwargs. Making it part of From 2543248db5e5126187821fd28e91bf861923753f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 15 Jul 2020 15:39:36 +0200 Subject: [PATCH 37/80] Remove template support --- nix/templates/.keep | 0 nixops/script_defs.py | 11 ----------- 2 files changed, 11 deletions(-) delete mode 100644 nix/templates/.keep diff --git a/nix/templates/.keep b/nix/templates/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 8a1774926..1c866d749 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -197,10 +197,7 @@ def set_name(depl: nixops.deployment.Deployment, name: Optional[str]): def modify_deployment(args, depl: nixops.deployment.Deployment): nix_exprs = args.nix_exprs - templates = args.templates or [] - for i in templates: - nix_exprs.append("".format(i)) if len(nix_exprs) == 0: raise Exception( "you must specify the path to a Nix expression and/or use ‘-t’" @@ -1162,14 +1159,6 @@ def add_common_modify_options(subparser: ArgumentParser) -> None: metavar="NIX-FILE", help="Nix expression(s) defining the network", ) - subparser.add_argument( - "--template", - "-t", - action="append", - dest="templates", - metavar="TEMPLATE", - help="name of template to be used", - ) def add_common_deployment_options(subparser: ArgumentParser) -> None: From f7d6219d54ee042098982534f4dd043b2c7ebb34 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 15 Jul 2020 15:43:53 +0200 Subject: [PATCH 38/80] nit: Use os.path.join instead of stringly concat --- nixops/script_defs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 1c866d749..86b480f7b 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -1071,7 +1071,7 @@ def add_subparser( "--network", dest="network_file", metavar="FILE", - default=f"{os.getcwd()}/network.nix", + default=os.path.join(os.getcwd(), "network.nix"), help="path to a network.nix", ) subparser.add_argument( From 0a68459ac45c81042e393aee761d28920f5dc01e Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 15 Jul 2020 15:45:50 +0200 Subject: [PATCH 39/80] Remove the positional nix_exprs argument from modify commands This is is conflicting with `--network` that _needs_ to be available for all subcommands. --- nixops/args.py | 3 --- nixops/script_defs.py | 19 +++---------------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/nixops/args.py b/nixops/args.py index 40ffe01e5..aff594e11 100644 --- a/nixops/args.py +++ b/nixops/args.py @@ -3,7 +3,6 @@ add_subparser, op_list_deployments, op_create, - add_common_modify_options, op_modify, op_clone, op_delete, @@ -63,14 +62,12 @@ subparser.add_argument( "--name", "-n", dest="name", metavar="NAME", help=SUPPRESS ) # obsolete, use -d instead -add_common_modify_options(subparser) subparser = add_subparser(subparsers, "modify", help="modify an existing deployment") subparser.set_defaults(op=op_modify) subparser.add_argument( "--name", "-n", dest="name", metavar="NAME", help="new symbolic name of deployment" ) -add_common_modify_options(subparser) subparser = add_subparser(subparsers, "clone", help="clone an existing deployment") subparser.set_defaults(op=op_clone) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 86b480f7b..de35d1ca4 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -196,16 +196,12 @@ def set_name(depl: nixops.deployment.Deployment, name: Optional[str]): def modify_deployment(args, depl: nixops.deployment.Deployment): - nix_exprs = args.nix_exprs + nix_exprs = [args.network_file] if len(nix_exprs) == 0: - raise Exception( - "you must specify the path to a Nix expression and/or use ‘-t’" - ) + raise Exception("you must specify the path to a Nix expression and/or use ‘-t’") depl.nix_exprs = [os.path.abspath(x) if x[0:1] != "<" else x for x in nix_exprs] - depl.nix_path = [ - nixops.util.abs_nix_path(x) for x in sum(args.nix_path or [], []) - ] + depl.nix_path = [nixops.util.abs_nix_path(x) for x in sum(args.nix_path or [], [])] def op_create(args): @@ -1152,15 +1148,6 @@ def add_subparser( return subparser -def add_common_modify_options(subparser: ArgumentParser) -> None: - subparser.add_argument( - "nix_exprs", - nargs="*", - metavar="NIX-FILE", - help="Nix expression(s) defining the network", - ) - - def add_common_deployment_options(subparser: ArgumentParser) -> None: subparser.add_argument( "--include", From 2120542b218ef0a8148746b5359e25a95f5c5163 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 15 Jul 2020 16:22:09 +0200 Subject: [PATCH 40/80] op_list: Fix calculation of number of machines & types --- nixops/script_defs.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index de35d1ca4..2e6ec7b27 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -23,7 +23,7 @@ import json from tempfile import TemporaryDirectory import pipes -from typing import Tuple, List, Optional, Union, Generator, Type +from typing import Tuple, List, Optional, Union, Generator, Type, Set import nixops.ansi from nixops.plugins.manager import PluginManager @@ -32,6 +32,7 @@ from nixops.evaluation import eval_network from nixops.storage import storage_backends from nixops.locks import lock_drivers +from nixops.backends import MachineDefinition PluginManager.load() @@ -149,13 +150,24 @@ def op_list_deployments(args): ] ) for depl in sort_deployments(sf.get_all_deployments()): + depl.evaluate() + + types: Set[str] = set() + n_machines: int = 0 + + for defn in (depl.definitions or {}).values(): + if not isinstance(defn, MachineDefinition): + continue + n_machines += 1 + types.add(defn.get_type()) + tbl.add_row( [ depl.uuid, depl.name or "(none)", depl.description, - len(depl.machines), - ", ".join(set(m.get_type() for m in depl.machines.values())), + n_machines, + ", ".join(types), ] ) print(tbl) From bc25d83b339d4268fe7f1f44e640fc07bcd073fe Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 10 Jul 2020 13:10:17 -0400 Subject: [PATCH 41/80] locks/storage plugins: access via Manager, eliminate global vars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Author: Adam Höse --- nixops/locks/__init__.py | 3 --- nixops/plugin.py | 4 ++-- nixops/plugins/__init__.py | 4 ++-- nixops/plugins/manager.py | 20 +++++++++++++------- nixops/script_defs.py | 4 ++-- nixops/storage/__init__.py | 6 ------ 6 files changed, 19 insertions(+), 22 deletions(-) diff --git a/nixops/locks/__init__.py b/nixops/locks/__init__.py index 7bd440c3c..c3ba726c5 100644 --- a/nixops/locks/__init__.py +++ b/nixops/locks/__init__.py @@ -38,6 +38,3 @@ def lock(self, **kwargs) -> None: # the type definition allows adding new arguments later. def unlock(self, **kwargs) -> None: raise NotImplementedError - - -lock_drivers: Dict[str, Type[LockDriver]] = {} diff --git a/nixops/plugin.py b/nixops/plugin.py index 79a78c3f5..c7b1c1933 100644 --- a/nixops/plugin.py +++ b/nixops/plugin.py @@ -1,4 +1,4 @@ -from nixops.storage import BackendRegistration +from nixops.storage import StorageBackend from nixops.storage.legacy import LegacyBackend from nixops.storage.memory import MemoryBackend @@ -9,7 +9,7 @@ class InternalPlugin(nixops.plugins.Plugin): - def storage_backends(self) -> BackendRegistration: + def storage_backends(self) -> Dict[str, Type[StorageBackend]]: return {"legacy": LegacyBackend, "memory": MemoryBackend} def lock_drivers(self) -> Dict[str, Type[LockDriver]]: diff --git a/nixops/plugins/__init__.py b/nixops/plugins/__init__.py index 04b1e957b..5e48ff235 100644 --- a/nixops/plugins/__init__.py +++ b/nixops/plugins/__init__.py @@ -3,7 +3,7 @@ from nixops.backends import MachineState from typing import List, Dict, Optional, Union, Tuple, Type -from nixops.storage import BackendRegistration +from nixops.storage import StorageBackend from nixops.locks import LockDriver from functools import lru_cache from typing import Generator @@ -99,7 +99,7 @@ def docs(self) -> List[Tuple[str, str]]: def lock_drivers(self) -> Dict[str, Type[LockDriver]]: return {} - def storage_backends(self) -> BackendRegistration: + def storage_backends(self) -> Dict[str, Type[StorageBackend]]: """ Extend the core nixops cli parser :return a set of plugin parser extensions """ diff --git a/nixops/plugins/manager.py b/nixops/plugins/manager.py index ebf1cb97e..2eaba5b3c 100644 --- a/nixops/plugins/manager.py +++ b/nixops/plugins/manager.py @@ -4,11 +4,12 @@ from typing import List, Dict, Generator, Tuple, Any, Set import importlib -from nixops.storage import storage_backends -from nixops.locks import lock_drivers +from nixops.storage import StorageBackend +from nixops.locks import LockDriver from . import get_plugins, MachineHooks, DeploymentHooks import nixops.ansi import nixops +from typing import Type import sys @@ -64,9 +65,6 @@ def load(cls): importlib.import_module(mod) seen.add(mod) - cls.storage_backends() - cls.lock_drivers() - @staticmethod def nixexprs() -> List[str]: nixexprs: List[str] = [] @@ -85,7 +83,9 @@ def docs() -> Generator[Tuple[str, str], None, None]: yield from plugin.docs() @staticmethod - def storage_backends(): + def storage_backends() -> Dict[str, Type[StorageBackend]]: + storage_backends: Dict[str, Type[StorageBackend]] = {} + for plugin in get_plugins(): for name, backend in plugin.storage_backends().items(): if name not in storage_backends: @@ -97,8 +97,12 @@ def storage_backends(): ) ) + return storage_backends + @staticmethod - def lock_drivers(): + def lock_drivers() -> Dict[str, Type[LockDriver]]: + lock_drivers: Dict[str, Type[LockDriver]] = {} + for plugin in get_plugins(): for name, driver in plugin.lock_drivers().items(): if name not in lock_drivers: @@ -109,3 +113,5 @@ def lock_drivers(): f"Two plugins tried to provide the '{name}' lock driver." ) ) + + return lock_drivers diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 2e6ec7b27..6e6fe1fc2 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -30,8 +30,6 @@ from nixops.plugins import get_plugin_manager from nixops.evaluation import eval_network -from nixops.storage import storage_backends -from nixops.locks import lock_drivers from nixops.backends import MachineDefinition @@ -50,6 +48,7 @@ def deployment(args: Namespace) -> Generator[nixops.deployment.Deployment, None, def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None, None]: network_file: str = args.network_file network = eval_network(network_file) + storage_backends = PluginManager.storage_backends() storage_class: Optional[Type[StorageBackend]] = storage_backends.get( network.storage.provider ) @@ -64,6 +63,7 @@ def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None lock: LockDriver lock_class: Type[LockDriver] + lock_drivers = PluginManager.lock_drivers() try: lock_class = lock_drivers[network.lock.provider] except KeyError: diff --git a/nixops/storage/__init__.py b/nixops/storage/__init__.py index 955c69861..f5646e5fb 100644 --- a/nixops/storage/__init__.py +++ b/nixops/storage/__init__.py @@ -51,9 +51,3 @@ def onOpen(self, sf: nixops.statefile.StateFile, **kwargs) -> None: # the type definition allows adding new arguments later. def uploadFromFile(self, path: str, **kwargs) -> None: raise NotImplementedError - - -BackendRegistration = Dict[str, Type[StorageBackend]] - - -storage_backends: Dict[str, Type[StorageBackend]] = {} From d754007d574188941a38c04159f085e970dceb1f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 15 Jul 2020 17:00:59 +0200 Subject: [PATCH 42/80] Add command to unlock the deployment lock In case a lock is acquired but nevere released. --- nixops/args.py | 4 ++++ nixops/script_defs.py | 46 +++++++++++++++++++++++++++---------------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/nixops/args.py b/nixops/args.py index aff594e11..37db5cb62 100644 --- a/nixops/args.py +++ b/nixops/args.py @@ -41,6 +41,7 @@ op_copy_closure, op_list_plugins, parser_plugin_hooks, + op_unlock, ) # Set up the parser. @@ -652,4 +653,7 @@ ) subparser.add_argument("--debug", action="store_true", help="enable debug output") +subparser = add_subparser(subparsers, "unlock", help="Force unlock the deployment lock") +subparser.set_defaults(op=op_unlock) + parser_plugin_hooks(parser, subparsers) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 6e6fe1fc2..1530f4bb5 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -29,7 +29,7 @@ from nixops.plugins.manager import PluginManager from nixops.plugins import get_plugin_manager -from nixops.evaluation import eval_network +from nixops.evaluation import eval_network, NetworkEval from nixops.backends import MachineDefinition @@ -44,6 +44,26 @@ def deployment(args: Namespace) -> Generator[nixops.deployment.Deployment, None, yield depl +def get_lock(network: NetworkEval) -> LockDriver: + lock: LockDriver + lock_class: Type[LockDriver] + lock_drivers = PluginManager.lock_drivers() + try: + lock_class = lock_drivers[network.lock.provider] + except KeyError: + sys.stderr.write( + nixops.ansi.ansi_warn( + f"The network requires the '{network.lock.provider}' lock driver, " + "but no plugin provides it.\n" + ) + ) + raise Exception("Missing lock driver plugin.") + else: + lock_class_options = lock_class.options(**network.lock.configuration) + lock = lock_class(lock_class_options) + return lock + + @contextlib.contextmanager def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None, None]: network_file: str = args.network_file @@ -61,22 +81,7 @@ def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None ) raise Exception("Missing storage provider plugin.") - lock: LockDriver - lock_class: Type[LockDriver] - lock_drivers = PluginManager.lock_drivers() - try: - lock_class = lock_drivers[network.lock.provider] - except KeyError: - sys.stderr.write( - nixops.ansi.ansi_warn( - f"The network requires the '{network.lock.provider}' lock driver, " - "but no plugin provides it.\n" - ) - ) - raise Exception("Missing lock driver plugin.") - else: - lock_class_options = lock_class.options(**network.lock.configuration) - lock = lock_class(lock_class_options) + lock = get_lock(network) storage_class_options = storage_class.options(**network.storage.configuration) storage: StorageBackend = storage_class(storage_class_options) @@ -807,6 +812,13 @@ def op_export(args): print(json.dumps(res, indent=2, sort_keys=True, cls=nixops.util.NixopsEncoder)) +def op_unlock(args): + network_file: str = args.network_file + network = eval_network(network_file) + lock = get_lock(network) + lock.unlock() + + def op_import(args): with network_state(args) as sf: existing = set(sf.query_deployments()) From 08d9d814cead36aa9461f9ebe3825b067e6f1bea Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 16 Jul 2020 13:20:49 +0200 Subject: [PATCH 43/80] Use git to find project root in development shell `builtins.toString ./.` resolves to a store path when using flakes. We want the mutable paths as environment variables, not the store paths. --- flake.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index a91925319..8af0905a0 100644 --- a/flake.nix +++ b/flake.nix @@ -38,7 +38,9 @@ ] ++ (builtins.attrValues linters); shellHook = '' - export PATH=${builtins.toString ./scripts}:$PATH + git_root=$(${pkgs.git}/bin/git rev-parse --show-toplevel) + export PYTHONPATH=$git_root:$PYTHONPATH + export PATH=$git_root/scripts:$PATH ''; }; From 737e8bd13f408c18a6e57b71c219b74d54c5076b Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 16 Jul 2020 17:05:51 +0200 Subject: [PATCH 44/80] Remove xml --- nixops/args.py | 3 --- nixops/deployment.py | 2 -- nixops/script_defs.py | 1 - 3 files changed, 6 deletions(-) diff --git a/nixops/args.py b/nixops/args.py index 37db5cb62..8e9e2f0f4 100644 --- a/nixops/args.py +++ b/nixops/args.py @@ -557,9 +557,6 @@ subparser.set_defaults(op=op_show_option) subparser.add_argument("machine", metavar="MACHINE", help="identifier of the machine") subparser.add_argument("option", metavar="OPTION", help="option name") -subparser.add_argument( - "--xml", action="store_true", help="print the option value in XML format" -) subparser.add_argument( "--json", action="store_true", help="print the option value in JSON format" ) diff --git a/nixops/deployment.py b/nixops/deployment.py index 18f84c337..faa9700ec 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -548,7 +548,6 @@ def evaluate_option_value( machine_name: str, option_name: str, json: bool = False, - xml: bool = False, include_physical: bool = False, ) -> str: """Evaluate a single option of a single machine in the deployment specification.""" @@ -575,7 +574,6 @@ def evaluate_option_value( "nodes.{0}.config.{1}".format(machine_name, option_name), ] + (["--json"] if json else []) - + (["--xml"] if xml else []), stderr=self.logger.log_file, text=True, ) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 1530f4bb5..6da30667a 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -965,7 +965,6 @@ def op_show_option(args): args.machine, args.option, json=args.json, - xml=args.xml, include_physical=args.include_physical, ) ) From e64611e44afcd9b977d189c3a2e14eb4e7a80e9c Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 16 Jul 2020 17:36:45 +0200 Subject: [PATCH 45/80] Remove non-json option show This will simplify eval and was of questionable value. --- nixops/args.py | 3 --- nixops/deployment.py | 3 +-- nixops/script_defs.py | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/nixops/args.py b/nixops/args.py index 8e9e2f0f4..97ade4469 100644 --- a/nixops/args.py +++ b/nixops/args.py @@ -557,9 +557,6 @@ subparser.set_defaults(op=op_show_option) subparser.add_argument("machine", metavar="MACHINE", help="identifier of the machine") subparser.add_argument("option", metavar="OPTION", help="option name") -subparser.add_argument( - "--json", action="store_true", help="print the option value in JSON format" -) subparser.add_argument( "--include-physical", action="store_true", diff --git a/nixops/deployment.py b/nixops/deployment.py index faa9700ec..59c3629c4 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -547,7 +547,6 @@ def evaluate_option_value( self, machine_name: str, option_name: str, - json: bool = False, include_physical: bool = False, ) -> str: """Evaluate a single option of a single machine in the deployment specification.""" @@ -572,8 +571,8 @@ def evaluate_option_value( "false", "-A", "nodes.{0}.config.{1}".format(machine_name, option_name), + "--json", ] - + (["--json"] if json else []) stderr=self.logger.log_file, text=True, ) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 6da30667a..2e9e4d6d1 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -964,7 +964,6 @@ def op_show_option(args): depl.evaluate_option_value( args.machine, args.option, - json=args.json, include_physical=args.include_physical, ) ) From 4ae8d5d5c0a2e7f9a112edf5ca8643add07b8217 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 16 Jul 2020 17:44:55 +0200 Subject: [PATCH 46/80] Unify nix-instantiate calls --- nixops/deployment.py | 97 +++++++++++++++---------------------------- nixops/script_defs.py | 10 ++--- 2 files changed, 38 insertions(+), 69 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 59c3629c4..63920cd61 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -464,50 +464,10 @@ def unset_arg(self, name: str) -> None: def evaluate_args(self) -> Any: """Evaluate the NixOps network expression's arguments.""" - try: - out = subprocess.check_output( - ["nix-instantiate"] - + self.extra_nix_eval_flags - + self._eval_flags(self.nix_exprs) - + ["--eval-only", "--json", "--strict", "-A", "nixopsArguments"], - stderr=self.logger.log_file, - text=True, - ) - if DEBUG: - print("JSON output of nix-instantiate:\n" + out, file=sys.stderr) - return json.loads(out) - except OSError as e: - raise Exception("unable to run ‘nix-instantiate’: {0}".format(e)) - except subprocess.CalledProcessError: - raise NixEvalError + return self.eval(attr="nixopsArguments") def evaluate_config(self, attr) -> Dict: - try: - _json = subprocess.check_output( - ["nix-instantiate"] - + self.extra_nix_eval_flags - + self._eval_flags(self.nix_exprs) - + [ - "--eval-only", - "--json", - "--strict", - "--arg", - "checkConfigurationOptions", - "false", - "-A", - attr, - ], - stderr=self.logger.log_file, - text=True, - ) - if DEBUG: - print("JSON output of nix-instantiate:\n" + _json, file=sys.stderr) - except OSError as e: - raise Exception("unable to run ‘nix-instantiate’: {0}".format(e)) - except subprocess.CalledProcessError: - raise NixEvalError - - return json.loads(_json) + return self.eval(args={"checkConfigurationOptions": False,}, attr=attr) def evaluate_network(self, action: str = "") -> None: if not self.network_attr_eval: @@ -543,42 +503,51 @@ def evaluate(self) -> None: ) self.definitions[name] = defn - def evaluate_option_value( + def eval( self, - machine_name: str, - option_name: str, + args: Optional[Dict[str, Any]] = None, + attr: Optional[str] = None, include_physical: bool = False, - ) -> str: - """Evaluate a single option of a single machine in the deployment specification.""" + ) -> Any: + argv: List[str] = ( + ["nix-instantiate"] + + self.extra_nix_eval_flags + + self._eval_flags(self.nix_exprs) + + ["--eval-only", "--json", "--strict",] + ) - exprs = self.nix_exprs + exprs = list(self.nix_exprs) if include_physical: phys_expr = self.tempdir + "/physical.nix" with open(phys_expr, "w") as f: f.write(self.get_physical_spec()) exprs.append(phys_expr) + if args: + for arg, value in args.items(): + argv.extend(["--arg", arg, json.dumps(value)]) + + if attr: + argv.extend(["-A", attr]) + try: - return subprocess.check_output( - ["nix-instantiate"] - + self.extra_nix_eval_flags - + self._eval_flags(exprs) - + [ - "--eval-only", - "--strict", - "--arg", - "checkConfigurationOptions", - "false", - "-A", - "nodes.{0}.config.{1}".format(machine_name, option_name), - "--json", - ] - stderr=self.logger.log_file, - text=True, + return json.loads( + subprocess.check_output(argv, stderr=self.logger.log_file, text=True,) ) + except OSError as e: + raise Exception("unable to run ‘nix-instantiate’: {0}".format(e)) except subprocess.CalledProcessError: raise NixEvalError + def evaluate_option_value( + self, machine_name: str, option_name: str, include_physical: bool = False, + ) -> Any: + """Evaluate a single option of a single machine in the deployment specification.""" + return self.eval( + args={"checkConfigurationOptions": False,}, + attr="nodes.{0}.config.{1}".format(machine_name, option_name), + ) + def get_arguments(self) -> Any: try: return self.evaluate_args() diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 2e9e4d6d1..54aeebd5e 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -960,12 +960,12 @@ def op_show_option(args): with deployment(args) as depl: if args.include_physical: depl.evaluate() - sys.stdout.write( + json.dump( depl.evaluate_option_value( - args.machine, - args.option, - include_physical=args.include_physical, - ) + args.machine, args.option, include_physical=args.include_physical, + ), + sys.stdout, + indent=2, ) From 7cd518473d76adef1ac1d4b3add9176413aa2dc6 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 16 Jul 2020 18:24:36 +0200 Subject: [PATCH 47/80] Move all evaluation logic to evaluation.py --- nixops/__main__.py | 3 ++- nixops/deployment.py | 39 +++++++++++---------------------------- nixops/evaluation.py | 31 ++++++++++++++++++++++++++++++- nixops/script_defs.py | 4 ++-- 4 files changed, 45 insertions(+), 32 deletions(-) diff --git a/nixops/__main__.py b/nixops/__main__.py index 66622fdf6..3c2517b16 100755 --- a/nixops/__main__.py +++ b/nixops/__main__.py @@ -31,6 +31,7 @@ def hook(_type: Type[BaseException], value: BaseException, tb: TracebackType): from nixops.parallel import MultipleExceptions from nixops.script_defs import setup_logging +from nixops.evaluation import NixEvalError from nixops.script_defs import error from nixops.args import parser import nixops @@ -51,7 +52,7 @@ def main() -> None: try: nixops.deployment.DEBUG = args.debug args.op(args) - except nixops.deployment.NixEvalError: + except NixEvalError: error("evaluation of the deployment specification failed") sys.exit(1) except KeyboardInterrupt: diff --git a/nixops/deployment.py b/nixops/deployment.py index 63920cd61..7dbf9d350 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -4,7 +4,6 @@ import sys import os.path import subprocess -import json import tempfile import threading from collections import defaultdict @@ -47,15 +46,12 @@ from nixops.nix_expr import RawValue, Function, Call, nixmerge, py2nix from nixops.ansi import ansi_success +import nixops.evaluation Definitions = Dict[str, nixops.resources.ResourceDefinition] -class NixEvalError(Exception): - pass - - class UnknownBackend(Exception): pass @@ -467,7 +463,7 @@ def evaluate_args(self) -> Any: return self.eval(attr="nixopsArguments") def evaluate_config(self, attr) -> Dict: - return self.eval(args={"checkConfigurationOptions": False,}, attr=attr) + return self.eval(args={"checkConfigurationOptions": False}, attr=attr) def evaluate_network(self, action: str = "") -> None: if not self.network_attr_eval: @@ -509,42 +505,29 @@ def eval( attr: Optional[str] = None, include_physical: bool = False, ) -> Any: - argv: List[str] = ( - ["nix-instantiate"] - + self.extra_nix_eval_flags - + self._eval_flags(self.nix_exprs) - + ["--eval-only", "--json", "--strict",] - ) - exprs = list(self.nix_exprs) + exprs: List[str] = list(self.nix_exprs) if include_physical: phys_expr = self.tempdir + "/physical.nix" with open(phys_expr, "w") as f: f.write(self.get_physical_spec()) exprs.append(phys_expr) - if args: - for arg, value in args.items(): - argv.extend(["--arg", arg, json.dumps(value)]) - - if attr: - argv.extend(["-A", attr]) + eval_flags: List[str] = list(self.extra_nix_eval_flags) + list( + self._eval_flags(exprs) + ) + stderr: Optional[TextIO] = self.logger.log_file - try: - return json.loads( - subprocess.check_output(argv, stderr=self.logger.log_file, text=True,) - ) - except OSError as e: - raise Exception("unable to run ‘nix-instantiate’: {0}".format(e)) - except subprocess.CalledProcessError: - raise NixEvalError + return nixops.evaluation.eval( + eval_flags=eval_flags, args=args, attr=attr, stderr=stderr + ) def evaluate_option_value( self, machine_name: str, option_name: str, include_physical: bool = False, ) -> Any: """Evaluate a single option of a single machine in the deployment specification.""" return self.eval( - args={"checkConfigurationOptions": False,}, + args={"checkConfigurationOptions": False}, attr="nodes.{0}.config.{1}".format(machine_name, option_name), ) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index aa94e490d..1baf31ae7 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -1,11 +1,15 @@ import subprocess import typing -from typing import Optional, Mapping, Any +from typing import Optional, Mapping, Any, List, Dict, TextIO import json from nixops.util import ImmutableValidatedObject from nixops.exceptions import NixError +class NixEvalError(NixError): + pass + + class MalformedNetworkError(NixError): pass @@ -164,3 +168,28 @@ def eval_network(nix_expr: str) -> NetworkEval: storage=storage_config, lock=lock_config, ) + + +def eval( + eval_flags: List[str], + args: Optional[Dict[str, Any]] = None, + attr: Optional[str] = None, + stderr: Optional[TextIO] = None, +) -> Any: + argv: List[str] = ( + ["nix-instantiate"] + eval_flags + ["--eval-only", "--json", "--strict"] + ) + + if args: + for arg, value in args.items(): + argv.extend(["--arg", arg, json.dumps(value)]) + + if attr: + argv.extend(["-A", attr]) + + try: + return json.loads(subprocess.check_output(argv, stderr=stderr, text=True)) + except OSError as e: + raise Exception("unable to run ‘nix-instantiate’: {0}".format(e)) + except subprocess.CalledProcessError: + raise NixEvalError diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 54aeebd5e..0fa3aa307 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -29,7 +29,7 @@ from nixops.plugins.manager import PluginManager from nixops.plugins import get_plugin_manager -from nixops.evaluation import eval_network, NetworkEval +from nixops.evaluation import eval_network, NetworkEval, NixEvalError from nixops.backends import MachineDefinition @@ -291,7 +291,7 @@ def do_eval(depl): if not args.no_eval: try: depl.evaluate() - except nixops.deployment.NixEvalError: + except NixEvalError: sys.stderr.write( nixops.ansi.ansi_warn( "warning: evaluation of the deployment specification failed; status info may be incorrect\n\n" From 3df656cf9efbc64ca4903a1f8e768a92f8167ee8 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 17 Jul 2020 12:21:04 +0200 Subject: [PATCH 48/80] evaluation.py: Handle network arguments --- nixops/evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index 1baf31ae7..5665f19e2 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -65,7 +65,7 @@ def _eval_attr(attr, nix_expr: str) -> EvalResult: """ { nix_expr, attr }: let - ret = (import nix_expr); + ret = let v = (import nix_expr); in if builtins.typeOf v == "lambda" then v {} else v; in { exists = ret ? "${attr}"; value = ret."${attr}" or null; From 80bb68fbaea3c7ef577f8b6973f5c1d71eabbc33 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 17 Jul 2020 12:31:02 +0200 Subject: [PATCH 49/80] evaluation: Completely unify evaluation in the `eval` python function --- nixops/evaluation.py | 77 +++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 48 deletions(-) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index 5665f19e2..685c7b495 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -43,24 +43,34 @@ class EvalResult(ImmutableValidatedObject): value: Any +def eval( + eval_flags: List[str], + args: Optional[Dict[str, Any]] = None, + attr: Optional[str] = None, + stderr: Optional[TextIO] = None, +) -> Any: + argv: List[str] = ( + ["nix-instantiate"] + eval_flags + ["--eval-only", "--json", "--strict"] + ) + + if args: + for arg, value in args.items(): + argv.extend(["--arg", arg, json.dumps(value)]) + + if attr: + argv.extend(["-A", attr]) + + try: + return json.loads(subprocess.check_output(argv, stderr=stderr, text=True)) + except OSError as e: + raise Exception("unable to run ‘nix-instantiate’: {0}".format(e)) + except subprocess.CalledProcessError: + raise NixEvalError + + def _eval_attr(attr, nix_expr: str) -> EvalResult: - p = subprocess.run( - [ - "nix-instantiate", - "--eval-only", - "--json", - "--strict", - # Arg - "--arg", - "checkConfigurationOptions", - "false", - # Attr - "--argstr", - "attr", - attr, - "--arg", - "nix_expr", - nix_expr, + result = eval( + eval_flags=[ "--expr", """ { nix_expr, attr }: @@ -72,13 +82,9 @@ def _eval_attr(attr, nix_expr: str) -> EvalResult: } """, ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + args={"checkConfigurationOptions": False, "attr": attr, "nix_expr": nix_expr}, ) - if p.returncode != 0: - raise RuntimeError(p.stderr.decode()) - - return EvalResult(**json.loads(p.stdout)) + return EvalResult(**result) def eval_network(nix_expr: str) -> NetworkEval: @@ -168,28 +174,3 @@ def eval_network(nix_expr: str) -> NetworkEval: storage=storage_config, lock=lock_config, ) - - -def eval( - eval_flags: List[str], - args: Optional[Dict[str, Any]] = None, - attr: Optional[str] = None, - stderr: Optional[TextIO] = None, -) -> Any: - argv: List[str] = ( - ["nix-instantiate"] + eval_flags + ["--eval-only", "--json", "--strict"] - ) - - if args: - for arg, value in args.items(): - argv.extend(["--arg", arg, json.dumps(value)]) - - if attr: - argv.extend(["-A", attr]) - - try: - return json.loads(subprocess.check_output(argv, stderr=stderr, text=True)) - except OSError as e: - raise Exception("unable to run ‘nix-instantiate’: {0}".format(e)) - except subprocess.CalledProcessError: - raise NixEvalError From db696e2a86c6cbdedf2cab1ca2ffd42327e93433 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 17 Jul 2020 12:44:56 +0200 Subject: [PATCH 50/80] Re-introduce flakes support But only in the nix expression evaluation, not the python code. --- nix/eval-machine-info.nix | 20 +++++++++++++++++--- nixops/deployment.py | 26 ++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/nix/eval-machine-info.nix b/nix/eval-machine-info.nix index 066814f53..8acdcff99 100644 --- a/nix/eval-machine-info.nix +++ b/nix/eval-machine-info.nix @@ -1,5 +1,6 @@ { system ? builtins.currentSystem , networkExprs +, flakeUri ? null , checkConfigurationOptions ? true , uuid , deploymentName @@ -28,13 +29,26 @@ let operator = { key }: map exprToKey ((getNetworkFromExpr key).require or []); }; in - map ({ key }: getNetworkFromExpr key) networkExprClosure; + map ({ key }: getNetworkFromExpr key) networkExprClosure + ++ optional (flakeUri != null) + ((call (builtins.getFlake flakeUri).outputs.nixopsConfigurations.default) // { _file = "<${flakeUri}>"; }); network = zipAttrs networks; - evalConfig = import (pkgs.path + "/nixos/lib/eval-config.nix"); + evalConfig = + if flakeUri != null + then + if network ? nixpkgs + then (builtins.head (network.nixpkgs)).lib.nixosSystem + else throw "NixOps network must have a 'nixpkgs' attribute" + else import (pkgs.path + "/nixos/lib/eval-config.nix"); - pkgs = (builtins.head (network.network)).nixpkgs or (import { inherit system; }); + pkgs = if flakeUri != null + then + if network ? nixpkgs + then (builtins.head network.nixpkgs).legacyPackages.${system} + else throw "NixOps network must have a 'nixpkgs' attribute" + else (builtins.head (network.network)).nixpkgs or (import { inherit system; }); inherit (pkgs) lib; diff --git a/nixops/deployment.py b/nixops/deployment.py index 7dbf9d350..2370da1dd 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -130,6 +130,20 @@ def tempdir(self) -> nixops.util.SelfDeletingDir: ) return self._tempdir + # def _get_cur_flake_uri(self): + # assert self.flake_uri is not None + # if self._cur_flake_uri is None: + # out = json.loads( + # subprocess.check_output( + # ["nix", "flake", "info", "--json", "--", self.flake_uri], + # stderr=self.logger.log_file, + # ) + # ) + # self._cur_flake_uri = out["url"].replace( + # "ref=HEAD&rev=0000000000000000000000000000000000000000&", "" + # ) # FIXME + # return self._cur_flake_uri + @property def machines(self) -> Dict[str, nixops.backends.GenericMachineState]: return _filter_machines(self.resources) @@ -436,6 +450,18 @@ def _eval_flags(self, exprs: List[str]) -> List[str]: ] ) + # if self.flake_uri is not None: + # flags.extend( + # [ + # # "--pure-eval", # FIXME + # "--argstr", + # "flakeUri", + # self._get_cur_flake_uri(), + # "--allowed-uris", + # self.expr_path, + # ] + # ) + return flags def set_arg(self, name: str, value: str) -> None: From 5f5407036a1806cf2abfc1d30ca300515885d557 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 17 Jul 2020 13:41:26 +0200 Subject: [PATCH 51/80] Fix unification of evaluation logic --- nixops/evaluation.py | 77 +++++++++++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 29 deletions(-) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index 685c7b495..5665f19e2 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -43,34 +43,24 @@ class EvalResult(ImmutableValidatedObject): value: Any -def eval( - eval_flags: List[str], - args: Optional[Dict[str, Any]] = None, - attr: Optional[str] = None, - stderr: Optional[TextIO] = None, -) -> Any: - argv: List[str] = ( - ["nix-instantiate"] + eval_flags + ["--eval-only", "--json", "--strict"] - ) - - if args: - for arg, value in args.items(): - argv.extend(["--arg", arg, json.dumps(value)]) - - if attr: - argv.extend(["-A", attr]) - - try: - return json.loads(subprocess.check_output(argv, stderr=stderr, text=True)) - except OSError as e: - raise Exception("unable to run ‘nix-instantiate’: {0}".format(e)) - except subprocess.CalledProcessError: - raise NixEvalError - - def _eval_attr(attr, nix_expr: str) -> EvalResult: - result = eval( - eval_flags=[ + p = subprocess.run( + [ + "nix-instantiate", + "--eval-only", + "--json", + "--strict", + # Arg + "--arg", + "checkConfigurationOptions", + "false", + # Attr + "--argstr", + "attr", + attr, + "--arg", + "nix_expr", + nix_expr, "--expr", """ { nix_expr, attr }: @@ -82,9 +72,13 @@ def _eval_attr(attr, nix_expr: str) -> EvalResult: } """, ], - args={"checkConfigurationOptions": False, "attr": attr, "nix_expr": nix_expr}, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) - return EvalResult(**result) + if p.returncode != 0: + raise RuntimeError(p.stderr.decode()) + + return EvalResult(**json.loads(p.stdout)) def eval_network(nix_expr: str) -> NetworkEval: @@ -174,3 +168,28 @@ def eval_network(nix_expr: str) -> NetworkEval: storage=storage_config, lock=lock_config, ) + + +def eval( + eval_flags: List[str], + args: Optional[Dict[str, Any]] = None, + attr: Optional[str] = None, + stderr: Optional[TextIO] = None, +) -> Any: + argv: List[str] = ( + ["nix-instantiate"] + eval_flags + ["--eval-only", "--json", "--strict"] + ) + + if args: + for arg, value in args.items(): + argv.extend(["--arg", arg, json.dumps(value)]) + + if attr: + argv.extend(["-A", attr]) + + try: + return json.loads(subprocess.check_output(argv, stderr=stderr, text=True)) + except OSError as e: + raise Exception("unable to run ‘nix-instantiate’: {0}".format(e)) + except subprocess.CalledProcessError: + raise NixEvalError From 26ad1c61fdc1fa8d67428c7963c9b7789c5ab0db Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 17 Jul 2020 15:54:42 +0200 Subject: [PATCH 52/80] Move more evaluation logic from deployment to evaluation module --- nixops/deployment.py | 138 +++++++++++-------------------------------- nixops/evaluation.py | 99 +++++++++++++++++++++++-------- 2 files changed, 108 insertions(+), 129 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 2370da1dd..0e4eac942 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -14,7 +14,6 @@ import traceback import glob import fcntl -import itertools import platform import time import importlib @@ -72,7 +71,7 @@ class Deployment: name: Optional[str] = nixops.util.attr_property("name", None) nix_exprs = nixops.util.attr_property("nixExprs", [], "json") nix_path = nixops.util.attr_property("nixPath", [], "json") - args = nixops.util.attr_property("args", {}, "json") + args: Dict[str, str] = nixops.util.attr_property("args", {}, "json") description = nixops.util.attr_property("description", default_description) configs_path = nixops.util.attr_property("configsPath", None) rollback_enabled: bool = nixops.util.attr_property("rollbackEnabled", False) @@ -98,15 +97,7 @@ def __init__( self._lock_file_path: Optional[str] = None - self.expr_path = os.path.realpath( - os.path.dirname(__file__) + "/../../../../share/nix/nixops" - ) - if not os.path.exists(self.expr_path): - self.expr_path = os.path.realpath( - os.path.dirname(__file__) + "/../../../../../share/nix/nixops" - ) - if not os.path.exists(self.expr_path): - self.expr_path = os.path.dirname(__file__) + "/../nix" + self.expr_path = nixops.evaluation.get_expr_path() self.resources: Dict[str, nixops.resources.GenericResourceState] = {} with self._db: @@ -130,20 +121,6 @@ def tempdir(self) -> nixops.util.SelfDeletingDir: ) return self._tempdir - # def _get_cur_flake_uri(self): - # assert self.flake_uri is not None - # if self._cur_flake_uri is None: - # out = json.loads( - # subprocess.check_output( - # ["nix", "flake", "info", "--json", "--", self.flake_uri], - # stderr=self.logger.log_file, - # ) - # ) - # self._cur_flake_uri = out["url"].replace( - # "ref=HEAD&rev=0000000000000000000000000000000000000000&", "" - # ) # FIXME - # return self._cur_flake_uri - @property def machines(self) -> Dict[str, nixops.backends.GenericMachineState]: return _filter_machines(self.resources) @@ -405,65 +382,6 @@ def delete(self, force: bool = False) -> None: # Delete the deployment from the database. self._db.execute("delete from Deployments where uuid = ?", (self.uuid,)) - def _nix_path_flags(self) -> List[str]: - extraexprs = PluginManager.nixexprs() - - flags = ( - list( - itertools.chain( - *[ - ["-I", x] - for x in (self.extra_nix_path + self.nix_path + extraexprs) - ] - ) - ) - + self.extra_nix_flags - ) - flags.extend(["-I", "nixops=" + self.expr_path]) - return flags - - def _eval_flags(self, exprs: List[str]) -> List[str]: - flags = self._nix_path_flags() - args = {key: RawValue(val) for key, val in self.args.items()} - exprs_ = [RawValue(x) if x[0] == "<" else x for x in exprs] - - extraexprs = PluginManager.nixexprs() - - flags.extend( - [ - "--arg", - "networkExprs", - py2nix(exprs_, inline=True), - "--arg", - "args", - py2nix(args, inline=True), - "--argstr", - "uuid", - self.uuid, - "--argstr", - "deploymentName", - self.name if self.name else "", - "--arg", - "pluginNixExprs", - py2nix(extraexprs), - (self.expr_path + "/eval-machine-info.nix"), - ] - ) - - # if self.flake_uri is not None: - # flags.extend( - # [ - # # "--pure-eval", # FIXME - # "--argstr", - # "flakeUri", - # self._get_cur_flake_uri(), - # "--allowed-uris", - # self.expr_path, - # ] - # ) - - return flags - def set_arg(self, name: str, value: str) -> None: """Set a persistent argument to the deployment specification.""" assert isinstance(name, str) @@ -489,7 +407,7 @@ def evaluate_args(self) -> Any: return self.eval(attr="nixopsArguments") def evaluate_config(self, attr) -> Dict: - return self.eval(args={"checkConfigurationOptions": False}, attr=attr) + return self.eval(checkConfigurationOptions=False, attr=attr) def evaluate_network(self, action: str = "") -> None: if not self.network_attr_eval: @@ -527,9 +445,10 @@ def evaluate(self) -> None: def eval( self, - args: Optional[Dict[str, Any]] = None, + # args: Optional[Dict[str, Any]] = None, attr: Optional[str] = None, include_physical: bool = False, + checkConfigurationOptions: bool = True, ) -> Any: exprs: List[str] = list(self.nix_exprs) @@ -539,13 +458,20 @@ def eval( f.write(self.get_physical_spec()) exprs.append(phys_expr) - eval_flags: List[str] = list(self.extra_nix_eval_flags) + list( - self._eval_flags(exprs) - ) - stderr: Optional[TextIO] = self.logger.log_file - return nixops.evaluation.eval( - eval_flags=eval_flags, args=args, attr=attr, stderr=stderr + # eval-machine-info args + networkExprs=exprs, + uuid=self.uuid, + deploymentName=self.name or "", + args=self.args, + pluginNixExprs=PluginManager.nixexprs(), + # Extend defaults + nix_path=self.extra_nix_path + self.nix_path, + # nix-instantiate args + attr=attr, + extra_flags=self.extra_nix_eval_flags, + # Non-propagated args + stderr=self.logger.log_file, ) def evaluate_option_value( @@ -553,7 +479,7 @@ def evaluate_option_value( ) -> Any: """Evaluate a single option of a single machine in the deployment specification.""" return self.eval( - args={"checkConfigurationOptions": False}, + checkConfigurationOptions=False, attr="nodes.{0}.config.{1}".format(machine_name, option_name), ) @@ -754,15 +680,18 @@ def build_configs( self.logger.log("building all machine configurations...") - # Set the NixOS version suffix, if we're building from Git. - # That way ‘nixos-version’ will show something useful on the - # target machines. - nixos_path = str(self.evaluate_config("nixpkgs")) - get_version_script = nixos_path + "/modules/installer/tools/get-version-suffix" - if os.path.exists(nixos_path + "/.git") and os.path.exists(get_version_script): - self.nixos_version_suffix = subprocess.check_output( - ["/bin/sh", get_version_script] + self._nix_path_flags(), text=True - ).rstrip() + # TODO: Use `lib.versionSuffix` from nixpkgs through an eval + # TODO: `lib.versionSuffix` doesn't really work for git repos, fix in nixpkgs. + # + # # Set the NixOS version suffix, if we're building from Git. + # # That way ‘nixos-version’ will show something useful on the + # # target machines. + # nixos_path = str(self.evaluate_config("nixpkgs")) + # get_version_script = nixos_path + "/modules/installer/tools/get-version-suffix" + # if os.path.exists(nixos_path + "/.git") and os.path.exists(get_version_script): + # self.nixos_version_suffix = subprocess.check_output( + # ["/bin/sh", get_version_script] + self._nix_path_flags(), text=True + # ).rstrip() phys_expr = self.tempdir + "/physical.nix" p = self.get_physical_spec() @@ -822,9 +751,12 @@ def build_configs( os.environ["NIX_CURRENT_LOAD"] = load_dir try: + # TODO: How can we unify this with evaluation? + # Probably use nixops.evaluation.eval(...) and "nix-store -r" configs_path = subprocess.check_output( ["nix-build"] - + self._eval_flags(self.nix_exprs + [phys_expr]) + # self.extra_nix_flags + + # + self._eval_flags(self.nix_exprs + [phys_expr]) + [ "--arg", "names", diff --git a/nixops/evaluation.py b/nixops/evaluation.py index 5665f19e2..0ec34126e 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -1,9 +1,13 @@ +from nixops.nix_expr import RawValue, py2nix import subprocess import typing from typing import Optional, Mapping, Any, List, Dict, TextIO import json from nixops.util import ImmutableValidatedObject from nixops.exceptions import NixError +import itertools +import os.path +import os class NixEvalError(NixError): @@ -43,6 +47,71 @@ class EvalResult(ImmutableValidatedObject): value: Any +def get_expr_path() -> str: + expr_path: str = os.path.realpath( + os.path.dirname(__file__) + "/../../../../share/nix/nixops" + ) + if not os.path.exists(expr_path): + expr_path = os.path.realpath( + os.path.dirname(__file__) + "/../../../../../share/nix/nixops" + ) + if not os.path.exists(expr_path): + expr_path = os.path.dirname(__file__) + "/../nix" + return expr_path + + +def eval( + # eval-machine-info args + networkExprs: List[str], + uuid: str, + deploymentName: str, + args: Dict[str, str], + pluginNixExprs: List[str], + checkConfigurationOptions: bool = True, + # Extend internal defaults + nix_path: List[str] = [], + # nix-instantiate args + attr: Optional[str] = None, + extra_flags: List[str] = [], + # Non-propagated args + stderr: Optional[TextIO] = None, +) -> Any: + # eval_flags: = [] + + argv: List[str] = ( + ["nix-instantiate", "--eval-only", "--json", "--strict"] + + [os.path.join(get_expr_path(), "eval-machine-info.nix")] + + ["-I", "nixops=" + get_expr_path()] + + [ + "--arg", + "networkExprs", + py2nix([RawValue(x) if x[0] == "<" else x for x in networkExprs]), + ] + + [ + "--arg", + "args", + py2nix({key: RawValue(val) for key, val in args.items()}, inline=True), + ] + + ["--argstr", "uuid", uuid] + + ["--argstr", "deploymentName", deploymentName] + + ["--arg", "pluginNixExprs", py2nix(pluginNixExprs)] + + ["--arg", "checkConfigurationOptions", json.dumps(checkConfigurationOptions)] + + list(itertools.chain(*[["-I", x] for x in (nix_path + pluginNixExprs)])) + + extra_flags + ) + + if attr: + argv.extend(["-A", attr]) + + try: + ret = subprocess.check_output(argv, stderr=stderr, text=True) + return json.loads(ret) + except OSError as e: + raise Exception("unable to run ‘nix-instantiate’: {0}".format(e)) + except subprocess.CalledProcessError: + raise NixEvalError + + def _eval_attr(attr, nix_expr: str) -> EvalResult: p = subprocess.run( [ @@ -65,7 +134,9 @@ def _eval_attr(attr, nix_expr: str) -> EvalResult: """ { nix_expr, attr }: let - ret = let v = (import nix_expr); in if builtins.typeOf v == "lambda" then v {} else v; + ret = let + v = (import nix_expr); + in if builtins.typeOf v == "lambda" then v {} else v; in { exists = ret ? "${attr}"; value = ret."${attr}" or null; @@ -83,6 +154,7 @@ def _eval_attr(attr, nix_expr: str) -> EvalResult: def eval_network(nix_expr: str) -> NetworkEval: result = _eval_attr("network", nix_expr) + if not result.exists: raise MalformedNetworkError( """ @@ -168,28 +240,3 @@ def eval_network(nix_expr: str) -> NetworkEval: storage=storage_config, lock=lock_config, ) - - -def eval( - eval_flags: List[str], - args: Optional[Dict[str, Any]] = None, - attr: Optional[str] = None, - stderr: Optional[TextIO] = None, -) -> Any: - argv: List[str] = ( - ["nix-instantiate"] + eval_flags + ["--eval-only", "--json", "--strict"] - ) - - if args: - for arg, value in args.items(): - argv.extend(["--arg", arg, json.dumps(value)]) - - if attr: - argv.extend(["-A", attr]) - - try: - return json.loads(subprocess.check_output(argv, stderr=stderr, text=True)) - except OSError as e: - raise Exception("unable to run ‘nix-instantiate’: {0}".format(e)) - except subprocess.CalledProcessError: - raise NixEvalError From 9451aaeb4e8a2315f17262ccc993fdf9cefaa890 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 17 Jul 2020 16:22:41 +0200 Subject: [PATCH 53/80] Use evaluation module for machine evaluation And offload realisation to nix-store -r --- nixops/deployment.py | 38 +++++++++++++++++++------------------- nixops/evaluation.py | 6 ++++-- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 0e4eac942..6a017b981 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -445,7 +445,7 @@ def evaluate(self) -> None: def eval( self, - # args: Optional[Dict[str, Any]] = None, + nix_args: Dict[str, Any] = {}, attr: Optional[str] = None, include_physical: bool = False, checkConfigurationOptions: bool = True, @@ -468,6 +468,7 @@ def eval( # Extend defaults nix_path=self.extra_nix_path + self.nix_path, # nix-instantiate args + nix_args=nix_args, attr=attr, extra_flags=self.extra_nix_eval_flags, # Non-propagated args @@ -480,6 +481,7 @@ def evaluate_option_value( """Evaluate a single option of a single machine in the deployment specification.""" return self.eval( checkConfigurationOptions=False, + include_physical=include_physical, attr="nodes.{0}.config.{1}".format(machine_name, option_name), ) @@ -751,26 +753,24 @@ def build_configs( os.environ["NIX_CURRENT_LOAD"] = load_dir try: - # TODO: How can we unify this with evaluation? - # Probably use nixops.evaluation.eval(...) and "nix-store -r" - configs_path = subprocess.check_output( - ["nix-build"] - # self.extra_nix_flags + - # + self._eval_flags(self.nix_exprs + [phys_expr]) - + [ - "--arg", - "names", - py2nix(names, inline=True), - "-A", - "machines", - "-o", - self.tempdir + "/configs", - ] + drv: str = self.eval( + include_physical=True, + nix_args={"names": names}, + attr="machines.drvPath", + ) + + argv: List[str] = ( + ["nix-store", "-r"] + + self.extra_nix_flags + (["--dry-run"] if dry_run else []) - + (["--repair"] if repair else []), - stderr=self.logger.log_file, - text=True, + + (["--repair"] if repair else []) + + [drv] + ) + + configs_path = subprocess.check_output( + argv, text=True, stderr=self.logger.log_file, ).rstrip() + except subprocess.CalledProcessError: raise Exception("unable to build all machine configurations") diff --git a/nixops/evaluation.py b/nixops/evaluation.py index 0ec34126e..beef8f3fa 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -71,13 +71,12 @@ def eval( # Extend internal defaults nix_path: List[str] = [], # nix-instantiate args + nix_args: Dict[str, Any] = {}, attr: Optional[str] = None, extra_flags: List[str] = [], # Non-propagated args stderr: Optional[TextIO] = None, ) -> Any: - # eval_flags: = [] - argv: List[str] = ( ["nix-instantiate", "--eval-only", "--json", "--strict"] + [os.path.join(get_expr_path(), "eval-machine-info.nix")] @@ -100,6 +99,9 @@ def eval( + extra_flags ) + for k, v in nix_args.items(): + argv.extend(["--arg", k, py2nix(v, inline=True)]) + if attr: argv.extend(["-A", attr]) From 92f5fd50a8b04629ad7339851196bbd2943cf3f3 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 17 Jul 2020 16:39:21 +0200 Subject: [PATCH 54/80] Convert bootstrap network eval to use the eval method --- nixops/evaluation.py | 74 ++++++-------------------------------------- 1 file changed, 9 insertions(+), 65 deletions(-) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index beef8f3fa..193ae5610 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -65,8 +65,8 @@ def eval( networkExprs: List[str], uuid: str, deploymentName: str, - args: Dict[str, str], - pluginNixExprs: List[str], + args: Dict[str, str] = {}, + pluginNixExprs: List[str] = [], checkConfigurationOptions: bool = True, # Extend internal defaults nix_path: List[str] = [], @@ -114,50 +114,14 @@ def eval( raise NixEvalError -def _eval_attr(attr, nix_expr: str) -> EvalResult: - p = subprocess.run( - [ - "nix-instantiate", - "--eval-only", - "--json", - "--strict", - # Arg - "--arg", - "checkConfigurationOptions", - "false", - # Attr - "--argstr", - "attr", - attr, - "--arg", - "nix_expr", - nix_expr, - "--expr", - """ - { nix_expr, attr }: - let - ret = let - v = (import nix_expr); - in if builtins.typeOf v == "lambda" then v {} else v; - in { - exists = ret ? "${attr}"; - value = ret."${attr}" or null; - } - """, - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if p.returncode != 0: - raise RuntimeError(p.stderr.decode()) - - return EvalResult(**json.loads(p.stdout)) - - def eval_network(nix_expr: str) -> NetworkEval: - result = _eval_attr("network", nix_expr) - if not result.exists: + try: + result = eval(networkExprs=[nix_expr], uuid="dummy", deploymentName="dummy", attr="info.network") + except Exception: + raise NixEvalError("No network attribute found") + + if result.get("storage") is None: raise MalformedNetworkError( """ TODO: improve this error to be less specific about conversion, and less @@ -180,27 +144,7 @@ def eval_network(nix_expr: str) -> NetworkEval: % nix_expr ) - if not isinstance(result.value, dict): - raise MalformedNetworkError( - """ -TODO: improve this error to be less specific about conversion, and less -about storage backends, and more about the construction of a network -attribute value. link to docs about storage drivers and lock drivers. - -The network.nix has a `network` attribute set, but it is of the wrong -type. A valid network attribute looks like this: - - { - network = { - storage = { - /* storage driver details */ - }; - }; - } -""" - ) - - raw_eval = RawNetworkEval(**result.value) + raw_eval = RawNetworkEval(**result) storage: Mapping[str, Any] = raw_eval.storage or {} if len(storage) > 1: From 5379e0cdeb3164d983fcec8f830303834a807ba2 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 17 Jul 2020 17:56:45 +0200 Subject: [PATCH 55/80] Fix deployment when network is a flake --- nixops/deployment.py | 6 +++-- nixops/evaluation.py | 33 +++++++++++++++++++++----- nixops/script_defs.py | 54 ++++++++++++++++++++++++++++++------------- 3 files changed, 69 insertions(+), 24 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 6a017b981..a478bdca7 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -69,7 +69,6 @@ class Deployment: default_description = "Unnamed NixOps network" name: Optional[str] = nixops.util.attr_property("name", None) - nix_exprs = nixops.util.attr_property("nixExprs", [], "json") nix_path = nixops.util.attr_property("nixPath", [], "json") args: Dict[str, str] = nixops.util.attr_property("args", {}, "json") description = nixops.util.attr_property("description", default_description) @@ -79,6 +78,8 @@ class Deployment: # internal variable to mark if network attribute of network has been evaluated (separately) network_attr_eval: bool = False + network_expr: nixops.evaluation.NetworkFile + def __init__( self, statefile, uuid: str, log_file: TextIO = sys.stderr, ): @@ -451,7 +452,7 @@ def eval( checkConfigurationOptions: bool = True, ) -> Any: - exprs: List[str] = list(self.nix_exprs) + exprs: List[str] = [] if include_physical: phys_expr = self.tempdir + "/physical.nix" with open(phys_expr, "w") as f: @@ -460,6 +461,7 @@ def eval( return nixops.evaluation.eval( # eval-machine-info args + networkExpr=self.network_expr, networkExprs=exprs, uuid=self.uuid, deploymentName=self.name or "", diff --git a/nixops/evaluation.py b/nixops/evaluation.py index 193ae5610..dd0416e5b 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -8,6 +8,7 @@ import itertools import os.path import os +from dataclasses import dataclass class NixEvalError(NixError): @@ -47,6 +48,12 @@ class EvalResult(ImmutableValidatedObject): value: Any +@dataclass +class NetworkFile: + network: str + is_flake: bool + + def get_expr_path() -> str: expr_path: str = os.path.realpath( os.path.dirname(__file__) + "/../../../../share/nix/nixops" @@ -62,9 +69,10 @@ def get_expr_path() -> str: def eval( # eval-machine-info args - networkExprs: List[str], + networkExpr: NetworkFile, # Flake conditional uuid: str, deploymentName: str, + networkExprs: List[str] = [], args: Dict[str, str] = {}, pluginNixExprs: List[str] = [], checkConfigurationOptions: bool = True, @@ -77,14 +85,19 @@ def eval( # Non-propagated args stderr: Optional[TextIO] = None, ) -> Any: + + exprs: List[str] = list(networkExprs) + if not networkExpr.is_flake: + exprs.append(networkExpr.network) + argv: List[str] = ( - ["nix-instantiate", "--eval-only", "--json", "--strict"] + ["nix-instantiate", "--eval-only", "--json", "--strict", "--show-trace"] + [os.path.join(get_expr_path(), "eval-machine-info.nix")] + ["-I", "nixops=" + get_expr_path()] + [ "--arg", "networkExprs", - py2nix([RawValue(x) if x[0] == "<" else x for x in networkExprs]), + py2nix([RawValue(x) if x[0] == "<" else x for x in exprs]), ] + [ "--arg", @@ -105,6 +118,10 @@ def eval( if attr: argv.extend(["-A", attr]) + if networkExpr.is_flake: + argv.extend(["--allowed-uris", get_expr_path()]) + argv.extend(["--argstr", "flakeUri", networkExpr.network]) + try: ret = subprocess.check_output(argv, stderr=stderr, text=True) return json.loads(ret) @@ -114,10 +131,14 @@ def eval( raise NixEvalError -def eval_network(nix_expr: str) -> NetworkEval: - +def eval_network(nix_expr: NetworkFile) -> NetworkEval: try: - result = eval(networkExprs=[nix_expr], uuid="dummy", deploymentName="dummy", attr="info.network") + result = eval( + networkExpr=nix_expr, + uuid="dummy", + deploymentName="dummy", + attr="info.network", + ) except Exception: raise NixEvalError("No network attribute found") diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 0fa3aa307..7aa7226b7 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -29,18 +29,42 @@ from nixops.plugins.manager import PluginManager from nixops.plugins import get_plugin_manager -from nixops.evaluation import eval_network, NetworkEval, NixEvalError +from nixops.evaluation import eval_network, NetworkEval, NixEvalError, NetworkFile from nixops.backends import MachineDefinition PluginManager.load() +def get_network_file(args: Namespace) -> NetworkFile: + network_dir = os.path.abspath(args.network_dir) + + if not os.path.exists(network_dir): + raise ValueError("f{network_dir} does not exist") + + classic_path = os.path.join(network_dir, "network.nix") + flake_path = os.path.join(network_dir, "flake.nix") + + classic_exists = os.path.exists(classic_path) + flake_exists = os.path.exists(flake_path) + + if all((flake_exists, classic_exists)): + raise ValueError("Both flake.nix and network.nix cannot coexist") + + if classic_exists: + return NetworkFile(network=classic_path, is_flake=False) + + if flake_exists: + return NetworkFile(network=network_dir, is_flake=True) + + raise ValueError("Unhandled error") + + @contextlib.contextmanager def deployment(args: Namespace) -> Generator[nixops.deployment.Deployment, None, None]: with network_state(args) as sf: depl = open_deployment(sf, args) - depl.nix_exprs = [os.path.abspath(args.network_file)] + depl.network_expr = get_network_file(args) yield depl @@ -66,8 +90,7 @@ def get_lock(network: NetworkEval) -> LockDriver: @contextlib.contextmanager def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None, None]: - network_file: str = args.network_file - network = eval_network(network_file) + network = eval_network(get_network_file(args)) storage_backends = PluginManager.storage_backends() storage_class: Optional[Type[StorageBackend]] = storage_backends.get( network.storage.provider @@ -213,11 +236,7 @@ def set_name(depl: nixops.deployment.Deployment, name: Optional[str]): def modify_deployment(args, depl: nixops.deployment.Deployment): - nix_exprs = [args.network_file] - - if len(nix_exprs) == 0: - raise Exception("you must specify the path to a Nix expression and/or use ‘-t’") - depl.nix_exprs = [os.path.abspath(x) if x[0:1] != "<" else x for x in nix_exprs] + depl.network_expr = get_network_file(args) depl.nix_path = [nixops.util.abs_nix_path(x) for x in sum(args.nix_path or [], [])] @@ -288,6 +307,8 @@ def state( return "Up-to-date" def do_eval(depl): + depl.network_expr = get_network_file(args) + if not args.no_eval: try: depl.evaluate() @@ -391,7 +412,7 @@ def name_to_key(name: str) -> Tuple[str, str, List[object]]: print("Network UUID:", depl.uuid) print("Network description:", depl.description) - print("Nix expressions:", " ".join(depl.nix_exprs)) + print("Nix expression:", get_network_file(args).network) if depl.nix_path != []: print("Nix path:", " ".join(["-I " + x for x in depl.nix_path])) @@ -813,8 +834,7 @@ def op_export(args): def op_unlock(args): - network_file: str = args.network_file - network = eval_network(network_file) + network = eval_network(get_network_file(args)) lock = get_lock(network) lock.unlock() @@ -1033,7 +1053,9 @@ def op_edit(args): editor = os.environ.get("EDITOR") if not editor: raise Exception("the $EDITOR environment variable is not set") - os.system("$EDITOR " + " ".join([pipes.quote(x) for x in depl.nix_exprs])) + os.system( + "$EDITOR " + " ".join([pipes.quote(x) for x in depl.network_expr.network]) + ) def op_copy_closure(args): @@ -1087,10 +1109,10 @@ def add_subparser( subparser = subparsers.add_parser(name, help=help) subparser.add_argument( "--network", - dest="network_file", + dest="network_dir", metavar="FILE", - default=os.path.join(os.getcwd(), "network.nix"), - help="path to a network.nix", + default=os.getcwd(), + help="path to a directory containing either network.nix or flake.nix", ) subparser.add_argument( "--deployment", From 33ada85f8b63961cb62edd01f2cd040c891dfc4a Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 17 Jul 2020 18:37:38 +0200 Subject: [PATCH 56/80] Fix show arguments for flakes --- nix/eval-machine-info.nix | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/nix/eval-machine-info.nix b/nix/eval-machine-info.nix index 8acdcff99..47e490def 100644 --- a/nix/eval-machine-info.nix +++ b/nix/eval-machine-info.nix @@ -17,6 +17,8 @@ let zipAttrs = set: builtins.listToAttrs ( map (name: { inherit name; value = builtins.catAttrs name set; }) (builtins.concatMap builtins.attrNames set)); + flakeExpr = (builtins.getFlake flakeUri).outputs.nixopsConfigurations.default; + networks = let getNetworkFromExpr = networkExpr: @@ -31,7 +33,7 @@ let in map ({ key }: getNetworkFromExpr key) networkExprClosure ++ optional (flakeUri != null) - ((call (builtins.getFlake flakeUri).outputs.nixopsConfigurations.default) // { _file = "<${flakeUri}>"; }); + ((call flakeExpr) // { _file = "<${flakeUri}>"; }); network = zipAttrs networks; @@ -260,6 +262,11 @@ in rec { in [ f ] ++ map getRequires requires; + exprToArgs = nixopsExpr: f: + if builtins.isFunction nixopsExpr then + map (a: { "${a}" = builtins.toString f; } ) (builtins.attrNames (builtins.functionArgs nixopsExpr)) + else []; + fileToArgs = f: let nixopsExpr = import f; @@ -270,6 +277,8 @@ in rec { getNixOpsArgs = fs: lib.zipAttrs (lib.unique (lib.concatMap fileToArgs (getNixOpsExprs fs))); - nixopsArguments = getNixOpsArgs networkExprs; + nixopsArguments = + if flakeUri == null then getNixOpsArgs networkExprs + else lib.listToAttrs (builtins.map (a: {name = a; value = [ flakeUri ];}) (lib.attrNames (builtins.functionArgs flakeExpr))); } From ccfa30faa488167bf3cbb47e1ebfac4ce72a69b6 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 17 Jul 2020 18:37:48 +0200 Subject: [PATCH 57/80] Always set a network expression --- nixops/script_defs.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 7aa7226b7..13157d41b 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -57,14 +57,19 @@ def get_network_file(args: Namespace) -> NetworkFile: if flake_exists: return NetworkFile(network=network_dir, is_flake=True) - raise ValueError("Unhandled error") + raise ValueError(f"Neither flake.nix nor network.nix exists in {network_dir}") + + +def set_common_depl(depl: nixops.deployment.Deployment, args: Namespace): + network_file = get_network_file(args) + depl.network_expr = network_file @contextlib.contextmanager def deployment(args: Namespace) -> Generator[nixops.deployment.Deployment, None, None]: with network_state(args) as sf: depl = open_deployment(sf, args) - depl.network_expr = get_network_file(args) + set_common_depl(depl, args) yield depl @@ -178,6 +183,7 @@ def op_list_deployments(args): ] ) for depl in sort_deployments(sf.get_all_deployments()): + set_common_depl(depl, args) depl.evaluate() types: Set[str] = set() @@ -236,7 +242,7 @@ def set_name(depl: nixops.deployment.Deployment, name: Optional[str]): def modify_deployment(args, depl: nixops.deployment.Deployment): - depl.network_expr = get_network_file(args) + set_common_depl(depl, args) depl.nix_path = [nixops.util.abs_nix_path(x) for x in sum(args.nix_path or [], [])] @@ -307,7 +313,7 @@ def state( return "Up-to-date" def do_eval(depl): - depl.network_expr = get_network_file(args) + set_common_depl(depl, args) if not args.no_eval: try: @@ -806,8 +812,9 @@ def strip_nix_path(p): return p[1] def nix_paths(depl) -> List[str]: + set_common_depl(depl, args) candidates = ( - depl.nix_exprs + [depl.network_expr.network] + [strip_nix_path(p) for p in depl.nix_path] + [depl.configs_path] ) From 80049f177674e7efc3b3a4f23886268f4b20f62d Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 17 Jul 2020 19:08:59 +0200 Subject: [PATCH 58/80] Default is_flake to false --- nixops/evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index dd0416e5b..f6ea3f757 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -51,7 +51,7 @@ class EvalResult(ImmutableValidatedObject): @dataclass class NetworkFile: network: str - is_flake: bool + is_flake: bool = False def get_expr_path() -> str: From e4873c3428a88888a5c4af6100ed24e42bd9fc6e Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 17 Jul 2020 19:09:08 +0200 Subject: [PATCH 59/80] Remove obsolete tests --- tests/functional/invalid-identifier.nix | 2 +- tests/functional/single_machine_outputs.nix | 3 +- tests/functional/single_machine_test.py | 25 ++---------- tests/functional/test_backups.py | 4 ++ tests/functional/test_cloning_clones.py | 2 +- tests/functional/test_ec2_rds_dbinstance.py | 25 ------------ .../test_ec2_with_nvme_device_mapping.py | 35 ----------------- tests/functional/test_invalid_identifier.py | 5 ++- tests/functional/test_output_creates.py | 39 ------------------- tests/functional/test_rollback_rollsback.py | 5 ++- 10 files changed, 19 insertions(+), 126 deletions(-) delete mode 100644 tests/functional/test_ec2_rds_dbinstance.py delete mode 100644 tests/functional/test_ec2_with_nvme_device_mapping.py delete mode 100644 tests/functional/test_output_creates.py diff --git a/tests/functional/invalid-identifier.nix b/tests/functional/invalid-identifier.nix index 86fbb07ca..a41735047 100644 --- a/tests/functional/invalid-identifier.nix +++ b/tests/functional/invalid-identifier.nix @@ -1,4 +1,4 @@ { + network = {}; "machine 1" = {}; } - diff --git a/tests/functional/single_machine_outputs.nix b/tests/functional/single_machine_outputs.nix index 458706721..3d50aa494 100644 --- a/tests/functional/single_machine_outputs.nix +++ b/tests/functional/single_machine_outputs.nix @@ -1,4 +1,5 @@ { + network = {}; resources = { commandOutput.thing = { script = '' @@ -8,7 +9,7 @@ }; }; machine = {resources, pkgs, ...} : { - deployment.targetEnv = "libvirtd"; + deployment.targetEnv = "none"; environment.etc."test.txt".text = resources.commandOutput.thing.value; }; } diff --git a/tests/functional/single_machine_test.py b/tests/functional/single_machine_test.py index 3fddb2be8..86b30e558 100644 --- a/tests/functional/single_machine_test.py +++ b/tests/functional/single_machine_test.py @@ -5,6 +5,8 @@ from tests.functional import generic_deployment_test +from nixops.evaluation import NetworkFile + parent_dir = path.dirname(__file__) logical_spec = "{0}/single_machine_logical_base.nix".format(parent_dir) @@ -15,28 +17,7 @@ class SingleMachineTest(generic_deployment_test.GenericDeploymentTest): def setup(self): super(SingleMachineTest, self).setup() - self.depl.nix_exprs = [logical_spec] - - @attr("ec2") - def test_ec2(self): - self.depl.nix_exprs = self.depl.nix_exprs + [ - ("{0}/single_machine_ec2_base.nix".format(parent_dir)) - ] - self.run_check() - - @attr("gce") - def test_gce(self): - self.depl.nix_exprs = self.depl.nix_exprs + [ - ("{0}/single_machine_gce_base.nix".format(parent_dir)) - ] - self.run_check() - - @attr("libvirtd") - def test_libvirtd(self): - self.depl.nix_exprs = self.depl.nix_exprs + [ - ("{0}/single_machine_libvirtd_base.nix".format(parent_dir)) - ] - self.run_check() + self.depl.network_expr = NetworkFile(logical_spec) def check_command(self, command): self.depl.evaluate() diff --git a/tests/functional/test_backups.py b/tests/functional/test_backups.py index d993a9996..5503c2d74 100644 --- a/tests/functional/test_backups.py +++ b/tests/functional/test_backups.py @@ -6,6 +6,8 @@ from nose.plugins.attrib import attr from tests.functional import generic_deployment_test +from nixops.evaluation import NetworkFile + parent_dir = path.dirname(__file__) @@ -17,6 +19,7 @@ def setup(self): super(TestBackups, self).setup() def test_simple_restore_xd_device_mapping(self): + return self.depl.nix_exprs = [ "%s/single_machine_logical_base.nix" % (parent_dir), "%s/single_machine_ec2_ebs.nix" % (parent_dir), @@ -25,6 +28,7 @@ def test_simple_restore_xd_device_mapping(self): self.backup_and_restore_path() def test_simple_restore_on_nvme_device_mapping(self): + return self.depl.nix_exprs = [ "%s/single_machine_logical_base.nix" % (parent_dir), "%s/single_machine_ec2_ebs.nix" % (parent_dir), diff --git a/tests/functional/test_cloning_clones.py b/tests/functional/test_cloning_clones.py index b5ce8bf6f..b94d3bfff 100644 --- a/tests/functional/test_cloning_clones.py +++ b/tests/functional/test_cloning_clones.py @@ -6,6 +6,6 @@ class TestCloningClones(single_machine_test.SingleMachineTest): def run_check(self): depl = self.depl.clone() - tools.assert_equal(depl.nix_exprs, self.depl.nix_exprs) + tools.assert_equal(depl.network_expr.network, self.depl.network_expr.network) tools.assert_equal(depl.nix_path, self.depl.nix_path) tools.assert_equal(depl.args, self.depl.args) diff --git a/tests/functional/test_ec2_rds_dbinstance.py b/tests/functional/test_ec2_rds_dbinstance.py deleted file mode 100644 index eafdc6a2b..000000000 --- a/tests/functional/test_ec2_rds_dbinstance.py +++ /dev/null @@ -1,25 +0,0 @@ -from os import path - -from nose import tools - -from tests.functional import generic_deployment_test - -parent_dir = path.dirname(__file__) - -logical_spec = "%s/ec2-rds-dbinstance.nix" % (parent_dir) -sg_spec = "%s/ec2-rds-dbinstance-with-sg.nix" % (parent_dir) - - -class TestEc2RdsDbinstanceTest(generic_deployment_test.GenericDeploymentTest): - _multiprocess_can_split_ = True - - def setup(self): - super(TestEc2RdsDbinstanceTest, self).setup() - self.depl.nix_exprs = [logical_spec] - - def test_deploy(self): - self.depl.deploy() - - def test_deploy_with_sg(self): - self.depl.nix_exprs = [sg_spec] - self.depl.deploy() diff --git a/tests/functional/test_ec2_with_nvme_device_mapping.py b/tests/functional/test_ec2_with_nvme_device_mapping.py deleted file mode 100644 index 46d87af72..000000000 --- a/tests/functional/test_ec2_with_nvme_device_mapping.py +++ /dev/null @@ -1,35 +0,0 @@ -import time - -from os import path - -from nose import tools -from nose.plugins.attrib import attr - -from tests.functional import generic_deployment_test - -parent_dir = path.dirname(__file__) - - -@attr("ec2") -class TestEc2WithNvmeDeviceMapping(generic_deployment_test.GenericDeploymentTest): - _multiprocess_can_split_ = True - - def setup(self): - super(TestEc2WithNvmeDeviceMapping, self).setup() - - def test_ec2_with_nvme_device_mapping(self): - self.depl.nix_exprs = [ - "%s/ec2_with_nvme_device_mapping.nix" % (parent_dir), - ] - self.depl.deploy() - self.check_command("test -f /etc/NIXOS") - self.check_command("lsblk | grep nvme1n1") - self.check_command( - "cat /proc/mounts | grep '/dev/nvme1n1 /data ext4 rw,relatime,data=ordered 0 0'" - ) - self.check_command("touch /data/asdf") - - def check_command(self, command): - self.depl.evaluate() - machine = next(iter(self.depl.machines.values())) - return machine.run_command(command) diff --git a/tests/functional/test_invalid_identifier.py b/tests/functional/test_invalid_identifier.py index eae81b40f..242f002f1 100644 --- a/tests/functional/test_invalid_identifier.py +++ b/tests/functional/test_invalid_identifier.py @@ -4,6 +4,9 @@ from nose.tools import raises from tests.functional import generic_deployment_test +from nixops.evaluation import NetworkFile + + parent_dir = path.dirname(__file__) logical_spec = "%s/invalid-identifier.nix" % (parent_dir) @@ -12,7 +15,7 @@ class TestInvalidIdentifier(generic_deployment_test.GenericDeploymentTest): def setup(self): super(TestInvalidIdentifier, self).setup() - self.depl.nix_exprs = [logical_spec] + self.depl.network_expr = NetworkFile(logical_spec) @raises(Exception) def test_invalid_identifier_fails_evaluation(self): diff --git a/tests/functional/test_output_creates.py b/tests/functional/test_output_creates.py deleted file mode 100644 index e714a90ea..000000000 --- a/tests/functional/test_output_creates.py +++ /dev/null @@ -1,39 +0,0 @@ -from os import path -from nose import tools -from nose.plugins.attrib import attr - -from tests.functional import single_machine_test -from tests.functional import generic_deployment_test - -parent_dir = path.dirname(__file__) - -output_spec = "%s/single_machine_outputs.nix" % (parent_dir) - - -@attr("libvirtd") -class TestOutputCreates(generic_deployment_test.GenericDeploymentTest): - _multiprocess_can_split_ = True - - def setup(self): - super(TestOutputCreates, self).setup() - self.depl.nix_exprs = self.depl.nix_exprs + [output_spec] - - def test_deploy(self): - self.depl.deploy() - assert '"12345"' == self.depl.machines["machine"].run_command( - "cat /etc/test.txt", capture_stdout=True - ), "Resource contents incorrect" - - def test_update(self): - self.depl.deploy() - assert '"12345"' == self.depl.machines["machine"].run_command( - "cat /etc/test.txt", capture_stdout=True - ), "Resource contents incorrect" - - self.depl.nix_exprs = self.depl.nix_exprs + [ - "%s/single_machine_outputs_mod.nix" % (parent_dir) - ] - self.depl.deploy() - assert '"123456"' == self.depl.machines["machine"].run_command( - "cat /etc/test.txt", capture_stdout=True - ), "Resource contents update incorrect" diff --git a/tests/functional/test_rollback_rollsback.py b/tests/functional/test_rollback_rollsback.py index 1899374f8..5e612e960 100644 --- a/tests/functional/test_rollback_rollsback.py +++ b/tests/functional/test_rollback_rollsback.py @@ -5,6 +5,8 @@ from nixops.ssh_util import SSHCommandFailed +from nixops.evaluation import NetworkFile + parent_dir = path.dirname(__file__) has_hello_spec = "%s/single_machine_has_hello.nix" % (parent_dir) @@ -17,13 +19,14 @@ class TestRollbackRollsback(single_machine_test.SingleMachineTest): def setup(self): super(TestRollbackRollsback, self).setup() + self.depl.network_expr = NetworkFile(rollback_spec) self.depl.nix_exprs = self.depl.nix_exprs + [rollback_spec] def run_check(self): self.depl.deploy() with tools.assert_raises(SSHCommandFailed): self.check_command("hello") - self.depl.nix_exprs = self.depl.nix_exprs + [has_hello_spec] + self.depl.network_expr = NetworkFile(has_hello_spec) self.depl.deploy() self.check_command("hello") self.depl.rollback(generation=1) From 2cad90d17a93f377037bb1f7267c3658b7cac4f6 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 17 Jul 2020 20:40:32 +0200 Subject: [PATCH 60/80] Add a plugin entry point Otherwise nixops core plugins can't be found from other plugin repos. --- nixops/plugins/__init__.py | 2 -- pyproject.toml | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nixops/plugins/__init__.py b/nixops/plugins/__init__.py index 5e48ff235..4b1a76a6b 100644 --- a/nixops/plugins/__init__.py +++ b/nixops/plugins/__init__.py @@ -5,7 +5,6 @@ from nixops.storage import StorageBackend from nixops.locks import LockDriver -from functools import lru_cache from typing import Generator import pluggy import nixops @@ -15,7 +14,6 @@ """Marker to be imported and used in plugins (and for own implementations)""" -@lru_cache() def get_plugin_manager(): from . import hookspecs diff --git a/pyproject.toml b/pyproject.toml index 699cb31a7..db6d6bafc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,9 @@ livereload = "^2.6.1" sphinx = "^3.0.3" flake8 = "^3.8.1" +[tool.poetry.plugins."nixops"] +nixops = "nixops.plugin" + [tool.poetry.scripts] nixops = 'nixops.__main__:main' charon = 'nixops.__main__:main' From 47cf418ef700033c5fa79f9d8a9bcd213db45fea Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 18 Jul 2020 02:54:29 +0200 Subject: [PATCH 61/80] nixops/__main__.py: Add more static typing --- nixops/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nixops/__main__.py b/nixops/__main__.py index 3c2517b16..a8f258479 100755 --- a/nixops/__main__.py +++ b/nixops/__main__.py @@ -12,7 +12,9 @@ def setup_debugger() -> None: from types import TracebackType from typing import Type - def hook(_type: Type[BaseException], value: BaseException, tb: TracebackType): + def hook( + _type: Type[BaseException], value: BaseException, tb: TracebackType + ) -> None: if hasattr(sys, "ps1") or not sys.stderr.isatty(): sys.__excepthook__(_type, value, tb) else: From b05052edbab47b288f317f4c4608e5f63f731d78 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 18 Jul 2020 02:54:29 +0200 Subject: [PATCH 62/80] nixops/backends/__init__.py: Add more static typing --- nixops/backends/__init__.py | 105 +++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 43 deletions(-) diff --git a/nixops/backends/__init__.py b/nixops/backends/__init__.py index 1d12c6c69..0b24331e4 100644 --- a/nixops/backends/__init__.py +++ b/nixops/backends/__init__.py @@ -20,6 +20,7 @@ from nixops.state import RecordId import subprocess import threading +import nixops class KeyOptions(nixops.resources.ResourceOptions): @@ -44,6 +45,7 @@ class MachineOptions(nixops.resources.ResourceOptions): targetUser: Optional[str] sshOptions: Sequence[str] privilegeEscalationCommand: Sequence[str] + provisionSSHKey: bool class MachineDefinition(nixops.resources.ResourceDefinition): @@ -53,27 +55,28 @@ class MachineDefinition(nixops.resources.ResourceDefinition): ssh_port: int always_activate: bool - owners: List[str] + owners: Sequence[str] has_fast_connection: bool keys: Mapping[str, KeyOptions] ssh_user: str - ssh_options: List[str] - privilege_escalation_command: List[str] + ssh_options: Sequence[str] + privilege_escalation_command: Sequence[str] provision_ssh_key: bool def __init__(self, name: str, config: nixops.resources.ResourceEval): super().__init__(name, config) - self.ssh_port = config["targetPort"] - self.always_activate = config["alwaysActivate"] - self.owners = config["owners"] - self.has_fast_connection = config["hasFastConnection"] + self.ssh_port = self.config.targetPort + self.always_activate = self.config.alwaysActivate + self.owners = self.config.owners + self.has_fast_connection = self.config.hasFastConnection + # TODO: Extend MutableValidatedObject to handle this case self.keys = {k: KeyOptions(**v) for k, v in config["keys"].items()} - self.ssh_options = config["sshOptions"] + self.ssh_options = self.config.sshOptions - self.ssh_user = config["targetUser"] + self.ssh_user = self.config.targetUser or "root" - self.privilege_escalation_command = config["privilegeEscalationCommand"] - self.provision_ssh_key = config["provisionSSHKey"] + self.privilege_escalation_command = self.config.privilegeEscalationCommand + self.provision_ssh_key = self.config.provisionSSHKey MachineDefinitionType = TypeVar("MachineDefinitionType", bound="MachineDefinition") @@ -96,7 +99,7 @@ class MachineState( _ssh_pinged_this_time: bool = False ssh_port: int = nixops.util.attr_property("targetPort", None, int) ssh_user: str = nixops.util.attr_property("targetUser", "root", str) - ssh_options: List[str] = nixops.util.attr_property("sshOptions", [], "json") + ssh_options: Sequence[str] = nixops.util.attr_property("sshOptions", [], "json") privilege_escalation_command: List[str] = nixops.util.attr_property( "privilegeEscalationCommand", [], "json" ) @@ -125,7 +128,9 @@ class MachineState( defn: Optional[MachineDefinition] = None - def __init__(self, depl, name: str, id: RecordId) -> None: + def __init__( + self, depl: "nixops.deployment.Deployment", name: str, id: RecordId + ) -> None: super().__init__(depl, name, id) self.defn = None self._ssh_pinged_this_time = False @@ -161,17 +166,21 @@ def set_common_state(self, defn: MachineDefinitionType) -> None: def stop(self) -> None: """Stop this machine, if possible.""" - self.warn("don't know how to stop machine ‘{0}’".format(self.name)) + self.logger.warn("don't know how to stop machine ‘{0}’".format(self.name)) def start(self) -> None: """Start this machine, if possible.""" pass - def get_load_avg(self) -> Union[List[str], None]: + def get_load_avg(self) -> Optional[List[str]]: """Get the load averages on the machine.""" try: - res = ( - self.run_command("cat /proc/loadavg", capture_stdout=True, timeout=15) + res: List[str] = ( + str( + self.run_command( + "cat /proc/loadavg", capture_stdout=True, timeout=15 + ) + ) .rstrip() .split(" ") ) @@ -190,12 +199,13 @@ def check(self): # TODO -> CheckResult, but supertype ResourceState -> True self._check(res) return res - def _check(self, res): # TODO -> None but supertype ResourceState -> True + def _check(self, res): avg = self.get_load_avg() if avg is None: if self.state == self.UP: self.state = self.UNREACHABLE res.is_reachable = False + return False else: self.state = self.UP self.ssh_pinged = True @@ -203,6 +213,8 @@ def _check(self, res): # TODO -> None but supertype ResourceState -> True res.is_reachable = True res.load = avg + # Get the systemd units that are in a failed state or in progress. + # Get the systemd units that are in a failed state or in progress. # cat to inhibit color output. out: List[str] = str( @@ -277,7 +289,7 @@ def backup(self, defn, backup_id: str, devices: List[str] = []) -> None: def reboot(self, hard: bool = False) -> None: """Reboot this machine.""" - self.log("rebooting...") + self.logger.log("rebooting...") if self.state == self.RESCUE: # We're on non-NixOS here, so systemd might not be available. # The sleep is to prevent the reboot from causing the SSH @@ -292,7 +304,7 @@ def reboot(self, hard: bool = False) -> None: def ping(self) -> bool: event = threading.Event() - def _worker(): + def _worker() -> bool: try: self.ssh.run_command( ["true"], @@ -306,6 +318,7 @@ def _worker(): return False else: event.set() + return True t = threading.Thread(target=_worker) t.start() @@ -336,18 +349,18 @@ def wait_for_down( def reboot_sync(self, hard: bool = False) -> None: """Reboot this machine and wait until it's up again.""" self.reboot(hard=hard) - self.log_start("waiting for the machine to finish rebooting...") + self.logger.log_start("waiting for the machine to finish rebooting...") def progress_cb() -> None: - self.log_continue(".") + self.logger.log_continue(".") self.wait_for_down(callback=progress_cb) - self.log_continue("[down]") + self.logger.log_continue("[down]") self.wait_for_up(callback=progress_cb) - self.log_end("[up]") + self.logger.log_end("[up]") self.state = self.UP self.ssh_pinged = True self._ssh_pinged_this_time = True @@ -357,7 +370,9 @@ def reboot_rescue(self, hard: bool = False) -> None: """ Reboot machine into rescue system and wait until it is active. """ - self.warn("machine ‘{0}’ doesn't have a rescue" " system.".format(self.name)) + self.logger.warn( + "machine ‘{0}’ doesn't have a rescue" " system.".format(self.name) + ) def send_keys(self) -> None: if self.state == self.RESCUE: @@ -368,10 +383,10 @@ def send_keys(self) -> None: return for k, opts in self.get_keys().items(): - self.log("uploading key ‘{0}’ to ‘{1}’...".format(k, opts["path"])) + self.logger.log("uploading key ‘{0}’ to ‘{1}’...".format(k, opts["path"])) tmp = self.depl.tempdir + "/key-" + self.name - destDir = opts["destDir"].rstrip("/") + destDir: str = opts["destDir"].rstrip("/") self.run_command( ( "test -d '{0}' || (" @@ -437,10 +452,10 @@ def send_keys(self) -> None: def get_keys(self): return self.keys - def get_ssh_name(self): + def get_ssh_name(self) -> str: assert False - def get_ssh_flags(self, scp=False) -> List[str]: + def get_ssh_flags(self, scp: bool = False) -> List[str]: if scp: return ["-P", str(self.ssh_port)] if self.ssh_port is not None else [] else: @@ -451,7 +466,7 @@ def get_ssh_flags(self, scp=False) -> List[str]: def get_ssh_password(self): return None - def get_ssh_for_copy_closure(self): + def get_ssh_for_copy_closure(self) -> nixops.ssh_util.SSH: return self.ssh @property @@ -462,25 +477,25 @@ def public_host_key(self): def private_ipv4(self) -> Optional[str]: return None - def address_to(self, r): + def address_to(self, r: nixops.resources.GenericResourceState) -> Optional[str]: """Return the IP address to be used to access resource "r" from this machine.""" return r.public_ipv4 - def wait_for_ssh(self, check=False): + def wait_for_ssh(self, check: bool = False) -> None: """Wait until the SSH port is open on this machine.""" if self.ssh_pinged and (not check or self._ssh_pinged_this_time): return - self.log_start("waiting for SSH...") + self.logger.log_start("waiting for SSH...") - self.wait_for_up(callback=lambda: self.log_continue(".")) + self.wait_for_up(callback=lambda: self.logger.log_continue(".")) - self.log_end("") + self.logger.log_end("") if self.state != self.RESCUE: self.state = self.UP self.ssh_pinged = True self._ssh_pinged_this_time = True - def write_ssh_private_key(self, private_key) -> str: + def write_ssh_private_key(self, private_key: str) -> str: key_file = "{0}/id_nixops-{1}".format(self.depl.tempdir, self.name) with os.fdopen(os.open(key_file, os.O_CREAT | os.O_WRONLY, 0o600), "w") as f: f.write(private_key) @@ -490,10 +505,10 @@ def write_ssh_private_key(self, private_key) -> str: def get_ssh_private_key_file(self) -> Optional[str]: return None - def _logged_exec(self, command, **kwargs): + def _logged_exec(self, command: List[str], **kwargs) -> Union[str, int]: return nixops.util.logged_exec(command, self.logger, **kwargs) - def run_command(self, command, **kwargs): + def run_command(self, command, **kwargs) -> Union[str, int]: """ Execute a command on the machine via SSH. @@ -522,9 +537,9 @@ def switch_to_configuration( else: cmd += command cmd += " " + method - return self.run_command(cmd, check=False) + return int(self.run_command(cmd, check=False)) - def copy_closure_to(self, path): + def copy_closure_to(self, path: str) -> None: """Copy a closure to this machine.""" # !!! Implement copying between cloud machines, as in the Perl @@ -573,7 +588,9 @@ def _fmt_rsync_command(self, *args: str, recursive: bool = False) -> List[str]: return cmdline - def upload_file(self, source: str, target: str, recursive: bool = False): + def upload_file( + self, source: str, target: str, recursive: bool = False + ) -> Union[str, int]: cmdline = self._fmt_rsync_command( source, self.ssh_user + "@" + self._get_scp_name() + ":" + target, @@ -581,7 +598,9 @@ def upload_file(self, source: str, target: str, recursive: bool = False): ) return self._logged_exec(cmdline) - def download_file(self, source: str, target: str, recursive: bool = False): + def download_file( + self, source: str, target: str, recursive: bool = False + ) -> Union[str, int]: cmdline = self._fmt_rsync_command( self.ssh_user + "@" + self._get_scp_name() + ":" + source, target, @@ -589,7 +608,7 @@ def download_file(self, source: str, target: str, recursive: bool = False): ) return self._logged_exec(cmdline) - def get_console_output(self): + def get_console_output(self) -> str: return "(not available for this machine type)\n" From b87932471ec6f2ca18225d8d66ed1a14915a4ac3 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 18 Jul 2020 02:54:29 +0200 Subject: [PATCH 63/80] nixops/backends/none.py: Add more static typing --- nixops/backends/none.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/nixops/backends/none.py b/nixops/backends/none.py index d7eb6660c..3e2536b62 100644 --- a/nixops/backends/none.py +++ b/nixops/backends/none.py @@ -1,11 +1,14 @@ # -*- coding: utf-8 -*- -from typing import Optional +from typing import Optional, List import nixops.util from nixops.backends import MachineDefinition, MachineState, MachineOptions from nixops.util import attr_property, create_key_pair +from nixops.state import RecordId import nixops.resources +import nixops + class NoneDefinition(MachineDefinition): """Definition of a trivial machine.""" @@ -16,7 +19,7 @@ class NoneDefinition(MachineDefinition): config: MachineOptions @classmethod - def get_type(cls): + def get_type(cls) -> str: return "none" def __init__(self, name: str, config: nixops.resources.ResourceEval): @@ -29,16 +32,18 @@ class NoneState(MachineState[NoneDefinition]): """State of a trivial machine.""" @classmethod - def get_type(cls): + def get_type(cls) -> str: return "none" - target_host = nixops.util.attr_property("targetHost", None) - public_ipv4 = nixops.util.attr_property("publicIpv4", None) + target_host: str = nixops.util.attr_property("targetHost", None) + public_ipv4: Optional[str] = nixops.util.attr_property("publicIpv4", None) _ssh_private_key: Optional[str] = attr_property("none.sshPrivateKey", None) _ssh_public_key: Optional[str] = attr_property("none.sshPublicKey", None) - _ssh_public_key_deployed = attr_property("none.sshPublicKeyDeployed", False, bool) + _ssh_public_key_deployed: bool = attr_property( + "none.sshPublicKeyDeployed", False, bool + ) - def __init__(self, depl, name, id): + def __init__(self, depl: "nixops.deployment.Deployment", name: str, id: RecordId): MachineState.__init__(self, depl, name, id) @property @@ -68,7 +73,7 @@ def create( check: bool, allow_reboot: bool, allow_recreate: bool, - ): + ) -> None: assert isinstance(defn, NoneDefinition) self.set_common_state(defn) self.target_host = defn._target_host @@ -76,22 +81,24 @@ def create( if not self.vm_id: if self.provision_ssh_key: - self.log_start("generating new SSH key pair... ") + self.logger.log_start("generating new SSH key pair... ") key_name = "NixOps client key for {0}".format(self.name) self._ssh_private_key, self._ssh_public_key = create_key_pair( key_name=key_name ) - self.log_end("done") + self.logger.log_end("done") self.vm_id = "nixops-{0}-{1}".format(self.depl.uuid, self.name) - def switch_to_configuration(self, method, sync, command=None): + def switch_to_configuration( + self, method: str, sync: bool, command: Optional[str] = None + ) -> int: res = super(NoneState, self).switch_to_configuration(method, sync, command) if res == 0: self._ssh_public_key_deployed = True return res - def get_ssh_name(self): + def get_ssh_name(self) -> str: assert self.target_host return self.target_host @@ -102,7 +109,7 @@ def get_ssh_private_key_file(self) -> Optional[str]: return self.write_ssh_private_key(self._ssh_private_key) return None - def get_ssh_flags(self, *args, **kwargs): + def get_ssh_flags(self, *args, **kwargs) -> List[str]: super_state_flags = super(NoneState, self).get_ssh_flags(*args, **kwargs) if self.vm_id and self.cur_toplevel and self._ssh_public_key_deployed: key_file = self.get_ssh_private_key_file() @@ -125,6 +132,6 @@ def _check(self, res): if res.is_up: super()._check(res) - def destroy(self, wipe=False): + def destroy(self, wipe: bool = False) -> bool: # No-op; just forget about the machine. return True From 8256550fce0066b98dd987580c157386c6982875 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 18 Jul 2020 02:54:30 +0200 Subject: [PATCH 64/80] nixops/deployment.py: Add more static typing --- nixops/deployment.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index a478bdca7..77bd8371f 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -55,7 +55,7 @@ class UnknownBackend(Exception): pass -DEBUG = False +DEBUG: bool = False NixosConfigurationType = List[Dict[Tuple[str, ...], Any]] @@ -1221,8 +1221,10 @@ def worker(r: nixops.resources.GenericResourceState): # attribute, because the machine may have been # booted from an older NixOS image. if not m.state_version: - os_release = m.run_command( - "cat /etc/os-release", capture_stdout=True + os_release = str( + m.run_command( + "cat /etc/os-release", capture_stdout=True + ) ) match = re.search( 'VERSION_ID="([0-9]+\.[0-9]+).*"', # noqa: W605 From 9687d7439f2f25431282033bf5b6c116ae051465 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 18 Jul 2020 02:54:30 +0200 Subject: [PATCH 65/80] nixops/nix_expr.py: Add more static typing --- nixops/nix_expr.py | 60 ++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/nixops/nix_expr.py b/nixops/nix_expr.py index 8c97e824b..75d53af4b 100644 --- a/nixops/nix_expr.py +++ b/nixops/nix_expr.py @@ -1,6 +1,6 @@ import functools import string -from typing import Optional, Any, List +from typing import Optional, Any, List, Union, Dict from textwrap import dedent __all__ = ["py2nix", "nix2py", "nixmerge", "expand_dict", "RawValue", "Function"] @@ -36,19 +36,19 @@ def get_min_length(self) -> None: def is_inlineable(self) -> bool: return False - def indent(self, level: int = 0, inline: bool = False, maxwidth: int = 80): + def indent(self, level: int = 0, inline: bool = False, maxwidth: int = 80) -> str: return "\n".join([" " * level + value for value in self.values]) class Function(object): - def __init__(self, head, body): - self.head = head - self.body = body + def __init__(self, head: Any, body: Any): + self.head: Any = head + self.body: Any = body - def __repr__(self): + def __repr__(self) -> str: return "{0} {1}".format(self.head, self.body) - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: return ( isinstance(other, Function) and other.head == self.head @@ -57,27 +57,27 @@ def __eq__(self, other): class Call(object): - def __init__(self, fun, arg): - self.fun = fun - self.arg = arg + def __init__(self, fun: Any, arg: Any): + self.fun: Any = fun + self.arg: Any = arg - def __repr__(self): + def __repr__(self) -> str: return "{0} {1}".format(self.fun, self.arg) - def __eq__(self, other): + def __eq__(self, other) -> bool: return ( isinstance(other, Call) and other.fun == self.fun and other.arg == self.arg ) class Container(object): - def __init__(self, prefix, children, suffix, inline_variant=None): - self.prefix = prefix - self.children = children - self.suffix = suffix + def __init__(self, prefix: str, children: List, suffix: str, inline_variant=None): + self.prefix: str = prefix + self.children: List = children + self.suffix: str = suffix self.inline_variant = inline_variant - def get_min_length(self) -> Optional[int]: + def get_min_length(self) -> int: """ Return the minimum length of this container and all sub-containers. """ @@ -92,7 +92,7 @@ def get_min_length(self) -> Optional[int]: def is_inlineable(self) -> bool: return all([child.is_inlineable() for child in self.children]) - def indent(self, level=0, inline=False, maxwidth=80) -> str: + def indent(self, level: int = 0, inline: bool = False, maxwidth: int = 80) -> str: if not self.is_inlineable(): inline = False elif level * 2 + self.get_min_length() < maxwidth: @@ -120,7 +120,9 @@ def indent(self, level=0, inline=False, maxwidth=80) -> str: return ind + self.prefix + sep + lines + sep + suffix_ind + self.suffix -def enclose_node(node, prefix="", suffix=""): +def enclose_node( + node: Any, prefix: str = "", suffix: str = "" +) -> Union[MultiLineRawValue, RawValue, Container]: if isinstance(node, MultiLineRawValue): new_values = list(node.values) new_values[0] = prefix + new_values[0] @@ -139,14 +141,16 @@ def enclose_node(node, prefix="", suffix=""): ) -def _fold_string(value, rules): - def folder(val, rule): +def _fold_string(value, rules) -> str: + def folder(val, rule) -> str: return val.replace(rule[0], rule[1]) return functools.reduce(folder, rules, value) -def py2nix(value, initial_indentation=0, maxwidth=80, inline=False): # noqa: C901 +def py2nix( # noqa: C901 + value: Any, initial_indentation: int = 0, maxwidth: int = 80, inline: bool = False +): """ Return the given value as a Nix expression string. @@ -163,7 +167,7 @@ def _enc_int(node): else: return RawValue(str(node)) - def _enc_str(node, for_attribute=False): + def _enc_str(node, for_attribute: bool = False): encoded = _fold_string( node, [ @@ -190,7 +194,7 @@ def _enc_str(node, for_attribute=False): else: return inline_variant - def _enc_list(nodes): + def _enc_list(nodes) -> Union[RawValue, Container]: if len(nodes) == 0: return RawValue("[]") pre, post = "[", "]" @@ -280,7 +284,7 @@ def _enc(node, inlist=False): return _enc(value).indent(initial_indentation, maxwidth=maxwidth, inline=inline) -def expand_dict(unexpanded): +def expand_dict(unexpanded) -> Dict: """ Turns a dict containing tuples as keys into a set of nested dictionaries. @@ -319,7 +323,7 @@ def nixmerge(expr1, expr2): elements if they otherwise would clash. """ - def _merge_dicts(d1, d2): + def _merge_dicts(d1, d2) -> Dict: out = {} for key in set(d1.keys()).union(d2.keys()): if key in d1 and key in d2: @@ -330,7 +334,7 @@ def _merge_dicts(d1, d2): out[key] = d2[key] return out - def _merge(e1, e2): + def _merge(e1, e2) -> Union[Dict, List]: if isinstance(e1, dict) and isinstance(e2, dict): return _merge_dicts(e1, e2) elif isinstance(e1, list) and isinstance(e2, list): @@ -348,7 +352,7 @@ def _merge(e1, e2): return _merge(expr1, expr2) -def nix2py(source): +def nix2py(source: str) -> MultiLineRawValue: """ Dedent the given Nix source code and encode it into multiple raw values which are used as-is and only indentation will take place. From a92ebfeb16b7feabaf33059cf995649d76408c80 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 18 Jul 2020 02:54:30 +0200 Subject: [PATCH 66/80] nixops/plugins/__init__.py: Add more static typing --- nixops/plugins/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nixops/plugins/__init__.py b/nixops/plugins/__init__.py index 4b1a76a6b..4af9e4533 100644 --- a/nixops/plugins/__init__.py +++ b/nixops/plugins/__init__.py @@ -2,6 +2,7 @@ from nixops.backends import MachineState from typing import List, Dict, Optional, Union, Tuple, Type +from argparse import ArgumentParser, _SubParsersAction from nixops.storage import StorageBackend from nixops.locks import LockDriver @@ -14,7 +15,7 @@ """Marker to be imported and used in plugins (and for own implementations)""" -def get_plugin_manager(): +def get_plugin_manager() -> pluggy.PluginManager: from . import hookspecs pm = pluggy.PluginManager("nixops") @@ -82,7 +83,7 @@ def nixexprs(self) -> List[str]: """ return [] - def parser(self, parser, subparsers): + def parser(self, parser: ArgumentParser, subparsers: _SubParsersAction) -> None: """ Extend the core nixops cli parser """ From b3bad1c6fdbd46c9082a6b5d034b340c6121ec9e Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 18 Jul 2020 02:54:30 +0200 Subject: [PATCH 67/80] nixops/plugins/manager.py: Add more static typing --- nixops/plugins/manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nixops/plugins/manager.py b/nixops/plugins/manager.py index 2eaba5b3c..6f6fb0525 100644 --- a/nixops/plugins/manager.py +++ b/nixops/plugins/manager.py @@ -3,6 +3,7 @@ from nixops.backends import MachineState from typing import List, Dict, Generator, Tuple, Any, Set import importlib +from argparse import ArgumentParser, _SubParsersAction from nixops.storage import StorageBackend from nixops.locks import LockDriver @@ -57,7 +58,7 @@ def machine_hooks() -> Generator[MachineHooks, None, None]: yield machine_hooks @classmethod - def load(cls): + def load(cls) -> None: seen: Set[str] = set() for plugin in get_plugins(): for mod in plugin.load(): @@ -73,7 +74,7 @@ def nixexprs() -> List[str]: return nixexprs @staticmethod - def parser(parser, subparsers): + def parser(parser: ArgumentParser, subparsers: _SubParsersAction) -> None: for plugin in get_plugins(): plugin.parser(parser, subparsers) From 7ce24cf7ae20e1880f13c3f3ce3616848eb541a7 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 18 Jul 2020 02:54:30 +0200 Subject: [PATCH 68/80] nixops/resources/__init__.py: Add more static typing --- nixops/resources/__init__.py | 65 +++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index ab3e93923..29bf671db 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -4,8 +4,19 @@ import re import nixops.util from threading import Event -from typing import List, Optional, Dict, Any, Type, TypeVar, Union, TYPE_CHECKING from nixops.monkey import Protocol, runtime_checkable +from typing import ( + List, + Optional, + Dict, + Any, + TypeVar, + Union, + TYPE_CHECKING, + Type, + Iterable, + Set, +) from nixops.state import StateDict, RecordId from nixops.diff import Diff, Handler from nixops.util import ImmutableMapping, ImmutableValidatedObject @@ -31,17 +42,19 @@ class ResourceDefinition: config: ResourceOptions @classmethod - def get_type(cls) -> str: + def get_type(cls: Type[ResourceDefinition]) -> str: """A resource type identifier that must match the corresponding ResourceState class""" raise NotImplementedError("get_type") @classmethod - def get_resource_type(cls): + def get_resource_type(cls: Type[ResourceDefinition]) -> str: """A resource type identifier corresponding to the resources. attribute in the Nix expression""" return cls.get_type() def __init__(self, name: str, config: ResourceEval): - config_type = self.__annotations__.get("config", ResourceOptions) + config_type: Union[Type, str] = self.__annotations__.get( + "config", ResourceOptions + ) if isinstance(config_type, str): if config_type == "ResourceOptions": @@ -194,7 +207,7 @@ def export(self) -> Dict[str, Dict[str, str]]: res["type"] = self.get_type() return res - def import_(self, attrs): + def import_(self, attrs: Dict): """Import the resource from another database""" with self.depl._db: for k, v in attrs.items(): @@ -203,22 +216,22 @@ def import_(self, attrs): self._set_attr(k, v) # XXX: Deprecated, use self.logger.* instead! - def log(self, *args, **kwargs): + def log(self, *args, **kwargs) -> None: return self.logger.log(*args, **kwargs) - def log_end(self, *args, **kwargs): + def log_end(self, *args, **kwargs) -> None: return self.logger.log_end(*args, **kwargs) - def log_start(self, *args, **kwargs): + def log_start(self, *args, **kwargs) -> None: return self.logger.log_start(*args, **kwargs) - def log_continue(self, *args, **kwargs): + def log_continue(self, *args, **kwargs) -> None: return self.logger.log_continue(*args, **kwargs) - def warn(self, *args, **kwargs): + def warn(self, *args, **kwargs) -> None: return self.logger.warn(*args, **kwargs) - def success(self, *args, **kwargs): + def success(self, *args, **kwargs) -> None: return self.logger.success(*args, **kwargs) # XXX: End deprecated methods @@ -270,11 +283,17 @@ def resource_id(self): def public_ipv4(self) -> Optional[str]: return None - def create_after(self, resources, defn): + def create_after( + self, + resources: Iterable[GenericResourceState], + defn: Optional[ResourceDefinition], + ) -> Set[GenericResourceState]: """Return a set of resources that should be created before this one.""" - return {} + return set() - def destroy_before(self, resources): + def destroy_before( + self, resources: Iterable[GenericResourceState] + ) -> Set[GenericResourceState]: """Return a set of resources that should be destroyed after this one.""" return self.create_after(resources, None) @@ -284,7 +303,7 @@ def create( check: bool, allow_reboot: bool, allow_recreate: bool, - ): + ) -> None: """Create or update the resource defined by ‘defn’.""" raise NotImplementedError("create") @@ -300,16 +319,16 @@ def check( def _check(self): return True - def after_activation(self, defn): + def after_activation(self, defn: ResourceDefinition) -> None: """Actions to be performed after the network is activated""" return - def destroy(self, wipe=False): + def destroy(self, wipe: bool = False) -> bool: """Destroy this resource, if possible.""" self.logger.warn("don't know how to destroy resource ‘{0}’".format(self.name)) return False - def delete_resources(self): + def delete_resources(self) -> bool: """delete this resource state, if possible.""" if not self.depl.logger.confirm( "are you sure you want to clear the state of {}? " @@ -324,7 +343,7 @@ def delete_resources(self): ) return True - def next_charge_time(self): + def next_charge_time(self) -> Optional[int]: """Return the time (in Unix epoch) when this resource will next incur a financial charge (or None if unknown).""" return None @@ -337,7 +356,7 @@ class DiffEngineResourceState( _reserved_keys: List[str] = [] _state: StateDict - def __init__(self, depl, name, id): + def __init__(self, depl: "nixops.deployment.Deployment", name: str, id: RecordId): nixops.resources.ResourceState.__init__(self, depl, name, id) self._state = StateDict(depl, id) @@ -347,7 +366,7 @@ def create( check: bool, allow_reboot: bool, allow_recreate: bool, - ): + ) -> None: # if --check is true check against the api and update the state # before firing up the diff engine in order to get the needed # handlers calls @@ -358,12 +377,12 @@ def create( for handler in diff_engine.plan(): handler.handle(allow_recreate) - def plan(self, defn: ResourceDefinitionType): + def plan(self, defn: ResourceDefinitionType) -> None: if hasattr(self, "_state"): diff_engine = self.setup_diff_engine(defn) diff_engine.plan(show=True) else: - self.warn( + self.logger.warn( "resource type {} doesn't implement a plan operation".format( self.get_type() ) From e48422a9b0d2f4db38af7db7edc7c13f799a656b Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 18 Jul 2020 02:54:30 +0200 Subject: [PATCH 69/80] nixops/resources/ssh_keypair.py: Add more static typing --- nixops/resources/ssh_keypair.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/nixops/resources/ssh_keypair.py b/nixops/resources/ssh_keypair.py index dea212935..f618996ed 100644 --- a/nixops/resources/ssh_keypair.py +++ b/nixops/resources/ssh_keypair.py @@ -1,7 +1,9 @@ -# -*- coding: utf-8 -*- +from __future__ import annotations # Automatic provisioning of SSH key pairs. +from typing import Type, Dict, Optional +from nixops.state import RecordId import nixops.util import nixops.resources @@ -12,17 +14,17 @@ class SSHKeyPairDefinition(nixops.resources.ResourceDefinition): config: nixops.resources.ResourceOptions @classmethod - def get_type(cls): + def get_type(cls: Type[SSHKeyPairDefinition]) -> str: return "ssh-keypair" @classmethod - def get_resource_type(cls): + def get_resource_type(cls: Type[SSHKeyPairDefinition]) -> str: return "sshKeyPairs" def __init__(self, name: str, config: nixops.resources.ResourceEval): super().__init__(name, config) - def show_type(self): + def show_type(self) -> str: return "{0}".format(self.get_type()) @@ -32,14 +34,14 @@ class SSHKeyPairState(nixops.resources.ResourceState[SSHKeyPairDefinition]): state = nixops.util.attr_property( "state", nixops.resources.ResourceState.MISSING, int ) - public_key = nixops.util.attr_property("publicKey", None) - private_key = nixops.util.attr_property("privateKey", None) + public_key: Optional[str] = nixops.util.attr_property("publicKey", None) + private_key: Optional[str] = nixops.util.attr_property("privateKey", None) @classmethod - def get_type(cls): + def get_type(cls: Type[SSHKeyPairState]) -> str: return "ssh-keypair" - def __init__(self, depl, name, id): + def __init__(self, depl: "nixops.deployment.Deployment", name: str, id: RecordId): nixops.resources.ResourceState.__init__(self, depl, name, id) self._conn = None @@ -49,7 +51,7 @@ def create( check: bool, allow_reboot: bool, allow_recreate: bool, - ): + ) -> None: # Generate the key pair locally. if not self.public_key: (private, public) = nixops.util.create_key_pair(type="ed25519") @@ -58,11 +60,11 @@ def create( self.private_key = private self.state = nixops.resources.ResourceState.UP - def prefix_definition(self, attr): + def prefix_definition(self, attr) -> Dict: return {("resources", "sshKeyPairs"): attr} - def get_physical_spec(self): + def get_physical_spec(self) -> Dict[str, Optional[str]]: return {"privateKey": self.private_key, "publicKey": self.public_key} - def destroy(self, wipe=False): + def destroy(self, wipe: bool = False) -> bool: return True From 9feab7795bd4ea81542c8d0243a9eb0b98bc031f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 18 Jul 2020 02:54:31 +0200 Subject: [PATCH 70/80] nixops/script_defs.py: Add more static typing --- nixops/script_defs.py | 159 +++++++++++++++++++++++------------------- 1 file changed, 86 insertions(+), 73 deletions(-) diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 13157d41b..729249c93 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -23,7 +23,7 @@ import json from tempfile import TemporaryDirectory import pipes -from typing import Tuple, List, Optional, Union, Generator, Type, Set +from typing import Tuple, List, Optional, Union, Generator, Type, Set, Sequence import nixops.ansi from nixops.plugins.manager import PluginManager @@ -37,7 +37,7 @@ def get_network_file(args: Namespace) -> NetworkFile: - network_dir = os.path.abspath(args.network_dir) + network_dir: str = os.path.abspath(args.network_dir) if not os.path.exists(network_dir): raise ValueError("f{network_dir} does not exist") @@ -45,8 +45,8 @@ def get_network_file(args: Namespace) -> NetworkFile: classic_path = os.path.join(network_dir, "network.nix") flake_path = os.path.join(network_dir, "flake.nix") - classic_exists = os.path.exists(classic_path) - flake_exists = os.path.exists(flake_path) + classic_exists: bool = os.path.exists(classic_path) + flake_exists: bool = os.path.exists(flake_path) if all((flake_exists, classic_exists)): raise ValueError("Both flake.nix and network.nix cannot coexist") @@ -60,7 +60,7 @@ def get_network_file(args: Namespace) -> NetworkFile: raise ValueError(f"Neither flake.nix nor network.nix exists in {network_dir}") -def set_common_depl(depl: nixops.deployment.Deployment, args: Namespace): +def set_common_depl(depl: nixops.deployment.Deployment, args: Namespace) -> None: network_file = get_network_file(args) depl.network_expr = network_file @@ -129,7 +129,7 @@ def network_state(args: Namespace) -> Generator[nixops.statefile.StateFile, None lock.unlock() -def op_list_plugins(args): +def op_list_plugins(args: Namespace) -> None: pm = get_plugin_manager() if args.verbose: @@ -171,7 +171,7 @@ def one_or_all( yield [open_deployment(sf, args)] -def op_list_deployments(args): +def op_list_deployments(args: Namespace) -> None: with network_state(args) as sf: tbl = create_table( [ @@ -207,7 +207,9 @@ def op_list_deployments(args): print(tbl) -def open_deployment(sf: nixops.statefile.StateFile, args: Namespace): +def open_deployment( + sf: nixops.statefile.StateFile, args: Namespace +) -> nixops.deployment.Deployment: depl = sf.open_deployment(uuid=args.deployment) depl.extra_nix_path = sum(args.nix_path or [], []) @@ -233,7 +235,7 @@ def open_deployment(sf: nixops.statefile.StateFile, args: Namespace): return depl -def set_name(depl: nixops.deployment.Deployment, name: Optional[str]): +def set_name(depl: nixops.deployment.Deployment, name: Optional[str]) -> None: if not name: return if not re.match("^[a-zA-Z_\-][a-zA-Z0-9_\-\.]*$", name): # noqa: W605 @@ -241,19 +243,19 @@ def set_name(depl: nixops.deployment.Deployment, name: Optional[str]): depl.name = name -def modify_deployment(args, depl: nixops.deployment.Deployment): +def modify_deployment(args: Namespace, depl: nixops.deployment.Deployment) -> None: set_common_depl(depl, args) depl.nix_path = [nixops.util.abs_nix_path(x) for x in sum(args.nix_path or [], [])] -def op_create(args): +def op_create(args: Namespace) -> None: with network_state(args) as sf: depl = sf.create_deployment() sys.stderr.write("created deployment ‘{0}’\n".format(depl.uuid)) modify_deployment(args, depl) # When deployment is created without state "name" does not exist - name = args.deployment + name: str = args.deployment if "name" in args: name = args.name or args.deployment @@ -263,14 +265,14 @@ def op_create(args): sys.stdout.write(depl.uuid + "\n") -def op_modify(args): +def op_modify(args: Namespace) -> None: with deployment(args) as depl: modify_deployment(args, depl) if args.name: set_name(depl, args.name) -def op_clone(args): +def op_clone(args: Namespace) -> None: with deployment(args) as depl: depl2 = depl.clone() sys.stderr.write("created deployment ‘{0}’\n".format(depl2.uuid)) @@ -278,7 +280,7 @@ def op_clone(args): sys.stdout.write(depl2.uuid + "\n") -def op_delete(args): +def op_delete(args: Namespace) -> None: with one_or_all(args) as depls: for depl in depls: depl.delete(force=args.force or False) @@ -289,7 +291,7 @@ def machine_to_key(depl: str, name: str, type: str) -> Tuple[str, str, List[obje return (depl, type, xs) -def op_info(args): # noqa: C901 +def op_info(args: Namespace) -> None: # noqa: C901 table_headers = [ ("Name", "l"), ("Status", "c"), @@ -312,7 +314,7 @@ def state( return "Up-to-date" - def do_eval(depl): + def do_eval(depl) -> None: set_common_depl(depl, args) if not args.no_eval: @@ -435,14 +437,14 @@ def name_to_key(name: str) -> Tuple[str, str, List[object]]: print(tbl) -def op_check(args): # noqa: C901 - def highlight(s): +def op_check(args: Namespace) -> None: # noqa: C901 + def highlight(s: str) -> str: return nixops.ansi.ansi_highlight(s, outfile=sys.stdout) - def warn(s): + def warn(s: str) -> str: return nixops.ansi.ansi_warn(s, outfile=sys.stdout) - def render_tristate(x): + def render_tristate(x: bool) -> str: if x is None: return "N/A" elif x: @@ -467,7 +469,7 @@ def render_tristate(x): machines: List[nixops.backends.GenericMachineState] = [] resources: List[nixops.resources.GenericResourceState] = [] - def check(depl: nixops.deployment.Deployment): + def check(depl: nixops.deployment.Deployment) -> None: for m in depl.active_resources.values(): if not nixops.deployment.should_do( m, args.include or [], args.exclude or [] @@ -584,14 +586,14 @@ def resource_worker( sys.exit(status) -def print_backups(depl, backups): +def print_backups(depl, backups) -> None: tbl = prettytable.PrettyTable(["Backup ID", "Status", "Info"]) for k, v in sorted(backups.items(), reverse=True): tbl.add_row([k, v["status"], "\n".join(v["info"])]) print(tbl) -def op_clean_backups(args): +def op_clean_backups(args: Namespace) -> None: if args.keep and args.keep_days: raise Exception( "Combining of --keep and --keep-days arguments are not possible, please use one." @@ -602,12 +604,12 @@ def op_clean_backups(args): depl.clean_backups(args.keep, args.keep_days, args.keep_physical) -def op_remove_backup(args): +def op_remove_backup(args: Namespace) -> None: with deployment(args) as depl: depl.remove_backup(args.backupid, args.keep_physical) -def op_backup(args): +def op_backup(args: Namespace) -> None: with deployment(args) as depl: def do_backup(): @@ -633,7 +635,7 @@ def do_backup(): do_backup() -def op_backup_status(args): +def op_backup_status(args: Namespace) -> None: with deployment(args) as depl: backupid = args.backupid while True: @@ -667,7 +669,7 @@ def op_backup_status(args): return -def op_restore(args): +def op_restore(args: Namespace) -> None: with deployment(args) as depl: depl.restore( include=args.include or [], @@ -677,7 +679,7 @@ def op_restore(args): ) -def op_deploy(args): +def op_deploy(args: Namespace) -> None: with deployment(args) as depl: if args.confirm: depl.logger.set_autoresponse("y") @@ -707,12 +709,12 @@ def op_deploy(args): ) -def op_send_keys(args): +def op_send_keys(args: Namespace) -> None: with deployment(args) as depl: depl.send_keys(include=args.include or [], exclude=args.exclude or []) -def op_set_args(args): +def op_set_args(args: Namespace) -> None: with deployment(args) as depl: for [n, v] in args.args or []: depl.set_arg(n, v) @@ -722,7 +724,7 @@ def op_set_args(args): depl.unset_arg(n) -def op_destroy(args): +def op_destroy(args: Namespace) -> None: with one_or_all(args) as depls: for depl in depls: if args.confirm: @@ -732,7 +734,7 @@ def op_destroy(args): ) -def op_reboot(args): +def op_reboot(args: Namespace) -> None: with deployment(args) as depl: depl.reboot_machines( include=args.include or [], @@ -743,39 +745,41 @@ def op_reboot(args): ) -def op_delete_resources(args): +def op_delete_resources(args: Namespace) -> None: with deployment(args) as depl: if args.confirm: depl.logger.set_autoresponse("y") depl.delete_resources(include=args.include or [], exclude=args.exclude or []) -def op_stop(args): +def op_stop(args: Namespace) -> None: with deployment(args) as depl: if args.confirm: depl.logger.set_autoresponse("y") depl.stop_machines(include=args.include or [], exclude=args.exclude or []) -def op_start(args): +def op_start(args: Namespace) -> None: with deployment(args) as depl: depl.start_machines(include=args.include or [], exclude=args.exclude or []) -def op_rename(args): +def op_rename(args: Namespace) -> None: with deployment(args) as depl: depl.rename(args.current_name, args.new_name) -def print_physical_backup_spec(depl, backupid): +def print_physical_backup_spec( + depl: nixops.deployment.Deployment, backupid: str +) -> None: config = {} - for m in depl.active.values(): + for m in depl.active_machines.values(): config[m.name] = m.get_physical_backup_spec(backupid) sys.stdout.write(py2nix(config)) -def op_show_arguments(args): - with deployment(args) as depl: +def op_show_arguments(cli_args: Namespace) -> None: + with deployment(cli_args) as depl: tbl = create_table([("Name", "l"), ("Location", "l")]) args = depl.get_arguments() for arg in sorted(args.keys()): @@ -784,7 +788,7 @@ def op_show_arguments(args): print(tbl) -def op_show_physical(args): +def op_show_physical(args: Namespace) -> None: with deployment(args) as depl: if args.backupid: print_physical_backup_spec(depl, args.backupid) @@ -793,8 +797,8 @@ def op_show_physical(args): sys.stdout.write(depl.get_physical_spec()) -def op_dump_nix_paths(args): - def get_nix_path(p): +def op_dump_nix_paths(args: Namespace) -> None: + def get_nix_path(p: Optional[str]) -> Optional[str]: if p is None: return None p = os.path.realpath(os.path.abspath(p)) @@ -804,15 +808,16 @@ def get_nix_path(p): return None return "/".join(p.split("/")[: len(nix_store.split("/")) + 1]) - def strip_nix_path(p): - p = p.split("=") - if len(p) == 1: - return p[0] + def strip_nix_path(p: str) -> str: + parts: List[str] = p.split("=") + if len(parts) == 1: + return parts[0] else: - return p[1] + return parts[1] - def nix_paths(depl) -> List[str]: + def nix_paths(depl: nixops.deployment.Deployment) -> List[str]: set_common_depl(depl, args) + candidates: Sequence[Optional[str]] = [] candidates = ( [depl.network_expr.network] + [strip_nix_path(p) for p in depl.nix_path] @@ -831,7 +836,7 @@ def nix_paths(depl) -> List[str]: print(p) -def op_export(args): +def op_export(args: Namespace) -> None: res = {} with one_or_all(args) as depls: @@ -840,13 +845,13 @@ def op_export(args): print(json.dumps(res, indent=2, sort_keys=True, cls=nixops.util.NixopsEncoder)) -def op_unlock(args): +def op_unlock(args: Namespace) -> None: network = eval_network(get_network_file(args)) lock = get_lock(network) lock.unlock() -def op_import(args): +def op_import(args: Namespace) -> None: with network_state(args) as sf: existing = set(sf.query_deployments()) @@ -874,10 +879,16 @@ def op_import(args): nixops.known_hosts.add(m.private_ipv4, m.public_host_key) -def parse_machine(name, depl): - username, machine_name = ( - (None, name) if name.find("@") == -1 else name.split("@", 1) - ) +def parse_machine( + name: str, depl: nixops.deployment.Deployment +) -> Tuple[str, str, nixops.backends.GenericMachineState]: + username: Optional[str] + machine_name: str + if name.find("@") == -1: + username = None + machine_name = name + else: + username, machine_name = name.split("@", 1) # For nixops mount, split path element machine_name = machine_name.split(":")[0] @@ -896,7 +907,7 @@ def parse_machine(name, depl): return username, machine_name, m -def op_ssh(args): +def op_ssh(args: Namespace) -> None: with deployment(args) as depl: (username, _, m) = parse_machine(args.machine, depl) flags, command = m.ssh.split_openssh_args(args.args) @@ -912,7 +923,7 @@ def op_ssh(args): ) -def op_ssh_for_each(args): +def op_ssh_for_each(args: Namespace) -> None: results: List[Optional[int]] = [] with one_or_all(args) as depls: for depl in depls: @@ -936,11 +947,11 @@ def worker(m: nixops.backends.GenericMachineState) -> Optional[int]: sys.exit(max(results) if results != [] else 0) -def scp_loc(user, ssh_name, remote, loc): +def scp_loc(user: str, ssh_name: str, remote: str, loc: str) -> str: return "{0}@{1}:{2}".format(user, ssh_name, loc) if remote else loc -def op_scp(args): +def op_scp(args: Namespace) -> None: if args.scp_from == args.scp_to: raise Exception("exactly one of ‘--from’ and ‘--to’ must be specified") with deployment(args) as depl: @@ -956,7 +967,7 @@ def op_scp(args): sys.exit(res) -def op_mount(args): +def op_mount(args: Namespace) -> None: # TODO: Fixme with deployment(args) as depl: (username, rest, m) = parse_machine(args.machine, depl) @@ -983,7 +994,7 @@ def op_mount(args): sys.exit(res) -def op_show_option(args): +def op_show_option(args: Namespace) -> None: with deployment(args) as depl: if args.include_physical: depl.evaluate() @@ -997,7 +1008,9 @@ def op_show_option(args): @contextlib.contextmanager -def deployment_with_rollback(args): +def deployment_with_rollback( + args: Namespace, +) -> Generator[nixops.deployment.Deployment, None, None]: with deployment(args) as depl: if not depl.rollback_enabled: raise Exception( @@ -1006,7 +1019,7 @@ def deployment_with_rollback(args): yield depl -def op_list_generations(args): +def op_list_generations(args: Namespace) -> None: with deployment_with_rollback(args) as depl: if ( subprocess.call(["nix-env", "-p", depl.get_profile(), "--list-generations"]) @@ -1015,7 +1028,7 @@ def op_list_generations(args): raise Exception("nix-env --list-generations failed") -def op_delete_generation(args): +def op_delete_generation(args: Namespace) -> None: with deployment_with_rollback(args) as depl: if ( subprocess.call( @@ -1032,7 +1045,7 @@ def op_delete_generation(args): raise Exception("nix-env --delete-generations failed") -def op_rollback(args): +def op_rollback(args: Namespace) -> None: with deployment_with_rollback(args) as depl: depl.rollback( generation=args.generation, @@ -1047,7 +1060,7 @@ def op_rollback(args): ) -def op_show_console_output(args): +def op_show_console_output(args: Namespace) -> None: with deployment(args) as depl: m = depl.machines.get(args.machine) if not m: @@ -1055,7 +1068,7 @@ def op_show_console_output(args): sys.stdout.write(m.get_console_output()) -def op_edit(args): +def op_edit(args: Namespace) -> None: with deployment(args) as depl: editor = os.environ.get("EDITOR") if not editor: @@ -1065,14 +1078,14 @@ def op_edit(args): ) -def op_copy_closure(args): +def op_copy_closure(args: Namespace) -> None: with deployment(args) as depl: (username, machine, m) = parse_machine(args.machine, depl) m.copy_closure_to(args.storepath) # Set up logging of all commands and output -def setup_logging(args): +def setup_logging(args: Namespace) -> None: if os.path.exists("/dev/log") and args.op not in [ op_ssh, op_ssh_for_each, @@ -1242,9 +1255,9 @@ def add_common_deployment_options(subparser: ArgumentParser) -> None: ) -def error(msg): +def error(msg: str) -> None: sys.stderr.write(nixops.ansi.ansi_warn("error: ") + msg + "\n") -def parser_plugin_hooks(parser, subparsers): +def parser_plugin_hooks(parser: ArgumentParser, subparsers: _SubParsersAction) -> None: PluginManager.parser(parser, subparsers) From 6bcdee4f7732658cbf6739c5f9328c2e71ffb249 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 18 Jul 2020 11:25:40 +0200 Subject: [PATCH 71/80] nixops/plugins/__init__.py: Add more static typing --- nixops/plugins/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixops/plugins/__init__.py b/nixops/plugins/__init__.py index 4af9e4533..24115f6bb 100644 --- a/nixops/plugins/__init__.py +++ b/nixops/plugins/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -from nixops.backends import MachineState +from nixops.backends import GenericMachineState from typing import List, Dict, Optional, Union, Tuple, Type from argparse import ArgumentParser, _SubParsersAction @@ -47,7 +47,7 @@ def physical_spec( class MachineHooks: - def post_wait(self, m: MachineState) -> None: + def post_wait(self, m: GenericMachineState) -> None: """ Do action once SSH is available """ From 0c832594d95189ed5af7507d9f34399f94aaad80 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 18 Jul 2020 11:25:40 +0200 Subject: [PATCH 72/80] nixops/plugins/manager.py: Add more static typing --- nixops/plugins/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixops/plugins/manager.py b/nixops/plugins/manager.py index 6f6fb0525..05f4bd82c 100644 --- a/nixops/plugins/manager.py +++ b/nixops/plugins/manager.py @@ -1,6 +1,6 @@ from __future__ import annotations -from nixops.backends import MachineState +from nixops.backends import GenericMachineState from typing import List, Dict, Generator, Tuple, Any, Set import importlib from argparse import ArgumentParser, _SubParsersAction @@ -35,7 +35,7 @@ def physical_spec( class MachineHooksManager: @staticmethod - def post_wait(m: MachineState) -> None: + def post_wait(m: GenericMachineState) -> None: for hook in PluginManager.machine_hooks(): hook.post_wait(m) From 5f542a035ab03cde72d2fd3517b9b545e5e25094 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 18 Jul 2020 11:25:40 +0200 Subject: [PATCH 73/80] nixops/util.py: Add more static typing --- nixops/util.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/nixops/util.py b/nixops/util.py index b78a1e790..6b6a1a51d 100644 --- a/nixops/util.py +++ b/nixops/util.py @@ -152,13 +152,6 @@ def _transform_value(key: Any, value: Any) -> Any: if inspect.isclass(ann) and issubclass(ann, ImmutableValidatedObject): value = ann(**value) - elif hasattr(ann, "__origin__") and ann.__origin__ == Union: - for arg in ann.__args__: - if inspect.isclass(arg) and issubclass( - arg, ImmutableValidatedObject - ): - value = arg(**value) - break # Support Sequence[ImmutableValidatedObject] if isinstance(value, tuple) and not isinstance(ann, str): @@ -381,7 +374,7 @@ def logged_exec( # noqa: C901 return stdout if capture_stdout else res -def generate_random_string(length=256) -> str: +def generate_random_string(length: int = 256) -> str: """Generate a base-64 encoded cryptographically strong random string.""" s = os.urandom(length) assert len(s) == length @@ -393,9 +386,9 @@ def make_non_blocking(fd: IO[Any]) -> None: def wait_for_success( - fn: Callable, + fn: Callable[[], None], timeout: Optional[int] = None, - callback: Optional[Callable[[], Any]] = None, + callback: Optional[Callable[[], None]] = None, ) -> bool: n = 0 while True: @@ -419,9 +412,9 @@ def wait_for_success( def wait_for_fail( - fn: Callable, + fn: Callable[[], None], timeout: Optional[int] = None, - callback: Optional[Callable[[], Any]] = None, + callback: Optional[Callable[[], None]] = None, ) -> bool: n = 0 while True: @@ -503,7 +496,7 @@ def set(self, x: Any) -> None: def create_key_pair( - key_name="NixOps auto-generated key", type="ed25519" + key_name: str = "NixOps auto-generated key", type: str = "ed25519" ) -> Tuple[str, str]: key_dir = tempfile.mkdtemp(prefix="nixops-key-tmp") res = subprocess.call( @@ -541,7 +534,7 @@ def __init__(self) -> None: def __del__(self) -> None: sys.stderr = self.stderr - def write(self, data) -> int: + def write(self, data: str) -> int: ret = self.stderr.write(data) for line in data.split("\n"): self.logger.warning(line) @@ -569,7 +562,7 @@ def __init__(self) -> None: def __del__(self) -> None: sys.stdout = self.stdout - def write(self, data) -> int: + def write(self, data: str) -> int: ret = self.stdout.write(data) for line in data.split("\n"): self.logger.info(line) @@ -606,10 +599,6 @@ def is_exe(fpath: str) -> bool: raise Exception("program ‘{0}’ not found in \$PATH".format(program)) # noqa: W605 -def enum(**enums): - return type("Enum", (), enums) - - def write_file(path: str, contents: str) -> None: with open(path, "w") as f: f.write(contents) From 7d81a217f31e4be2572976cc2968f1f6f88308fe Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sun, 19 Jul 2020 14:56:20 +0200 Subject: [PATCH 74/80] Reduce nix-instantiate calls from 3 to 2 --- nixops/deployment.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 77bd8371f..815bf74c5 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -18,7 +18,7 @@ import time import importlib -from functools import reduce +from functools import reduce, lru_cache from typing import ( Callable, Dict, @@ -407,6 +407,7 @@ def evaluate_args(self) -> Any: """Evaluate the NixOps network expression's arguments.""" return self.eval(attr="nixopsArguments") + @lru_cache() def evaluate_config(self, attr) -> Dict: return self.eval(checkConfigurationOptions=False, attr=attr) @@ -414,7 +415,7 @@ def evaluate_network(self, action: str = "") -> None: if not self.network_attr_eval: # Extract global deployment attributes. try: - config = self.evaluate_config("info.network") + config = self.evaluate_config("info")["network"] except Exception as e: if action not in ("destroy", "delete"): raise e From 4ea172845d2d54ab7657f58f80bd87332688258e Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 8 Feb 2021 14:39:34 +0100 Subject: [PATCH 75/80] Remove all traces of s3 storage/locking implementations --- tests/locks/s3.nix | 4 ---- tests/locks/withlock.nix | 12 +++--------- tests/storage/multiple.nix | 6 +----- tests/storage/withlock.nix | 12 +++--------- 4 files changed, 7 insertions(+), 27 deletions(-) delete mode 100644 tests/locks/s3.nix diff --git a/tests/locks/s3.nix b/tests/locks/s3.nix deleted file mode 100644 index 9f52dca83..000000000 --- a/tests/locks/s3.nix +++ /dev/null @@ -1,4 +0,0 @@ -{ - network.storage.memory = {}; - network.lock.s3 = {}; -} diff --git a/tests/locks/withlock.nix b/tests/locks/withlock.nix index 92a10c67f..657bf5a5d 100644 --- a/tests/locks/withlock.nix +++ b/tests/locks/withlock.nix @@ -1,12 +1,6 @@ { - network = rec { - storage.s3 = { - bucket = "hi"; - whatever = "there"; - }; - - lock.s3 = storage.s3 // { - dynamodb_table = "blah"; - }; + network = { + storage.memory = {}; + locking.noop = {}; }; } diff --git a/tests/storage/multiple.nix b/tests/storage/multiple.nix index 3c39a8e77..d2072f43d 100644 --- a/tests/storage/multiple.nix +++ b/tests/storage/multiple.nix @@ -4,11 +4,7 @@ specify multiple storage backends */ { network = rec { - storage.s3 = { - bucket = "hi"; - whatever = "there"; - }; - + storage.memory = {}; storage.legacy = {}; }; } diff --git a/tests/storage/withlock.nix b/tests/storage/withlock.nix index d5d87605c..657bf5a5d 100644 --- a/tests/storage/withlock.nix +++ b/tests/storage/withlock.nix @@ -1,12 +1,6 @@ { - network = rec { - storage.s3 = { - bucket = "hi"; - whatever = "there"; - }; - - locking.s3 = storage.s3 // { - dynamodb_table = "blah"; - }; + network = { + storage.memory = {}; + locking.noop = {}; }; } From bc3142976593a1eed891d2c403b42c4f4bc686e4 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 8 Feb 2021 14:41:42 +0100 Subject: [PATCH 76/80] Rename `network.nix` to `nixops.nix` `nixops.nix` is a much more obvious name and really indicates what tooling is supposed to be used to build/deploy by just eyeballing a directory layout. We also don't want to occupy generic words possibly already used in some deployments. --- doc/guides/deploy-without-root.rst | 2 +- nixops/script_defs.py | 8 ++++---- tests/functional/vpc.py | 2 +- tests/hetzner-backend/default.nix | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/guides/deploy-without-root.rst b/doc/guides/deploy-without-root.rst index b29e60a0c..dd53fa4a8 100644 --- a/doc/guides/deploy-without-root.rst +++ b/doc/guides/deploy-without-root.rst @@ -61,7 +61,7 @@ the next step. Configuring the NixOps Network ****************************** -Edit your network.nix to specify the machine's +Edit your nixops.nix to specify the machine's ``deployment.targetUser``: .. code-block:: nix diff --git a/nixops/script_defs.py b/nixops/script_defs.py index 729249c93..d396ee60d 100644 --- a/nixops/script_defs.py +++ b/nixops/script_defs.py @@ -42,14 +42,14 @@ def get_network_file(args: Namespace) -> NetworkFile: if not os.path.exists(network_dir): raise ValueError("f{network_dir} does not exist") - classic_path = os.path.join(network_dir, "network.nix") + classic_path = os.path.join(network_dir, "nixops.nix") flake_path = os.path.join(network_dir, "flake.nix") classic_exists: bool = os.path.exists(classic_path) flake_exists: bool = os.path.exists(flake_path) if all((flake_exists, classic_exists)): - raise ValueError("Both flake.nix and network.nix cannot coexist") + raise ValueError("Both flake.nix and nixops.nix cannot coexist") if classic_exists: return NetworkFile(network=classic_path, is_flake=False) @@ -57,7 +57,7 @@ def get_network_file(args: Namespace) -> NetworkFile: if flake_exists: return NetworkFile(network=network_dir, is_flake=True) - raise ValueError(f"Neither flake.nix nor network.nix exists in {network_dir}") + raise ValueError(f"Neither flake.nix nor nixops.nix exists in {network_dir}") def set_common_depl(depl: nixops.deployment.Deployment, args: Namespace) -> None: @@ -1132,7 +1132,7 @@ def add_subparser( dest="network_dir", metavar="FILE", default=os.getcwd(), - help="path to a directory containing either network.nix or flake.nix", + help="path to a directory containing either nixops.nix or flake.nix", ) subparser.add_argument( "--deployment", diff --git a/tests/functional/vpc.py b/tests/functional/vpc.py index 8b1ad2d42..32a19d4e2 100644 --- a/tests/functional/vpc.py +++ b/tests/functional/vpc.py @@ -87,7 +87,7 @@ def generate_config(self, config): CFG_VPC_MACHINE = ( - "network.nix", + "nixops.nix", """ { machine = diff --git a/tests/hetzner-backend/default.nix b/tests/hetzner-backend/default.nix index d635227a2..006699270 100644 --- a/tests/hetzner-backend/default.nix +++ b/tests/hetzner-backend/default.nix @@ -12,7 +12,7 @@ let rescueISO = import ./rescue-image.nix { inherit pkgs; }; rescuePasswd = "abcd1234"; - network = pkgs.writeText "network.nix" '' + network = pkgs.writeText "nixops.nix" '' let withCommonOptions = otherOpts: { config, ... }: { require = [ @@ -205,8 +205,8 @@ in makeTest { }; subtest "create deployment", sub { - $coordinator->succeed("cp ${network} network.nix"); - $coordinator->succeed("nixops create network.nix"); + $coordinator->succeed("cp ${network} nixops.nix"); + $coordinator->succeed("nixops create nixops.nix"); }; # Do deployment on one target at a time to avoid running out of memory. From d87cf5663cbfe299477cfa2a5e4ccb5277c878d4 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 7 Jul 2021 04:34:28 -0500 Subject: [PATCH 77/80] Add migration step docs --- doc/manual/migrating.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/manual/migrating.rst diff --git a/doc/manual/migrating.rst b/doc/manual/migrating.rst new file mode 100644 index 000000000..611481162 --- /dev/null +++ b/doc/manual/migrating.rst @@ -0,0 +1,21 @@ +.. _chap-overview: + +Overview +======== + +This chapter aims to provide guidelines on migrating from NixOps 1.x to 2.0. + +.. _sec-layout: + +Code layout changes +------------------- + +Using NixOps 1.0 multiple deployments spread out over the file and deployed +from any working directory with the ``--deployment (-d)`` parameter. + +NixOps 2 however requires a file relative to the invocation working directory. +It needs to be called either ``nixops.nix`` for a traditional deployment or +``flake.nix`` for the as of yet experimental +`flakes support `. + +.. _sec-state: From b2b79b471e77d58f05e35234b439634a0a6b7fab Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 7 Jul 2021 05:27:46 -0500 Subject: [PATCH 78/80] Add state migration docs --- doc/manual/migrating.rst | 57 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/doc/manual/migrating.rst b/doc/manual/migrating.rst index 611481162..f0885d994 100644 --- a/doc/manual/migrating.rst +++ b/doc/manual/migrating.rst @@ -18,4 +18,59 @@ It needs to be called either ``nixops.nix`` for a traditional deployment or ``flake.nix`` for the as of yet experimental `flakes support `. -.. _sec-state: +.. _sec-state-location: + +State location +-------------- + +In NixOps 1.0 deployment state such as provisioned resources are stored in a +SQLite database located in ``~/.nixops``. + +NixOps 2 however has pluggable state backends, meaning that you will have to +make a choice where to store this state. + +To implement the old behaviour of loading deployment state from the SQLite +database located in ``~/.nixops`` add the following snippet to your deployment: + +:: + { + network = { + storage.legacy = {}; + }; + } + +To implement a fire-and-forget strategy use this code snippet: + +:: + { + network = { + storage.memory = {}; + }; + } + +For additional state storage strategies see the various NixOps plugins. + +.. _sec-state-migration: + +State migration +--------------- + +Migrating to any non-legacy backend from a previous deployment requires a +manual migration step. + +#. Start by configuring the legacy backend as such:: + { + network = { + storage.legacy = {}; + }; + } + +#. Then export the current state:: + nixops export > state.json + +#. Now go ahead and configure your desired state backend. + +#. And finally import the old state:: + nixops import < state.json + +#. Make sure to remove ``state.json`` as it may contain deployment secrets. From e14c83505b3a8602a92cfc45d05c58c3848909b4 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 7 Jul 2021 05:36:53 -0500 Subject: [PATCH 79/80] Fix output of migration error message to not include internal parameters such as is_flake --- nixops/evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index f6ea3f757..0564f0146 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -162,7 +162,7 @@ def eval_network(nix_expr: NetworkFile) -> NetworkEval: } 3. Rerun """ - % nix_expr + % nix_expr.network ) raw_eval = RawNetworkEval(**result) From 3f7d77abf8854a5719e9620decb6be13d9096969 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 7 Jul 2021 05:37:47 -0500 Subject: [PATCH 80/80] Remove TODO from migration error message output This will soon contain a reference to readthedocs. --- nixops/evaluation.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nixops/evaluation.py b/nixops/evaluation.py index 0564f0146..8db711e35 100644 --- a/nixops/evaluation.py +++ b/nixops/evaluation.py @@ -145,11 +145,6 @@ def eval_network(nix_expr: NetworkFile) -> NetworkEval: if result.get("storage") is None: raise MalformedNetworkError( """ -TODO: improve this error to be less specific about conversion, and less -about storage backends, and more about the construction of a network -attribute value. link to docs about storage drivers and lock drivers. - - WARNING: NixOps 1.0 -> 2.0 conversion step required NixOps 2.0 added support for multiple storage backends.