Add system tests for librepcb-cli

.gitignore

@@ -29,3 +29,9 @@ i18n/*.qm
 
 # Vim
 *.swp
+
+# Python
+.idea/
+.pytest_cache/
+__pycache__/
+*.pyc

tests/README.md

@@ -4,3 +4,4 @@ This directory contains tests for LibrePCB. Purpose of subdirectories:
 
 - `data`: Data files (for example LibrePCB projects) used for the tests.
 - `unittests`: Unit/integration tests for all static libraries of LibrePCB.
+- `cli`: System tests for the LibrePCB CLI.

tests/cli/.gitignore

@@ -0,0 +1 @@
+pytest.ini

tests/cli/README.md

@@ -0,0 +1,20 @@
+# System Tests for the LibrePCB CLI
+
+This directory contains system tests for `librepcb-cli` using
+[pytest](https://docs.pytest.org).
+
+## Create Virtualenv
+
+    mkvirtualenv -p `which python3` librepcb-cli
+
+## Install Requirements
+
+    pip install -r requirements.txt
+
+## Run Tests
+
+    pytest -v --librepcb-executable=/path/to/librepcb-cli
+
+## Links
+
+- [Documentation of `pytest`](https://docs.pytest.org/en/latest/contents.html)

tests/cli/conftest.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import shutil
+import pytest
+import subprocess
+
+
+CLI_DIR = os.path.dirname(__file__)
+TESTS_DIR = os.path.dirname(CLI_DIR)
+REPO_DIR = os.path.dirname(TESTS_DIR)
+DATA_DIR = os.path.join(TESTS_DIR, 'data')
+CLI_DATA_DIR = os.path.join(DATA_DIR, 'cli')
+FIXTURES_DATA_DIR = os.path.join(CLI_DATA_DIR, 'fixtures')
+
+
+def pytest_addoption(parser):
+    parser.addoption("--librepcb-executable",
+                     action="store",
+                     help="Path to librepcb-cli executable to test")
+
+
+class CliExecutor(object):
+    def __init__(self, config, tmpdir):
+        super(CliExecutor, self).__init__()
+        self.executable = os.path.abspath(config.getoption('--librepcb-executable'))
+        if not os.path.exists(self.executable):
+            raise Exception("Executable '{}' not found. Please pass it with "
+                            "'--librepcb-executable'.".format(self.executable))
+        self.tmpdir = tmpdir
+        # Copy test data to temporary directory to avoid modifications in original data
+        self.data_dir = os.path.join(self.tmpdir, 'data')
+        shutil.copytree(CLI_DATA_DIR, self.data_dir)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    def abspath(self, relpath):
+        return os.path.join(self.tmpdir, relpath)
+
+    def run(self, *args):
+        p = subprocess.Popen([self.executable] + list(args), cwd=self.tmpdir,
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                             universal_newlines=True, env=self._env())
+        stdout, stderr = p.communicate()
+        # output to stdout/stderr because it helps debugging failed tests
+        sys.stdout.write(stdout)
+        sys.stderr.write(stderr)
+        return p.returncode, stdout.splitlines(), stderr.splitlines()
+
+    def _env(self):
+        env = os.environ
+        # Make output independent from the system's language
+        env['LC_ALL'] = 'C'
+        # Override configuration location to make tests independent of existing configs
+        env['LIBREPCB_CONFIG_DIR'] = os.path.join(self.tmpdir, 'config')
+        # Use a neutral username
+        env['USERNAME'] = 'testuser'
+        return env
+
+
+@pytest.fixture
+def cli(request, tmpdir):
+    """
+    Fixture to start the LibrePCB CLI
+    """
+    with CliExecutor(request.config, str(tmpdir)) as executor:
+        yield executor

tests/cli/open-project/test_everything.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import pytest
+
+"""
+Test command "open-project" with all available options
+"""
+
+PROJECT_DIR = 'data/Empty Project/'
+PROJECT_PATH = PROJECT_DIR + 'Empty Project.lpp'
+OUTPUT_DIR = PROJECT_DIR + 'output/v1/gerber'
+
+PROJECT_2_DIR = 'data/Project With Two Boards/'
+PROJECT_2_PATH = PROJECT_2_DIR + 'Project With Two Boards.lpp'
+OUTPUT_2_DIR = PROJECT_2_DIR + 'output/v1/gerber'
+
+
+@pytest.mark.parametrize("project_path,output_dir,gerber_count", [
+    (PROJECT_PATH, OUTPUT_DIR, 8),
+    (PROJECT_2_PATH, OUTPUT_2_DIR, 16),
+], ids=[
+    'EmptyProject',
+    'ProjectWithTwoBoards'
+])
+def test_everything(cli, project_path, output_dir, gerber_count):
+    # prepare "--export-schematics"
+    schematic = cli.abspath('schematic.pdf')
+    assert not os.path.exists(schematic)
+
+    # prepare "--export-pcb-fabrication-data"
+    output = cli.abspath(output_dir)
+    assert not os.path.exists(output)
+
+    # prepare "--save"
+    path = cli.abspath(project_path)
+    original_filesize = os.path.getsize(path)
+    with open(path, 'ab') as f:
+        f.write(b'\n\n')
+    assert os.path.getsize(path) == original_filesize + 2
+
+    # run the CLI
+    code, stdout, stderr = cli.run('open-project',
+                                   '--erc',
+                                   '--export-schematics={}'.format(schematic),
+                                   '--export-pcb-fabrication-data',
+                                   '--save',
+                                   project_path)
+    assert code == 0
+    assert len(stderr) == 0
+    assert len(stdout) > 0
+    assert stdout[-1] == 'SUCCESS'
+
+    # check "--export-schematics"
+    assert os.path.exists(schematic)
+
+    # check "--export-pcb-fabrication-data"
+    assert os.path.exists(output)
+    assert len(os.listdir(output)) == gerber_count
+
+    # check "--save"
+    assert os.path.getsize(path) == original_filesize

tests/cli/open-project/test_export_pcb_fabrication_data.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import fileinput
+
+"""
+Test command "open-project --export-pcb-fabrication-data"
+"""
+
+PROJECT_DIR = 'data/Empty Project/'
+PROJECT_PATH = PROJECT_DIR + 'Empty Project.lpp'
+OUTPUT_DIR = PROJECT_DIR + 'output/v1/gerber'
+
+PROJECT_2_DIR = 'data/Project With Two Boards/'
+PROJECT_2_PATH = PROJECT_2_DIR + 'Project With Two Boards.lpp'
+OUTPUT_2_DIR = PROJECT_2_DIR + 'output/v1/gerber'
+
+
+def test_if_project_without_boards_succeeds(cli):
+    # remove all boards first
+    with open(cli.abspath(PROJECT_DIR + 'boards/boards.lp'), 'w') as f:
+        f.write('(librepcb_boards)')
+    dir = cli.abspath(OUTPUT_DIR)
+    assert not os.path.exists(dir)
+    code, stdout, stderr = cli.run('open-project',
+                                   '--export-pcb-fabrication-data',
+                                   PROJECT_PATH)
+    assert code == 0
+    assert len(stderr) == 0
+    assert len(stdout) > 0
+    assert stdout[-1] == 'SUCCESS'
+    assert not os.path.exists(dir)  # nothing was exported ;)
+
+
+def test_export_project_with_one_board_implicit(cli):
+    dir = cli.abspath(OUTPUT_DIR)
+    assert not os.path.exists(dir)
+    code, stdout, stderr = cli.run('open-project',
+                                   '--export-pcb-fabrication-data',
+                                   PROJECT_PATH)
+    assert code == 0
+    assert len(stderr) == 0
+    assert len(stdout) > 0
+    assert stdout[-1] == 'SUCCESS'
+    assert os.path.exists(dir)
+    assert len(os.listdir(dir)) == 8
+
+
+def test_export_project_with_one_board_explicit(cli):
+    dir = cli.abspath(OUTPUT_DIR)
+    assert not os.path.exists(dir)
+    code, stdout, stderr = cli.run('open-project',
+                                   '--export-pcb-fabrication-data',
+                                   '--board=default',
+                                   PROJECT_PATH)
+    assert code == 0
+    assert len(stderr) == 0
+    assert len(stdout) > 0
+    assert stdout[-1] == 'SUCCESS'
+    assert os.path.exists(dir)
+    assert len(os.listdir(dir)) == 8
+
+
+def test_if_exporting_invalid_board_fails(cli):
+    dir = cli.abspath(OUTPUT_DIR)
+    assert not os.path.exists(dir)
+    code, stdout, stderr = cli.run('open-project',
+                                   '--export-pcb-fabrication-data',
+                                   '--board=foo',
+                                   PROJECT_PATH)
+    assert code == 1
+    assert len(stderr) == 1
+    assert "No board with the name 'foo' found." in stderr[0]
+    assert len(stdout) > 0
+    assert stdout[-1] == 'Finished with errors!'
+    assert not os.path.exists(dir)
+
+
+def test_export_project_with_two_boards_implicit(cli):
+    dir = cli.abspath(OUTPUT_2_DIR)
+    assert not os.path.exists(dir)
+    code, stdout, stderr = cli.run('open-project',
+                                   '--export-pcb-fabrication-data',
+                                   PROJECT_2_PATH)
+    assert code == 0
+    assert len(stderr) == 0
+    assert len(stdout) > 0
+    assert stdout[-1] == 'SUCCESS'
+    assert os.path.exists(dir)
+    assert len(os.listdir(dir)) == 16
+
+
+def test_export_project_with_two_boards_explicit_one(cli):
+    dir = cli.abspath(OUTPUT_2_DIR)
+    assert not os.path.exists(dir)
+    code, stdout, stderr = cli.run('open-project',
+                                   '--export-pcb-fabrication-data',
+                                   '--board=copy',
+                                   PROJECT_2_PATH)
+    assert code == 0
+    assert len(stderr) == 0
+    assert len(stdout) > 0
+    assert stdout[-1] == 'SUCCESS'
+    assert os.path.exists(dir)
+    assert len(os.listdir(dir)) == 8
+
+
+def test_export_project_with_two_boards_explicit_two(cli):
+    dir = cli.abspath(OUTPUT_2_DIR)
+    assert not os.path.exists(dir)
+    code, stdout, stderr = cli.run('open-project',
+                                   '--export-pcb-fabrication-data',
+                                   '--board=copy',
+                                   '--board=default',
+                                   PROJECT_2_PATH)
+    assert code == 0
+    assert len(stderr) == 0
+    assert len(stdout) > 0
+    assert stdout[-1] == 'SUCCESS'
+    assert os.path.exists(dir)
+    assert len(os.listdir(dir)) == 16
+
+
+def test_export_project_with_two_conflicting_boards_fails(cli):
+    # change gerber output path to the same for both boards
+    boardfile = cli.abspath(PROJECT_2_DIR + 'boards/copy/board.lp')
+    for line in fileinput.input(boardfile, inplace=1):
+        print(line.replace("_copy", ""))
+    dir = cli.abspath(OUTPUT_2_DIR)
+    assert not os.path.exists(dir)
+    code, stdout, stderr = cli.run('open-project',
+                                   '--export-pcb-fabrication-data',
+                                   PROJECT_2_PATH)
+    assert code == 1
+    assert len(stderr) > 0
+    assert 'Some files were written multiple times' in stderr[0]
+    assert len(stdout) > 0
+    assert stdout[-1] == 'Finished with errors!'
+
+
+def test_export_project_with_two_conflicting_boards_succeeds_explicit(cli):
+    # change gerber output path to the same for both boards
+    boardfile = cli.abspath(PROJECT_2_DIR + 'boards/copy/board.lp')
+    for line in fileinput.input(boardfile, inplace=1):
+        print(line.replace("_copy", ""))
+    dir = cli.abspath(OUTPUT_2_DIR)
+    assert not os.path.exists(dir)
+    code, stdout, stderr = cli.run('open-project',
+                                   '--export-pcb-fabrication-data',
+                                   '--board=copy',
+                                   PROJECT_2_PATH)
+    assert code == 0
+    assert len(stderr) == 0
+    assert len(stdout) > 0
+    assert stdout[-1] == 'SUCCESS'
+    assert os.path.exists(dir)
+    assert len(os.listdir(dir)) == 8

tests/cli/open-project/test_export_schematics.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+
+"""
+Test command "open-project --export-schematics"
+"""
+
+PROJECT_DIR = 'data/Empty Project/'
+PROJECT_PATH = PROJECT_DIR + 'Empty Project.lpp'
+
+
+def test_if_unknown_file_extension_fails(cli):
+    code, stdout, stderr = cli.run('open-project',
+                                   '--export-schematics=foo.bar',
+                                   PROJECT_PATH)
+    assert code == 1
+    assert len(stderr) == 1
+    assert 'Unknown extension' in stderr[0]
+    assert len(stdout) > 0
+    assert stdout[-1] == 'Finished with errors!'
+
+
+def test_exporting_pdf_with_relative_path(cli):
+    path = cli.abspath('sch.pdf')
+    assert not os.path.exists(path)
+    code, stdout, stderr = cli.run('open-project',
+                                   '--export-schematics=sch.pdf',
+                                   PROJECT_PATH)
+    assert code == 0
+    assert len(stderr) == 0
+    assert len(stdout) > 0
+    assert stdout[-1] == 'SUCCESS'
+    assert os.path.exists(path)
+
+
+def test_exporting_pdf_with_absolute_path(cli):
+    path = cli.abspath('schematic with spaces.pdf')
+    assert not os.path.exists(path)
+    code, stdout, stderr = cli.run('open-project',
+                                   '--export-schematics={}'.format(path),
+                                   PROJECT_PATH)
+    assert code == 0
+    assert len(stderr) == 0
+    assert len(stdout) > 0
+    assert stdout[-1] == 'SUCCESS'
+    assert os.path.exists(path)
+
+
+def test_if_output_directories_are_created(cli):
+    dir = cli.abspath('nonexistent directory/nested')
+    path = os.path.join(dir, 'schematic.pdf')
+    assert not os.path.exists(dir)
+    code, stdout, stderr = cli.run('open-project',
+                                   '--export-schematics={}'.format(path),
+                                   PROJECT_PATH)
+    assert code == 0
+    assert len(stderr) == 0
+    assert len(stdout) > 0
+    assert stdout[-1] == 'SUCCESS'
+    assert os.path.exists(dir)
+    assert os.path.exists(path)