diff --git a/.github/workflows/dev-test.yml b/.github/workflows/dev-test.yml new file mode 100644 index 000000000..38adadd16 --- /dev/null +++ b/.github/workflows/dev-test.yml @@ -0,0 +1,32 @@ +name: Dev Test/Deploy + +on: + push: {} + pull_request: + branches: dev + +jobs: + test: + uses: openworm/owmeta/.github/workflows/test.yml@github-workflows + with: + coveralls: ${{github.event_name != 'schedule'}} + ref: ${{ github.ref }} + secrets: + my_github_token: ${{secrets.GITHUB_TOKEN}} + pubmed_api_key: ${{secrets.PUBMED_API_KEY}} + dev-deploy: + if: github.event_name == 'push' && github.ref == 'refs/heads/dev' + uses: openworm/owmeta-core/.github/workflows/deploy.yml@github-workflows + needs: test + with: + ref: refs/heads/dev + secrets: + twine_token: ${{secrets.TWINE_PASSWORD}} + + coveralls: + if: github.event_name == 'push' + needs: test + uses: openworm/owmeta/.github/workflows/coveralls-upload.yml@github-workflows + secrets: + my_github_token: ${{secrets.GITHUB_TOKEN}} + diff --git a/.github/workflows/scheduled-dev-build.yml b/.github/workflows/scheduled-dev-build.yml new file mode 100644 index 000000000..20889a3f4 --- /dev/null +++ b/.github/workflows/scheduled-dev-build.yml @@ -0,0 +1,13 @@ +name: Build develop + +on: + schedule: + - cron: '39 12 10,23 * *' + +jobs: + test: + uses: openworm/owmeta/.github/workflows/test.yml@github-workflows + with: + ref: refs/heads/dev + secrets: + pubmed_api_key: ${{secrets.PUBMED_API_KEY}} diff --git a/.github/workflows/scheduled-master-build.yml b/.github/workflows/scheduled-master-build.yml new file mode 100644 index 000000000..d70e2a38f --- /dev/null +++ b/.github/workflows/scheduled-master-build.yml @@ -0,0 +1,13 @@ +name: Build master + +on: + schedule: + - cron: '13 2 10,23 * *' + +jobs: + test: + uses: openworm/owmeta/.github/workflows/test.yml@github-workflows + with: + ref: refs/heads/master + secrets: + pubmed_api_key: ${{secrets.PUBMED_API_KEY}} diff --git a/.travis.yml b/.travis.yml index af87dd68c..8e258c868 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ before_install: install: - pip install . -- owm clone https://github.com/openworm/OpenWormData.git +- owm clone https://github.com/openworm/OpenWormData.git --branch owmeta - | if [ $BUNDLE_TESTS ] ; then owm bundle register ./owmeta-schema-bundle.yml diff --git a/README.md b/README.md index 40add0ed8..4720c5d36 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -[![Build Status](https://travis-ci.org/openworm/owmeta.png?branch=dev)](https://travis-ci.org/openworm/owmeta/builds) +[![Build Status](https://github.com/openworm/owmeta/actions/workflows/dev-test.yml/badge.svg)](https://github.com/openworm/owmeta/actions/workflows/dev-test.yml) [![Docs](https://readthedocs.org/projects/owmeta/badge/?version=latest)](https://owmeta.readthedocs.io/en/latest) [![Coverage Status](https://coveralls.io/repos/github/openworm/owmeta/badge.svg?branch=dev)](https://coveralls.io/github/openworm/owmeta?branch=dev) -owmeta +owmeta ====== pyow_in_overview @@ -82,7 +82,7 @@ This project holds a working-copy of the database. You can retrieve it by executing the following command line after owmeta installation: ```bash -owm clone https://github.com/openworm/OpenWormData.git +owm clone https://github.com/openworm/OpenWormData.git --branch owmeta ``` This command should create a directory `.owm` in your current working @@ -170,7 +170,7 @@ data and models to corresponding articles from peer-reviewed literature: >>> evctx = conn(Context)(ident='http://example.org/evidence/context') # Make a context for defining domain knowledge ->>> dctx = conn(Context)(ident='http://example.org/data/context') +>>> dctx = evctx(Context)(ident='http://example.org/data/context') >>> doc = evctx(Document)(key="Sulston83", author='Sulston et al.', date='1983') >>> e = evctx(Evidence)(key="Sulston83", reference=doc) >>> avdl = dctx(Neuron)(name="AVDL") @@ -178,8 +178,9 @@ data and models to corresponding articles from peer-reviewed literature: owmeta_core.statement.Statement(subj=Neuron(ident=rdflib.term.URIRef('http://data.openworm.org/sci/bio/Neuron#AVDL')), prop=owmeta.cell.Cell_lineageName(owner=Neuron(ident=rdflib.term.URIRef('http://data.openworm.org/sci/bio/Neuron#AVDL'))), obj=owmeta_core.dataobject_property.ContextualizedPropertyValue(rdflib.term.Literal('AB alaaapalr')), context=owmeta_core.context.Context(ident="http://example.org/data/context")) >>> e.supports(dctx.rdf_object) owmeta_core.statement.Statement(subj=Evidence(ident=rdflib.term.URIRef('http://data.openworm.org/Evidence#Sulston83')), prop=owmeta.evidence.Evidence_supports(owner=Evidence(ident=rdflib.term.URIRef('http://data.openworm.org/Evidence#Sulston83'))), obj=ContextDataObject(ident=rdflib.term.URIRef('http://example.org/data/context')), context=owmeta_core.context.Context(ident="http://example.org/evidence/context")) ->>> dctx.save_context() ->>> evctx.save_context() +>>> with conn.transaction_manager: +... dctx.save_context() +... evctx.save_context() ``` @@ -267,9 +268,10 @@ Finally, when you're done accessing the database, be sure to disconnect from it: ``` -More examples can be found -[here](http://owm-doc.readthedocs.org/en/latest/making_dataObjects.html) and -[here](https://github.com/openworm/owmeta/tree/master/examples). +More examples can be found [in the owmeta-core +documentation](https://owmeta-core.readthedocs.io/en/latest/making_dataObjects.html) +and [in the ./examples directory of the owmeta Git +repository](https://github.com/openworm/owmeta/tree/master/examples). Documentation ------------- diff --git a/ci-install.sh b/ci-install.sh new file mode 100755 index 000000000..f3941884e --- /dev/null +++ b/ci-install.sh @@ -0,0 +1,25 @@ +#!/bin/sh -ex + +pip install -e . +pip install coveralls +pip install -r test-requirements.txt +owm clone https://github.com/openworm/OpenWormData.git --branch owmeta +if [ $BUNDLE_TESTS ] ; then + owm bundle register ./owmeta-schema-bundle.yml + owm bundle register ./owmeta-data-bundle.yml + owm bundle install openworm/owmeta-schema + owm bundle install openworm/owmeta-data +fi +if [ "$PYTHON_VERSION" = "3.6" ] ; then + # pyparsing 3.0 no longer allows setting the .name attribute for + # TokenConverter, but the fix for that only goes in RDFLib 6 which + # does not support Python 3.6. See + # https://github.com/RDFLib/rdflib/issues/1190 and + # https://github.com/rdflib/rdflib/issues/1370 for details on the issue. + # + # Since we're a library, it's up to the user how to resolve this + # (e.g., ignore and do not use SPARQL) so we do not impose any + # version constraints in package metadata, just adjust so the build + # passes + pip install 'pyparsing<3.0.0' +fi diff --git a/ci-test.sh b/ci-test.sh index afe07331f..45919e079 100755 --- a/ci-test.sh +++ b/ci-test.sh @@ -4,7 +4,6 @@ pt () { sh -c "pytest --cov=owmeta $*" } - COVERAGES="" add_coverage () { diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 000000000..2969d6199 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,12 @@ +#!/bin/bash -xe +HEAD_REV=$(git rev-parse HEAD) +git ls-remote origin refs/heads/dev | grep -q -E "^$HEAD_REV" || \ + ( echo "Not deploying since we aren't on the 'dev' branch" >&2 && \ + exit 0 ) + +git log --format=%s -n 1 "$HEAD_REV" | grep -E -q '(^MINOR:)|(\[skip-deploy\])' && exit 0 + +date=$(date +"%Y%m%d%H%M%S") +sed -i -r "s/__version__ = '([^']+)\\.dev0'/__version__ = '\\1.dev$date'/" owmeta/__init__.py +python setup.py egg_info sdist bdist_wheel +twine upload -c "Built by CI. Uploaded after $(date +"%Y-%m-%d %H:%M:%S")" dist/owmeta*tar.gz dist/owmeta*whl diff --git a/docs/adding_data.rst b/docs/adding_data.rst index 92ad8fe4b..09e54d29b 100644 --- a/docs/adding_data.rst +++ b/docs/adding_data.rst @@ -152,7 +152,7 @@ using contexts. The code below shows how to do that:: >>> # Serialize the data in the nquads format so we can see that all of our >>> # statements are in the proper context - >>> print(g.serialize(format='nquads').decode('UTF-8')) + >>> print(g.serialize(format='nquads', encoding='UTF-8').decode('UTF-8')) . <... diff --git a/docs/conf.py b/docs/conf.py index 38785258b..00ac7c644 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -37,6 +37,7 @@ intersphinx_mapping = {'python': ('https://docs.python.org/3', None), 'rdflib': ('https://rdflib.readthedocs.io/en/stable/', None), 'owmeta-core': ('https://owmeta-core.readthedocs.io/en/stable/', None), + 'requests': ('https://requests.readthedocs.io/en/stable/', None), 'bibtexparser': ('https://bibtexparser.readthedocs.io/en/master/', None)} # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/examples/add_reference.py b/examples/add_reference.py index 3cf334d9f..6fdfe61e4 100644 --- a/examples/add_reference.py +++ b/examples/add_reference.py @@ -10,7 +10,8 @@ import owmeta_core as P from owmeta_core.data import Data -from owmeta_core.context import Context +from owmeta_core.dataobject import DataObject +from owmeta_core.context import Context, IMPORTS_CONTEXT_KEY from owmeta.evidence import Evidence from owmeta.neuron import Neuron @@ -18,13 +19,18 @@ # Create dummy database configuration. d = Data() +d[IMPORTS_CONTEXT_KEY] = 'http://example.org/imports' # Connect to database with dummy configuration conn = P.connect(conf=d) conn.mapper.add_class(Evidence) conn.mapper.add_class(Document) -ctx = conn(Context)(ident='http://example.org/data') evctx = conn(Context)(ident='http://example.org/meta') +ctx = evctx(Context)(ident='http://example.org/data') + +# Add the Context RDF class to the mapper -- normally you'd get these from the +# openworm/owmeta-core bundle as a dependency +conn.mapper.add_class(type(ctx.rdf_object)) # Create a new Neuron object to work with n = ctx(Neuron)(name='AVAL') @@ -46,9 +52,18 @@ ctx.save_context() evctx.save_context() +evctx.add_import(Document.definition_context) +evctx.save_imports() + +# Add a couple contexts to the store so we can resolve needed types. normally you'd get +# these from the openworm/owmeta-core bundle as a dependency, so they don't have to be +# added to the database again. +Document.definition_context.save_context(conn.rdf) +DataObject.definition_context.save_context(conn.rdf) + # What does my evidence object contain? for e_i in evctx.stored(Evidence)().load(): print(e_i.reference(), e_i.supports()) # Disconnect from the database. -P.disconnect(conn) +conn.disconnect() diff --git a/examples/rmgr.py b/examples/rmgr.py index 9fc16be98..5fbfd0e54 100644 --- a/examples/rmgr.py +++ b/examples/rmgr.py @@ -113,7 +113,7 @@ def setup(sctx, ctx, name, type): return n -with OWM('../.owm').connect() as conn: +with OWM('../.owm').connect().transaction() as conn: ctx = conn(Context)('http://example.org/data') evctx = conn(Context)('http://example.org/evidence') diff --git a/owmeta/__init__.py b/owmeta/__init__.py index 84f21260d..1905f81b2 100644 --- a/owmeta/__init__.py +++ b/owmeta/__init__.py @@ -13,7 +13,7 @@ """ from __future__ import print_function -__version__ = '0.12.3.dev0' +__version__ = '0.12.4.dev0' __author__ = 'OpenWorm.org authors and contributors' import logging diff --git a/owmeta/bibtex.py b/owmeta/bibtex.py index 801655135..a03a37392 100644 --- a/owmeta/bibtex.py +++ b/owmeta/bibtex.py @@ -1,4 +1,3 @@ -import re import bibtexparser from .evidence import Evidence @@ -15,7 +14,7 @@ def bibtex_to_document(bibtex_entry, context=None): def update_document_with_bibtex(document, bibtex_entry): - document.set_key(bibtex_entry['ID']) + document.key = bibtex_entry['ID'] for ath in bibtex_entry.get('author', tuple()): document.author(ath) diff --git a/owmeta/cell.py b/owmeta/cell.py index 48142ad83..802513c58 100644 --- a/owmeta/cell.py +++ b/owmeta/cell.py @@ -56,7 +56,9 @@ class Cell(BiologyType): parentOf = ObjectProperty(value_type=This, multiple=True) - key_property = {'property': name, 'type': 'direct'} + key_property = 'name' + + direct_key = True def __init__(self, name=None, lineageName=None, **kwargs): # NOTE: We name the `name` and `lineageName` as positional parameters for diff --git a/owmeta/data_trans/bibtex.py b/owmeta/data_trans/bibtex.py index 120ba0cbe..99d1f1d5d 100644 --- a/owmeta/data_trans/bibtex.py +++ b/owmeta/data_trans/bibtex.py @@ -1,4 +1,3 @@ -from rdflib.namespace import Namespace from owmeta_core.datasource import Informational, DataTranslator, DataSource from owmeta_core.data_trans.local_file_ds import LocalFileDataSource from owmeta_core.data_trans.context_datasource import VariableIdentifierContext @@ -23,7 +22,8 @@ def __init__(self, *args, **kwargs): imported=(CONTEXT,)) self.context_property(self.evidence_context.rdf_object) - def commit_augment(self): + def after_transform(self): + super(EvidenceDataSource, self).after_transform() saved_contexts = set([]) self.data_context.save_context(inline_imports=True, saved_contexts=saved_contexts) self.context.save_context(inline_imports=True, saved_contexts=saved_contexts) diff --git a/owmeta/data_trans/connections.py b/owmeta/data_trans/connections.py index 297cdf6a6..c540e140a 100644 --- a/owmeta/data_trans/connections.py +++ b/owmeta/data_trans/connections.py @@ -1,9 +1,6 @@ -import re -import traceback import csv import logging -from owmeta_core.context import Context from owmeta_core.datasource import GenericTranslation from owmeta_core.dataobject import ObjectProperty from owmeta_core.data_trans.csv_ds import CSVDataTranslator, CSVDataSource diff --git a/owmeta/data_trans/data_with_evidence_ds.py b/owmeta/data_trans/data_with_evidence_ds.py index 3aecd75f1..d51249d30 100644 --- a/owmeta/data_trans/data_with_evidence_ds.py +++ b/owmeta/data_trans/data_with_evidence_ds.py @@ -1,8 +1,7 @@ -from rdflib.namespace import Namespace - from owmeta_core.context import Context from owmeta_core.datasource import Informational, DataSource -from owmeta_core.data_trans.context_datasource import VariableIdentifierContext +from owmeta_core.data_trans.context_datasource import (VariableIdentifierContext, + VariableIdentifierContextDataObject) from .. import CONTEXT, SCI_CTX @@ -12,8 +11,10 @@ class DataWithEvidenceDataSource(DSMixin, DataSource): ''' A data source that has an "evidence context" containing statements which support those - in its "data context". The data source also has a combined context which imports both - the data and evidence contexts. + in its "data context". The data source also has a combined context which imports both + the data and evidence contexts. The data and evidence contexts have identifiers based + on the data source's identifier and the combined context has the same identifier as + the data source. ''' class_context = SCI_CTX @@ -22,6 +23,7 @@ class DataWithEvidenceDataSource(DSMixin, DataSource): property_name='evidence_context', property_type='ObjectProperty', multiple=False, + value_type=VariableIdentifierContextDataObject, description='The context in which evidence' ' for the "Data context" is defined') @@ -29,6 +31,7 @@ class DataWithEvidenceDataSource(DSMixin, DataSource): property_name='data_context', property_type='ObjectProperty', multiple=False, + value_type=VariableIdentifierContextDataObject, description='The context in which primary data' ' for this data source is defined') @@ -36,6 +39,7 @@ class DataWithEvidenceDataSource(DSMixin, DataSource): property_name='combined_context', property_type='ObjectProperty', multiple=False, + value_type=VariableIdentifierContextDataObject, description='Context importing both the data and evidence contexts') def __init__(self, *args, **kwargs): @@ -80,7 +84,8 @@ def context_for(self, ident=None, **kwargs): res = self.__ad_hoc_contexts[key] return res - def commit_augment(self): + def after_transform(self): + super(DataWithEvidenceDataSource, self).after_transform() for ctx in self.__ad_hoc_contexts.values(): ctx.save() diff --git a/owmeta/document.py b/owmeta/document.py index b5add9f28..200569e1c 100644 --- a/owmeta/document.py +++ b/owmeta/document.py @@ -1,16 +1,19 @@ from six.moves.urllib.parse import urlparse, urlencode -from six.moves.urllib.request import Request, urlopen -from six.moves.urllib.error import HTTPError, URLError import re import logging from owmeta_core.graph_object import IdentifierMissingException from owmeta_core.context import Context -from owmeta_core.dataobject import DataObject, DatatypeProperty, Alias +import owmeta_core.dataobject_property as DP +from owmeta_core.dataobject import DataObject, DatatypeProperty, Alias, BaseDataObject +import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry from . import SCI_CTX from . import bibtex as BIB + logger = logging.getLogger(__name__) @@ -159,11 +162,17 @@ def identifier_augment(self): return self.make_identifier(s) raise IdentifierMissingException(self) - # TODO: Provide a way to override modification of already set values. - def update_from_wormbase(self, replace_existing=False): - """ Queries wormbase for additional data to fill in the Document. + def update_from_wormbase(self, replace_existing=False, **kwargs): + """ Queries WormBase.org for additional data to fill in the `Document`. If replace_existing is set to `True`, then existing values will be cleared. + + Parameters + ---------- + replace_existing : bool + Whether to replace values that are already set for a given property + **kwargs + Passed on as arguments to `requests.Session.get` """ # XXX: wormbase's REST API is pretty sparse in terms of data provided. @@ -176,8 +185,8 @@ def update_from_wormbase(self, replace_existing=False): # get the author try: root = self.conf.get('wormbase_api_root_url', 'http://rest.wormbase.org') - url = root + '/rest/widget/paper/' + str(wbid) + '/overview?content-type=application%2Fjson' - j = _json_request(url) + url = f'{root}/rest/widget/paper/{wbid}/overview?content-type=application%2Fjson' + j = _json_request(url, **kwargs) if 'fields' in j: f = j['fields'] if 'authors' in f: @@ -239,22 +248,45 @@ def crRequest(doi): if 'year' in r: self.year(r['year']) - def update_from_pubmed(self): + def update_from_pubmed(self, read_size=2**16, **kwargs): + ''' + Update the document attributes from NCBI Entrez API using the pubmed attribute + + Parameters + ---------- + chunk_size : int + The number of bytes to pass to `requests.Response.iter_content`. This *may* + reduce runtime memory requirements for the request. + **kwargs + Passed on as arguments to `requests.Session.get` + ''' + def pmRequest(pmid): import xml.etree.ElementTree as ET # Python 2.5 and up - url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=' + str(pmid) + + url = ('https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?' + f'db=pubmed&id={pmid}') key = self.get('pubmed.api_key', None) if key: - url += '&api_key=' + key + url += f'&api_key={key}' else: logger.warning("PubMed API key not defined. API calls will be limited.") - s = _url_request(url) + + if 'do_retries' not in kwargs: + kwargs['do_retries'] = True + + kwargs['stream'] = True + + s = _url_request(url, **kwargs) if hasattr(s, 'charset'): parser = ET.XMLParser(encoding=s.charset) else: - parser = None + parser = ET.XMLParser(encoding='UTF-8') - return ET.parse(s, parser) + with s: + for chunk in s.iter_content(read_size): + parser.feed(chunk) + return parser.close() pmid = self.pmid.defined_values if len(pmid) == 1: @@ -283,6 +315,18 @@ def pmRequest(pmid): ' Please try with just one Pubmed ID') +class SourcedFrom(DP.ObjectProperty): + ''' + Indicates which document provided the source for an object + ''' + class_context = SCI_CTX + link_name = "sourced_from" + value_type = BaseDocument + owner_type = BaseDataObject + multiple = False + lazy = True + + def _wormbase_uri_to_wbid(uri): return str(urlparse(uri).path.split("/")[2]) @@ -302,40 +346,48 @@ def _doi_uri_to_doi(uri): return doi -class EmptyRes(object): - def read(self): - return bytes() +def _url_request(url, requests_session=None, do_retries=False, **kwargs): + if requests_session is None: + sess = requests.Session() + else: + sess = requests_session + + if do_retries: + retries = Retry() + adapter = HTTPAdapter(max_retries=retries) + sess.mount('http://', adapter) + sess.mount('https://', adapter) + + if 'timeout' not in kwargs: + kwargs['timeout'] = 1 -def _url_request(url, headers={}): try: - r = Request(url, headers=headers) - s = urlopen(r, timeout=1) - info = dict(s.info()) - content_type = {k.lower(): info[k] for k in info}['content-type'] - md = re.search("charset *= *([^ ]+)", content_type) - if md: - s.charset = md.group(1) - - return s - except HTTPError: - logger.error("Error in request for {}".format(url), exc_info=True) - return EmptyRes() - except URLError: - logger.error("Error in request for {}".format(url), exc_info=True) - return EmptyRes() - - -def _json_request(url): - import json - headers = {'Accept': 'application/json'} + resp = sess.get(url, **kwargs) + if resp.status_code != 200: + raise Exception(f'Service returned status code {resp.status_code}') + content_type = resp.headers.get('content-type') + if content_type: + md = re.search("charset *= *([^ ]+)", content_type) + if md: + resp.charset = md.group(1) + + return resp + except Exception: + logger.error("Error in request for %s", url, exc_info=True) + raise + + +def _json_request(url, **kwargs): + if 'headers' in kwargs: + headers = kwargs['headers'] + else: + headers = {} + kwargs['headers'] = headers + headers['Accept'] = 'application/json' try: - data = _url_request(url, headers).read().decode('UTF-8') - if hasattr(data, 'charset'): - return json.loads(data, encoding=data.charset) - else: - return json.loads(data) + return _url_request(url, **kwargs).json() except BaseException: - logger.warning("Couldn't retrieve JSON data from " + url, + logger.warning("Couldn't retrieve JSON data from %s", url, exc_info=True) return {} diff --git a/setup.py b/setup.py index 997d39095..e55c3a67c 100644 --- a/setup.py +++ b/setup.py @@ -2,8 +2,6 @@ # from setuptools import setup -import os -import sys long_description = """ @@ -34,20 +32,13 @@ setup( name='owmeta', zip_safe=False, - setup_requires=['pytest-runner'], - tests_require=[ - 'pytest>=3.4.0', - 'pytest-cov>=2.5.1', - 'discover==0.4.0', - 'requests', - 'pytest-parallel' - ], install_requires=[ - 'owmeta-core', + 'owmeta-core>=0.14.0.dev0', 'bibtexparser~=1.1.0', 'libneuroml', 'rdflib>=4.1.2', - 'six~=1.10' + 'six~=1.10', + 'requests', ], version=version, packages=['owmeta', @@ -75,6 +66,8 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Topic :: Scientific/Engineering' ] ) diff --git a/test-requirements.txt b/test-requirements.txt index 6ef09b1af..ee75c3362 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,3 +3,5 @@ pytest-cov>=2.5.1 discover==0.4.0 requests pytest-parallel +owmeta-pytest-plugin +http-server-pytest-fixtures diff --git a/tests/ConnectionNeurotransmitterDataTranslatorTest.py b/tests/ConnectionNeurotransmitterDataTranslatorTest.py index 6b39139af..023494a2c 100644 --- a/tests/ConnectionNeurotransmitterDataTranslatorTest.py +++ b/tests/ConnectionNeurotransmitterDataTranslatorTest.py @@ -44,18 +44,15 @@ def setUp(self): self.conn_ds.data_context(Connection)(pre_cell=Neuron('PreCell'), post_cell=Neuron('PostCell'), syntype='send') - self.nt_ds.commit() - self.conn_ds.commit() + self.conn_ds.data_context.save() def test_connection_exists(self): res = self.cut(self.conn_ds, self.nt_ds) - res.commit() conn = res.data_context.stored(Connection).query() self.assertEqual(len(list(conn.load())), 1) def test_adds_nt(self): res = self.cut(self.conn_ds, self.nt_ds) - res.commit() conn = res.data_context.stored(Connection).query() self.assertEqual(list(conn.load())[0].synclass(), 'neurotransmitter') @@ -73,16 +70,14 @@ def setUp(self): post_cell=Neuron('PostCell'), syntype='send', number=3) - self.conn_ds.commit() + self.conn_ds.data_context.save() def test_connection_exists(self): res = self.cut(self.conn_ds, self.nt_ds) - res.commit() conn = res.data_context.stored(Connection).query() self.assertEqual(len(list(conn.load())), 1) def test_adds_nt(self): res = self.cut(self.conn_ds, self.nt_ds) - res.commit() conn = res.data_context.stored(Connection).query() self.assertEqual(list(conn.load())[0].synclass(), 'neurotransmitter') diff --git a/tests/DataTestTemplate.py b/tests/DataTestTemplate.py index c55dc9f17..eb29f9302 100644 --- a/tests/DataTestTemplate.py +++ b/tests/DataTestTemplate.py @@ -7,6 +7,7 @@ from owmeta_core.context import Context from owmeta_core.data import Data +from owmeta_core.mapper import CLASS_REGISTRY_CONTEXT_KEY from .GraphDBInit import delete_zodb_data_store, TEST_CONFIG @@ -34,6 +35,7 @@ def setUp(self): x = z[len(td):] h = tempfile.mkdtemp() self.TestConfig['rdf.store_conf'] = h + x + self.TestConfig[CLASS_REGISTRY_CONTEXT_KEY] = 'http://example.org/class_registry' self.delete_dir() self.connection = owmeta_core.connect(conf=self.TestConfig) self.mapper = self.connection.mapper diff --git a/tests/DocumentSourcedFromTest.py b/tests/DocumentSourcedFromTest.py new file mode 100644 index 000000000..b762774f0 --- /dev/null +++ b/tests/DocumentSourcedFromTest.py @@ -0,0 +1,43 @@ +from os.path import join as p +import json + +import pytest +from rdflib.namespace import Namespace + +from owmeta_core.datasource import DataSource +from owmeta_pytest_plugin import bundle_versions + +from owmeta.document import SourcedFrom + + +EX = Namespace('http://example.org/') + + +@pytest.mark.inttest +@bundle_versions('owmeta_schema_bundle', [3]) +@pytest.mark.bundle_remote('ow') +def test_augment_cmd_load(owmeta_schema_bundle, owm_project): + owm_project.fetch(owmeta_schema_bundle) + owm = owm_project.owm() + deps = [{'id': owmeta_schema_bundle.id, 'version': owmeta_schema_bundle.version}] + owm.config.set('dependencies', json.dumps(deps)) + + modpath = owm_project.make_module('test_module') + owm_project.writefile(p(modpath, 'asource.py'), + 'tests/test_modules/evidence_ds_mixin_source.py') + owm_project.sh('owm save test_module.asource') + + owm_project.sh(f'owm namespace bind ex {EX}') + ctxid = owm.default_context.identifier + owm_project.sh( + f'owm contexts add-import {ctxid} http://schema.openworm.org/2020/07/sci', + 'owm declare owmeta.document:Document doi=10.1101/398370 --id=ex:doc', + 'owm declare test_module.asource:ASource' + ' owmeta.document:SourcedFrom=ex:doc' + ' --id=ex:ds',) + + owm = owm_project.owm() + with owm.connect(): + ds = owm.default_context.stored(DataSource)(ident=EX['ds']).load_one() + doc = ds.attach_property(SourcedFrom).one() + assert doc.identifier == EX['doc'] diff --git a/tests/DocumentationTest.py b/tests/DocumentationTest.py index 98e907fee..edb0a7a16 100644 --- a/tests/DocumentationTest.py +++ b/tests/DocumentationTest.py @@ -72,7 +72,8 @@ class Widget(DataObject): hardiness = DatatypeProperty() fullness = DatatypeProperty() part_number = DatatypeProperty() - key_property = {'name': 'part_number', 'type': 'direct'} + key_property = 'part_number' + direct_key = True ctx = Context(ident='http://example.org/data/imports/BDW_Widgets_2018-2019') diff --git a/tests/EvidenceTest.py b/tests/EvidenceTest.py index 5cb47dc8d..4024b1536 100644 --- a/tests/EvidenceTest.py +++ b/tests/EvidenceTest.py @@ -4,6 +4,7 @@ from owmeta_core.dataobject import DataObject from owmeta_core.configure import Configurable +from owmeta_core.context import Context from owmeta.evidence import Evidence @@ -14,6 +15,7 @@ except ImportError: from mock import patch +import rdflib class EvidenceTest(_DataTest): ctx_classes = (Evidence,) @@ -32,7 +34,8 @@ def test_asserts(self): Asserting something should allow us to get it back. """ e = self.ctx.Evidence(key='WBPaper00044600') - r = DataObject(key="context_data_object") + ctx = self.context(Context)(rdflib.URIRef('https://example.org/context')) + r = ctx.rdf_object e.supports(r) s = list(e.supports.get()) self.assertIn(r, s) diff --git a/tests/conftest.py b/tests/conftest.py index 02c227e98..21f231e56 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,7 @@ import tempfile import requests +from owmeta_pytest_plugin import bundle_fixture_helper from pytest import fixture @@ -119,3 +120,7 @@ def wait_for_started(server_data, max_tries=10): done = True except Exception: L.info("Unable to connect to the bundle server. Trying again.", exc_info=True) + + +data_bundle = fixture(bundle_fixture_helper('openworm/owmeta-data')) +owmeta_schema_bundle = fixture(bundle_fixture_helper('openworm/owmeta-schema')) diff --git a/tests/test_default.conf b/tests/test_default.conf index 972e1db26..7a0d34b20 100644 --- a/tests/test_default.conf +++ b/tests/test_default.conf @@ -6,5 +6,6 @@ "user.email" : "jerry@cn.com", "rdf.upload_block_statement_count" : 50, "test_variable" : "test_value", - "pubmed.api_key" : "$PUBMED_API_KEY" + "pubmed.api_key" : "$PUBMED_API_KEY", + "imports_context_id" : "https://example.org/imports-context" } diff --git a/tests/test_modules/evidence_ds_mixin_source.py b/tests/test_modules/evidence_ds_mixin_source.py new file mode 100644 index 000000000..e1beb1658 --- /dev/null +++ b/tests/test_modules/evidence_ds_mixin_source.py @@ -0,0 +1,6 @@ +from owmeta_core.datasource import DataSource, Informational + + +class ASource(DataSource): + class_context = 'http://example.org/a_context' + a = Informational(display_name='A Value')