diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000..a6af790 --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,34 @@ +[bumpversion] +current_version = 0.1.0-dev +commit = False +tag = False +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P\w+))? +serialize = + {major}.{minor}.{patch}-{release} + {major}.{minor}.{patch} + +[bumpversion:part:release] +optional_value = placeholder +values = + dev + placeholder + +[bumpversion:file:VERSION] +search = {current_version} +replace = {new_version} + +[bumpversion:file:src/dask_elasticsearch/__init__.py] +search = __version__ = '{current_version}' +replace = __version__ = '{new_version}' + +[bumpversion:file:.cookiecutterrc] +search = version: {current_version} +replace = version: {new_version} + +[bumpversion:file:HISTORY.rst] +search = .. comment:: bumpversion marker +replace = .. comment:: bumpversion marker + + {new_version} ({now:%Y-%m-%d}) + ------------------ + diff --git a/.cookiecutterrc b/.cookiecutterrc new file mode 100644 index 0000000..698ca22 --- /dev/null +++ b/.cookiecutterrc @@ -0,0 +1,33 @@ +# This file exists so you can easily regenerate your project. +# +# `cookiepatcher` is a convenient shim around `cookiecutter` +# for regenerating projects (it will generate a .cookiecutterrc +# automatically for any template). To use it: +# +# pip install cookiepatcher +# cookiepatcher gh:ionelmc/cookiecutter-pylibrary project-path +# +# See: +# https://pypi.python.org/pypi/cookiecutter +# +# Alternatively, you can run: +# +# cookiecutter --overwrite-if-exists --config-file=project-path/.cookiecutterrc gh:ionelmc/cookiecutter-pylibrary + +default_context: + email: rolando at rmax.io + full_name: Rolando (Max) Espinoza + github_username: rolando + project_name: Dask-Elasticsearch + project_package: dask_elasticsearch + project_short_description: Elasticsearch reader for Dask. + project_slug: dask-elasticsearch + pypi_username: rolando + use_codecov: y + use_cython: y + use_landscape: y + use_pypi_deployment_with_travis: y + use_pytest: y + use_requiresio: y + version: 0.1.0-dev + year: 2017 diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..c311d57 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,24 @@ +[paths] +source = + src + +[run] +branch = true +source = + dask_elasticsearch + tests +parallel = true + +[report] +show_missing = true +precision = 2 +omit = +exclude_lines = + pragma: no cover + def __repr__ + if self.debug: + if settings.DEBUG + raise AssertionError + raise NotImplementedError + if 0: + if __name__ == .__main__.: diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d4a2c44 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,21 @@ +# http://editorconfig.org + +root = true + +[*] +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true +charset = utf-8 +end_of_line = lf + +[*.bat] +indent_style = tab +end_of_line = crlf + +[LICENSE] +insert_final_newline = false + +[Makefile] +indent_style = tab diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cf3102d --- /dev/null +++ b/.gitignore @@ -0,0 +1,91 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..3ae8df7 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,79 @@ +# Config file for automatic testing at travis-ci.org +# This file will be regenerated if you run travis_pypi_setup.py + +language: python +python: 3.5 +sudo: false + +env: + - TOXENV=py35 + - TOXENV=py34 + - TOXENV=py27 + +matrix: + fast_finish: true + + +before_install: + - python --version + - uname -a + - lsb_release -a + +# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors +install: + - pip install -U pip wheel + - pip install -U tox + - pip install -U coverage + - pip install -U requires.io + - pip install -U twine + - virtualenv --version + - easy_install --version + - pip --version + - tox --version + +# command to run tests, e.g. python setup.py test +script: + - tox --workdir $HOME/.tox + +after_success: + # Codecov requires a single .coverage and will run 'coverage xml' to + # generate the report. + - coverage combine + - bash <(curl -s https://codecov.io/bash) + - requires.io update-site + + + +after_failure: + - more $HOME/.tox/log/* | cat + - more $HOME/.tox/*/log/* | cat + +before_cache: + - rm -fr $HOME/.cache/pip/log + - rm -fr $HOME/.tox/log/* + - rm -fr $HOME/.tox/*/log/* + +cache: + directories: + - $HOME/.cache/pip + - $HOME/.tox + +notifications: + email: + on_sucess: never + on_failure: always + +# FIXME: After you create the Github repo and add it to Travis, run the +# travis_pypi_setup.py script to finish PyPI deployment setup. Then you can +# remove the script travis_pypi_setup.py. +deploy: + provider: pypi + distributions: "sdist bdist_wheel" + user: darkrho + password: + secure: PLEASE_REPLACE_ME + on: + tags: true + all_branches: true + repo: rolando/dask-elasticsearch + condition: "$TOXENV == py35" diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 0000000..0cfaf33 --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,13 @@ +======= +Credits +======= + +Development Lead +---------------- + +* Rolando (Max) Espinoza + +Contributors +------------ + +None yet. Why not be the first? diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..4e60f15 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,113 @@ +.. highlight:: shell + +============ +Contributing +============ + +Contributions are welcome, and they are greatly appreciated! Every +little bit helps, and credit will always be given. + +You can contribute in many ways: + +Types of Contributions +---------------------- + +Report Bugs +~~~~~~~~~~~ + +Report bugs at https://github.com/rolando/dask-elasticsearch/issues. + +If you are reporting a bug, please include: + +* Your operating system name and version. +* Any details about your local setup that might be helpful in troubleshooting. +* Detailed steps to reproduce the bug. + +Fix Bugs +~~~~~~~~ + +Look through the GitHub issues for bugs. Anything tagged with "bug" +is open to whoever wants to implement it. + +Implement Features +~~~~~~~~~~~~~~~~~~ + +Look through the GitHub issues for features. Anything tagged with "feature" +is open to whoever wants to implement it. + +Write Documentation +~~~~~~~~~~~~~~~~~~~ + +Dask-Elasticsearch could always use more documentation, whether as part of the +official Dask-Elasticsearch docs, in docstrings, or even on the web in blog posts, +articles, and such. + +Submit Feedback +~~~~~~~~~~~~~~~ + +The best way to send feedback is to file an issue at https://github.com/rolando/dask-elasticsearch/issues. + +If you are proposing a feature: + +* Explain in detail how it would work. +* Keep the scope as narrow as possible, to make it easier to implement. +* Remember that this is a volunteer-driven project, and that contributions + are welcome :) + +Get Started! +------------ + +Ready to contribute? Here's how to set up `dask-elasticsearch` for local development. + +1. Fork the `dask-elasticsearch` repo on GitHub. +2. Clone your fork locally:: + + $ git clone git@github.com:your_name_here/dask-elasticsearch.git + +3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: + + $ mkvirtualenv dask-elasticsearch + $ cd dask-elasticsearch/ + $ python setup.py develop + +4. Create a branch for local development:: + + $ git checkout -b name-of-your-bugfix-or-feature + + Now you can make your changes locally. + +5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: + + $ flake8 dask_elasticsearch tests + $ python setup.py test or py.test + $ tox + + To get flake8 and tox, just pip install them into your virtualenv. + +6. Commit your changes and push your branch to GitHub:: + + $ git add . + $ git commit -m "Your detailed description of your changes." + $ git push origin name-of-your-bugfix-or-feature + +7. Submit a pull request through the GitHub website. + +Pull Request Guidelines +----------------------- + +Before you submit a pull request, check that it meets these guidelines: + +1. The pull request should include tests. +2. If the pull request adds functionality, the docs should be updated. Put + your new functionality into a function with a docstring, and add the + feature to the list in README.rst. +3. The pull request should work for Python 2.6, 2.7, 3.3, 3.4 and 3.5, and for PyPy. Check + https://travis-ci.org/rolando/dask-elasticsearch/pull_requests + and make sure that the tests pass for all supported Python versions. + +Tips +---- + +To run a subset of tests:: + + $ py.test tests.test_dask_elasticsearch diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 0000000..7664a58 --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,10 @@ +======= +History +======= + +.. comment:: bumpversion marker + +0.1.0-dev (unreleased) +---------------------- + +* First release on PyPI. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..288c441 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2017, Rolando (Max) Espinoza + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..e04fb3d --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,17 @@ +graft docs +graft requirements +graft src +graft tests + +include *.in +include *.ini +include *.rst +include *.txt + +include LICENSE +include VERSION +include Makefile + +global-exclude __pycache__ *.py[cod] +global-exclude *.so *.dylib +global-exclude *.c *.cpp diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8218ad8 --- /dev/null +++ b/Makefile @@ -0,0 +1,151 @@ +.PHONY: clean-so clean-test clean-pyc clean-build clean-docs clean +.PHONY: docs check check-manifest check-setup check-history lint +.PHONY: test test-all coverage +.PHONY: compile-reqs install-reqs +.PHONY: release dist install build-inplace +define BROWSER_PYSCRIPT +import os, webbrowser, sys +try: + from urllib import pathname2url +except: + from urllib.request import pathname2url + +webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) +endef +export BROWSER_PYSCRIPT +BROWSER := python -c "$$BROWSER_PYSCRIPT" + +SPHINX_BUILD := html + +help: + @echo "check - check setup, code style, setup, etc" + @echo "check-manifest - check manifest" + @echo "check-setup - check setup" + @echo "check-history - check history" + @echo "clean - remove all build, test, coverage and Python artifacts" + @echo "clean-build - remove build artifacts" + @echo "clean-docs - remove docs artifacts" + @echo "clean-pyc - remove Python file artifacts" + @echo "clean-test - remove test and coverage artifacts" + @echo "clean-so - remove compiled extensions" + @echo "lint - check style with flake8" + @echo "test - run tests quickly with the default Python" + @echo "test-all - run tests on every Python version with tox" + @echo "coverage - check code coverage quickly with the default Python" + @echo "compile-reqs - compile requirements" + @echo "install-reqs - install requirements" + @echo "docs - generate Sphinx HTML documentation, including API docs" + @echo "release - package and upload a release" + @echo "dist - package" + @echo "develop - install package in develop mode" + @echo "install - install the package to the active Python's site-packages" + +check: check-setup check-manifest check-history lint + +check-setup: + @echo "Checking package metadata (name, description, etc)" + python setup.py check --strict --metadata --restructuredtext + +check-manifest: + @echo "Checking MANIFEST.in" + check-manifest --ignore ".*" + +check-history: + @echo "Checking latest version in HISTORY" + VERSION=`cat VERSION`; grep "^$${VERSION}\b" HISTORY.rst + +clean: clean-build clean-docs clean-pyc clean-test clean-so + +clean-build: + rm -fr build/ + rm -fr dist/ + rm -fr .eggs/ + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + + +clean-docs: + $(MAKE) -C docs clean + +clean-pyc: + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + +clean-test: + rm -fr .tox/ + rm -f .coverage + rm -fr htmlcov/ + +clean-so: + find . -name '*.so' -exec rm -f {} + + +lint: + flake8 src tests + +build-inplace: + python setup.py build_ext --inplace + +develop: clean + pip install -e . + +TEST_ARGS ?= +test: develop + py.test $(TEST_ARGS) + +test-all: + tox -v + +coverage: develop + coverage run -m py.test + coverage combine + coverage report + +coverage-html: coverage + coverage html + $(BROWSER) htmlcov/index.html + +docs-build: develop + rm -f docs/dask_elasticsearch.rst + rm -f docs/modules.rst + sphinx-apidoc -o docs/ src/dask_elasticsearch + $(MAKE) -C docs clean + $(MAKE) -C docs $(SPHINX_BUILD) + +docs: docs-build + $(BROWSER) docs/_build/$(SPHINX_BUILD)/index.html + +servedocs: docs + watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . + +release: clean check dist + git branch | grep '* master' + # Tagging release. + VERSION=`cat VERSION`; git tag -a v$$VERSION + git push --follow-tags + twine upload dist/* + +dist: clean + python setup.py sdist + ls -l dist + +install: clean + pip install . + +REQUIREMENTS_IN := $(wildcard requirements*.in) +.PHONY: $(REQUIREMENTS_IN) + +requirements%.txt: requirements%.in + pip-compile -v $< -o $@ + +REQUIREMENTS_TXT := $(REQUIREMENTS_IN:.in=.txt) +ifndef REQUIREMENTS_TXT +REQUIREMENTS_TXT := $(wildcard requirements*.txt) +endif + +compile-reqs: $(REQUIREMENTS_TXT) + @test -z "$$REQUIREMENTS_TXT" && echo "No 'requirements*.in' files. Nothing to do" + +install-reqs: + @test -z "$$REQUIREMENTS_TXT" && echo "No 'requirements*.txt' files. Nothing to do" + $(foreach req,$(REQUIREMENTS_TXT),pip install -r $(req);) diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..871696a --- /dev/null +++ b/README.rst @@ -0,0 +1,52 @@ +================== +Dask-Elasticsearch +================== + +.. image:: https://img.shields.io/pypi/v/dask-elasticsearch.svg + :target: https://pypi.python.org/pypi/dask-elasticsearch + +.. image:: https://img.shields.io/pypi/pyversions/dask-elasticsearch.svg + :target: https://pypi.python.org/pypi/dask-elasticsearch + +.. image:: https://readthedocs.org/projects/dask-elasticsearch/badge/?version=latest + :target: https://readthedocs.org/projects/dask-elasticsearch/?badge=latest + :alt: Documentation Status + +.. image:: https://img.shields.io/travis/rolando/dask-elasticsearch.svg + :target: https://travis-ci.org/rolando/dask-elasticsearch + +.. image:: https://codecov.io/github/rolando/dask-elasticsearch/coverage.svg?branch=master + :alt: Coverage Status + :target: https://codecov.io/github/rolando/dask-elasticsearch + +.. image:: https://landscape.io/github/rolando/dask-elasticsearch/master/landscape.svg?style=flat + :target: https://landscape.io/github/rolando/dask-elasticsearch/master + :alt: Code Quality Status + +.. image:: https://requires.io/github/rolando/dask-elasticsearch/requirements.svg?branch=master + :alt: Requirements Status + :target: https://requires.io/github/rolando/dask-elasticsearch/requirements/?branch=master + +Elasticsearch reader for Dask. + +* Free software: MIT license +* Documentation: https://dask-elasticsearch.readthedocs.org. +* Python versions: 2.7, 3.4+ + +Features +-------- + +FIXME + +Quickstart +---------- + +FIXME + +Credits +------- + +This package was created with Cookiecutter_ and the `rolando/cookiecutter-pypackage`_ project template. + +.. _Cookiecutter: https://github.com/audreyr/cookiecutter +.. _`rolando/cookiecutter-pypackage`: https://github.com/rolando/cookiecutter-pypackage diff --git a/TODO.rst b/TODO.rst new file mode 100644 index 0000000..2ef024f --- /dev/null +++ b/TODO.rst @@ -0,0 +1,4 @@ +TODO +==== + +* Make a TODO diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..0d4d124 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1.0-dev diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..bc5ad03 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,177 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/dask-elasticsearch.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/dask-elasticsearch.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/dask-elasticsearch" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/dask-elasticsearch" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/authors.rst b/docs/authors.rst new file mode 100644 index 0000000..e122f91 --- /dev/null +++ b/docs/authors.rst @@ -0,0 +1 @@ +.. include:: ../AUTHORS.rst diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..b6e43ff --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# dask-elasticsearch documentation build configuration file, created by +# sphinx-quickstart on Tue Jul 9 22:26:36 2013. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import os +import re + +# If extensions (or modules to document with autodoc) are in another +# directory, add these directories to sys.path here. If the directory is +# relative to the documentation root, use os.path.abspath to make it +# absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# Get the project root dir, which is the parent dir of this +project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# -- General configuration --------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Dask-Elasticsearch' +copyright = u'2017, Rolando (Max) Espinoza' + +# The version info for the project you're documenting, acts as replacement +# for |version| and |release|, also used in various other places throughout +# the built documents. +# +# The full version, including alpha/beta/rc tags. +release = open(os.path.join(project_root, 'VERSION')).read().strip() +# The short X.Y version. +version = re.findall(r'\d+\.\d+\.\d+', release)[0] + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to +# some non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built +# documents. +#keep_warnings = False + + +# -- Options for HTML output ------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a +# theme further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as +# html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the +# top of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon +# of the docs. This file should be a Windows icon file (.ico) being +# 16x16 or 32x32 pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) +# here, relative to this directory. They are copied after the builtin +# static files, so a file named "default.css" will overwrite the builtin +# "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page +# bottom, using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names +# to template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. +# Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. +# Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages +# will contain a tag referring to it. The value of this option +# must be the base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'dask_elasticsearchdoc' + + +# -- Options for LaTeX output ------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + #'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +latex_documents = [ + ('index', 'dask_elasticsearch.tex', + u'Dask-Elasticsearch Documentation', + u'Rolando (Max) Espinoza', 'manual'), +] + +# The name of an image file (relative to this directory) to place at +# the top of the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings +# are parts, not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output ------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'dask_elasticsearch', + u'Dask-Elasticsearch Documentation', + [u'Rolando (Max) Espinoza'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ---------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'dask_elasticsearch', + u'Dask-Elasticsearch Documentation', + u'Rolando (Max) Espinoza', + 'dask-elasticsearch', + 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..e582053 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1 @@ +.. include:: ../CONTRIBUTING.rst diff --git a/docs/history.rst b/docs/history.rst new file mode 100644 index 0000000..2506499 --- /dev/null +++ b/docs/history.rst @@ -0,0 +1 @@ +.. include:: ../HISTORY.rst diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..3e31406 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,24 @@ +.. dask-elasticsearch documentation master file, created by + sphinx-quickstart on Tue Jul 9 22:26:36 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Dask-Elasticsearch's documentation! +====================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + readme + installation + history + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..8bb93a1 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,49 @@ +.. highlight:: shell + +============ +Installation +============ + + +Stable release +-------------- + +To install Dask-Elasticsearch, run this command in your terminal: + +.. code-block:: console + + $ pip install dask-elasticsearch + +If you don't have `pip`_ installed, this `Python installation guide`_ can guide +you through the process. + +.. _pip: https://pip.pypa.io +.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ + + +From sources +------------ + +The sources for Dask-Elasticsearch can be downloaded from the `Github repo`_. + +You can either clone the public repository: + +.. code-block:: console + + $ git clone git://github.com/rolando/dask-elasticsearch + +Or download the `tarball`_: + +.. code-block:: console + + $ curl -OL https://github.com/rolando/dask-elasticsearch/tarball/master + +Once you have a copy of the source, you can install it with: + +.. code-block:: console + + $ pip install -e . + + +.. _Github repo: https://github.com/rolando/dask-elasticsearch +.. _tarball: https://github.com/rolando/dask-elasticsearch/tarball/master diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..f5b41a0 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,242 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\dask-elasticsearch.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\dask-elasticsearch.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/docs/readme.rst b/docs/readme.rst new file mode 100644 index 0000000..72a3355 --- /dev/null +++ b/docs/readme.rst @@ -0,0 +1 @@ +.. include:: ../README.rst diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..678eff3 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +# Readthedocs workaround. +# This should be installed using pip from the root directory. +-e . diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..4b5760a --- /dev/null +++ b/pytest.ini @@ -0,0 +1,16 @@ +[pytest] +norecursedirs = + .* + dist + build +python_classes = + *Suite +python_functions = + test_* +python_files = + test_*.py + tests.py +ignore = + setup.py +addopts = + -rxEfsw -v diff --git a/requirements/dev.txt b/requirements/dev.txt new file mode 100644 index 0000000..c13985a --- /dev/null +++ b/requirements/dev.txt @@ -0,0 +1,8 @@ +# This packages are requires only for development and release management. +Sphinx +bumpversion +check-manifest +pip-tools +twine +watchdog +wheel diff --git a/requirements/install.txt b/requirements/install.txt new file mode 100644 index 0000000..997d22c --- /dev/null +++ b/requirements/install.txt @@ -0,0 +1,3 @@ +# This packages are required to install and run our package. +dask[imperative] +elasticsearch diff --git a/requirements/setup.txt b/requirements/setup.txt new file mode 100644 index 0000000..51c1dd4 --- /dev/null +++ b/requirements/setup.txt @@ -0,0 +1,2 @@ +# This packages are required before running setup (i.e. build commands require +# to import this packages). diff --git a/requirements/tests.txt b/requirements/tests.txt new file mode 100644 index 0000000..003944b --- /dev/null +++ b/requirements/tests.txt @@ -0,0 +1,4 @@ +# This packages are required to run all the tests and perform checks. +coverage +flake8 +pytest diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..6a4d7a4 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[flake8] +exclude = docs, tests +max-line-length = 120 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..736e2da --- /dev/null +++ b/setup.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from pkgutil import walk_packages +from setuptools import setup + + +def find_packages(path): + # This method returns packages and subpackages as well. + return [name for _, name, is_pkg in walk_packages([path]) if is_pkg] + + +def read_file(filename): + with open(filename) as fp: + return fp.read().strip() + + +def read_rst(filename): + # Ignore unsupported directives by pypi. + return ''.join(line for line in read_file(filename).splitlines() + if not line.startswith('.. comment::')) + + +def read_requirements(filename): + return [line.strip() for line in read_file(filename).splitlines() + if not line.startswith('#')] + + +setup_attrs = dict( + name='dask-elasticsearch', + version=read_file('VERSION'), + description="Elasticsearch reader for Dask.", + long_description=read_rst('README.rst') + '\n\n' + read_rst('HISTORY.rst'), + author="Rolando (Max) Espinoza", + author_email='rolando at rmax.io', + url='https://github.com/rolando/dask-elasticsearch', + packages=list(find_packages('src')), + package_dir={'': 'src'}, + setup_requires=read_requirements('requirements/setup.txt'), + install_requires=read_requirements('requirements/install.txt'), + include_package_data=True, + license="MIT", + zip_safe=False, + keywords='dask-elasticsearch', + classifiers=[ + 'Development Status :: 2 - Pre-Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', + "Programming Language :: Python :: 2", + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + ], +) + + +if __name__ == "__main__": + setup(**setup_attrs) diff --git a/src/dask_elasticsearch/__init__.py b/src/dask_elasticsearch/__init__.py new file mode 100644 index 0000000..8ea4f9f --- /dev/null +++ b/src/dask_elasticsearch/__init__.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +"""An Elasticsearch reader for Dask. +""" +from dask import delayed +from elasticsearch import Elasticsearch +from elasticsearch.helpers import scan + + +__author__ = 'Rolando (Max) Espinoza' +__email__ = 'rolando at rmax.io' +__version__ = '0.1.0-dev' + + +def _elasticsearch_scan(client_cls, client_kwargs, **params): + # This method is executed in the worker's process and here we instantiate + # the ES client as it cannot be serialized. + client = client_cls(**(client_kwargs or {})) + return list(scan(client, **params)) + + +def read_elasticsearch(query=None, npartitions=8, client_cls=None, + client_kwargs=None, **kwargs): + """Reads documents from Elasticsearch. + + By default, documents are sorted by ``_doc``. For more information see the + scrolling section in Elasticsearch documentation. + + Parameters + ---------- + query : dict, optional + Search query. + npartitions : int, optional + Number of partitions, default is 8. + client_cls : elasticsearch.Elasticsearch, optional + Elasticsearch client class. + client_kwargs : dict, optional + Elasticsearch client parameters. + **params + Additional keyword arguments are passed to the the + ``elasticsearch.helpers.scan`` function. + + Returns + ------- + out : List[Delayed] + A list of ``dask.Delayed`` objects. + + Examples + -------- + + Get all documents in elasticsearch. + + >>> docs = dask.bag.from_delayed(read_elasticsearch()) + + Get documents matching a given query. + + >>> query = {"query": {"match_all": {}}} + >>> docs = dask.bag.from_delayed(read_elasticsearch(query, index="myindex", doc_type="stuff")) + + + """ + query = query or {} + # Sorting by _doc is preferred for scrolling. + query.setdefault('sort', ['_doc']) + if client_cls is None: + client_cls = Elasticsearch + values = [] + # We load documents in parallel using the scrolling + slicing feature. + for idx in range(npartitions): + slice = {'id': idx, 'max': npartitions} + scan_kwargs = dict(kwargs, query=dict(query, slice=slice)) + values.append( + delayed(_elasticsearch_scan)(client_cls, client_kwargs, **scan_kwargs) + ) + return values diff --git a/tests/test_package_import.py b/tests/test_package_import.py new file mode 100644 index 0000000..aaed9b9 --- /dev/null +++ b/tests/test_package_import.py @@ -0,0 +1,7 @@ +import dask_elasticsearch + + +def test_package_metadata(): + assert dask_elasticsearch.__author__ + assert dask_elasticsearch.__email__ + assert dask_elasticsearch.__version__ diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..092b44b --- /dev/null +++ b/tox.ini @@ -0,0 +1,10 @@ +[tox] +envlist = py27, py34, py35 + +[testenv] +deps = + -rrequirements/setup.txt + -rrequirements/install.txt + -rrequirements/tests.txt +commands = + {posargs:coverage run -m py.test}