diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9246d1e --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.o +*.so +Makefile +mkmf.log +/pkg/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..aefa78b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,15 @@ +notifications: + email: + recipients: + - kou@cozmixng.org + +sudo: required + +services: + - docker + +install: + - docker build -t red-data-tools/red-arrow-nmatrix . + +script: + - docker run red-data-tools/red-arrow-nmatrix /bin/sh -c "bundle exec rake" diff --git a/.yardopts b/.yardopts new file mode 100644 index 0000000..9663abf --- /dev/null +++ b/.yardopts @@ -0,0 +1,6 @@ +--output-dir doc/reference/en +--markup markdown +--markup-provider kramdown +lib/**/*.rb +- +doc/text/**/* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..12455ac --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM ruby:2.4.1 + +MAINTAINER Kouhei Sutou + +RUN \ + apt update && \ + apt install -y apt-transport-https && \ + echo "deb https://packages.groonga.org/debian/ jessie main" > \ + /etc/apt/sources.list.d/groonga.list && \ + apt update && \ + apt install -y --allow-unauthenticated groonga-keyring && \ + apt update && \ + apt install -y libarrow-glib-dev + +RUN mkdir /app +WORKDIR /app +COPY . /app +RUN bundle install diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..4832f31 --- /dev/null +++ b/Gemfile @@ -0,0 +1,19 @@ +# -*- ruby -*- +# +# Copyright 2017 Kouhei Sutou +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source "https://rubygems.org/" + +gemspec diff --git a/LICENSE b/LICENSE new file mode 120000 index 0000000..67b44f6 --- /dev/null +++ b/LICENSE @@ -0,0 +1 @@ +doc/text/apache-2.0.txt \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..ee1c3ea --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# README + +## Name + +Red Arrow NMatrix + +## Description + +Red Arrow NMatrix is a library that provides converters between Apache Arrow's tensor data (`Arrow::Tensor`) and NMatrix's matrix data (`NMatrix`). + +Red Arrow NMatrix adds `Arrow::Tensor#to_nmatrix` for Apache Arrow to NMatrix conversion. Red Arrow NMatrix adds `NMatrix#to_arrow` for NMatrix to Apache Arrow conversion. + +## Install + +```text +% gem install red-arrow-nmatrix +``` + +## Usage + +```ruby +require "arrow-nmatrix" + +tensor.to_nmatrix # -> An object of NMatrix + +matrix.to_arrow # -> An object of Arrow::Tensor +``` + +## Dependencies + +* [Red Arrow](https://github.com/red-data-tools/red-arrow) + +* [NMatrix](https://github.com/SciRuby/nmatrix) + +## Authors + +* Kouhei Sutou \ + +## License + +Apache License 2.0. See doc/text/apache-2.0.txt for details. + +(Kouhei Sutou has a right to change the license including contributed +patches.) diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..2139ec9 --- /dev/null +++ b/Rakefile @@ -0,0 +1,44 @@ +# -*- ruby -*- +# +# Copyright 2017 Kouhei Sutou +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "rubygems" +require "bundler/gem_helper" +require "packnga" + +base_dir = File.join(File.dirname(__FILE__)) + +helper = Bundler::GemHelper.new(base_dir) +def helper.version_tag + version +end + +helper.install +spec = helper.gemspec + +Packnga::DocumentTask.new(spec) do |task| + task.original_language = "en" + task.translate_language = "ja" +end + +Packnga::ReleaseTask.new(spec) do +end + +desc "Run tests" +task :test => :compile do + ruby("test/run-test.rb") +end + +task default: :test diff --git a/doc/text/apache-2.0.txt b/doc/text/apache-2.0.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/doc/text/apache-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/doc/text/news.md b/doc/text/news.md new file mode 100644 index 0000000..c7b08e2 --- /dev/null +++ b/doc/text/news.md @@ -0,0 +1,5 @@ +# News + +## 0.0.1 - 2017-04-XX + +Initial release!!! diff --git a/ext/arrow-nmatrix/arrow-nmatrix.c b/ext/arrow-nmatrix/arrow-nmatrix.c new file mode 100644 index 0000000..3455736 --- /dev/null +++ b/ext/arrow-nmatrix/arrow-nmatrix.c @@ -0,0 +1,214 @@ +/* + * Copyright 2017 Kouhei Sutou + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include + +/* TODO: NMatrix should extern them. */ +extern const size_t DTYPE_SIZES[NM_NUM_DTYPES]; +/* extern size_t nm_storage_count_max_elements(const STORAGE* storage); */ + +void Init_arrow_nmatrix(void); + +static nm_dtype_t +garrow_type_to_nmatrix_dtype(GArrowType arrow_type) +{ + nm_dtype_t nmatrix_type = (nm_dtype_t)-1; + + switch (arrow_type) { + case GARROW_TYPE_UINT8: + nmatrix_type = BYTE; + break; + case GARROW_TYPE_INT8: + nmatrix_type = INT8; + break; + case GARROW_TYPE_INT16: + nmatrix_type = INT16; + break; + case GARROW_TYPE_INT32: + nmatrix_type = INT32; + break; + case GARROW_TYPE_INT64: + nmatrix_type = INT64; + break; + case GARROW_TYPE_FLOAT: + nmatrix_type = FLOAT32; + break; + case GARROW_TYPE_DOUBLE: + nmatrix_type = FLOAT64; + break; + case GARROW_TYPE_NA: + case GARROW_TYPE_BOOL: + case GARROW_TYPE_UINT16: + case GARROW_TYPE_UINT32: + case GARROW_TYPE_UINT64: + case GARROW_TYPE_HALF_FLOAT: + case GARROW_TYPE_STRING: + case GARROW_TYPE_BINARY: + case GARROW_TYPE_DATE32: + case GARROW_TYPE_DATE64: + case GARROW_TYPE_TIMESTAMP: + case GARROW_TYPE_TIME32: + case GARROW_TYPE_TIME64: + case GARROW_TYPE_INTERVAL: + case GARROW_TYPE_DECIMAL: + case GARROW_TYPE_LIST: + case GARROW_TYPE_STRUCT: + case GARROW_TYPE_UNION: + case GARROW_TYPE_DICTIONARY: + default: + break; + } + + return nmatrix_type; +} + +static VALUE +rb_arrow_tensor_to_nmatrix(VALUE self) +{ + GArrowTensor *tensor; + GArrowType value_type; + nm_dtype_t nmatrix_data_type; + gint64 *shape; + gint n_dimensions; + GArrowBuffer *buffer; + GBytes *data; + gconstpointer data_raw; + gsize data_size; + VALUE rb_nmatrix = Qnil; + + tensor = RVAL2GOBJ(self); + value_type = garrow_tensor_get_value_type(tensor); + nmatrix_data_type = garrow_type_to_nmatrix_dtype(value_type); + if (nmatrix_data_type == (nm_dtype_t)-1) { + GArrowDataType *data_type; + VALUE rb_data_type; + data_type = garrow_tensor_get_value_data_type(tensor); + rb_data_type = GOBJ2RVAL(data_type); + g_object_unref(data_type); + rb_raise(rb_eArgError, + "Arrow::Tensor data type must be uint8, int*, float or double: " + "<%" PRIsVALUE ">", + rb_data_type); + } + + shape = garrow_tensor_get_shape(tensor, &n_dimensions); + buffer = garrow_tensor_get_buffer(tensor); + data = garrow_buffer_get_data(buffer); + data_raw = g_bytes_get_data(data, &data_size); + + rb_nmatrix = rb_nmatrix_dense_create(nmatrix_data_type, + (size_t *)shape, + n_dimensions, + (void *)data_raw, + data_size); + g_bytes_unref(data); + g_object_unref(buffer); + g_free(shape); + + return rb_nmatrix; +} + +static GArrowDataType * +nmatrix_dtype_to_garrow_data_type(nm_dtype_t nmatrix_type) +{ + GArrowDataType *arrow_data_type = NULL; + + switch (nmatrix_type) { + case BYTE: + arrow_data_type = garrow_uint8_data_type_new(); + break; + case INT8: + arrow_data_type = garrow_int8_data_type_new(); + break; + case INT16: + arrow_data_type = garrow_int16_data_type_new(); + break; + case INT32: + arrow_data_type = garrow_int32_data_type_new(); + break; + case INT64: + arrow_data_type = garrow_int64_data_type_new(); + break; + case FLOAT32: + arrow_data_type = garrow_float_data_type_new(); + break; + case FLOAT64: + arrow_data_type = garrow_double_data_type_new(); + break; + case COMPLEX64: + case COMPLEX128: + case RUBYOBJ: + default: + break; + } + + return arrow_data_type; +} + +static VALUE +rb_nmatrix_to_arrow(VALUE self) +{ + GArrowDoubleDataType *data_type; + GArrowBuffer *data; + GArrowTensor *tensor; + VALUE rb_tensor; + + data_type = nmatrix_dtype_to_garrow_data_type(NM_DTYPE(self)); + if (!data_type) { + rb_raise(rb_eArgError, + "NMatrix data type must be " + ":byte, :int8, :int16, :int32, :int64, :float32 or :float64: " + "<%" PRIsVALUE ">", + self); + } + data = garrow_buffer_new((const guint8 *)NM_DENSE_ELEMENTS(self), + NM_SIZEOF_DTYPE(self) * NM_DENSE_COUNT(self)); + tensor = garrow_tensor_new(GARROW_DATA_TYPE(data_type), + data, + (gint64 *)(NM_STORAGE(self)->shape), + NM_DIM(self), + NULL, + 0, + NULL, + 0); + g_object_unref(data); + g_object_unref(data_type); + + rb_tensor = GOBJ2RVAL(tensor); + g_object_unref(tensor); + + return rb_tensor; +} + +void +Init_arrow_nmatrix(void) +{ + VALUE rb_Arrow; + VALUE rb_ArrowTensor; + + rb_Arrow = rb_const_get(rb_cObject, rb_intern("Arrow")); + rb_ArrowTensor = rb_const_get(rb_Arrow, rb_intern("Tensor")); + + rb_define_method(rb_ArrowTensor, "to_nmatrix", + rb_arrow_tensor_to_nmatrix, 0); + + rb_define_method(cNMatrix, "to_arrow", + rb_nmatrix_to_arrow, 0); +} diff --git a/ext/arrow-nmatrix/extconf.rb b/ext/arrow-nmatrix/extconf.rb new file mode 100644 index 0000000..bc22622 --- /dev/null +++ b/ext/arrow-nmatrix/extconf.rb @@ -0,0 +1,31 @@ +# Copyright 2017 Kouhei Sutou +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "mkmf-gnome2" + +required_pkg_config_package("arrow-glib") + +[ + ["glib2", "ext/glib2"], + ["nmatrix", "ext/nmatrix"], +].each do |name, source_dir| + spec = find_gem_spec(name) + source_dir = File.join(spec.full_gem_path, source_dir) + build_dir = source_dir + add_depend_package_path(name, source_dir, build_dir) +end + +have_func("rb_array_const_ptr", "ruby.h") + +create_makefile("arrow_nmatrix") diff --git a/lib/arrow-nmatrix.rb b/lib/arrow-nmatrix.rb new file mode 100644 index 0000000..d37d424 --- /dev/null +++ b/lib/arrow-nmatrix.rb @@ -0,0 +1,25 @@ +# Copyright 2017 Kouhei Sutou +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "arrow" +require "nmatrix" + +require "arrow-nmatrix/version" + +begin + major, minor, _ = RUBY_VERSION.split(/\./) + require "#{major}.#{minor}/arrow_nmatrix.so" +rescue LoadError + require "arrow_nmatrix.so" +end diff --git a/lib/arrow-nmatrix/version.rb b/lib/arrow-nmatrix/version.rb new file mode 100644 index 0000000..3933f73 --- /dev/null +++ b/lib/arrow-nmatrix/version.rb @@ -0,0 +1,17 @@ +# Copyright 2017 Kouhei Sutou +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module ArrowNMatrix + VERSION = "0.0.1" +end diff --git a/red-arrow-nmatrix.gemspec b/red-arrow-nmatrix.gemspec new file mode 100644 index 0000000..e48d7fd --- /dev/null +++ b/red-arrow-nmatrix.gemspec @@ -0,0 +1,53 @@ +# -*- ruby -*- +# +# Copyright 2017 Kouhei Sutou +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +clean_white_space = lambda do |entry| + entry.gsub(/(\A\n+|\n+\z)/, '') + "\n" +end + +$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "lib")) +require "arrow-nmatrix/version" + +Gem::Specification.new do |spec| + spec.name = "red-arrow-nmatrix" + spec.version = ArrowNMatrix::VERSION + spec.homepage = "https://github.com/red-data-tools/red-arrow-nmatrix" + spec.authors = ["Kouhei Sutou"] + spec.email = ["kou@clear-code.com"] + + readme = File.read("README.md") + readme.force_encoding("UTF-8") + entries = readme.split(/^\#\#\s(.*)$/) + clean_white_space.call(entries[entries.index("Description") + 1]) + description = clean_white_space.call(entries[entries.index("Description") + 1]) + spec.summary, spec.description, = description.split(/\n\n+/, 3) + spec.license = "Apache-2.0" + spec.files = ["README.md", "Rakefile", "Gemfile", "#{spec.name}.gemspec"] + spec.files += [".yardopts"] + spec.files += Dir.glob("lib/**/*.rb") + spec.files += Dir.glob("doc/text/*") + spec.extensions = FileList["ext/arrow-nmatrix/extconf.rb"] + spec.test_files += Dir.glob("test/**/*") + + spec.add_runtime_dependency("red-arrow") + spec.add_runtime_dependency("nmatrix") + + spec.add_development_dependency("bundler") + spec.add_development_dependency("rake") + spec.add_development_dependency("test-unit") + spec.add_development_dependency("packnga") + spec.add_development_dependency("kramdown") +end diff --git a/test/helper.rb b/test/helper.rb new file mode 100644 index 0000000..1ef06c5 --- /dev/null +++ b/test/helper.rb @@ -0,0 +1,17 @@ +# Copyright 2017 Kouhei Sutou +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "arrow-nmatrix" + +require "test-unit" diff --git a/test/run-test.rb b/test/run-test.rb new file mode 100755 index 0000000..828e5ad --- /dev/null +++ b/test/run-test.rb @@ -0,0 +1,38 @@ +#!/usr/bin/env ruby +# +# Copyright 2017 Kouhei Sutou +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +$VERBOSE = true + +require "pathname" + +base_dir = Pathname.new(__FILE__).dirname.parent.expand_path + +ext_dir = base_dir + "ext" + "arrow-nmatrix" +lib_dir = base_dir + "lib" +test_dir = base_dir + "test" + +$LOAD_PATH.unshift(ext_dir.to_s) +$LOAD_PATH.unshift(lib_dir.to_s) + +if system("type make > /dev/null") + Dir.chdir(ext_dir) do + system("make -j8 > /dev/null") or exit(false) + end +end + +require_relative "helper" + +exit(Test::Unit::AutoRunner.run(true, test_dir.to_s)) diff --git a/test/test-to-arrow.rb b/test/test-to-arrow.rb new file mode 100644 index 0000000..003caa1 --- /dev/null +++ b/test/test-to-arrow.rb @@ -0,0 +1,211 @@ +# Copyright 2017 Kouhei Sutou +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class ToArrowTest < Test::Unit::TestCase + test(":byte") do + data = [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + ], + [ + [9, 10, 11, 12], + [13, 14, 15, 16], + ], + [ + [17, 18, 19, 20], + [21, 22, 23, 24], + ], + ] + shape = [3, 2, 4] + nmatrix = NMatrix.new(shape, data.flatten, dtype: :byte) + tensor = nmatrix.to_arrow + assert_equal([ + Arrow::Type::UINT8, + data.flatten, + ], + [ + tensor.value_type, + tensor.buffer.data.to_s.unpack("C*"), + ]) + end + + test(":int8") do + data = [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + ], + [ + [9, 10, 11, 12], + [-1, -2, -3, -4], + ], + [ + [-5, -6, -7, -8], + [-9, -10, -11, -12], + ], + ] + shape = [3, 2, 4] + nmatrix = NMatrix.new(shape, data.flatten, dtype: :int8) + tensor = nmatrix.to_arrow + assert_equal([ + Arrow::Type::INT8, + data.flatten, + ], + [ + tensor.value_type, + tensor.buffer.data.to_s.unpack("c*"), + ]) + end + + test(":int16") do + data = [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + ], + [ + [9, 10, 11, 12], + [-1, -2, -3, -4], + ], + [ + [-5, -6, -7, -8], + [-9, -10, -11, -12], + ], + ] + shape = [3, 2, 4] + nmatrix = NMatrix.new(shape, data.flatten, dtype: :int16) + tensor = nmatrix.to_arrow + assert_equal([ + Arrow::Type::INT16, + data.flatten, + ], + [ + tensor.value_type, + tensor.buffer.data.to_s.unpack("s*"), + ]) + end + + test(":int32") do + data = [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + ], + [ + [9, 10, 11, 12], + [-1, -2, -3, -4], + ], + [ + [-5, -6, -7, -8], + [-9, -10, -11, -12], + ], + ] + shape = [3, 2, 4] + nmatrix = NMatrix.new(shape, data.flatten, dtype: :int32) + tensor = nmatrix.to_arrow + assert_equal([ + Arrow::Type::INT32, + data.flatten, + ], + [ + tensor.value_type, + tensor.buffer.data.to_s.unpack("l*"), + ]) + end + + test(":int64") do + data = [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + ], + [ + [9, 10, 11, 12], + [-1, -2, -3, -4], + ], + [ + [-5, -6, -7, -8], + [-9, -10, -11, -12], + ], + ] + shape = [3, 2, 4] + nmatrix = NMatrix.new(shape, data.flatten, dtype: :int64) + tensor = nmatrix.to_arrow + assert_equal([ + Arrow::Type::INT64, + data.flatten, + ], + [ + tensor.value_type, + tensor.buffer.data.to_s.unpack("q*"), + ]) + end + + test(":float32") do + data = [ + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + ], + [ + [9.0, 10.0, 11.0, 12.0], + [13.0, 14.0, 15.0, 16.0], + ], + [ + [17.0, 18.0, 19.0, 20.0], + [21.0, 22.0, 23.0, 24.0], + ], + ] + shape = [3, 2, 4] + nmatrix = NMatrix.new(shape, data.flatten, dtype: :float32) + tensor = nmatrix.to_arrow + assert_equal([ + Arrow::Type::FLOAT, + data.flatten, + ], + [ + tensor.value_type, + tensor.buffer.data.to_s.unpack("f*"), + ]) + end + + test(":float64") do + data = [ + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + ], + [ + [9.0, 10.0, 11.0, 12.0], + [13.0, 14.0, 15.0, 16.0], + ], + [ + [17.0, 18.0, 19.0, 20.0], + [21.0, 22.0, 23.0, 24.0], + ], + ] + shape = [3, 2, 4] + nmatrix = NMatrix.new(shape, data.flatten, dtype: :float64) + tensor = nmatrix.to_arrow + assert_equal([ + Arrow::Type::DOUBLE, + data.flatten, + ], + [ + tensor.value_type, + tensor.buffer.data.to_s.unpack("d*"), + ]) + end +end diff --git a/test/test-to-nmatrix.rb b/test/test-to-nmatrix.rb new file mode 100644 index 0000000..ec9d119 --- /dev/null +++ b/test/test-to-nmatrix.rb @@ -0,0 +1,190 @@ +# Copyright 2017 Kouhei Sutou +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class ToNMatrixTest < Test::Unit::TestCase + test("UInt8") do + data = [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + ], + [ + [9, 10, 11, 12], + [13, 14, 15, 16], + ], + [ + [17, 18, 19, 20], + [21, 22, 23, 24], + ], + ] + shape = [3, 2, 4] + tensor = Arrow::Tensor.new(Arrow::UInt8DataType.new, + Arrow::Buffer.new(data.flatten.pack("C*")), + shape, + nil, + nil) + assert_equal(NMatrix.new(shape, data.flatten, dtype: :byte), + tensor.to_nmatrix) + end + + test("Int8") do + data = [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + ], + [ + [9, 10, 11, 12], + [-1, -2, -3, -4], + ], + [ + [-5, -6, -7, -8], + [-9, -10, -11, -12], + ], + ] + shape = [3, 2, 4] + tensor = Arrow::Tensor.new(Arrow::Int8DataType.new, + Arrow::Buffer.new(data.flatten.pack("c*")), + shape, + nil, + nil) + assert_equal(NMatrix.new(shape, data.flatten, dtype: :int8), + tensor.to_nmatrix) + end + + test("Int16") do + data = [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + ], + [ + [9, 10, 11, 12], + [-1, -2, -3, -4], + ], + [ + [-5, -6, -7, -8], + [-9, -10, -11, -12], + ], + ] + shape = [3, 2, 4] + tensor = Arrow::Tensor.new(Arrow::Int16DataType.new, + Arrow::Buffer.new(data.flatten.pack("s*")), + shape, + nil, + nil) + assert_equal(NMatrix.new(shape, data.flatten, dtype: :int16), + tensor.to_nmatrix) + end + + test("Int32") do + data = [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + ], + [ + [9, 10, 11, 12], + [-1, -2, -3, -4], + ], + [ + [-5, -6, -7, -8], + [-9, -10, -11, -12], + ], + ] + shape = [3, 2, 4] + tensor = Arrow::Tensor.new(Arrow::Int32DataType.new, + Arrow::Buffer.new(data.flatten.pack("l*")), + shape, + nil, + nil) + assert_equal(NMatrix.new(shape, data.flatten, dtype: :int32), + tensor.to_nmatrix) + end + + test("Int64") do + data = [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + ], + [ + [9, 10, 11, 12], + [-1, -2, -3, -4], + ], + [ + [-5, -6, -7, -8], + [-9, -10, -11, -12], + ], + ] + shape = [3, 2, 4] + tensor = Arrow::Tensor.new(Arrow::Int64DataType.new, + Arrow::Buffer.new(data.flatten.pack("q*")), + shape, + nil, + nil) + assert_equal(NMatrix.new(shape, data.flatten, dtype: :int64), + tensor.to_nmatrix) + end + + test("Float") do + data = [ + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + ], + [ + [9.0, 10.0, 11.0, 12.0], + [13.0, 14.0, 15.0, 16.0], + ], + [ + [17.0, 18.0, 19.0, 20.0], + [21.0, 22.0, 23.0, 24.0], + ], + ] + shape = [3, 2, 4] + tensor = Arrow::Tensor.new(Arrow::FloatDataType.new, + Arrow::Buffer.new(data.flatten.pack("f*")), + shape, + nil, + nil) + assert_equal(NMatrix.new(shape, data.flatten, dtype: :float32), + tensor.to_nmatrix) + end + + test("Double") do + data = [ + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + ], + [ + [9.0, 10.0, 11.0, 12.0], + [13.0, 14.0, 15.0, 16.0], + ], + [ + [17.0, 18.0, 19.0, 20.0], + [21.0, 22.0, 23.0, 24.0], + ], + ] + shape = [3, 2, 4] + tensor = Arrow::Tensor.new(Arrow::DoubleDataType.new, + Arrow::Buffer.new(data.flatten.pack("d*")), + shape, + nil, + nil) + assert_equal(NMatrix.new(shape, data.flatten, dtype: :float64), + tensor.to_nmatrix) + end +end