Pin 3.0.x branch against Mapnik 3.0.x mason package

.travis.yml

@@ -1,30 +1,22 @@
 language: generic
 
-git:
-  submodules: true
-  depth: 10
-
-addons:
-  postgresql: "9.4"
-
 matrix:
   include:
     - os: linux
       sudo: false
-      compiler: ": clang"
+      compiler: clang
       # note: only using ccache for CC is intentional here to
       # workaround an odd bug in distutils that manifests when only `ccache` is used to link
       # because distutils also has a bug whereby CC is used to compile instead of CXX, this works :)
-      env: JOBS=8 CXX="clang++-3.8 -Qunused-arguments" CC="ccache clang-3.8"
+      env: JOBS=8 CXX="clang++-3.9 -Qunused-arguments" CC="ccache clang-3.9 -Qunused-arguments"
       addons:
         apt:
           sources: [ 'ubuntu-toolchain-r-test']
           packages: [ 'libstdc++-5-dev', 'gdb', 'apport']
-    # disabled until I have time to fix https://github.com/mapnik/python-mapnik/issues/95
-    #- os: osx
-    #  osx_image: xcode7.3 # upgrades clang from 6 -> 7
-    #  compiler: clang
-    #  env: JOBS=4
+    - os: osx
+      osx_image: xcode8.2
+      compiler: clang
+      env: JOBS=4
 
 cache:
   directories:
@@ -55,23 +47,14 @@ before_install:
  - export COMMIT_MESSAGE=$(git show -s --format=%B $TRAVIS_COMMIT | tr -d '\n')
  - |
    if [[ $(uname -s) == 'Linux' ]]; then
-     export LDSHARED=$(python -c "import os;from distutils import sysconfig;print sysconfig.get_config_var('LDSHARED').replace('cc ','clang++-3.8 ')");
-     ./.mason/mason install clang 3.8.0
-     export PATH=$(./.mason/mason prefix clang 3.8.0)/bin:${PATH}
+     export LDSHARED=$(python -c "import os;from distutils import sysconfig;print sysconfig.get_config_var('LDSHARED').replace('cc ','clang++-3.9 ')");
+     mason install clang++ 3.9.1
+     export PATH=$(mason prefix clang++ 3.9.1)/bin:${PATH}
      which clang++
    else
      sudo easy_install pip;
-     brew info postgres;
-     export DATA_PATH=$(brew --prefix)/var/postgres;
-     rm -rf ${DATA_PATH};
-     initdb ${DATA_PATH} -E utf8;
-     pg_ctl -w start -l postgres.log --pgdata ${DATA_PATH};
-     cat postgres.log;
-     createuser -s postgres;
      export LDSHARED=$(python -c "import os;from distutils import sysconfig;print sysconfig.get_config_var('LDSHARED').replace('cc ','clang++ ')");
    fi
- - psql -c 'create database template_postgis;' -U postgres
- - psql -c 'create extension postgis;' -d template_postgis -U postgres
  - pip install --upgrade --user nose
  - pip install --upgrade --user wheel
  - pip install --upgrade --user twine
@@ -84,15 +67,15 @@ install:
  - python setup.py install --prefix ${PYTHONUSERBASE}
 
 before_script:
- - ulimit -c unlimited -S
- - RESULT=0
- - python test/run_tests.py || RESULT=$?
- - ls -l .
- - for i in $(find ./ -maxdepth 1 -name 'core*' -print); do gdb $(which python) $i -ex "thread apply all bt" -ex "set pagination 0" -batch; done;
- - if [[ ${RESULT} != 0 ]]; then exit $RESULT; fi
+ # start postgres/postgis
+ - source mason-config.env
+ - ./mason_packages/.link/bin/postgres -k ${PGHOST} > postgres.log &
 
 script:
+ - python test/run_tests.py
  - python test/visual.py -q
+ # stop postgres
+ - ./mason_packages/.link/bin/pg_ctl -w stop
  - |
    if [[ ${COMMIT_MESSAGE} =~ "[publish]" ]]; then
      python setup.py bdist_wheel

bootstrap.sh

@@ -7,64 +7,68 @@ function install() {
     MASON_PLATFORM_ID=$(mason env MASON_PLATFORM_ID)
     if [[ ! -d ./mason_packages/${MASON_PLATFORM_ID}/${1}/ ]]; then
         mason install $1 $2
-        # the rm here is to workaround https://github.com/mapbox/mason/issues/230
-        rm -f ./mason_packages/.link/mason.ini
         mason link $1 $2
     fi
 }
 
-ICU_VERSION="55.1"
+ICU_VERSION="57.1"
 
 function install_mason_deps() {
-    FAIL=0
-    install mapnik latest &
-    install ccache 3.2.4 &
-    install jpeg_turbo 1.5.0 libjpeg &
-    install libpng 1.6.24 libpng &
-    install libtiff 4.0.6 libtiff &
-    install libpq 9.5.2 &
-    install sqlite 3.14.1 libsqlite3 &
-    install expat 2.2.0 libexpat &
-    install icu ${ICU_VERSION} &
-    install proj 4.9.2 libproj &
-    install pixman 0.34.0 libpixman-1 &
-    install cairo 1.14.6 libcairo &
-    install protobuf 2.6.1 &
-    # technically protobuf is not a mapnik core dep, but installing
-    # here by default helps make mapnik-vector-tile builds easier
-    install webp 0.5.1 libwebp &
-    install gdal 2.1.1 libgdal &
-    install boost 1.61.0 &
-    install boost_libsystem 1.61.0 &
-    install boost_libfilesystem 1.61.0 &
-    install boost_libprogram_options 1.61.0 &
-    install boost_libregex_icu 1.61.0 &
-    # technically boost thread and python are not a core dep, but installing
-    # here by default helps make python-mapnik builds easier
-    install boost_libthread 1.61.0 &
-    install boost_libpython 1.61.0 &
-    install freetype 2.6.5 libfreetype &
-    install harfbuzz 1.3.0 libharfbuzz &
-    for job in $(jobs -p)
-    do
-        wait $job || let "FAIL+=1"
-    done
-    if [[ "$FAIL" != "0" ]]; then
-        exit ${FAIL}
-    fi
+    install mapnik 3.0.15
+    install jpeg_turbo 1.5.1
+    install libpng 1.6.28
+    install libtiff 4.0.7
+    install libpq 9.6.2
+    install sqlite 3.17.0
+    install expat 2.2.0
+    install icu ${ICU_VERSION}
+    install proj 4.9.3
+    install pixman 0.34.0
+    install cairo 1.14.8
+    install webp 0.6.0
+    install libgdal 2.1.3
+    install boost 1.63.0
+    install boost_libsystem 1.63.0
+    install boost_libfilesystem 1.63.0
+    install boost_libprogram_options 1.63.0
+    install boost_libregex_icu57 1.63.0
+    install freetype 2.7.1
+    install harfbuzz 1.4.2-ft
+    # deps needed by python-mapnik (not mapnik core)
+    install boost_libthread 1.63.0
+    install boost_libpython 1.63.0
+    install postgis 2.3.2-1
 }
 
 function setup_runtime_settings() {
     local MASON_LINKED_ABS=$(pwd)/mason_packages/.link
-    export PROJ_LIB=${MASON_LINKED_ABS}/share/proj
-    export ICU_DATA=${MASON_LINKED_ABS}/share/icu/${ICU_VERSION}
-    export GDAL_DATA=${MASON_LINKED_ABS}/share/gdal
-    if [[ $(uname -s) == 'Darwin' ]]; then
-        export DYLD_LIBRARY_PATH=$(pwd)/mason_packages/.link/lib:${DYLD_LIBRARY_PATH}
-    else
-        export LD_LIBRARY_PATH=$(pwd)/mason_packages/.link/lib:${LD_LIBRARY_PATH}
-    fi
-    export PATH=$(pwd)/mason_packages/.link/bin:${PATH}
+    echo "export PROJ_LIB=${MASON_LINKED_ABS}/share/proj" > mason-config.env
+    echo "export ICU_DATA=${MASON_LINKED_ABS}/share/icu/${ICU_VERSION}" >> mason-config.env
+    echo "export GDAL_DATA=${MASON_LINKED_ABS}/share/gdal" >> mason-config.env
+    echo "export PATH=$(pwd)/mason_packages/.link/bin:${PATH}" >> mason-config.env
+    echo "export PGTEMP_DIR=$(pwd)/local-tmp" >> mason-config.env
+    echo "export PGDATA=$(pwd)/local-postgres" >> mason-config.env
+    echo "export PGHOST=$(pwd)/local-unix-socket" >> mason-config.env
+    echo "export PGPORT=1111" >> mason-config.env
+
+    source mason-config.env
+    rm -rf ${PGHOST}
+    mkdir -p ${PGHOST}
+    rm -rf ${PGDATA}
+    mkdir -p ${PGDATA}
+    rm -rf ${PGTEMP_DIR}
+    mkdir -p ${PGTEMP_DIR}
+    ./mason_packages/.link/bin/initdb
+    sleep 2
+    ./mason_packages/.link/bin/postgres -k ${PGHOST} > postgres.log &
+    sleep 2
+    ./mason_packages/.link/bin/createdb template_postgis -T postgres
+    ./mason_packages/.link/bin/psql template_postgis -c "CREATE TABLESPACE temp_disk LOCATION '${PGTEMP_DIR}';"
+    ./mason_packages/.link/bin/psql template_postgis -c "SET temp_tablespaces TO 'temp_disk';"
+    ./mason_packages/.link/bin/psql template_postgis -c "CREATE PROCEDURAL LANGUAGE 'plpythonu' HANDLER plpython_call_handler;"
+    ./mason_packages/.link/bin/psql template_postgis -c "CREATE EXTENSION postgis;"
+    ./mason_packages/.link/bin/psql template_postgis -c "SELECT PostGIS_Full_Version();"
+    ./mason_packages/.link/bin/pg_ctl -w stop
 }
 
 function main() {

scripts/setup_mason.sh

@@ -4,18 +4,13 @@ set -eu
 set -o pipefail
 
 # we pin the mason version to avoid changes in mason breaking builds
-MASON_VERSION="new-pkgs"
+MASON_VERSION="1150c38"
 
 function setup_mason() {
-    if [[ ! -d ./.mason ]]; then
-        git clone https://github.com/mapbox/mason.git ./.mason
-        (cd ./.mason && git checkout ${MASON_VERSION})
-    else
-        echo "Updating to latest mason"
-        (cd ./.mason && git fetch && git checkout ${MASON_VERSION})
-    fi
+    mkdir -p ./mason
+    curl -sSfL https://github.com/mapbox/mason/archive/${MASON_VERSION}.tar.gz | tar --gunzip --extract --strip-components=1 --exclude="*md" --exclude="test*" --directory=./mason
     export MASON_HOME=$(pwd)/mason_packages/.link
-    export PATH=$(pwd)/.mason:$PATH
+    export PATH=$(pwd)/mason:${PATH}
     export CXX=${CXX:-clang++}
     export CC=${CC:-clang}
 }

setup.py

@@ -120,14 +120,10 @@ def run(self):
 '-lmapnik-json',
 ] + ['-l%s' % i for i in get_boost_library_names()])
 
-# Dynamically make the mapnik/paths.py file if it doesn't exist.
-if os.path.isfile('mapnik/paths.py'):
-    create_paths = False
-else:
-    create_paths = True
-    f_paths = open('mapnik/paths.py', 'w')
-    f_paths.write('import os\n')
-    f_paths.write('\n')
+# Dynamically make the mapnik/paths.py file
+f_paths = open('mapnik/paths.py', 'w')
+f_paths.write('import os\n')
+f_paths.write('\n')
 
 input_plugin_path = check_output([mapnik_config, '--input-plugins'])
 font_path = check_output([mapnik_config, '--fonts'])
@@ -137,7 +133,7 @@ def run(self):
         if sys.platform == 'darwin':
             base_f = 'libmapnik.dylib'
         else:
-            base_f = 'libmapnik.so.3.0'
+            base_f = 'libmapnik.so'
         f = os.path.join(lib_path, base_f)
         if not os.path.exists(os.path.join('mapnik', 'lib')):
             os.makedirs(os.path.join('mapnik', 'lib'))
@@ -147,30 +143,32 @@ def run(self):
     input_plugin_files = os.listdir(input_plugin_path)
     input_plugin_files = [os.path.join(
         input_plugin_path, f) for f in input_plugin_files]
-    if not os.path.exists(os.path.join('mapnik', 'plugins', 'input')):
-        os.makedirs(os.path.join('mapnik', 'plugins', 'input'))
+    if not os.path.exists(os.path.join('mapnik', 'lib', 'mapnik', 'input')):
+        os.makedirs(os.path.join('mapnik', 'lib', 'mapnik', 'input'))
     for f in input_plugin_files:
         try:
             shutil.copyfile(f, os.path.join(
-                'mapnik', 'plugins', 'input', os.path.basename(f)))
+                'mapnik', 'lib', 'mapnik', 'input', os.path.basename(f)))
         except shutil.Error:
             pass
     font_files = os.listdir(font_path)
     font_files = [os.path.join(font_path, f) for f in font_files]
-    if not os.path.exists(os.path.join('mapnik', 'plugins', 'fonts')):
-        os.makedirs(os.path.join('mapnik', 'plugins', 'fonts'))
+    if not os.path.exists(os.path.join('mapnik', 'lib', 'mapnik', 'fonts')):
+        os.makedirs(os.path.join('mapnik', 'lib', 'mapnik', 'fonts'))
     for f in font_files:
         try:
             shutil.copyfile(f, os.path.join(
-                'mapnik', 'plugins', 'fonts', os.path.basename(f)))
+                'mapnik', 'lib', 'mapnik', 'fonts', os.path.basename(f)))
         except shutil.Error:
             pass
-    if create_paths:
-        f_paths.write(
-            'mapniklibpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "plugins")\n')
-        f_paths.write("inputpluginspath = os.path.join(mapniklibpath,'input')\n")
-        f_paths.write("fontscollectionpath = os.path.join(mapniklibpath,'fonts')\n")
-elif create_paths:
+    f_paths.write(
+        'mapniklibpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "lib")\n')
+    f_paths.write("inputpluginspath = os.path.join(mapniklibpath, 'mapnik', 'input')\n")
+    f_paths.write("fontscollectionpath = os.path.join(mapniklibpath, 'mapnik', 'fonts')\n")
+    f_paths.write(
+        "__all__ = [mapniklibpath,inputpluginspath,fontscollectionpath]\n")
+    f_paths.close()
+else:
     if os.environ.get('LIB_DIR_NAME'):
         mapnik_lib_path = lib_path + os.environ.get('LIB_DIR_NAME')
     else:
@@ -181,8 +179,6 @@ def run(self):
         "inputpluginspath = '{path}'\n".format(path=input_plugin_path))
     f_paths.write(
         "fontscollectionpath = '{path}'\n".format(path=font_path))
-
-if create_paths:
     f_paths.write(
         "__all__ = [mapniklibpath,inputpluginspath,fontscollectionpath]\n")
     f_paths.close()
@@ -269,7 +265,7 @@ def run(self):
         'nose',
     ],
     package_data={
-        'mapnik': ['lib/*', 'plugins/*/*', 'share/*/*'],
+        'mapnik': ['lib/*.*', 'lib/*/*/*', 'share/*/*'],
     },
     test_suite='nose.collector',
     cmdclass={

test/python_tests/image_tiff_test.py

@@ -256,7 +256,7 @@ def test_tiff_rgba8_compare_stripped():
 
 def test_tiff_rgba8_compare_tiled():
     filepath1 = '../data/tiff/ndvi_256x256_rgba8_striped.tif'
-    filepath2 = '/tmp/mapnik-tiff-rgba8-stripped.tiff'
+    filepath2 = '/tmp/mapnik-tiff-rgba8-tiled.tiff'
     im = mapnik.Image.open(filepath1)
     im.save(filepath2, 'tiff:method=tiled')
     im2 = mapnik.Image.open(filepath2)