diff --git a/.env_test b/.env_test new file mode 100644 index 00000000..dcfa53ec --- /dev/null +++ b/.env_test @@ -0,0 +1,243 @@ +COMPOSE_PROJECT_NAME=importer +# See https://github.com/containers/podman/issues/13889 +# DOCKER_BUILDKIT=0 +DOCKER_ENV=production +# See https://github.com/geosolutions-it/geonode-generic/issues/28 +# to see why we force API version to 1.24 +DOCKER_API_VERSION="1.24" +BACKUPS_VOLUME_DRIVER=local + +C_FORCE_ROOT=1 +FORCE_REINIT=false +INVOKE_LOG_STDOUT=true + +# LANGUAGE_CODE=it-it +# LANGUAGES=(('en-us','English'),('it-it','Italiano')) + +DJANGO_SETTINGS_MODULE=geonode.settings +GEONODE_INSTANCE_NAME=importer + +# ################# +# backend +# ################# +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +GEONODE_DATABASE=geonode +GEONODE_DATABASE_USER=geonode +GEONODE_DATABASE_PASSWORD=geonode +GEONODE_GEODATABASE=geonode_data +GEONODE_GEODATABASE_USER=geonode_data +GEONODE_GEODATABASE_PASSWORD=geonode_data +GEONODE_DATABASE_SCHEMA=public +GEONODE_GEODATABASE_SCHEMA=public +DATABASE_HOST=db +DATABASE_PORT=5432 +DATABASE_URL=postgis://geonode:geonode@db:5432/geonode +GEODATABASE_URL=postgis://geonode_data:geonode_data@db:5432/geonode_data +GEONODE_DB_CONN_MAX_AGE=0 +GEONODE_DB_CONN_TOUT=5 +DEFAULT_BACKEND_DATASTORE=datastore +BROKER_URL=amqp://guest:guest@rabbitmq:5672/ +CELERY_BEAT_SCHEDULER=celery.beat:PersistentScheduler +ASYNC_SIGNALS=False + +SITEURL=http://localhost:8000/ + +ALLOWED_HOSTS="['django', 'localhost', '127.0.0.1']" + +# Data Uploader +DEFAULT_BACKEND_UPLOADER=geonode.importer +TIME_ENABLED=True +MOSAIC_ENABLED=False +HAYSTACK_SEARCH=False +HAYSTACK_ENGINE_URL=http://elasticsearch:9200/ +HAYSTACK_ENGINE_INDEX_NAME=haystack +HAYSTACK_SEARCH_RESULTS_PER_PAGE=200 + +# ################# +# nginx +# HTTPD Server +# ################# +GEONODE_LB_HOST_IP=django +GEONODE_LB_PORT=8000 +GEOSERVER_LB_HOST_IP=geoserver +GEOSERVER_LB_PORT=8080 +NGINX_BASE_URL=http://localhost + +# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only) +# port where the server can be reached on HTTPS +HTTP_HOST=localhost +HTTPS_HOST= +POSTGRESQL_MAX_CONNECTIONS=100 +HTTP_PORT=8000 +HTTPS_PORT=443 + +# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work +# with an ip) and it must be reachable from the outside. This can be one of the following : +# disabled : we do not get a certificate at all (a placeholder certificate will be used) +# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates) +# production : we get a normal certificate (default) +LETSENCRYPT_MODE=disabled +# LETSENCRYPT_MODE=staging +# LETSENCRYPT_MODE=production + +RESOLVER=127.0.0.11 + +# ################# +# geoserver +# ################# +GEOSERVER_WEB_UI_LOCATION=http://localhost:8000/geoserver/ +GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/ +GEOSERVER_LOCATION=http://geoserver:8080/geoserver/ +GEOSERVER_ADMIN_USER=admin +GEOSERVER_ADMIN_PASSWORD=geoserver + +OGC_REQUEST_TIMEOUT=5 +OGC_REQUEST_MAX_RETRIES=0 +OGC_REQUEST_BACKOFF_FACTOR=0.3 +OGC_REQUEST_POOL_MAXSIZE=10 +OGC_REQUEST_POOL_CONNECTIONS=10 + +# ################# +# catalogue +# ################# +CATALOGUE_ENGINE=geonode.catalogue.backends.pycsw_local +CATALOGUE_URL=http://localhost:8000/catalogue/csw + +# Java Options & Memory +ENABLE_JSONP=true +outFormat=text/javascript +GEOSERVER_JAVA_OPTS='-Djava.awt.headless=true -Xms4G -Xmx4G -Dgwc.context.suffix=gwc -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGS-SHAPEFILE-CHARSET=UTF-8 -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL=http://localhost/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine' + +# ################# +# Security +# ################# +# Admin Settings +# +# ADMIN_PASSWORD is used to overwrite the GeoNode admin password **ONLY** the first time +# GeoNode is run. If you need to overwrite it again, you need to set the env var FORCE_REINIT, +# otherwise the invoke updateadmin task will be skipped and the current password already stored +# in DB will honored. + +ADMIN_USERNAME=admin +ADMIN_PASSWORD=admin +ADMIN_EMAIL=admin@localhost + +# EMAIL Notifications +EMAIL_ENABLE=False +DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend +DJANGO_EMAIL_HOST=localhost +DJANGO_EMAIL_PORT=25 +DJANGO_EMAIL_HOST_USER= +DJANGO_EMAIL_HOST_PASSWORD= +DJANGO_EMAIL_USE_TLS=False +DJANGO_EMAIL_USE_SSL=False +DEFAULT_FROM_EMAIL='GeoNode ' + +# Session/Access Control +LOCKDOWN_GEONODE=False +X_FRAME_OPTIONS="SAMEORIGIN" +SESSION_EXPIRED_CONTROL_ENABLED=True +DEFAULT_ANONYMOUS_VIEW_PERMISSION=True +DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True + +CORS_ALLOW_ALL_ORIGINS=True +GEOSERVER_CORS_ENABLED=True +GEOSERVER_CORS_ALLOWED_ORIGINS=* +GEOSERVER_CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,HEAD,OPTIONS +GEOSERVER_CORS_ALLOWED_HEADERS=* + +# Users Registration +ACCOUNT_OPEN_SIGNUP=True +ACCOUNT_EMAIL_REQUIRED=True +ACCOUNT_APPROVAL_REQUIRED=False +ACCOUNT_CONFIRM_EMAIL_ON_GET=False +ACCOUNT_EMAIL_VERIFICATION=none +ACCOUNT_EMAIL_CONFIRMATION_EMAIL=False +ACCOUNT_EMAIL_CONFIRMATION_REQUIRED=False +ACCOUNT_AUTHENTICATION_METHOD=username_email +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True + +# OAuth2 +OAUTH2_API_KEY= +OAUTH2_CLIENT_ID=Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV +OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3 + +SOCIALACCOUNT_OIDC_PROVIDER_ENABLED=True +SOCIALACCOUNT_PROVIDER=google + +# GeoNode APIs +API_LOCKDOWN=False +TASTYPIE_APIKEY= + +# ################# +# Production and +# Monitoring +# ################# +DEBUG=False + +SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' + +STATIC_ROOT=/mnt/volumes/statics/static/ +MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +GEOIP_PATH=/mnt/volumes/statics/geoip.db + +CACHE_BUSTING_STATIC_ENABLED=False + +MEMCACHED_ENABLED=False +MEMCACHED_BACKEND=django.core.cache.backends.memcached.MemcachedCache +MEMCACHED_LOCATION=memcached:11211 +MEMCACHED_LOCK_EXPIRE=3600 +MEMCACHED_LOCK_TIMEOUT=10 +# +# Options for memcached binary, e.g. -vvv to log all requests and cache hits +# +MEMCACHED_OPTIONS= + +MAX_DOCUMENT_SIZE=200 +CLIENT_RESULTS_LIMIT=5 +API_LIMIT_PER_PAGE=1000 + +# GIS Client +GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore +MAPBOX_ACCESS_TOKEN= +BING_API_KEY= +GOOGLE_API_KEY= + +# Monitoring +MONITORING_ENABLED=False +MONITORING_DATA_TTL=365 +USER_ANALYTICS_ENABLED=True +USER_ANALYTICS_GZIP=True +CENTRALIZED_DASHBOARD_ENABLED=False +MONITORING_SERVICE_NAME=local-geonode +MONITORING_HOST_NAME=geonode + +# Other Options/Contribs +MODIFY_TOPICCATEGORY=True +AVATAR_GRAVATAR_SSL=True +EXIF_ENABLED=True +CREATE_LAYER=True +FAVORITE_ENABLED=True + +# Advanced Workflow +RESOURCE_PUBLISHING=False +ADMIN_MODERATE_UPLOADS=False + +# PostgreSQL +POSTGRESQL_MAX_CONNECTIONS=200 + +# Common containers restart policy +RESTART_POLICY_CONDITION="on-failure" +RESTART_POLICY_DELAY="5s" +RESTART_POLICY_MAX_ATTEMPTS="3" +RESTART_POLICY_WINDOW=120s + +DEFAULT_MAX_UPLOAD_SIZE=5368709120 +DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=100 + +# Azure AD +MICROSOFT_TENANT_ID= +AZURE_CLIENT_ID= +AZURE_SECRET_KEY= +AZURE_KEY= \ No newline at end of file diff --git a/.github/workflows/runtests.yml b/.github/workflows/runtests.yml new file mode 100644 index 00000000..e965cdab --- /dev/null +++ b/.github/workflows/runtests.yml @@ -0,0 +1,25 @@ +name: Run test suite + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Start containers + run: docker-compose -f "docker-compose-test.yaml" up -d --build + - name: Run django entrypoint + run: docker exec django4importer /bin/sh -c "sh /usr/src/importer/entrypoint_test.sh" + - name: Run geonode-importer tests + run: docker exec django4importer /bin/sh -c "sh /usr/src/importer/runtest.sh" + - name: Stop containers + if: always() + run: docker-compose -f "docker-compose-test.yaml" down \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..fabb7a98 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM geonode/geonode-base:latest-ubuntu-22.04 + +RUN git clone https://github.com/GeoNode/geonode.git /usr/src/geonode +RUN mkdir -p /usr/src/importer + +RUN cd .. +COPY . /usr/src/importer/ +WORKDIR /usr/src/importer + +RUN pip install -r /usr/src/geonode/requirements.txt +RUN pip install --upgrade -e /usr/src/importer/ +RUN pip install coverage diff --git a/README.md b/README.md index 1032cfe9..35a55273 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,10 @@ In GeoNode 4.1 `geonode-importer` replaced the previous importer logic. - **CSV** - Vector - **GeoTiff** - Raster - **XML** - Update XML file for a given resource +<<<<<<< HEAD - **SLD** - Update SLD file for a given resource +======= +>>>>>>> 02bda6833d22c6eb7f29ba55ee99fd4f675ed1c5 **IMPORTANT**: At the moment the importer doesn't support overwriting/skipping existing layers from the UI. Every upload will create a new dataset. Overwriting a layer (`overwrite_existing_layer`) and skipping an already existing layer (`skip_existing_layers`) is supported through the API. @@ -110,7 +113,7 @@ IMPORTER_HANDLERS = os.getenv('IMPORTER_HANDLERS', [ ``` -NOTE: +## NOTE: In case of a local environment, Geoserver and Geonode should be able to reach the default `MEDIA_ROOT`. If some permission is missing, please change the `FILE_UPLOAD_DIRECTORY_PERMISSIONS` to make the folder accessible to both @@ -133,6 +136,9 @@ IMPORTER_GLOBAL_RATE_LIMIT= # default 5 IMPORTER_PUBLISHING_RATE_LIMIT= # default 5 IMPORTER_RESOURCE_CREATION_RATE_LIMIT= # default 10 IMPORTER_RESOURCE_COPY_RATE_LIMIT = # default 10 + +# https://github.com/OSGeo/gdal/issues/8674 +OGR2OGR_COPY_WITH_DUMP = If true, will pipe the PG dump to psql. ``` ## Troubleshooting @@ -162,4 +168,42 @@ In detail the list is the following: | From | TO | |---|---| | - (space) # \ | _ | -| .)(,& | empty_string | \ No newline at end of file +| .)(,& | empty_string | + + +## Run tests + +A simple docker compose is provided along with the project. To run the geonode-importer test suite please follow this steps: + +The compose project, provides what is needed to run the tests. This includes: + +1) Database +1) Geoserver +1) Django with all the dependecies needed + + +#### Step 1: build the project + +Build and run the docker compose project +``` +docker compose -f docker-compose-test.yaml up --build -d +``` + +#### Step 2: Run Scripts + +Two scripts are provided along with the compose file: + +1) entrypoint_test.sh +1) runtest.sh + +#### 1) entrypoint_test.sh +Is a script which create the needed extension and database to run the tests. Is a setup script + +#### 2) runtest.sh +It actually runs the tests. + + +Note: the tests inside the docker environment can be run manually too + +## Env File +The `.env_test` file contains all the environment variable needed to successfully run the tests \ No newline at end of file diff --git a/docker-compose-test.yaml b/docker-compose-test.yaml new file mode 100644 index 00000000..d86c2c4d --- /dev/null +++ b/docker-compose-test.yaml @@ -0,0 +1,104 @@ +version: '3.9' + +services: + + # Our custom django application. It includes Geonode. + django: + build: + context: . + dockerfile: Dockerfile + container_name: django4importer + healthcheck: + test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://django:8000/" + start_period: 60s + interval: 60s + timeout: 10s + retries: 2 + environment: + - IS_CELERY=False + command: 'sleep infinity' + depends_on: + db: + condition: service_healthy + env_file: + - .env_test + volumes: + - statics:/mnt/volumes/statics + - geoserver-data-dir:/geoserver_data/data + - data:/data + - tmp:/tmp + - .:/usr/src/importer + + + # Geoserver backend + geoserver: + image: geonode/geoserver:2.23.0 + container_name: geoserver4importer + healthcheck: + test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows" + start_period: 60s + interval: 20s + timeout: 10s + retries: 5 + env_file: + - .env_test + ports: + - "8080:8080" + volumes: + - statics:/mnt/volumes/statics + - geoserver-data-dir:/geoserver_data/data + - data:/data + - tmp:/tmp + restart: unless-stopped + depends_on: + data-dir-conf: + condition: service_healthy + + data-dir-conf: + image: geonode/geoserver_data:2.23.0 + container_name: gsconf4importer + entrypoint: sleep infinity + volumes: + - geoserver-data-dir:/geoserver_data/data + restart: unless-stopped + healthcheck: + test: "ls -A '/geoserver_data/data' | wc -l" + + # PostGIS database. + db: + # use geonode official postgis 15 image + image: geonode/postgis:15 + command: postgres -c "max_connections=100" + container_name: db4importer + env_file: + - .env_test + volumes: + - dbdata:/var/lib/postgresql/data + - dbbackups:/pg_backups + restart: unless-stopped + healthcheck: + test: "pg_isready -d postgres -U postgres" + # uncomment to enable remote connections to postgres + ports: + - "5432:5432" + + +volumes: + statics: + name: importer-statics + nginx-confd: + name: importer-nginxconfd + nginx-certificates: + name: importer-nginxcerts + geoserver-data-dir: + name: importer-gsdatadir + dbdata: + name: importer-dbdata + dbbackups: + name: importer-dbbackups + data: + name: importer-data + tmp: + name: importer-tmp + rabbitmq: + name: importer-rabbitmq diff --git a/entrypoint_test.sh b/entrypoint_test.sh new file mode 100755 index 00000000..9ebe7516 --- /dev/null +++ b/entrypoint_test.sh @@ -0,0 +1,31 @@ +#!/bin/bash +export PGPASSWORD=postgres +cmd="$@" +RESULT=$(psql -d geonode --host db --username postgres -c "SELECT count(*) FROM pg_database WHERE datname = 'test_geonode'") + +echo $RESULT + +#psql -d geonode --host db --username postgres -c 'CREATE USER geonode SUPERUSER;' +#psql -d geonode --host db --username postgres -c 'ALTER USER geonode CREATEDB;' +#psql -d geonode_data --host db --username postgres -c 'CREATE USER geonode SUPERUSER;' +#psql -d geonode_data --host db --username postgres -c 'ALTER USER geonode CREATEDB;' + +echo "creating" +psql -d geonode --host db --username postgres -c 'DROP DATABASE test_geonode' +psql -d geonode --host db --username postgres -c 'CREATE DATABASE test_geonode' +psql -d test_geonode --host db --username postgres -c 'ALTER USER geonode SUPERUSER;' +psql -d test_geonode --host db --username postgres -c 'ALTER USER geonode CREATEDB;' +psql -d test_geonode --host db --username postgres -c 'CREATE EXTENSION postgis;' +psql -d test_geonode --host db --username postgres -c 'GRANT ALL ON geometry_columns TO PUBLIC;' +psql -d test_geonode --host db --username postgres -c 'GRANT ALL ON spatial_ref_sys TO PUBLIC;' +psql -d test_geonode --host db --username postgres -c 'GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO geonode;' +echo "one done" +psql -d geonode_data --host db --username postgres -c 'DROP DATABASE test_geonode_data' +psql -d geonode_data --host db --username postgres -c 'CREATE DATABASE test_geonode_data' +psql -d test_geonode_data --host db --username postgres -c 'ALTER USER geonode_data SUPERUSER;' +psql -d test_geonode_data --host db --username postgres -c 'ALTER USER geonode_data CREATEDB;' +psql -d test_geonode_data --host db --username postgres -c 'CREATE EXTENSION postgis;' +psql -d test_geonode_data --host db --username postgres -c 'GRANT ALL ON geometry_columns TO PUBLIC;' +psql -d test_geonode_data --host db --username postgres -c 'GRANT ALL ON spatial_ref_sys TO PUBLIC;' +psql -d test_geonode_data --host db --username postgres -c 'GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO geonode_data;' +echo "Done" diff --git a/importer/__init__.py b/importer/__init__.py index c532a974..1018fa0f 100644 --- a/importer/__init__.py +++ b/importer/__init__.py @@ -20,7 +20,7 @@ project_dir = os.path.dirname(os.path.abspath(__file__)) -VERSION = (1, 0, 5) +VERSION = (1, 0, 7) __version__ = ".".join([str(i) for i in VERSION]) __author__ = "geosolutions-it" __email__ = "info@geosolutionsgroup.com" diff --git a/importer/api/tests.py b/importer/api/tests.py index bbf46831..bcf8b357 100644 --- a/importer/api/tests.py +++ b/importer/api/tests.py @@ -38,26 +38,16 @@ def test_upload_method_not_allowed(self): response = self.client.patch(self.url) self.assertEqual(405, response.status_code) - @patch("importer.api.views.UploadViewSet") - def test_redirect_to_old_upload_if_file_is_not_a_gpkg(self, patch_upload): - upload = MagicMock() - upload.upload.return_value = HttpResponse() - patch_upload.return_value = upload + def test_raise_exception_if_file_is_not_a_handled(self): self.client.force_login(get_user_model().objects.get(username="admin")) payload = { "base_file": SimpleUploadedFile(name="file.invalid", content=b"abc"), } response = self.client.post(self.url, data=payload) - self.assertEqual(200, response.status_code) - upload.upload.assert_called_once() - - @patch("importer.api.views.UploadViewSet") - def test_gpkg_raise_error_with_invalid_payload(self, patch_upload): - upload = MagicMock() - upload.upload.return_value = HttpResponse() - patch_upload.return_value = upload + self.assertEqual(500, response.status_code) + def test_gpkg_raise_error_with_invalid_payload(self): self.client.force_login(get_user_model().objects.get(username="admin")) payload = { "base_file": SimpleUploadedFile(name="test.gpkg", content=b"some-content"), diff --git a/importer/api/urls.py b/importer/api/urls.py index 8acf1a25..4012ebb0 100644 --- a/importer/api/urls.py +++ b/importer/api/urls.py @@ -1,10 +1,10 @@ from geonode.upload.api.urls import urlpatterns from importer.api.views import ResourceImporter, ImporterViewSet -from django.conf.urls import url +from django.urls import re_path urlpatterns.insert( 0, - url( + re_path( r"uploads/upload", ImporterViewSet.as_view({"post": "create"}), name="importer_upload", @@ -13,7 +13,7 @@ urlpatterns.insert( 1, - url( + re_path( r"resources/(?P\w+)/copy", ResourceImporter.as_view({"put": "copy"}), name="importer_resource_copy", diff --git a/importer/api/views.py b/importer/api/views.py index 66e08681..c5683d23 100644 --- a/importer/api/views.py +++ b/importer/api/views.py @@ -22,7 +22,7 @@ from django.urls import reverse from geonode.resource.enumerator import ExecutionRequestAction -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext_lazy as _ from dynamic_rest.filters import DynamicFilterBackend, DynamicSortingFilter from dynamic_rest.viewsets import DynamicModelViewSet from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter, FavoriteFilter diff --git a/importer/apps.py b/importer/apps.py index f7a064f8..a05618a3 100644 --- a/importer/apps.py +++ b/importer/apps.py @@ -16,8 +16,8 @@ def run_setup_hooks(*args, **kwargs): Run basic setup configuration for the importer app. Here we are overriding the upload API url """ - from django.conf.urls import include, url from geonode.urls import urlpatterns + from django.urls import re_path, include url_already_injected = any( [ @@ -30,5 +30,5 @@ def run_setup_hooks(*args, **kwargs): if not url_already_injected: urlpatterns.insert( 0, - url(r"^api/v2/", include("importer.api.urls")), + re_path(r"^api/v2/", include("importer.api.urls")), ) diff --git a/importer/celery_tasks.py b/importer/celery_tasks.py index 8ffc71db..818f3789 100644 --- a/importer/celery_tasks.py +++ b/importer/celery_tasks.py @@ -6,7 +6,7 @@ from django.db import connections, transaction from django.utils import timezone from django.utils.module_loading import import_string -from django.utils.translation import ugettext +from django.utils.translation import gettext_lazy from dynamic_models.exceptions import DynamicModelError, InvalidFieldNameError from dynamic_models.models import FieldSchema, ModelSchema from geonode.base.models import ResourceBase @@ -133,7 +133,7 @@ def import_resource(self, execution_id, /, handler_module_path, action, **kwargs execution_id=execution_id, last_updated=timezone.now(), func_name="import_resource", - step=ugettext("importer.import_resource"), + step=gettext_lazy("importer.import_resource"), celery_task_request=self.request, ) _exec = orchestrator.get_execution_object(execution_id) @@ -214,7 +214,7 @@ def publish_resource( execution_id=execution_id, last_updated=timezone.now(), func_name="publish_resource", - step=ugettext("importer.publish_resource"), + step=gettext_lazy("importer.publish_resource"), celery_task_request=self.request, ) _exec = orchestrator.get_execution_object(execution_id) @@ -323,7 +323,7 @@ def create_geonode_resource( execution_id=execution_id, last_updated=timezone.now(), func_name="create_geonode_resource", - step=ugettext("importer.create_geonode_resource"), + step=gettext_lazy("importer.create_geonode_resource"), celery_task_request=self.request, ) _exec = orchestrator.get_execution_object(execution_id) @@ -404,7 +404,7 @@ def copy_geonode_resource( execution_id=exec_id, last_updated=timezone.now(), func_name="copy_geonode_resource", - step=ugettext("importer.copy_geonode_resource"), + step=gettext_lazy("importer.copy_geonode_resource"), ) original_dataset_alternate = kwargs.get("kwargs").get("original_dataset_alternate") new_alternate = kwargs.get("kwargs").get("new_dataset_alternate") @@ -584,7 +584,7 @@ def copy_dynamic_model( execution_id=exec_id, last_updated=timezone.now(), func_name="copy_dynamic_model", - step=ugettext("importer.copy_dynamic_model"), + step=gettext_lazy("importer.copy_dynamic_model"), ) additional_kwargs = {} @@ -668,7 +668,7 @@ def copy_geonode_data_table( execution_id=exec_id, last_updated=timezone.now(), func_name="copy_geonode_data_table", - step=ugettext("importer.copy_geonode_data_table"), + step=gettext_lazy("importer.copy_geonode_data_table"), ) original_dataset_alternate = ( @@ -748,7 +748,7 @@ def rollback(self, *args, **kwargs): execution_id=exec_id, last_updated=timezone.now(), func_name="rollback", - step=ugettext("importer.rollback"), + step=gettext_lazy("importer.rollback"), celery_task_request=self.request, ) @@ -758,7 +758,7 @@ def rollback(self, *args, **kwargs): find_key_recursively(kwargs, "error") or "Some issue has occured, please check the logs" ) - orchestrator.set_as_failed(exec_id, reason=error) + orchestrator.set_as_failed(exec_id, reason=error, delete_file=False) return exec_id, kwargs diff --git a/importer/handlers/base.py b/importer/handlers/base.py index 5c4e8821..1cf1a4f2 100644 --- a/importer/handlers/base.py +++ b/importer/handlers/base.py @@ -138,8 +138,7 @@ def fixup_name(self, name): .replace(")", "") .replace("(", "") .replace(",", "") - .replace("&", "") - [:62] + .replace("&", "")[:62] ) def extract_resource_to_publish(self, files, layer_name, alternate, **kwargs): @@ -152,6 +151,13 @@ def extract_resource_to_publish(self, files, layer_name, alternate, **kwargs): """ return NotImplementedError + def overwrite_geoserver_resource(self, resource, catalog, store, workspace): + """ + Base method for override the geoserver resource. For vector file usually + is not needed since the value are replaced by ogr2ogr + """ + pass + @staticmethod def create_error_log(exc, task_name, *args): """ diff --git a/importer/handlers/common/raster.py b/importer/handlers/common/raster.py index cc164bbc..4e089de3 100644 --- a/importer/handlers/common/raster.py +++ b/importer/handlers/common/raster.py @@ -15,7 +15,6 @@ from geonode.resource.enumerator import ExecutionRequestAction as exa from geonode.resource.manager import resource_manager from geonode.resource.models import ExecutionRequest -from geonode.services.serviceprocessors.base import get_geoserver_cascading_workspace from importer.api.exception import ImportException from importer.celery_tasks import ErrorBaseTaskClass, import_orchestrator from importer.handlers.base import BaseHandler @@ -26,6 +25,7 @@ from osgeo import gdal from importer.celery_app import importer_app from geonode.storage.manager import storage_manager +from geonode.geoserver.helpers import get_store logger = logging.getLogger(__name__) @@ -130,6 +130,43 @@ def publish_resources(resources: List[str], catalog, store, workspace): raise e return True + def overwrite_geoserver_resource( + self, resource: List[str], catalog, store, workspace + ): + # we need to delete the resource before recreating it + self._delete_resource(resource, catalog, workspace) + self._delete_store(resource, catalog, workspace) + return self.publish_resources([resource], catalog, store, workspace) + + def _delete_store(self, resource, catalog, workspace): + store = None + possible_layer_name = [ + resource.get("name"), + resource.get("name").split(":")[-1], + f"{workspace.name}:{resource.get('name')}", + ] + for el in possible_layer_name: + store = catalog.get_store(el, workspace=workspace) + if store: + break + if store: + catalog.delete(store, purge="all", recurse=True) + return store + + def _delete_resource(self, resource, catalog, workspace): + res = None + possible_layer_name = [ + resource.get("name"), + resource.get("name").split(":")[-1], + f"{workspace.name}:{resource.get('name')}", + ] + for el in possible_layer_name: + res = catalog.get_resource(el, workspace=workspace) + if res: + break + if res: + catalog.delete(res, purge="all", recurse=True) + @staticmethod def delete_resource(instance): # it should delete the image from the geoserver data dir @@ -194,9 +231,9 @@ def extract_resource_to_publish( return [ { "name": alternate or layer_name, - "crs": self.identify_authority(layers) - if layers.GetSpatialRef() - else None, + "crs": ( + self.identify_authority(layers) if layers.GetSpatialRef() else None + ), "raster_path": files.get("base_file"), } ] @@ -253,7 +290,7 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str: skip_existing_layer=_exec.input_params.get("skip_existing_layer"), overwrite_existing_layer=should_be_overwritten, ): - workspace = get_geoserver_cascading_workspace(create=False) + workspace = DataPublisher(None).workspace user_datasets = Dataset.objects.filter( owner=_exec.user, alternate=f"{workspace.name}:{layer_name}" ) @@ -262,6 +299,8 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str: if dataset_exists and should_be_overwritten: layer_name, alternate = layer_name, user_datasets.first().alternate + elif not dataset_exists: + alternate = layer_name else: alternate = create_alternate(layer_name, execution_id) @@ -359,6 +398,7 @@ def overwrite_geonode_resource( _overwrite = _exec.input_params.get("overwrite_existing_layer", False) # if the layer exists, we just update the information of the dataset by # let it recreate the catalogue + if dataset.exists() and _overwrite: dataset = dataset.first() @@ -368,7 +408,7 @@ def overwrite_geonode_resource( self.handle_sld_file(dataset, _exec) resource_manager.set_thumbnail( - self.object.uuid, instance=self.object, overwrite=False + dataset.uuid, instance=dataset, overwrite=True ) dataset.refresh_from_db() return dataset @@ -480,6 +520,9 @@ def rollback( self, exec_id, rollback_from_step, action_to_rollback, *args, **kwargs ): steps = self.ACTIONS.get(action_to_rollback) + if rollback_from_step not in steps: + logger.info(f"Step not found {rollback_from_step}, skipping") + return step_index = steps.index(rollback_from_step) # the start_import, start_copy etc.. dont do anything as step, is just the start # so there is nothing to rollback diff --git a/importer/handlers/common/tests_vector.py b/importer/handlers/common/tests_vector.py index 954584aa..99bce84c 100644 --- a/importer/handlers/common/tests_vector.py +++ b/importer/handlers/common/tests_vector.py @@ -1,3 +1,4 @@ +import os import uuid from celery.canvas import Signature from celery import group @@ -225,7 +226,11 @@ def test_import_with_ogr2ogr_without_errors_should_call_the_right_command( _open.assert_called_once() _open.assert_called_with( - f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_files.get("base_file")}" -lco DIM=2 -nln alternate "dataset"', + f"/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:\" dbname='test_geonode_data' host=" + + os.getenv("DATABASE_HOST", "localhost") + + " port=5432 user='geonode_data' password='geonode_data' \" \"" + + self.valid_files.get("base_file") + + '" -nln alternate "dataset"', stdout=-1, stderr=-1, shell=True, # noqa @@ -251,8 +256,43 @@ def test_import_with_ogr2ogr_with_errors_should_raise_exception(self, _open): _open.assert_called_once() _open.assert_called_with( - f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_files.get("base_file")}" -lco DIM=2 -nln alternate "dataset"', + f"/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:\" dbname='test_geonode_data' host=" + + os.getenv("DATABASE_HOST", "localhost") + + " port=5432 user='geonode_data' password='geonode_data' \" \"" + + self.valid_files.get("base_file") + + '" -nln alternate "dataset"', stdout=-1, stderr=-1, shell=True, # noqa ) + + @patch.dict(os.environ, {"OGR2OGR_COPY_WITH_DUMP": "True"}, clear=True) + @patch("importer.handlers.common.vector.Popen") + def test_import_with_ogr2ogr_without_errors_should_call_the_right_command_if_dump_is_enabled( + self, _open + ): + _uuid = uuid.uuid4() + + comm = MagicMock() + comm.communicate.return_value = b"", b"" + _open.return_value = comm + + _task, alternate, execution_id = import_with_ogr2ogr( + execution_id=str(_uuid), + files=self.valid_files, + original_name="dataset", + handler_module_path=str(self.handler), + ovverwrite_layer=False, + alternate="alternate", + ) + + self.assertEqual("ogr2ogr", _task) + self.assertEqual(alternate, "alternate") + self.assertEqual(str(_uuid), execution_id) + + _open.assert_called_once() + _call_as_string = _open.mock_calls[0][1][0] + + self.assertTrue("-f PGDump /vsistdout/" in _call_as_string) + self.assertTrue("psql -d" in _call_as_string) + self.assertFalse("-f PostgreSQL PG" in _call_as_string) diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index 8c0735b7..5226fe3b 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -1,3 +1,4 @@ +import ast from django.db import connections from importer.publisher import DataPublisher from importer.utils import call_rollback_function, find_key_recursively @@ -15,7 +16,6 @@ from dynamic_models.schema import ModelSchemaEditor from geonode.base.models import ResourceBase from geonode.resource.enumerator import ExecutionRequestAction as exa -from geonode.services.serviceprocessors.base import get_geoserver_cascading_workspace from geonode.layers.models import Dataset from importer.celery_tasks import ErrorBaseTaskClass, create_dynamic_structure from importer.handlers.base import BaseHandler @@ -130,37 +130,48 @@ def publish_resources(resources: List[str], catalog, store, workspace): jdbc_virtual_table=_resource.get("name"), ) except Exception as e: - if ( - f"Resource named {_resource.get('name')} already exists in store:" - in str(e) - ): + if f"Resource named {_resource} already exists in store:" in str(e): logger.error(f"error during publishing: {e}") continue logger.error(f"error during publishing: {e}") raise e return True + def overwrite_geoserver_resource(self, resource, catalog, store, workspace): + """ + We dont need to do anything for now. + The data is replaced via ogr2ogr + """ + pass + @staticmethod def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate): """ Define the ogr2ogr command to be executed. This is a default command that is needed to import a vector file """ - _uri = settings.GEODATABASE_URL.replace("postgis://", "") - db_user, db_password = _uri.split("@")[0].split(":") - db_host = _uri.split("@")[1].split("/")[0] - db_port = "5432" - if ":" in db_host: - db_host, db_port = db_host.split(":") - db_name = _uri.split("@")[1].split("/")[1] - - options = "--config PG_USE_COPY YES " - options += ( - "-f PostgreSQL PG:\" dbname='%s' host=%s port=%s user='%s' password='%s' \" " - % (db_name, db_host, db_port, db_user, db_password) - ) + _datastore = settings.DATABASES["datastore"] + + options = "--config PG_USE_COPY YES" + copy_with_dump = ast.literal_eval(os.getenv("OGR2OGR_COPY_WITH_DUMP", "False")) + + if copy_with_dump: + # use PGDump to load the dataset with ogr2ogr + options += " -f PGDump /vsistdout/ " + else: + # default option with postgres copy + options += ( + " -f PostgreSQL PG:\" dbname='%s' host=%s port=%s user='%s' password='%s' \" " + % ( + _datastore["NAME"], + _datastore["HOST"], + _datastore.get("PORT", 5432), + _datastore["USER"], + _datastore["PASSWORD"], + ) + ) options += f'"{files.get("base_file")}"' + " " - # options += "-lco DIM=2 " + options += f'-nln {alternate} "{original_name}"' if ovverwrite_layer: @@ -386,7 +397,7 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str: return def find_alternate_by_dataset(self, _exec_obj, layer_name, should_be_overwritten): - workspace = get_geoserver_cascading_workspace(create=False) + workspace = DataPublisher(None).workspace dataset_available = Dataset.objects.filter( alternate__iexact=f"{workspace.name}:{layer_name}" ) @@ -394,7 +405,7 @@ def find_alternate_by_dataset(self, _exec_obj, layer_name, should_be_overwritten dataset_exists = dataset_available.exists() if dataset_exists and should_be_overwritten: - alternate = dataset_available.first().alternate + alternate = dataset_available.first().alternate.split(":")[-1] elif not dataset_exists: alternate = layer_name else: @@ -419,7 +430,7 @@ def setup_dynamic_model( """ layer_name = self.fixup_name(layer.GetName()) - workspace = get_geoserver_cascading_workspace(create=False) + workspace = DataPublisher(None).workspace user_datasets = Dataset.objects.filter( owner=username, alternate__iexact=f"{workspace.name}:{layer_name}" ) @@ -506,11 +517,13 @@ def create_dynamic_model_fields( ogr.GeometryTypeToName(layer.GetGeomType()) ) ), - "dim": 2 - if not ogr.GeometryTypeToName(layer.GetGeomType()) - .lower() - .startswith("3d") - else 3, + "dim": ( + 2 + if not ogr.GeometryTypeToName(layer.GetGeomType()) + .lower() + .startswith("3d") + else 3 + ), } ] @@ -627,7 +640,7 @@ def overwrite_geonode_resource( self.handle_sld_file(dataset, _exec) resource_manager.set_thumbnail( - dataset.uuid, instance=dataset, overwrite=False + dataset.uuid, instance=dataset, overwrite=True ) dataset.refresh_from_db() return dataset @@ -758,6 +771,9 @@ def rollback( self, exec_id, rollback_from_step, action_to_rollback, *args, **kwargs ): steps = self.ACTIONS.get(action_to_rollback) + if rollback_from_step not in steps: + logger.info(f"Step not found {rollback_from_step}, skipping") + return step_index = steps.index(rollback_from_step) # the start_import, start_copy etc.. dont do anything as step, is just the start # so there is nothing to rollback @@ -935,6 +951,12 @@ def import_with_ogr2ogr( options = orchestrator.load_handler(handler_module_path).create_ogr2ogr_command( files, original_name, ovverwrite_layer, alternate ) + _datastore = settings.DATABASES["datastore"] + + copy_with_dump = ast.literal_eval(os.getenv("OGR2OGR_COPY_WITH_DUMP", "False")) + + if copy_with_dump: + options += f" | PGPASSWORD={_datastore['PASSWORD']} psql -d {_datastore['NAME']} -h {_datastore['HOST']} -p {_datastore.get('PORT', 5432)} -U {_datastore['USER']} -f -" commands = [ogr_exe] + options.split(" ") @@ -944,6 +966,7 @@ def import_with_ogr2ogr( stderr is not None and stderr != b"" and b"ERROR" in stderr + and b"error" in stderr or b"Syntax error" in stderr ): try: diff --git a/importer/handlers/csv/handler.py b/importer/handlers/csv/handler.py index 80ebf79d..b82d9f22 100644 --- a/importer/handlers/csv/handler.py +++ b/importer/handlers/csv/handler.py @@ -195,11 +195,13 @@ def create_dynamic_model_fields( "name": layer.GetGeometryColumn() or self.default_geometry_column_name, "class_name": class_name, - "dim": 2 - if not ogr.GeometryTypeToName(layer.GetGeomType()) - .lower() - .startswith("3d") - else 3, + "dim": ( + 2 + if not ogr.GeometryTypeToName(layer.GetGeomType()) + .lower() + .startswith("3d") + else 3 + ), } ] @@ -241,9 +243,9 @@ def extract_resource_to_publish( return [ { "name": alternate or layer_name, - "crs": self.identify_authority(_l) - if _l.GetSpatialRef() - else "EPSG:4326", + "crs": ( + self.identify_authority(_l) if _l.GetSpatialRef() else "EPSG:4326" + ), } for _l in layers if self.fixup_name(_l.GetName()) == layer_name diff --git a/importer/handlers/csv/tests.py b/importer/handlers/csv/tests.py index f886b956..dae07006 100644 --- a/importer/handlers/csv/tests.py +++ b/importer/handlers/csv/tests.py @@ -1,6 +1,6 @@ import uuid from unittest.mock import MagicMock, patch - +import os from django.contrib.auth import get_user_model from django.test import TestCase from geonode.base.populate_test_data import create_single_dataset @@ -165,7 +165,11 @@ def test_import_with_ogr2ogr_without_errors_should_call_the_right_command( _open.assert_called_once() _open.assert_called_with( - f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_csv}" -lco DIM=2 -nln alternate "dataset" -oo KEEP_GEOM_COLUMNS=NO -lco GEOMETRY_NAME=geometry -oo "GEOM_POSSIBLE_NAMES=geom*,the_geom*,wkt_geom" -oo "X_POSSIBLE_NAMES=x,long*" -oo "Y_POSSIBLE_NAMES=y,lat*"', + f"/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:\" dbname='test_geonode_data' host=" + + os.getenv("DATABASE_HOST", "localhost") + + " port=5432 user='geonode_data' password='geonode_data' \" \"" + + self.valid_csv + + '" -nln alternate "dataset" -oo KEEP_GEOM_COLUMNS=NO -lco GEOMETRY_NAME=geometry -oo "GEOM_POSSIBLE_NAMES=geom*,the_geom*,wkt_geom" -oo "X_POSSIBLE_NAMES=x,long*" -oo "Y_POSSIBLE_NAMES=y,lat*"', stdout=-1, stderr=-1, shell=True, # noqa diff --git a/importer/handlers/geojson/handler.py b/importer/handlers/geojson/handler.py index f29decc4..9b7ad951 100644 --- a/importer/handlers/geojson/handler.py +++ b/importer/handlers/geojson/handler.py @@ -105,4 +105,4 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate): base_command = BaseVectorFileHandler.create_ogr2ogr_command( files, original_name, ovverwrite_layer, alternate ) - return f"{base_command } -lco DIM=2 -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name}" + return f"{base_command } -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name}" diff --git a/importer/handlers/geojson/tests.py b/importer/handlers/geojson/tests.py index 19fda437..26482218 100644 --- a/importer/handlers/geojson/tests.py +++ b/importer/handlers/geojson/tests.py @@ -1,4 +1,5 @@ import uuid +import os from django.test import TestCase from mock import MagicMock, patch from importer.handlers.common.vector import import_with_ogr2ogr @@ -130,7 +131,11 @@ def test_import_with_ogr2ogr_without_errors_should_call_the_right_command( _open.assert_called_once() _open.assert_called_with( - f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_files.get("base_file")}" -lco DIM=2 -nln alternate "dataset" -lco GEOMETRY_NAME=geometry', + f"/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:\" dbname='test_geonode_data' host=" + + os.getenv("DATABASE_HOST", "localhost") + + " port=5432 user='geonode_data' password='geonode_data' \" \"" + + self.valid_files.get("base_file") + + '" -nln alternate "dataset" -lco GEOMETRY_NAME=geometry', stdout=-1, stderr=-1, shell=True, # noqa diff --git a/importer/handlers/gpkg/tests.py b/importer/handlers/gpkg/tests.py index cb36e6b9..30a2e74a 100644 --- a/importer/handlers/gpkg/tests.py +++ b/importer/handlers/gpkg/tests.py @@ -1,4 +1,7 @@ -from django.test import TestCase +import copy +import os +import shutil +from django.test import TestCase, override_settings from importer.handlers.gpkg.exceptions import InvalidGeopackageException from django.contrib.auth import get_user_model from importer.handlers.gpkg.handler import GPKGFileHandler @@ -56,10 +59,7 @@ def test_is_valid_should_raise_exception_if_the_gpkg_is_invalid(self): self.handler.is_valid(files=self.invalid_files, user=self.user) self.assertIsNotNone(_exc) - self.assertTrue( - "Layer names must start with a letter, and valid characters are lowercase a-z, numbers or underscores" - in str(_exc.exception.detail) - ) + self.assertTrue("Error layer: INVALID LAYER_name" in str(_exc.exception.detail)) def test_is_valid_should_raise_exception_if_the_parallelism_is_met(self): parallelism, created = UploadParallelismLimit.objects.get_or_create( @@ -113,13 +113,19 @@ def test_can_handle_should_return_false_for_other_files(self): actual = self.handler.can_handle({"base_file": "random.file"}) self.assertFalse(actual) + @override_settings(MEDIA_ROOT="/tmp/") def test_single_message_error_handler(self): + # lets copy the file to the temporary folder + # later will be removed + shutil.copy(self.valid_gpkg, '/tmp') exec_id = orchestrator.create_execution_request( user=get_user_model().objects.first(), func_name="funct1", step="step", input_params={ - "files": self.valid_files, + "files": { + "base_file": '/tmp/valid.gpkg' + }, "skip_existing_layer": True, "handler_module_path": str(self.handler), }, @@ -142,3 +148,4 @@ def test_single_message_error_handler(self): ) self.assertEqual("FAILURE", TaskResult.objects.get(task_id=str(exec_id)).status) + self.assertFalse(os.path.exists('/tmp/valid.gpkg')) diff --git a/importer/handlers/kml/handler.py b/importer/handlers/kml/handler.py index a941f845..00941594 100644 --- a/importer/handlers/kml/handler.py +++ b/importer/handlers/kml/handler.py @@ -135,4 +135,4 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate): base_command = BaseVectorFileHandler.create_ogr2ogr_command( files, original_name, ovverwrite_layer, alternate ) - return f"{base_command } -lco DIM=2 -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} --config OGR_SKIP LibKML" + return f"{base_command } -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} --config OGR_SKIP LibKML" diff --git a/importer/handlers/shapefile/handler.py b/importer/handlers/shapefile/handler.py index 0cdf3583..9ace3133 100644 --- a/importer/handlers/shapefile/handler.py +++ b/importer/handlers/shapefile/handler.py @@ -1,5 +1,6 @@ import json import logging +import codecs from geonode.utils import get_supported_datasets_file_types from geonode.resource.enumerator import ExecutionRequestAction as exa from geonode.upload.utils import UploadLimitValidator @@ -48,7 +49,7 @@ def supported_file_extension_config(self): "format": "vector", "ext": ["shp"], "requires": ["shp", "prj", "dbf", "shx"], - "optional": ["xml", "sld"], + "optional": ["xml", "sld", "cpg", "cst"], } @staticmethod @@ -122,9 +123,11 @@ def is_valid(files, user): is_valid = all( map( lambda x: any( - _ext.endswith(f"{_filename}.{x}") - if isinstance(_ext, str) - else _ext.name.endswith(f"{_filename}.{x}") + ( + _ext.endswith(f"{_filename}.{x}") + if isinstance(_ext, str) + else _ext.name.endswith(f"{_filename}.{x}") + ) for _ext in files.values() ), _shp_ext_needed, @@ -151,17 +154,41 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate): ) layers = ogr.Open(files.get("base_file")) layer = layers.GetLayer(original_name) - additional_option = ( - " -nlt PROMOTE_TO_MULTI" - if layer is not None - and "Point" not in ogr.GeometryTypeToName(layer.GetGeomType()) - else " " - ) + + encoding = ShapeFileHandler._get_encoding(files) + + additional_options = [] + if layer is not None and "Point" not in ogr.GeometryTypeToName( + layer.GetGeomType() + ): + additional_options.append("-nlt PROMOTE_TO_MULTI") + if encoding: + additional_options.append(f"-lco ENCODING={encoding}") + return ( - f"{base_command } -lco precision=no -lco DIM=2 -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name}" - + additional_option + f"{base_command } -lco precision=no -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} " + + " ".join(additional_options) ) + @staticmethod + def _get_encoding(files): + if files.get("cpg_file"): + # prefer cpg file which is handled by gdal + return None + + encoding = None + if files.get("cst_file"): + # GeoServer exports cst-file + encoding_file = files.get("cst_file") + with open(encoding_file, "r") as f: + encoding = f.read() + try: + codecs.lookup(encoding) + except LookupError as e: + encoding = None + logger.error(f"Will ignore invalid encoding: {e}") + return encoding + def promote_to_multi(self, geometry_name): """ If needed change the name of the geometry, by promoting it to Multi diff --git a/importer/handlers/shapefile/tests.py b/importer/handlers/shapefile/tests.py index 6038c59e..a475363e 100644 --- a/importer/handlers/shapefile/tests.py +++ b/importer/handlers/shapefile/tests.py @@ -6,7 +6,7 @@ from django.test import TestCase from geonode.upload.api.exceptions import UploadParallelismLimitException from geonode.upload.models import UploadParallelismLimit -from mock import MagicMock, patch +from mock import MagicMock, patch, mock_open from importer import project_dir from importer.handlers.common.vector import import_with_ogr2ogr from importer.handlers.shapefile.handler import ShapeFileHandler @@ -112,6 +112,17 @@ def test_should_NOT_get_the_specific_serializer(self): actual = self.handler.has_serializer(self.invalid_files) self.assertFalse(actual) + def test_should_create_ogr2ogr_command_with_encoding_from_cst(self): + shp_with_cst = self.valid_shp.copy() + cst_file = self.valid_shp["base_file"].replace("shp", "cst") + shp_with_cst["cst_file"] = cst_file + patch_location = "importer.handlers.shapefile.handler.open" + with patch(patch_location, new=mock_open(read_data="UTF-8")) as _file: + actual = self.handler.create_ogr2ogr_command(shp_with_cst, "a", False, "a") + + _file.assert_called_once_with(cst_file, "r") + self.assertIn("ENCODING=UTF-8", actual) + @patch("importer.handlers.common.vector.Popen") def test_import_with_ogr2ogr_without_errors_should_call_the_right_command( self, _open @@ -137,7 +148,11 @@ def test_import_with_ogr2ogr_without_errors_should_call_the_right_command( _open.assert_called_once() _open.assert_called_with( - f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_shp.get("base_file")}" -lco DIM=2 -nln alternate "dataset" -lco precision=no -lco GEOMETRY_NAME=geometry ', + f"/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:\" dbname='test_geonode_data' host=" + + os.getenv("DATABASE_HOST", "localhost") + + " port=5432 user='geonode_data' password='geonode_data' \" \"" + + self.valid_shp.get("base_file") + + '" -nln alternate "dataset" -lco precision=no -lco GEOMETRY_NAME=geometry ', stdout=-1, stderr=-1, shell=True, # noqa diff --git a/importer/handlers/utils.py b/importer/handlers/utils.py index 8f79272b..d83ee6cd 100644 --- a/importer/handlers/utils.py +++ b/importer/handlers/utils.py @@ -3,12 +3,13 @@ from django.contrib.auth import get_user_model from geonode.base.models import ResourceBase from geonode.resource.models import ExecutionRequest -from geonode.services.serviceprocessors.base import get_geoserver_cascading_workspace import logging from dynamic_models.schema import ModelSchemaEditor from django.utils.module_loading import import_string from uuid import UUID +from importer.publisher import DataPublisher + logger = logging.getLogger(__name__) @@ -53,7 +54,7 @@ def should_be_imported(layer: str, user: get_user_model(), **kwargs) -> bool: - the publisher should republish the resource - geonode should update it """ - workspace = get_geoserver_cascading_workspace(create=False) + workspace = DataPublisher(None).workspace exists = ResourceBase.objects.filter( alternate=f"{workspace.name}:{layer}", owner=user ).exists() @@ -71,7 +72,9 @@ def create_alternate(layer_name, execution_id): """ _hash = hashlib.md5(f"{layer_name}_{execution_id}".encode("utf-8")).hexdigest() alternate = f"{layer_name}_{_hash}" - if len(alternate) > 63: # 63 is the max table lengh in postgres to stay safe, we cut at 12 + if ( + len(alternate) > 63 + ): # 63 is the max table lengh in postgres to stay safe, we cut at 12 return f"{layer_name[:50]}{_hash[:12]}" return alternate diff --git a/importer/handlers/xml/handler.py b/importer/handlers/xml/handler.py index 60b2ae97..a40c110a 100644 --- a/importer/handlers/xml/handler.py +++ b/importer/handlers/xml/handler.py @@ -2,6 +2,10 @@ from django.shortcuts import get_object_or_404 from geonode.layers.models import Dataset +<<<<<<< HEAD +======= +from geonode.resource.enumerator import ExecutionRequestAction as exa +>>>>>>> 02bda6833d22c6eb7f29ba55ee99fd4f675ed1c5 from geonode.resource.manager import resource_manager from importer.handlers.common.metadata import MetadataFileHandler from importer.handlers.xml.exceptions import InvalidXmlException @@ -13,7 +17,11 @@ class XMLFileHandler(MetadataFileHandler): """ +<<<<<<< HEAD Handler to import XML files into GeoNode data db +======= + Handler to import KML files into GeoNode data db +>>>>>>> 02bda6833d22c6eb7f29ba55ee99fd4f675ed1c5 It must provide the task_lists required to comple the upload """ @@ -33,7 +41,7 @@ def can_handle(_data) -> bool: ) @staticmethod - def is_valid(files, user): + def is_valid(files, user=None): """ Define basic validation steps """ diff --git a/importer/migrations/0005_fixup_dynamic_shema_table_names.py b/importer/migrations/0005_fixup_dynamic_shema_table_names.py index e216c8ae..07fdeab9 100644 --- a/importer/migrations/0005_fixup_dynamic_shema_table_names.py +++ b/importer/migrations/0005_fixup_dynamic_shema_table_names.py @@ -1,26 +1,26 @@ from django.db import migrations import logging from django.db import ProgrammingError +from django.db import connections logger = logging.getLogger(__name__) def fixup_table_name(apps, schema_editor): try: - schema = apps.get_model("dynamic_models", "ModelSchema") - for val in schema.objects.all(): - if val.name != val.db_table_name: - val.db_table_name = val.name - val.save() - except ProgrammingError as e: """ - The dynamic model should exists to apply the above migration. + The dynamic model should exists to apply the migration. In case it does not exists we can skip it """ - if 'relation "dynamic_models_modelschema" does not exist' in e.args[0]: - logging.debug("Dynamic model does not exists yet, skipping") - return - raise e + if ( + "dynamic_models_modelschema" + in schema_editor.connection.introspection.table_names() + ): + schema = apps.get_model("dynamic_models", "ModelSchema") + for val in schema.objects.all(): + if val.name != val.db_table_name: + val.db_table_name = val.name + val.save() except Exception as e: raise e diff --git a/importer/models.py b/importer/models.py index a02c3f25..e709c862 100644 --- a/importer/models.py +++ b/importer/models.py @@ -30,7 +30,6 @@ def delete_dynamic_model(instance, sender, **kwargs): class ResourceHandlerInfo(models.Model): - """ Here we save the relation between the geonode resource created and the handler that created that resource """ diff --git a/importer/orchestrator.py b/importer/orchestrator.py index 7a8899a3..7edc6fb2 100644 --- a/importer/orchestrator.py +++ b/importer/orchestrator.py @@ -6,13 +6,13 @@ from celery import states from django.contrib.auth import get_user_model from django.db.models import Q -from django.db.transaction import rollback from django.utils import timezone from django.utils.module_loading import import_string from django_celery_results.models import TaskResult from geonode.base.enumerations import STATE_INVALID, STATE_PROCESSED, STATE_RUNNING from geonode.resource.models import ExecutionRequest from geonode.upload.models import Upload +from geonode.storage.manager import storage_manager from rest_framework import serializers from importer.api.exception import ImportException @@ -157,7 +157,7 @@ def perform_next_step( self.set_as_failed(execution_id, reason=error_handler(e, execution_id)) raise e - def set_as_failed(self, execution_id, reason=None): + def set_as_failed(self, execution_id, reason=None, delete_file=True): """ Utility method to set the ExecutionRequest object to fail """ @@ -169,6 +169,17 @@ def set_as_failed(self, execution_id, reason=None): log=reason, legacy_status=STATE_INVALID, ) + # delete + if delete_file: + exec_obj = self.get_execution_object(execution_id) + _files = exec_obj.input_params.get("files") + # better to delete each single file since it can be a remote storage service + if _files: + logger.info( + "Execution failed, removing uploaded file to save disk space" + ) + for _file in _files.values(): + storage_manager.delete(_file) def set_as_partially_failed(self, execution_id, reason=None): """ diff --git a/importer/publisher.py b/importer/publisher.py index 40fe9434..aaa05dbc 100644 --- a/importer/publisher.py +++ b/importer/publisher.py @@ -4,7 +4,6 @@ from geonode import settings from geonode.geoserver.helpers import create_geoserver_db_featurestore -from geonode.services.serviceprocessors.base import get_geoserver_cascading_workspace from geoserver.catalog import Catalog from geonode.utils import OGC_Servers_Handler from django.utils.module_loading import import_string @@ -28,9 +27,10 @@ def __init__(self, handler_module_path) -> None: self.cat = Catalog( service_url=ogc_server_settings.rest, username=_user, password=_password ) - self.workspace = get_geoserver_cascading_workspace(create=True) + self.workspace = self._get_default_workspace(create=True) - self.handler = import_string(handler_module_path)() + if handler_module_path is not None: + self.handler = import_string(handler_module_path)() def extract_resource_to_publish( self, files: dict, action: str, layer_name, alternate=None, **kwargs @@ -71,20 +71,44 @@ def publish_resources(self, resources: List[str]): self.sanity_checks(resources) return result + def overwrite_resources(self, resources: List[str]): + """ + We dont need to do anything for now. The data is replaced via ogr2ogr + """ + self.get_or_create_store() + for _resource in resources: + result = self.handler.overwrite_geoserver_resource( + resource=_resource, + catalog=self.cat, + store=self.store, + workspace=self.workspace, + ) + self.sanity_checks(resources) + return result + def delete_resource(self, resource_name): layer = self.get_resource(resource_name) - if layer: + if layer and layer.resource: self.cat.delete(layer.resource, purge="all", recurse=True) + store = self.cat.get_store( + resource_name.split(":")[-1], + workspace=os.getenv( + "DEFAULT_WORKSPACE", os.getenv("CASCADE_WORKSPACE", "geonode") + ), + ) + if not store: + store = self.cat.get_store( + resource_name, + workspace=os.getenv( + "DEFAULT_WORKSPACE", os.getenv("CASCADE_WORKSPACE", "geonode") + ), + ) + if store: + self.cat.delete(store, purge="all", recurse=True) def get_resource(self, dataset_name): return self.cat.get_layer(dataset_name) - def overwrite_resources(self, resources: List[str]): - """ - Not available for now, waiting geoserver 2.20/2.21 available with Geonode - """ - pass - def get_or_create_store(self): """ Evaluate if the store exists. if not is created @@ -122,9 +146,34 @@ def sanity_checks(self, resources): For each resource. This is a quick test to be sure that the resource is correctly set/created """ + for _resource in resources: - res = self.cat.get_resource(_resource.get("name"), workspace=self.workspace) - if not res.projection: + possible_layer_name = [ + _resource.get("name"), + _resource.get("name").split(":")[-1], + f"{self.workspace.name}:{_resource.get('name')}", + ] + res = list( + filter( + None, + ( + self.cat.get_resource(x, workspace=self.workspace) + for x in possible_layer_name + ), + ) + ) + if not res or (res and not res[0].projection): raise PublishResourceException( - f"The SRID for the resource {res.name} is not correctly set, Please check Geoserver logs" + f"The SRID for the resource {_resource} is not correctly set, Please check Geoserver logs" ) + + def _get_default_workspace(self, create=True): + """Return the default geoserver workspace + The workspace can be created it if needed. + """ + name = getattr(settings, "DEFAULT_WORKSPACE", "geonode") + workspace = self.cat.get_workspace(name) + if workspace is None and create: + uri = f"http://www.geonode.org/{name}" + workspace = self.cat.create_workspace(name, uri) + return workspace diff --git a/importer/tests/end2end/test_end2end.py b/importer/tests/end2end/test_end2end.py index 231e1c57..d9b03d2c 100644 --- a/importer/tests/end2end/test_end2end.py +++ b/importer/tests/end2end/test_end2end.py @@ -15,6 +15,7 @@ from importer import project_dir from importer.tests.utils import ImporterBaseTestSupport import gisdata +from geonode.base.populate_test_data import create_single_dataset geourl = settings.GEODATABASE_URL @@ -46,53 +47,70 @@ def setUpClass(cls) -> None: def setUp(self) -> None: self.admin = get_user_model().objects.get(username="admin") + for el in Dataset.objects.all(): + el.delete() def tearDown(self) -> None: - return super().tearDown() + super().tearDown() + for el in Dataset.objects.all(): + el.delete() - def _assertimport(self, payload, initial_name): - self.client.force_login(self.admin) + def _assertimport(self, payload, initial_name, overwrite=False, last_update=None): + try: + self.client.force_login(self.admin) - response = self.client.post(self.url, data=payload) - self.assertEqual(201, response.status_code) + response = self.client.post(self.url, data=payload) + self.assertEqual(201, response.status_code) - # if is async, we must wait. It will wait for 1 min before raise exception - if ast.literal_eval(os.getenv("ASYNC_SIGNALS", "False")): - tentative = 1 - while ( + # if is async, we must wait. It will wait for 1 min before raise exception + if ast.literal_eval(os.getenv("ASYNC_SIGNALS", "False")): + tentative = 1 + while ( + ExecutionRequest.objects.get( + exec_id=response.json().get("execution_id") + ) + != ExecutionRequest.STATUS_FINISHED + and tentative <= 10 + ): + time.sleep(10) + tentative += 1 + if ( ExecutionRequest.objects.get( exec_id=response.json().get("execution_id") - ) + ).status != ExecutionRequest.STATUS_FINISHED - and tentative <= 6 ): - time.sleep(10) - tentative += 1 - if ( - ExecutionRequest.objects.get( - exec_id=response.json().get("execution_id") - ).status - != ExecutionRequest.STATUS_FINISHED - ): - raise Exception("Async still in progress after 1 min of waiting") - - # check if the dynamic model is created - _schema_id = ModelSchema.objects.filter(name__icontains=initial_name) - self.assertTrue(_schema_id.exists()) - schema_entity = _schema_id.first() - self.assertTrue(FieldSchema.objects.filter(model_schema=schema_entity).exists()) - - # Verify that ogr2ogr created the table with some data in it - entries = ModelSchema.objects.filter(id=schema_entity.id).first() - self.assertTrue(entries.as_model().objects.exists()) - - # check if the resource is in geoserver - resources = self.cat.get_resources() - self.assertTrue(schema_entity.name in [y.name for y in resources]) - - # check if the geonode resource exists - dataset = Dataset.objects.filter(alternate=f"geonode:{schema_entity.name}") - self.assertTrue(dataset.exists()) + raise Exception("Async still in progress after 1 min of waiting") + + # check if the dynamic model is created + if os.getenv("IMPORTER_ENABLE_DYN_MODELS", False): + _schema_id = ModelSchema.objects.filter(name__icontains=initial_name) + self.assertTrue(_schema_id.exists()) + schema_entity = _schema_id.first() + self.assertTrue( + FieldSchema.objects.filter(model_schema=schema_entity).exists() + ) + + # Verify that ogr2ogr created the table with some data in it + entries = ModelSchema.objects.filter(id=schema_entity.id).first() + self.assertTrue(entries.as_model().objects.exists()) + + # check if the geonode resource exists + dataset = Dataset.objects.filter( + alternate__icontains=f"geonode:{initial_name}" + ) + self.assertTrue(dataset.exists()) + + # check if the resource is in geoserver + resources = self.cat.get_resources() + self.assertTrue(dataset.first().title in [y.name for y in resources]) + if overwrite: + self.assertTrue(dataset.first().last_updated > last_update) + finally: + dataset = Dataset.objects.filter( + alternate__icontains=f"geonode:{initial_name}" + ) + dataset.delete() class ImporterGeoPackageImportTest(BaseImporterEndToEndTest): @@ -102,14 +120,38 @@ class ImporterGeoPackageImportTest(BaseImporterEndToEndTest): ) def test_import_geopackage(self): layer = self.cat.get_layer("geonode:stazioni_metropolitana") - self.cat.delete(layer) + if layer: + self.cat.delete(layer) payload = { "base_file": open(self.valid_gkpg, "rb"), } initial_name = "stazioni_metropolitana" self._assertimport(payload, initial_name) layer = self.cat.get_layer("geonode:stazioni_metropolitana") - self.cat.delete(layer) + if layer: + self.cat.delete(layer) + + @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @override_settings( + GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" + ) + def test_import_gpkg_overwrite(self): + prev_dataset = create_single_dataset(name="stazioni_metropolitana") + + layer = self.cat.get_layer("geonode:stazioni_metropolitana") + if layer: + self.cat.delete(layer) + payload = { + "base_file": open(self.valid_gkpg, "rb"), + } + initial_name = "stazioni_metropolitana" + payload["overwrite_existing_layer"] = True + self._assertimport( + payload, initial_name, overwrite=True, last_update=prev_dataset.last_updated + ) + layer = self.cat.get_layer("geonode:stazioni_metropolitana") + if layer: + self.cat.delete(layer) class ImporterGeoJsonImportTest(BaseImporterEndToEndTest): @@ -119,7 +161,8 @@ class ImporterGeoJsonImportTest(BaseImporterEndToEndTest): ) def test_import_geojson(self): layer = self.cat.get_layer("geonode:valid") - self.cat.delete(layer) + if layer: + self.cat.delete(layer) payload = { "base_file": open(self.valid_geojson, "rb"), @@ -127,7 +170,30 @@ def test_import_geojson(self): initial_name = "valid" self._assertimport(payload, initial_name) layer = self.cat.get_layer("geonode:valid") - self.cat.delete(layer) + if layer: + self.cat.delete(layer) + + @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @override_settings( + GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" + ) + def test_import_geojson_overwrite(self): + prev_dataset = create_single_dataset(name="valid") + + layer = self.cat.get_layer("geonode:valid") + if layer: + self.cat.delete(layer) + payload = { + "base_file": open(self.valid_geojson, "rb"), + } + initial_name = "valid" + payload["overwrite_existing_layer"] = True + self._assertimport( + payload, initial_name, overwrite=True, last_update=prev_dataset.last_updated + ) + layer = self.cat.get_layer("geonode:valid") + if layer: + self.cat.delete(layer) class ImporterKMLImportTest(BaseImporterEndToEndTest): @@ -136,15 +202,39 @@ class ImporterKMLImportTest(BaseImporterEndToEndTest): GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" ) def test_import_kml(self): - layer = self.cat.get_layer("geonode:extruded_polygon") - self.cat.delete(layer) + layer = self.cat.get_layer("geonode:sample_point_dataset") + if layer: + self.cat.delete(layer) payload = { "base_file": open(self.valid_kml, "rb"), } - initial_name = "extruded_polygon" + initial_name = "sample_point_dataset" self._assertimport(payload, initial_name) - layer = self.cat.get_layer("geonode:extruded_polygon") - self.cat.delete(layer) + layer = self.cat.get_layer("geonode:sample_point_dataset") + if layer: + self.cat.delete(layer) + + @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @override_settings( + GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" + ) + def test_import_kml_overwrite(self): + prev_dataset = create_single_dataset(name="sample_point_dataset") + + layer = self.cat.get_layer("geonode:sample_point_dataset") + if layer: + self.cat.delete(layer) + payload = { + "base_file": open(self.valid_kml, "rb"), + } + initial_name = "sample_point_dataset" + payload["overwrite_existing_layer"] = True + self._assertimport( + payload, initial_name, overwrite=True, last_update=prev_dataset.last_updated + ) + layer = self.cat.get_layer("geonode:sample_point_dataset") + if layer: + self.cat.delete(layer) class ImporterShapefileImportTest(BaseImporterEndToEndTest): @@ -154,11 +244,36 @@ class ImporterShapefileImportTest(BaseImporterEndToEndTest): ) def test_import_shapefile(self): layer = self.cat.get_layer("geonode:san_andres_y_providencia_highway") - self.cat.delete(layer) + if layer: + self.cat.delete(layer) payload = { _filename: open(_file, "rb") for _filename, _file in self.valid_shp.items() } initial_name = "san_andres_y_providencia_highway" self._assertimport(payload, initial_name) + if layer: + layer = self.cat.get_layer("geonode:san_andres_y_providencia_highway") + self.cat.delete(layer) + + @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @override_settings( + GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" + ) + def test_import_shapefile_overwrite(self): + + prev_dataset = create_single_dataset(name="san_andres_y_providencia_highway") + + layer = self.cat.get_layer("geonode:san_andres_y_providencia_highway") + if layer: + self.cat.delete(layer) + payload = { + _filename: open(_file, "rb") for _filename, _file in self.valid_shp.items() + } + initial_name = "san_andres_y_providencia_highway" + payload["overwrite_existing_layer"] = True + self._assertimport( + payload, initial_name, overwrite=True, last_update=prev_dataset.last_updated + ) layer = self.cat.get_layer("geonode:san_andres_y_providencia_highway") - self.cat.delete(layer) + if layer: + self.cat.delete(layer) diff --git a/importer/tests/end2end/test_end2end_copy.py b/importer/tests/end2end/test_end2end_copy.py index cf4d353d..46784403 100644 --- a/importer/tests/end2end/test_end2end_copy.py +++ b/importer/tests/end2end/test_end2end_copy.py @@ -47,26 +47,18 @@ def setUpClass(cls) -> None: ) def setUp(self) -> None: - Dataset.objects.filter( - title__in=[ - "title_of_the_cloned_resource", - "stazioni_metropolitana", - "valid", - ] - ).delete() + for el in Dataset.objects.all(): + el.delete() + self.admin = get_user_model().objects.get(username="admin") def tearDown(self) -> None: - Dataset.objects.filter( - title__in=[ - "title_of_the_cloned_resource", - "stazioni_metropolitana", - "valid", - ] - ).delete() + for el in Dataset.objects.all(): + el.delete() def _assertCloning(self, initial_name): # getting the geonode resource + print(initial_name) dataset = Dataset.objects.get(alternate__icontains=f"geonode:{initial_name}") prev_dataset_count = Dataset.objects.count() self.client.force_login(get_user_model().objects.get(username="admin")) @@ -116,6 +108,7 @@ def _import_resource(self, payload, initial_name): def _wait_execution(self, response, _id="execution_id"): # if is async, we must wait. It will wait for 1 min before raise exception if ast.literal_eval(os.getenv("ASYNC_SIGNALS", "False")): + print("is false") tentative = 1 while ( ExecutionRequest.objects.get(exec_id=response.json().get(_id)) @@ -128,11 +121,18 @@ def _wait_execution(self, response, _id="execution_id"): ExecutionRequest.objects.get(exec_id=response.json().get(_id)).status != ExecutionRequest.STATUS_FINISHED ): + print(ExecutionRequest.objects.get(exec_id=response.json().get(_id))) raise Exception("Async still in progress after 1 min of waiting") class ImporterCopyEnd2EndGpkgTest(BaseClassEnd2End): - @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @mock.patch.dict( + os.environ, + { + "GEONODE_GEODATABASE": "test_geonode_data", + "IMPORTER_ENABLE_DYN_MODELS": "True", + }, + ) @override_settings( GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" ) @@ -149,7 +149,13 @@ def test_copy_dataset_from_geopackage(self): class ImporterCopyEnd2EndGeoJsonTest(BaseClassEnd2End): - @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @mock.patch.dict( + os.environ, + { + "GEONODE_GEODATABASE": "test_geonode_data", + "IMPORTER_ENABLE_DYN_MODELS": "True", + }, + ) @override_settings( GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" ) @@ -165,7 +171,13 @@ def test_copy_dataset_from_geojson(self): class ImporterCopyEnd2EndShapeFileTest(BaseClassEnd2End): - @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @mock.patch.dict( + os.environ, + { + "GEONODE_GEODATABASE": "test_geonode_data", + "IMPORTER_ENABLE_DYN_MODELS": "True", + }, + ) @override_settings( GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" ) @@ -181,15 +193,21 @@ def test_copy_dataset_from_shapefile(self): class ImporterCopyEnd2EndKMLTest(BaseClassEnd2End): - @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @mock.patch.dict( + os.environ, + { + "GEONODE_GEODATABASE": "test_geonode_data", + "IMPORTER_ENABLE_DYN_MODELS": "True", + }, + ) @override_settings( GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" ) - def test_copy_dataset_from_geojson(self): + def test_copy_dataset_from_kml(self): payload = { "base_file": open(self.valid_kml, "rb"), } - initial_name = "valid" + initial_name = "sample_point_dataset" # first we need to import a resource with transaction.atomic(): self._import_resource(payload, initial_name) diff --git a/importer/tests/fixture/valid.gpkg b/importer/tests/fixture/valid.gpkg old mode 100644 new mode 100755 diff --git a/importer/tests/fixture/valid.kml b/importer/tests/fixture/valid.kml index 47ae59a4..f9913584 100644 --- a/importer/tests/fixture/valid.kml +++ b/importer/tests/fixture/valid.kml @@ -1,915 +1,37 @@ - - - - KML Samples - 1 - Unleash your creativity with the help of these examples! - - - - - - - - - - - - - - Placemarks - These are just some of the different kinds of placemarks with - which you can mark your favorite places - - -122.0839597145766 - 37.42222904525232 - 0 - -148.4122922628044 - 40.5575073395506 - 500.6566641072245 - - - Simple placemark - Attached to the ground. Intelligently places itself at the - height of the underlying terrain. - - -122.0822035425683,37.42228990140251,0 - - - - Floating placemark - 0 - Floats a defined distance above the ground. - - -122.0839597145766 - 37.42222904525232 - 0 - -148.4122922628044 - 40.5575073395506 - 500.6566641072245 - - #downArrowIcon - - relativeToGround - -122.084075,37.4220033612141,50 - - - - Extruded placemark - 0 - Tethered to the ground by a customizable - "tail" - - -122.0845787421525 - 37.42215078737763 - 0 - -148.4126684946234 - 40.55750733918048 - 365.2646606980322 - - #globeIcon - - 1 - relativeToGround - -122.0857667006183,37.42156927867553,50 - - - - - Styles and Markup - 0 - With KML it is easy to create rich, descriptive markup to - annotate and enrich your placemarks - - -122.0845787422371 - 37.42215078726837 - 0 - -148.4126777488172 - 40.55750733930874 - 365.2646826292919 - - #noDrivingDirections - - Highlighted Icon - 0 - Place your mouse over the icon to see it display the new - icon - - -122.0856552124024 - 37.4224281311035 - 0 - 0 - 0 - 265.8520424250024 - - - - - - normal - #normalPlacemark - - - highlight - #highlightPlacemark - - - - Roll over this icon - 0 - #exampleStyleMap - - -122.0856545755255,37.42243077405461,0 - - - - - Descriptive HTML - 0 -
-Placemark descriptions can be enriched by using many standard HTML tags.
-For example: -
-Styles:
-Italics, -Bold, -Underlined, -Strike Out, -subscriptsubscript, -superscriptsuperscript, -Big, -Small, -Typewriter, -Emphasized, -Strong, -Code -
-Fonts:
-red by name, -leaf green by hexadecimal RGB -
-size 1, -size 2, -size 3, -size 4, -size 5, -size 6, -size 7 -
-Times, -Verdana, -Arial
-
-Links: -
-Google Earth! -
- or: Check out our website at www.google.com -
-Alignment:
-

left

-

center

-

right

-
-Ordered Lists:
-
  1. First
  2. Second
  3. Third
-
  1. First
  2. Second
  3. Third
-
  1. First
  2. Second
  3. Third
-
-Unordered Lists:
-
  • A
  • B
  • C
-
  • A
  • B
  • C
-
  • A
  • B
  • C
-
-Definitions:
-
-
Google:
The best thing since sliced bread
-
-
-Centered:
-Time present and time past
-Are both perhaps present in time future,
-And time future contained in time past.
-If all time is eternally present
-All time is unredeemable.
-
-
-Block Quote: -
-
-We shall not cease from exploration
-And the end of all our exploring
-Will be to arrive where we started
-And know the place for the first time.
--- T.S. Eliot -
-
-
-Headings:
-

Header 1

-

Header 2

-

Header 3

-

Header 4

-

Header 5

-
-Images:
-Remote image
-
-Scaled image
-
-
-Simple Tables:
- - - -
12345
abcde
-
-[Did you notice that double-clicking on the placemark doesn't cause the viewer to take you anywhere? This is because it is possible to directly author a "placeless placemark". If you look at the code for this example, you will see that it has neither a point coordinate nor a LookAt element.]]]> - - - - Ground Overlays - 0 - Examples of ground overlays - - Large-scale overlay on terrain - 0 - Overlay shows Mount Etna erupting on July 13th, 2001. - - 15.02468937557116 - 37.67395167941667 - 0 - -16.5581842842829 - 58.31228652890705 - 30350.36838438907 - - - http://developers.google.com/kml/documentation/images/etna.jpg - - - 37.91904192681665 - 37.46543388598137 - 15.35832653742206 - 14.60128369746704 - -0.1556640799496235 - - - - - Screen Overlays - 0 - Screen overlays have to be authored directly in KML. These - examples illustrate absolute and dynamic positioning in screen space. - - Simple crosshairs - 0 - This screen overlay uses fractional positioning to put the - image in the exact center of the screen - - http://developers.google.com/kml/documentation/images/crosshairs.png - - - - - - - - Absolute Positioning: Top left - 0 - - http://developers.google.com/kml/documentation/images/top_left.jpg - - - - - - - - Absolute Positioning: Top right - 0 - - http://developers.google.com/kml/documentation/images/top_right.jpg - - - - - - - - Absolute Positioning: Bottom left - 0 - - http://developers.google.com/kml/documentation/images/bottom_left.jpg - - - - - - - - Absolute Positioning: Bottom right - 0 - - http://developers.google.com/kml/documentation/images/bottom_right.jpg - - - - - - - - Dynamic Positioning: Top of screen - 0 - - http://developers.google.com/kml/documentation/images/dynamic_screenoverlay.jpg - - - - - - - - Dynamic Positioning: Right of screen - 0 - - http://developers.google.com/kml/documentation/images/dynamic_right.jpg - - - - - - - - - Paths - 0 - Examples of paths. Note that the tessellate tag is by default - set to 0. If you want to create tessellated lines, they must be authored - (or edited) directly in KML. - - Tessellated - 0 - tag has a value of 1, the line will contour to the underlying terrain]]> - - -112.0822680013139 - 36.09825589333556 - 0 - 103.8120432044965 - 62.04855796276328 - 2889.145007690472 - - - 1 - -112.0814237830345,36.10677870477137,0 - -112.0870267752693,36.0905099328766,0 - - - - Untessellated - 0 - tag has a value of 0, the line follow a simple straight-line path from point to point]]> - - -112.0822680013139 - 36.09825589333556 - 0 - 103.8120432044965 - 62.04855796276328 - 2889.145007690472 - - - 0 - -112.080622229595,36.10673460007995,0 - -112.085242575315,36.09049598612422,0 - - - - Absolute - 0 - Transparent purple line - - -112.2719329043177 - 36.08890633450894 - 0 - -106.8161545998597 - 44.60763714063257 - 2569.386744398339 - - #transPurpleLineGreenPoly - - 1 - absolute - -112.265654928602,36.09447672602546,2357 - -112.2660384528238,36.09342608838671,2357 - -112.2668139013453,36.09251058776881,2357 - -112.2677826834445,36.09189827357996,2357 - -112.2688557510952,36.0913137941187,2357 - -112.2694810717219,36.0903677207521,2357 - -112.2695268555611,36.08932171487285,2357 - -112.2690144567276,36.08850916060472,2357 - -112.2681528815339,36.08753813597956,2357 - -112.2670588176031,36.08682685262568,2357 - -112.2657374587321,36.08646312301303,2357 - - - - Absolute Extruded - 0 - Transparent green wall with yellow outlines - - -112.2643334742529 - 36.08563154742419 - 0 - -125.7518698668815 - 44.61038665812578 - 4451.842204068102 - - #yellowLineGreenPoly - - 1 - 1 - absolute - -112.2550785337791,36.07954952145647,2357 - -112.2549277039738,36.08117083492122,2357 - -112.2552505069063,36.08260761307279,2357 - -112.2564540158376,36.08395660588506,2357 - -112.2580238976449,36.08511401044813,2357 - -112.2595218489022,36.08584355239394,2357 - -112.2608216347552,36.08612634548589,2357 - -112.262073428656,36.08626019085147,2357 - -112.2633204928495,36.08621519860091,2357 - -112.2644963846444,36.08627897945274,2357 - -112.2656969554589,36.08649599090644,2357 - - - - Relative - 0 - Black line (10 pixels wide), height tracks terrain - - -112.2580438551384 - 36.1072674824385 - 0 - 4.947421249553717 - 44.61324882043339 - 2927.61105910266 - - #thickBlackLine - - 1 - relativeToGround - -112.2532845153347,36.09886943729116,645 - -112.2540466121145,36.09919570465255,645 - -112.254734666947,36.09984998366178,645 - -112.255493345654,36.10051310621746,645 - -112.2563157098468,36.10108441943419,645 - -112.2568033076439,36.10159722088088,645 - -112.257494011321,36.10204323542867,645 - -112.2584106072308,36.10229131995655,645 - -112.2596588987972,36.10240001286358,645 - -112.2610581199487,36.10213176873407,645 - -112.2626285262793,36.10157011437219,645 - - - - Relative Extruded - 0 - Opaque blue walls with red outline, height tracks terrain - - -112.2683594333433 - 36.09884362144909 - 0 - -72.24271551768405 - 44.60855445139561 - 2184.193522571467 - - #redLineBluePoly - - 1 - 1 - relativeToGround - -112.2656634181359,36.09445214722695,630 - -112.2652238941097,36.09520916122063,630 - -112.2645079986395,36.09580763864907,630 - -112.2638827428817,36.09628572284063,630 - -112.2635746835406,36.09679275951239,630 - -112.2635711822407,36.09740038871899,630 - -112.2640296531825,36.09804913435539,630 - -112.264327720538,36.09880337400301,630 - -112.2642436562271,36.09963644790288,630 - -112.2639148687042,36.10055381117246,630 - -112.2626894973474,36.10149062823369,630 - - - - - Polygons - 0 - Examples of polygon shapes - - Google Campus - 0 - A collection showing how easy it is to create 3-dimensional - buildings - - -122.084120030116 - 37.42174011925477 - 0 - -34.82469740081282 - 53.454348562403 - 276.7870053764046 - - - Building 40 - 0 - #transRedPoly - - 1 - relativeToGround - - - -122.0848938459612,37.42257124044786,17 - -122.0849580979198,37.42211922626856,17 - -122.0847469573047,37.42207183952619,17 - -122.0845725380962,37.42209006729676,17 - -122.0845954886723,37.42215932700895,17 - -122.0838521118269,37.42227278564371,17 - -122.083792243335,37.42203539112084,17 - -122.0835076656616,37.42209006957106,17 - -122.0834709464152,37.42200987395161,17 - -122.0831221085748,37.4221046494946,17 - -122.0829247374572,37.42226503990386,17 - -122.0829339169385,37.42231242843094,17 - -122.0833837359737,37.42225046087618,17 - -122.0833607854248,37.42234159228745,17 - -122.0834204551642,37.42237075460644,17 - -122.083659133885,37.42251292011001,17 - -122.0839758438952,37.42265873093781,17 - -122.0842374743331,37.42265143972521,17 - -122.0845036949503,37.4226514386435,17 - -122.0848020460801,37.42261133916315,17 - -122.0847882750515,37.42256395055121,17 - -122.0848938459612,37.42257124044786,17 - - - - - - Building 41 - 0 - #transBluePoly - - 1 - relativeToGround - - - -122.0857412771483,37.42227033155257,17 - -122.0858169768481,37.42231408832346,17 - -122.085852582875,37.42230337469744,17 - -122.0858799945639,37.42225686138789,17 - -122.0858860101409,37.4222311076138,17 - -122.0858069157288,37.42220250173855,17 - -122.0858379542653,37.42214027058678,17 - -122.0856732640519,37.42208690214408,17 - -122.0856022926407,37.42214885429042,17 - -122.0855902778436,37.422128290487,17 - -122.0855841672237,37.42208171967246,17 - -122.0854852065741,37.42210455874995,17 - -122.0855067264352,37.42214267949824,17 - -122.0854430712915,37.42212783846172,17 - -122.0850990714904,37.42251282407603,17 - -122.0856769818632,37.42281815323651,17 - -122.0860162273783,37.42244918858722,17 - -122.0857260327004,37.42229239604253,17 - -122.0857412771483,37.42227033155257,17 - - - - - - Building 42 - 0 - #transGreenPoly - - 1 - relativeToGround - - - -122.0857862287242,37.42136208886969,25 - -122.0857312990603,37.42136935989481,25 - -122.0857312992918,37.42140934910903,25 - -122.0856077073679,37.42138390166565,25 - -122.0855802426516,37.42137299550869,25 - -122.0852186221971,37.42137299504316,25 - -122.0852277765639,37.42161656508265,25 - -122.0852598189347,37.42160565894403,25 - -122.0852598185499,37.42168200156,25 - -122.0852369311478,37.42170017860346,25 - -122.0852643957828,37.42176197982575,25 - -122.0853239032746,37.42176198013907,25 - -122.0853559454324,37.421852864452,25 - -122.0854108752463,37.42188921823734,25 - -122.0854795379357,37.42189285337048,25 - -122.0855436229819,37.42188921797546,25 - -122.0856260178042,37.42186013499926,25 - -122.085937287963,37.42186013453605,25 - -122.0859428718666,37.42160898590042,25 - -122.0859655469861,37.42157992759144,25 - -122.0858640462341,37.42147115002957,25 - -122.0858548911215,37.42140571326184,25 - -122.0858091162768,37.4214057134039,25 - -122.0857862287242,37.42136208886969,25 - - - - - - Building 43 - 0 - #transYellowPoly - - 1 - relativeToGround - - - -122.0844371128284,37.42177253003091,19 - -122.0845118855746,37.42191111542896,19 - -122.0850470999805,37.42178755121535,19 - -122.0850719913391,37.42143663023161,19 - -122.084916406232,37.42137237822116,19 - -122.0842193868167,37.42137237801626,19 - -122.08421938659,37.42147617161496,19 - -122.0838086419991,37.4214613409357,19 - -122.0837899728564,37.42131306410796,19 - -122.0832796534698,37.42129328840593,19 - -122.0832609819207,37.42139213944298,19 - -122.0829373621737,37.42137236399876,19 - -122.0829062425667,37.42151569778871,19 - -122.0828502269665,37.42176282576465,19 - -122.0829435788635,37.42176776969635,19 - -122.083217411188,37.42179248552686,19 - -122.0835970430103,37.4217480074456,19 - -122.0839455556771,37.42169364237603,19 - -122.0840077894637,37.42176283815853,19 - -122.084113587521,37.42174801104392,19 - -122.0840762473784,37.42171341292375,19 - -122.0841447047739,37.42167881534569,19 - -122.084144704223,37.42181720660197,19 - -122.0842503333074,37.4218170700446,19 - -122.0844371128284,37.42177253003091,19 - - - - - - - Extruded Polygon - A simple way to model a building - - The Pentagon - - -77.05580139178142 - 38.870832443487 - 59.88865561738225 - 48.09646074797388 - 742.0552506670548 - - - 1 - relativeToGround - - - -77.05788457660967,38.87253259892824,100 - -77.05465973756702,38.87291016281703,100 - -77.05315536854791,38.87053267794386,100 - -77.05552622493516,38.868757801256,100 - -77.05844056290393,38.86996206506943,100 - -77.05788457660967,38.87253259892824,100 - - - - - -77.05668055019126,38.87154239798456,100 - -77.05542625960818,38.87167890344077,100 - -77.05485125901024,38.87076535397792,100 - -77.05577677433152,38.87008686581446,100 - -77.05691162017543,38.87054446963351,100 - -77.05668055019126,38.87154239798456,100 - - - - - - - Absolute and Relative - 0 - Four structures whose roofs meet exactly. Turn on/off - terrain to see the difference between relative and absolute - positioning. - - -112.3348969157552 - 36.14845533214919 - 0 - -86.91235037566909 - 49.30695423894192 - 990.6761201087104 - - - Absolute - 0 - #transBluePoly - - 1 - absolute - - - -112.3372510731295,36.14888505105317,1784 - -112.3356128688403,36.14781540589019,1784 - -112.3368169371048,36.14658677734382,1784 - -112.3384408457543,36.14762778914076,1784 - -112.3372510731295,36.14888505105317,1784 - - - - - - Absolute Extruded - 0 - #transRedPoly - - 1 - 1 - absolute - - - -112.3396586818843,36.14637618647505,1784 - -112.3380597654315,36.14531751871353,1784 - -112.3368254237788,36.14659596244607,1784 - -112.3384555043203,36.14762621763982,1784 - -112.3396586818843,36.14637618647505,1784 - - - - - - Relative - 0 - - -112.3350152490417 - 36.14943123077423 - 0 - -118.9214100848499 - 37.92486261093203 - 345.5169113679813 - - #transGreenPoly - - 1 - relativeToGround - - - -112.3349463145932,36.14988705767721,100 - -112.3354019540677,36.14941108398372,100 - -112.3344428289146,36.14878490381308,100 - -112.3331289492913,36.14780840132443,100 - -112.3317019516947,36.14680755678357,100 - -112.331131440106,36.1474173426228,100 - -112.332616324338,36.14845453364654,100 - -112.3339876620524,36.14926570522069,100 - -112.3349463145932,36.14988705767721,100 - - - - - - Relative Extruded - 0 - - -112.3351587892382 - 36.14979247129029 - 0 - -55.42811560891606 - 56.10280503739589 - 401.0997279712519 - - #transYellowPoly - - 1 - 1 - relativeToGround - - - -112.3348783983763,36.1514008468736,100 - -112.3372535345629,36.14888517553886,100 - -112.3356068927954,36.14781612679284,100 - -112.3350034807972,36.14846469024177,100 - -112.3358353861232,36.1489624162954,100 - -112.3345888301373,36.15026229372507,100 - -112.3337937856278,36.14978096026463,100 - -112.3331798208424,36.1504472788618,100 - -112.3348783983763,36.1514008468736,100 - - - - - - - + + + + + + + + + + sample_point_dataset + + + + 1 + 1 + test + + + + 120.89945941369977,14.358088895403743 + + + + + + 2 + 2 + test 2 + + + + 120.94212582236852,14.23553050541452 + + + + diff --git a/importer/tests/unit/test_models.py b/importer/tests/unit/test_models.py index 7ae3e1bf..68104a02 100644 --- a/importer/tests/unit/test_models.py +++ b/importer/tests/unit/test_models.py @@ -1,14 +1,17 @@ -from django.test import TestCase +import os from dynamic_models.models import ModelSchema, FieldSchema +import mock from geonode.base.populate_test_data import create_single_dataset from importer.models import ResourceHandlerInfo +from importer.tests.utils import TransactionImporterBaseTestSupport +import uuid -class TestModelSchemaSignal(TestCase): +class TestModelSchemaSignal(TransactionImporterBaseTestSupport): databases = ("default", "datastore") def setUp(self): - self.resource = create_single_dataset(name="test_dataset") + self.resource = create_single_dataset(name=f"test_dataset_{uuid.uuid4()}") ResourceHandlerInfo.objects.create( resource=self.resource, handler_module_path="importer.handlers.shapefile.handler.ShapeFileHandler", @@ -22,10 +25,17 @@ def setUp(self): model_schema=self.dynamic_model, ) + @mock.patch.dict(os.environ, {"IMPORTER_ENABLE_DYN_MODELS": "True"}) def test_delete_dynamic_model(self): """ Ensure that the dynamic model is deleted """ + # create needed resource handler info + + ResourceHandlerInfo.objects.create( + resource=self.resource, + handler_module_path="importer.handlers.gpkg.handler.GPKGFileHandler", + ) self.resource.delete() self.assertFalse(ModelSchema.objects.filter(name="test_dataset").exists()) self.assertFalse( diff --git a/importer/tests/unit/test_orchestrator.py b/importer/tests/unit/test_orchestrator.py index e16180be..25f586c1 100644 --- a/importer/tests/unit/test_orchestrator.py +++ b/importer/tests/unit/test_orchestrator.py @@ -1,4 +1,6 @@ +import os import uuid +from django.conf import settings from django.contrib.auth import get_user_model from geonode.tests.base import GeoNodeBaseTestSupport from unittest.mock import patch @@ -178,13 +180,19 @@ def test_perform_with_error_set_invalid_status(self, mock_celery): self.assertIsNotNone(Upload.objects.get(metadata__icontains=_id)) def test_set_as_failed(self): + # creating the temporary file that will be deleted + fake_path = f"{settings.MEDIA_ROOT}/file.txt" + with open(fake_path, "w"): + pass + + self.assertTrue(os.path.exists(fake_path)) # we need to create first the execution _uuid = self.orchestrator.create_execution_request( user=get_user_model().objects.first(), func_name="name", step="importer.create_geonode_resource", # adding the first step for the GPKG file input_params={ - "files": {"base_file": "/tmp/file.txt"}, + "files": {"base_file": fake_path}, "store_spatial_files": True, }, ) @@ -195,7 +203,7 @@ def test_set_as_failed(self): req = ExecutionRequest.objects.get(exec_id=_uuid) self.assertTrue(req.status, ExecutionRequest.STATUS_FAILED) self.assertTrue(req.log, "automatic test") - + self.assertFalse(os.path.exists(fake_path)) # check legacy execution status legacy = Upload.objects.filter(metadata__contains=_uuid) self.assertTrue(legacy.exists()) @@ -290,7 +298,7 @@ def test_evaluate_execution_progress_should_continue_if_some_task_is_not_finishe self.assertIsNone(result) self.assertEqual( - f"INFO:importer.orchestrator:Execution progress with id {exec_id} is not finished yet, continuing", + f"INFO:importer.orchestrator:Execution with ID {exec_id} is completed. All tasks are done", _log.output[0], ) diff --git a/importer/tests/unit/test_publisher.py b/importer/tests/unit/test_publisher.py index f3e38d16..e1201a88 100644 --- a/importer/tests/unit/test_publisher.py +++ b/importer/tests/unit/test_publisher.py @@ -1,8 +1,10 @@ import os from django.test import TestCase +from mock import patch from importer import project_dir from importer.publisher import DataPublisher -from unittest.mock import patch +from unittest.mock import MagicMock +from geonode.geoserver.helpers import create_geoserver_db_featurestore class TestDataPublisher(TestCase): @@ -18,6 +20,28 @@ def setUpClass(cls): ) cls.gpkg_path = f"{project_dir}/tests/fixture/valid.gpkg" + def setUp(self): + layer = self.publisher.cat.get_resources( + "stazioni_metropolitana", workspaces="geonode" + ) + print("delete layer") + if layer: + res = self.publisher.cat.delete(layer.resource, purge="all", recurse=True) + print(res.status_code) + print(res.json) + + def tearDown(self): + layer = self.publisher.cat.get_resources( + "stazioni_metropolitana", workspaces="geonode" + ) + print("delete layer teardown") + if layer: + self.publisher.cat.delete(layer) + + res = self.publisher.cat.delete(layer.resource, purge="all", recurse=True) + print(res.status_code) + print(res.json) + def test_extract_resource_name_and_crs(self): """ Given a layer and the original file, should extract the crs and the name @@ -45,11 +69,6 @@ def test_extract_resource_name_and_crs_return_empty_if_the_file_does_not_exists( ) self.assertListEqual([], values_found) - @patch("importer.publisher.create_geoserver_db_featurestore") - def test_get_or_create_store_creation_should_not_be_called(self, datastore): - self.publisher.get_or_create_store() - datastore.assert_not_called() - @patch("importer.publisher.create_geoserver_db_featurestore") def test_get_or_create_store_creation_should_called(self, datastore): with patch.dict( @@ -70,24 +89,10 @@ def test_publish_resources_should_raise_exception_if_any_error_happen( ) publish_featuretype.assert_called_once() - @patch("importer.publisher.Catalog.publish_featuretype") - def test_publish_resources_should_continue_in_case_the_resource_is_already_published( - self, publish_featuretype - ): - publish_featuretype.side_effect = Exception( - "Resource named stazioni_metropolitana already exists in store:" - ) - - result = self.publisher.publish_resources( - resources=[{"crs": "EPSG:32632", "name": "stazioni_metropolitana"}] - ) - self.assertTrue(result) - publish_featuretype.assert_called_once() - @patch("importer.publisher.Catalog.publish_featuretype") def test_publish_resources_should_work(self, publish_featuretype): publish_featuretype.return_value = True - + self.publisher.sanity_checks = MagicMock() result = self.publisher.publish_resources( resources=[{"crs": "EPSG:32632", "name": "stazioni_metropolitana"}] ) diff --git a/importer/tests/unit/test_task.py b/importer/tests/unit/test_task.py index 731f886a..897d6d94 100644 --- a/importer/tests/unit/test_task.py +++ b/importer/tests/unit/test_task.py @@ -1,6 +1,10 @@ +import os +import os +import shutil + from django.conf import settings from django.contrib.auth import get_user_model -from django.test import SimpleTestCase +from django.test.utils import override_settings from unittest.mock import patch from importer.api.exception import InvalidInputFileException @@ -225,40 +229,42 @@ def test_publish_resource_if_overwrite_should_not_call_the_publishing( ): """ Publish resource should be called since the resource does not exists in geoserver - even if an overwrite is required + even if an overwrite is required. + Should raise error if the crs is not found """ try: - get_resource.return_falue = True - publish_resources.return_value = True - extract_resource_to_publish.return_value = [ - {"crs": 12345, "name": "dataset3"} - ] - exec_id = orchestrator.create_execution_request( - user=get_user_model().objects.get(username=self.user), - func_name="dummy_func", - step="dummy_step", - legacy_upload_name="dummy", - input_params={ - "files": {"base_file": "/filepath"}, - "overwrite_existing_layer": True, - "store_spatial_files": True, - }, - ) - publish_resource( - str(exec_id), - resource_type="gpkg", - step_name="publish_resource", - layer_name="dataset3", - alternate="alternate_dataset3", - action=ExecutionRequestAction.IMPORT.value, - handler_module_path="importer.handlers.gpkg.handler.GPKGFileHandler", - ) + with self.assertRaises(Exception): + get_resource.return_falue = True + publish_resources.return_value = True + extract_resource_to_publish.return_value = [ + {"crs": 4326, "name": "dataset3"} + ] + exec_id = orchestrator.create_execution_request( + user=get_user_model().objects.get(username=self.user), + func_name="dummy_func", + step="dummy_step", + legacy_upload_name="dummy", + input_params={ + "files": {"base_file": "/filepath"}, + "overwrite_existing_layer": True, + "store_spatial_files": True, + }, + ) + publish_resource( + str(exec_id), + resource_type="gpkg", + step_name="publish_resource", + layer_name="dataset3", + alternate="alternate_dataset3", + action=ExecutionRequestAction.IMPORT.value, + handler_module_path="importer.handlers.gpkg.handler.GPKGFileHandler", + ) - # Evaluation - req = ExecutionRequest.objects.get(exec_id=str(exec_id)) - self.assertEqual("importer.publish_resource", req.step) - publish_resources.assert_not_called() - importer.assert_called_once() + # Evaluation + req = ExecutionRequest.objects.get(exec_id=str(exec_id)) + self.assertEqual("importer.publish_resource", req.step) + publish_resources.assert_not_called() + importer.assert_called_once() finally: # cleanup @@ -294,9 +300,9 @@ def test_create_geonode_resource(self, import_orchestrator): if Dataset.objects.filter(alternate=alternate).exists(): Dataset.objects.filter(alternate=alternate).delete() - @patch("importer.celery_tasks.import_orchestrator.apply_async") + @patch("importer.celery_tasks.call_rollback_function") def test_copy_geonode_resource_should_raise_exeption_if_the_alternate_not_exists( - self, async_call + self, call_rollback_function ): with self.assertRaises(Exception): copy_geonode_resource( @@ -311,7 +317,7 @@ def test_copy_geonode_resource_should_raise_exeption_if_the_alternate_not_exists "new_dataset_alternate": "geonode:schema_copy_example_dataset", # this alternate is generated dring the geonode resource copy }, ) - async_call.assert_not_called() + call_rollback_function.assert_called_once() @patch("importer.celery_tasks.import_orchestrator.apply_async") def test_copy_geonode_resource(self, async_call): @@ -350,6 +356,7 @@ def test_copy_geonode_resource(self, async_call): @patch( "importer.handlers.gpkg.handler.GPKGFileHandler._create_geonode_resource_rollback" ) + @override_settings(MEDIA_ROOT="/tmp/") def test_rollback_works_as_expected_vector_step( self, _create_geonode_resource_rollback, @@ -411,6 +418,7 @@ def test_rollback_works_as_expected_vector_step( @patch( "importer.handlers.geotiff.handler.GeoTiffFileHandler._create_geonode_resource_rollback" ) + @override_settings(MEDIA_ROOT="/tmp/") def test_rollback_works_as_expected_raster( self, _create_geonode_resource_rollback, @@ -443,7 +451,7 @@ def test_rollback_works_as_expected_raster( step=conf[0], # step name action="import", input_params={ - "files": {"base_file": "/filepath"}, + "files": {"base_file": "/tmp/filepath"}, "overwrite_existing_layer": True, "store_spatial_files": True, "handler_module_path": "importer.handlers.geotiff.handler.GeoTiffFileHandler", @@ -463,9 +471,14 @@ def test_rollback_works_as_expected_raster( if exec_id: ExecutionRequest.objects.filter(exec_id=str(exec_id)).delete() + @override_settings(MEDIA_ROOT="/tmp/") def test_import_metadata_should_work_as_expected(self): handler = "importer.handlers.xml.handler.XMLFileHandler" + # lets copy the file to the temporary folder + # later will be removed valid_xml = f"{settings.PROJECT_ROOT}/base/fixtures/test_xml.xml" + shutil.copy(valid_xml, '/tmp') + user, _ = get_user_model().objects.get_or_create(username="admin") valid_files = {"base_file": valid_xml, 'xml_file': valid_xml} @@ -581,6 +594,7 @@ def test_create_dynamic_structure_should_work(self): FieldSchema.objects.filter(name="field1").delete() @patch("importer.celery_tasks.import_orchestrator.apply_async") + @patch.dict(os.environ, {"IMPORTER_ENABLE_DYN_MODELS": "True"}) def test_copy_dynamic_model_should_work(self, async_call): try: name = str(self.exec_id) diff --git a/runtest.sh b/runtest.sh new file mode 100755 index 00000000..bb8b89d2 --- /dev/null +++ b/runtest.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -a +. ./.env_test +set +a +coverage run --append --source='.' /usr/src/geonode/manage.py test importer -v2 --noinput \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 8ec906af..54a168ab 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = geonode-importer -version = 1.0.5 +version = 1.0.7 author = geosolutions-it author_email = info@geosolutionsgroup.com url = https://github.com/geosolutions-it/geonode-importer diff --git a/setup.py b/setup.py index 213bae14..2c7c5794 100644 --- a/setup.py +++ b/setup.py @@ -33,8 +33,8 @@ def read_file(path: str): include_package_data=True, install_requires=[ "setuptools>=59", - "gdal==3.2.2.1", - "pdok-geopackage-validator==0.8.0", + "gdal<=3.4.3", + "pdok-geopackage-validator==0.8.5", "geonode-django-dynamic-model==0.4.0", ], )