Merge with master

.env_test

@@ -0,0 +1,243 @@
+COMPOSE_PROJECT_NAME=importer
+# See https://github.com/containers/podman/issues/13889
+# DOCKER_BUILDKIT=0
+DOCKER_ENV=production
+# See https://github.com/geosolutions-it/geonode-generic/issues/28
+# to see why we force API version to 1.24
+DOCKER_API_VERSION="1.24"
+BACKUPS_VOLUME_DRIVER=local
+
+C_FORCE_ROOT=1
+FORCE_REINIT=false
+INVOKE_LOG_STDOUT=true
+
+# LANGUAGE_CODE=it-it
+# LANGUAGES=(('en-us','English'),('it-it','Italiano'))
+
+DJANGO_SETTINGS_MODULE=geonode.settings
+GEONODE_INSTANCE_NAME=importer
+
+# #################
+# backend
+# #################
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+GEONODE_DATABASE=geonode
+GEONODE_DATABASE_USER=geonode
+GEONODE_DATABASE_PASSWORD=geonode
+GEONODE_GEODATABASE=geonode_data
+GEONODE_GEODATABASE_USER=geonode_data
+GEONODE_GEODATABASE_PASSWORD=geonode_data
+GEONODE_DATABASE_SCHEMA=public
+GEONODE_GEODATABASE_SCHEMA=public
+DATABASE_HOST=db
+DATABASE_PORT=5432
+DATABASE_URL=postgis://geonode:geonode@db:5432/geonode
+GEODATABASE_URL=postgis://geonode_data:geonode_data@db:5432/geonode_data
+GEONODE_DB_CONN_MAX_AGE=0
+GEONODE_DB_CONN_TOUT=5
+DEFAULT_BACKEND_DATASTORE=datastore
+BROKER_URL=amqp://guest:guest@rabbitmq:5672/
+CELERY_BEAT_SCHEDULER=celery.beat:PersistentScheduler
+ASYNC_SIGNALS=False
+
+SITEURL=http://localhost:8000/
+
+ALLOWED_HOSTS="['django', 'localhost', '127.0.0.1']"
+
+# Data Uploader
+DEFAULT_BACKEND_UPLOADER=geonode.importer
+TIME_ENABLED=True
+MOSAIC_ENABLED=False
+HAYSTACK_SEARCH=False
+HAYSTACK_ENGINE_URL=http://elasticsearch:9200/
+HAYSTACK_ENGINE_INDEX_NAME=haystack
+HAYSTACK_SEARCH_RESULTS_PER_PAGE=200
+
+# #################
+# nginx
+# HTTPD Server
+# #################
+GEONODE_LB_HOST_IP=django
+GEONODE_LB_PORT=8000
+GEOSERVER_LB_HOST_IP=geoserver
+GEOSERVER_LB_PORT=8080
+NGINX_BASE_URL=http://localhost
+
+# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only)
+# port where the server can be reached on HTTPS
+HTTP_HOST=localhost
+HTTPS_HOST=
+POSTGRESQL_MAX_CONNECTIONS=100
+HTTP_PORT=8000
+HTTPS_PORT=443
+
+# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work
+# with an ip) and it must be reachable from the outside. This can be one of the following :
+# disabled : we do not get a certificate at all (a placeholder certificate will be used)
+# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates)
+# production : we get a normal certificate (default)
+LETSENCRYPT_MODE=disabled
+# LETSENCRYPT_MODE=staging
+# LETSENCRYPT_MODE=production
+
+RESOLVER=127.0.0.11
+
+# #################
+# geoserver
+# #################
+GEOSERVER_WEB_UI_LOCATION=http://localhost:8000/geoserver/
+GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/
+GEOSERVER_LOCATION=http://geoserver:8080/geoserver/
+GEOSERVER_ADMIN_USER=admin
+GEOSERVER_ADMIN_PASSWORD=geoserver
+
+OGC_REQUEST_TIMEOUT=5
+OGC_REQUEST_MAX_RETRIES=0
+OGC_REQUEST_BACKOFF_FACTOR=0.3
+OGC_REQUEST_POOL_MAXSIZE=10
+OGC_REQUEST_POOL_CONNECTIONS=10
+
+# #################
+# catalogue
+# #################
+CATALOGUE_ENGINE=geonode.catalogue.backends.pycsw_local
+CATALOGUE_URL=http://localhost:8000/catalogue/csw
+
+# Java Options & Memory
+ENABLE_JSONP=true
+outFormat=text/javascript
+GEOSERVER_JAVA_OPTS='-Djava.awt.headless=true -Xms4G -Xmx4G -Dgwc.context.suffix=gwc -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGS-SHAPEFILE-CHARSET=UTF-8 -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL=http://localhost/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine'
+
+# #################
+# Security
+# #################
+# Admin Settings
+#
+# ADMIN_PASSWORD is used to overwrite the GeoNode admin password **ONLY** the first time
+# GeoNode is run. If you need to overwrite it again, you need to set the env var FORCE_REINIT,
+# otherwise the invoke updateadmin task will be skipped and the current password already stored
+# in DB will honored.
+
+ADMIN_USERNAME=admin
+ADMIN_PASSWORD=admin
+ADMIN_EMAIL=admin@localhost
+
+# EMAIL Notifications
+EMAIL_ENABLE=False
+DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend
+DJANGO_EMAIL_HOST=localhost
+DJANGO_EMAIL_PORT=25
+DJANGO_EMAIL_HOST_USER=
+DJANGO_EMAIL_HOST_PASSWORD=
+DJANGO_EMAIL_USE_TLS=False
+DJANGO_EMAIL_USE_SSL=False
+DEFAULT_FROM_EMAIL='GeoNode <[email protected]>'
+
+# Session/Access Control
+LOCKDOWN_GEONODE=False
+X_FRAME_OPTIONS="SAMEORIGIN"
+SESSION_EXPIRED_CONTROL_ENABLED=True
+DEFAULT_ANONYMOUS_VIEW_PERMISSION=True
+DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True
+
+CORS_ALLOW_ALL_ORIGINS=True
+GEOSERVER_CORS_ENABLED=True
+GEOSERVER_CORS_ALLOWED_ORIGINS=*
+GEOSERVER_CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,HEAD,OPTIONS
+GEOSERVER_CORS_ALLOWED_HEADERS=*
+
+# Users Registration
+ACCOUNT_OPEN_SIGNUP=True
+ACCOUNT_EMAIL_REQUIRED=True
+ACCOUNT_APPROVAL_REQUIRED=False
+ACCOUNT_CONFIRM_EMAIL_ON_GET=False
+ACCOUNT_EMAIL_VERIFICATION=none
+ACCOUNT_EMAIL_CONFIRMATION_EMAIL=False
+ACCOUNT_EMAIL_CONFIRMATION_REQUIRED=False
+ACCOUNT_AUTHENTICATION_METHOD=username_email
+AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True
+
+# OAuth2
+OAUTH2_API_KEY=
+OAUTH2_CLIENT_ID=Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV
+OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3
+
+SOCIALACCOUNT_OIDC_PROVIDER_ENABLED=True
+SOCIALACCOUNT_PROVIDER=google
+
+# GeoNode APIs
+API_LOCKDOWN=False
+TASTYPIE_APIKEY=
+
+# #################
+# Production and
+# Monitoring
+# #################
+DEBUG=False
+
+SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a'
+
+STATIC_ROOT=/mnt/volumes/statics/static/
+MEDIA_ROOT=/mnt/volumes/statics/uploaded/
+GEOIP_PATH=/mnt/volumes/statics/geoip.db
+
+CACHE_BUSTING_STATIC_ENABLED=False
+
+MEMCACHED_ENABLED=False
+MEMCACHED_BACKEND=django.core.cache.backends.memcached.MemcachedCache
+MEMCACHED_LOCATION=memcached:11211
+MEMCACHED_LOCK_EXPIRE=3600
+MEMCACHED_LOCK_TIMEOUT=10
+#
+# Options for memcached binary, e.g. -vvv to log all requests and cache hits
+#
+MEMCACHED_OPTIONS=
+
+MAX_DOCUMENT_SIZE=200
+CLIENT_RESULTS_LIMIT=5
+API_LIMIT_PER_PAGE=1000
+
+# GIS Client
+GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore
+MAPBOX_ACCESS_TOKEN=
+BING_API_KEY=
+GOOGLE_API_KEY=
+
+# Monitoring
+MONITORING_ENABLED=False
+MONITORING_DATA_TTL=365
+USER_ANALYTICS_ENABLED=True
+USER_ANALYTICS_GZIP=True
+CENTRALIZED_DASHBOARD_ENABLED=False
+MONITORING_SERVICE_NAME=local-geonode
+MONITORING_HOST_NAME=geonode
+
+# Other Options/Contribs
+MODIFY_TOPICCATEGORY=True
+AVATAR_GRAVATAR_SSL=True
+EXIF_ENABLED=True
+CREATE_LAYER=True
+FAVORITE_ENABLED=True
+
+# Advanced Workflow
+RESOURCE_PUBLISHING=False
+ADMIN_MODERATE_UPLOADS=False
+
+# PostgreSQL
+POSTGRESQL_MAX_CONNECTIONS=200
+
+# Common containers restart policy
+RESTART_POLICY_CONDITION="on-failure"
+RESTART_POLICY_DELAY="5s"
+RESTART_POLICY_MAX_ATTEMPTS="3"
+RESTART_POLICY_WINDOW=120s
+
+DEFAULT_MAX_UPLOAD_SIZE=5368709120
+DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=100
+
+# Azure AD
+MICROSOFT_TENANT_ID=
+AZURE_CLIENT_ID=
+AZURE_SECRET_KEY=
+AZURE_KEY=

.github/workflows/runtests.yml

@@ -0,0 +1,25 @@
+name: Run test suite
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Start containers
+      run: docker-compose -f "docker-compose-test.yaml" up -d --build
+    - name: Run django entrypoint
+      run: docker exec django4importer /bin/sh -c "sh /usr/src/importer/entrypoint_test.sh"
+    - name: Run geonode-importer tests
+      run: docker exec django4importer /bin/sh -c "sh /usr/src/importer/runtest.sh"
+    - name: Stop containers
+      if: always()
+      run: docker-compose -f "docker-compose-test.yaml" down

Dockerfile

@@ -0,0 +1,12 @@
+FROM geonode/geonode-base:latest-ubuntu-22.04
+
+RUN git clone https://github.com/GeoNode/geonode.git /usr/src/geonode
+RUN mkdir -p /usr/src/importer
+
+RUN cd ..
+COPY . /usr/src/importer/
+WORKDIR /usr/src/importer
+
+RUN pip install -r /usr/src/geonode/requirements.txt
+RUN pip install --upgrade  -e /usr/src/importer/
+RUN pip install coverage

README.md

@@ -16,7 +16,10 @@ In GeoNode 4.1 `geonode-importer` replaced the previous importer logic.
 - **CSV** - Vector
 - **GeoTiff** - Raster
 - **XML** - Update XML file for a given resource
+<<<<<<< HEAD
 - **SLD** - Update SLD file for a given resource
+=======
+>>>>>>> 02bda6833d22c6eb7f29ba55ee99fd4f675ed1c5
 
 **IMPORTANT**: At the moment the importer doesn't support overwriting/skipping existing layers from the UI. Every upload will create a new dataset.
 Overwriting a layer (`overwrite_existing_layer`) and skipping an already existing layer (`skip_existing_layers`) is supported through the API. 
@@ -110,7 +113,7 @@ IMPORTER_HANDLERS = os.getenv('IMPORTER_HANDLERS', [
 
 ```
 
-NOTE:
+## NOTE:
 In case of a local environment, Geoserver and Geonode should be able to reach the default `MEDIA_ROOT`.
 
 If some permission is missing, please change the `FILE_UPLOAD_DIRECTORY_PERMISSIONS` to make the folder accessible to both
@@ -133,6 +136,9 @@ IMPORTER_GLOBAL_RATE_LIMIT= # default 5
 IMPORTER_PUBLISHING_RATE_LIMIT= # default 5
 IMPORTER_RESOURCE_CREATION_RATE_LIMIT= # default 10
 IMPORTER_RESOURCE_COPY_RATE_LIMIT = # default 10
+
+# https://github.com/OSGeo/gdal/issues/8674
+OGR2OGR_COPY_WITH_DUMP = If true, will pipe the PG dump to psql.
 ```
 
 ## Troubleshooting
@@ -162,4 +168,42 @@ In detail the list is the following:
 | From | TO  |
 |---|---|
 | - (space) # \  | _  |
-| .)(,&  | empty_string  |
+| .)(,&  | empty_string  |
+
+
+## Run tests
+
+A simple docker compose is provided along with the project. To run the geonode-importer test suite please follow this steps:
+
+The compose project, provides what is needed to run the tests. This includes:
+
+1) Database
+1) Geoserver
+1) Django with all the dependecies needed
+
+
+#### Step 1: build the project
+
+Build and run the docker compose project
+```
+docker compose -f docker-compose-test.yaml up --build -d
+```
+
+#### Step 2: Run Scripts
+
+Two scripts are provided along with the compose file:
+
+1) entrypoint_test.sh
+1) runtest.sh
+
+#### 1) entrypoint_test.sh
+Is a script which create the needed extension and database to run the tests. Is a setup script
+
+#### 2) runtest.sh
+It actually runs the tests.
+
+
+Note: the tests inside the docker environment can be run manually too
+
+## Env File
+The `.env_test` file contains all the environment variable needed to successfully run the tests