diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c04a86..de8b3d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,9 @@ env: EMDAT_AUTHORIZATION_KEY: dummy-value IDMC_CLIENT_ID: dummy-value EOAPI_DOMAIN: https://montandon-eoapi.dummy.com + GFD_CREDENTIAL: dummy-value + GFD_SERVICE_ACCOUNT: dummy-value + jobs: pre_commit_checks: diff --git a/README.md b/README.md index b06ce63..c119bfe 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,18 @@ ```bash docker-compose exec web python manage.py extract_emdat_data ``` +- Command to import IDU data. + ```bash + docker-compose exec web python manage.py extract_idu_data + ``` +- Command to import GIDD data. + ```bash + docker-compose exec web python manage.py extract_gidd_data + ``` +- Command to import Global Flood Database data. + ```bash + docker-compose exec web python manage.py extract_gfd_data + ``` - To view the imported data in the admin panel you need to create yourself as a superuser: ```bash docker-compose exec web python manage.py createsuperuser diff --git a/apps/etl/etl_tasks/gfd.py b/apps/etl/etl_tasks/gfd.py new file mode 100644 index 0000000..31d5329 --- /dev/null +++ b/apps/etl/etl_tasks/gfd.py @@ -0,0 +1,25 @@ +from datetime import datetime, timedelta + +from celery import chain, shared_task + +from apps.etl.extraction.sources.gfd.extract import GFDExtraction +from apps.etl.transform.sources.gfd import GFDTransformHandler + + +@shared_task +def ext_and_transform_gfd_historical_data(): + chain( + GFDExtraction.task.s(), + GFDTransformHandler.task.s(), + ).apply_async() + + +@shared_task +def ext_and_transform_gfd_latest_data(): + end_date = datetime.now().date() + start_date = end_date - timedelta(days=1) + + chain( + GFDExtraction.task.s(start_date, end_date), + GFDTransformHandler.task.s(), + ).apply_async() diff --git a/apps/etl/extraction/sources/gfd/__init__.py b/apps/etl/extraction/sources/gfd/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/etl/extraction/sources/gfd/extract.py b/apps/etl/extraction/sources/gfd/extract.py new file mode 100644 index 0000000..03817a8 --- /dev/null +++ b/apps/etl/extraction/sources/gfd/extract.py @@ -0,0 +1,180 @@ +import base64 +import hashlib +import json +import logging +import tempfile +from typing import Any, Callable + +import ee +import requests +from django.conf import settings + +from apps.etl.extraction.sources.base.handler import BaseExtraction +from apps.etl.extraction.sources.base.utils import manage_duplicate_file_content +from apps.etl.models import ExtractionData +from main.celery import app + +logger = logging.getLogger(__name__) + + +DATA_URL = "https://earthengine.googleapis.com/v1alpha/projects/earthengine-legacy/assets/GLOBAL_FLOOD_DB/MODIS_EVENTS/V1" + + +class GFDExtraction(BaseExtraction): + + @classmethod + def decode_json(cls, encoded_str): + """Decodes a Base64 string back to a JSON object.""" + decoded_data = base64.urlsafe_b64decode(encoded_str.encode()).decode() + return json.loads(decoded_data) + + @classmethod + def get_json_credentials(cls, content): + with tempfile.NamedTemporaryFile(delete=False, mode="w") as temp_file: + json_string = json.dumps(content, sort_keys=True) + temp_file.write(json_string) + temp_path = temp_file.name + return temp_path + + @classmethod + def hash_json_content(cls, json_data): + """Hashes a JSON object using SHA256.""" + json_string = json.dumps(json_data, sort_keys=True) + return hashlib.sha256(json_string.encode()).hexdigest() + + @classmethod + def store_extraction_data( + cls, + validate_source_func: Callable[[Any], None], + source: int, + response: dict, + instance_id: int = None, + ): + """ + Save extracted data into database. Checks for duplicate content using hashing. + """ + file_extension = "json" + file_name = f"{source}.{file_extension}" + resp_data_content = json.dumps(response) + + # save the additional response data after the data is fetched from api. + extraction_instance = ExtractionData.objects.get(id=instance_id) + extraction_instance.resp_data_type = "application/json" + extraction_instance.save(update_fields=["resp_data_type"]) + + # Validate the non empty response data. + if resp_data_content: + # Source validation + if validate_source_func: + extraction_instance.source_validation_status = validate_source_func(resp_data_content)["status"] + extraction_instance.content_validation = validate_source_func(resp_data_content)["validation_error"] + + # manage duplicate file content. + hash_content = cls.hash_json_content(resp_data_content) + manage_duplicate_file_content( + source=extraction_instance.source, + hash_content=hash_content, + instance=extraction_instance, + response_data=resp_data_content, + file_name=file_name, + ) + return extraction_instance + + @classmethod + def _save_response_data(cls, instance: ExtractionData, response: requests.Response) -> dict: + """ + Save the response data to the extraction instance. + Args: + instance: ExtractionData instance to save to + response: Response object containing the data + Returns: + dict: Parsed JSON response content + """ + instance = cls.store_extraction_data( + response=response, + source=instance.source, + validate_source_func=None, + instance_id=instance.id, + ) + + return response + + @classmethod + def get_flood_data(cls, collection, batch_size=1000): + """Retrieve flood metadata in batches to avoid memory issues.""" + total_size = collection.size().getInfo() + + all_data = [] + for i in range(0, total_size, batch_size): + batch = collection.toList(batch_size, i).getInfo() + all_data.extend([feature for feature in batch]) + + return all_data + + @classmethod + def handle_extraction(cls, url: str, source: int, start_date, end_date) -> int: + """ + Process data extraction. + Returns: + int: ID of the extraction instance + """ + logger.info("Starting data extraction") + instance = cls._create_extraction_instance(url=url, source=source) + + try: + cls._update_instance_status(instance, ExtractionData.Status.IN_PROGRESS) + response = cls.extract_data(start_date, end_date) + response_data = cls._save_response_data(instance, response) + # Check if response contains data + if response_data: + cls._update_instance_status(instance, ExtractionData.Status.SUCCESS) + logger.info("Data extracted successfully") + else: + cls._update_instance_status( + instance, + ExtractionData.Status.SUCCESS, + ExtractionData.ValidationStatus.NO_DATA, + update_validation=True, + ) + logger.warning("No hazard data found in response") + + return instance.id + + except requests.exceptions.RequestException: + cls._update_instance_status(instance, ExtractionData.Status.FAILED) + logger.error( + "extraction failed", + exc_info=True, + extra={ + "source": instance.source, + }, + ) + raise + + @classmethod + def extract_data(cls, start_date=None, end_date=None): + # Set up authentication + service_account = settings.GFD_SERVICE_ACCOUNT + + # # Decode the earthengine credential + decoded_json = cls.decode_json(settings.GFD_CREDENTIAL) + credential_file_path = cls.get_json_credentials(decoded_json) + + # Authenticate + credentials = ee.ServiceAccountCredentials(service_account, credential_file_path) + ee.Initialize(credentials) + + # Load Global Flood Database (GFD) + gfd_data = ee.ImageCollection("GLOBAL_FLOOD_DB/MODIS_EVENTS/V1") + + # Filter flood events by date + if start_date and end_date: + gfd_data = gfd_data.filterDate(str(start_date), str(end_date)) + + flood_data = cls.get_flood_data(gfd_data, batch_size=500) + return flood_data + + @staticmethod + @app.task + def task(start_date=None, end_date=None): + return GFDExtraction().handle_extraction(DATA_URL, ExtractionData.Source.GIDD, start_date, end_date) diff --git a/apps/etl/extraction/sources/idu/extract.py b/apps/etl/extraction/sources/idu/extract.py index 96a216e..36d5af8 100644 --- a/apps/etl/extraction/sources/idu/extract.py +++ b/apps/etl/extraction/sources/idu/extract.py @@ -133,7 +133,6 @@ def handle_extraction(url) -> dict: int: ID of the extraction instance """ logger.info("Starting IDU data extraction") - print("Starting IDU data extraction") instance = IDUExtraction._create_extraction_instance(url=url) try: diff --git a/apps/etl/management/commands/extract_gfd_data.py b/apps/etl/management/commands/extract_gfd_data.py new file mode 100644 index 0000000..53823b1 --- /dev/null +++ b/apps/etl/management/commands/extract_gfd_data.py @@ -0,0 +1,10 @@ +from django.core.management.base import BaseCommand + +from apps.etl.etl_tasks.gfd import ext_and_transform_gfd_historical_data + + +class Command(BaseCommand): + help = "Import data from gfd api" + + def handle(self, *args, **options): + ext_and_transform_gfd_historical_data() diff --git a/apps/etl/tasks.py b/apps/etl/tasks.py index fb81295..abd2788 100644 --- a/apps/etl/tasks.py +++ b/apps/etl/tasks.py @@ -9,6 +9,7 @@ fetch_event_data, fetch_gdacs_geometry_data, ) +from apps.etl.extraction.sources.gfd.extract import GFDExtraction from apps.etl.extraction.sources.gidd.extract import GIDDExtraction from apps.etl.extraction.sources.glide.extract import ( # noqa: F401 import_hazard_data as import_glide_data, @@ -20,6 +21,7 @@ transform_geo_data, transform_impact_data, ) +from apps.etl.transform.sources.gfd import GFDTransformHandler # noqa: F401 from apps.etl.transform.sources.gidd import GIDDTransformHandler # noqa: F401 from apps.etl.transform.sources.glide import transform_glide_event_data # noqa: F401 from apps.etl.transform.sources.handler import BaseTransformerHandler @@ -27,6 +29,7 @@ IDUExtraction.handle_extraction GIDDExtraction.handle_extraction +GFDExtraction.handle_extraction BaseTransformerHandler.handle_transformation @@ -56,6 +59,11 @@ def extract_gidd_data(): call_command("extract_gidd_data") +@shared_task +def extract_gfd_data(): + call_command("extract_gfd_data") + + @shared_task def load_data(): call_command("load_data_to_stac") diff --git a/apps/etl/transform/sources/gfd.py b/apps/etl/transform/sources/gfd.py new file mode 100644 index 0000000..e6a9fc8 --- /dev/null +++ b/apps/etl/transform/sources/gfd.py @@ -0,0 +1,22 @@ +from pystac_monty.sources.gfd import GFDDataSource, GFDTransformer + +from apps.etl.models import ExtractionData +from apps.etl.transform.sources.handler import BaseTransformerHandler +from main.celery import app + + +class GFDTransformHandler(BaseTransformerHandler): + transformer = GFDTransformer + transformer_schema = GFDDataSource + + @classmethod + def get_schema_data(cls, extraction_obj: ExtractionData): + with extraction_obj.resp_data.open() as file_data: + data = file_data.read() + + return cls.transformer_schema(source_url=extraction_obj.url, data=data) + + @staticmethod + @app.task + def task(extraction_id): + return GFDTransformHandler().handle_transformation(extraction_id) diff --git a/apps/etl/transform/sources/handler.py b/apps/etl/transform/sources/handler.py index 781eb24..1e7eb2c 100644 --- a/apps/etl/transform/sources/handler.py +++ b/apps/etl/transform/sources/handler.py @@ -13,6 +13,9 @@ "idu-impacts": PyStacLoadData.ItemType.IMPACT, "idmc-events": PyStacLoadData.ItemType.EVENT, "idmc-gidd-impacts": PyStacLoadData.ItemType.IMPACT, + "gfd-events": PyStacLoadData.ItemType.EVENT, + "gfd-impacts": PyStacLoadData.ItemType.IMPACT, + "gfd-hazards": PyStacLoadData.ItemType.HAZARD, } diff --git a/docker-compose.yml b/docker-compose.yml index 679516a..ae17068 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,8 @@ x-server: &base_server_setup # ETL Sources EMDAT_AUTHORIZATION_KEY: ${EMDAT_AUTHORIZATION_KEY?error} IDMC_CLIENT_ID: ${IDMC_CLIENT_ID?error} + GFD_CREDENTIAL: ${GFD_CREDENTIAL?error} + GFD_SERVICE_ACCOUNT: ${GFD_SERVICE_ACCOUNT?error} # ETL Load EOAPI_DOMAIN: ${EOAPI_DOMAIN?error} volumes: diff --git a/helm/templates/config/secret.yaml b/helm/templates/config/secret.yaml index 1fe33ec..d61bca0 100644 --- a/helm/templates/config/secret.yaml +++ b/helm/templates/config/secret.yaml @@ -18,6 +18,8 @@ stringData: # ETL Sources EMDAT_AUTHORIZATION_KEY: {{ required "secrets.EMDAT_AUTHORIZATION_KEY" .Values.secrets.EMDAT_AUTHORIZATION_KEY | quote }} IDMC_CLIENT_ID: {{ required "secrets.IDMC_CLIENT_ID" .Values.secrets.IDMC_CLIENT_ID | quote }} + GFD_CREDENTIAL: {{ required "secrets.GFD_CREDENTIAL" .Values.secrets.GFD_CREDENTIAL | quote }} + GFD_SERVICE_ACCOUNT: {{ required "secrets.GFD_SERVICE_ACCOUNT" .Values.secrets.GFD_SERVICE_ACCOUNT | quote }} # Database {{- if .Values.postgresql.enabled }} diff --git a/helm/values-test.yaml b/helm/values-test.yaml index afb5fb7..09d27ff 100644 --- a/helm/values-test.yaml +++ b/helm/values-test.yaml @@ -36,6 +36,8 @@ secrets: # Sources EMDAT_AUTHORIZATION_KEY: dummy-key IDMC_CLIENT_ID: dummy-client-id + GFD_CREDENTIAL: dummy-gfd-cred + GFD_SERVICE_ACCOUNT: dummy-gfd-service-ac secretsAdditional: ENABLE_MAGIC_SECRET: "true" MAGIC_KEY: to-much-fun diff --git a/helm/values.yaml b/helm/values.yaml index 20ca229..c68dcc5 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -179,6 +179,8 @@ secrets: # ETL Sources EMDAT_AUTHORIZATION_KEY: IDMC_CLIENT_ID: + GFD_CREDENDIAL: + GFD_SERVICE_ACCOUNT: # NOTE: Used to pass additional secrets to api/worker containers # NOTE: Not used by azure vault secretsAdditional: diff --git a/libs/pystac-monty b/libs/pystac-monty index f8625a0..c3b72ea 160000 --- a/libs/pystac-monty +++ b/libs/pystac-monty @@ -1 +1 @@ -Subproject commit f8625a06b747568944e216d880c15adb68605d9b +Subproject commit c3b72ea1dba6b2b550f503069974ca8588f06473 diff --git a/main/settings.py b/main/settings.py index f772207..adf87e3 100644 --- a/main/settings.py +++ b/main/settings.py @@ -72,8 +72,14 @@ IDMC_DATA_URL=(str, "https://helix-tools-api.idmcdb.org"), # ETL Load configs EOAPI_DOMAIN=str, # http://montandon-eoapi.ifrc.org + GFD_CREDENTIAL=str, + GFD_SERVICE_ACCOUNT=str, ) +GFD_SERVICE_ACCOUNT = env("GFD_SERVICE_ACCOUNT") + +GFD_CREDENTIAL = env("GFD_CREDENTIAL") + EMDAT_AUTHORIZATION_KEY = env("EMDAT_AUTHORIZATION_KEY") IDMC_CLIENT_ID = env("IDMC_CLIENT_ID") @@ -328,6 +334,10 @@ "task": "apps.etl.tasks.extract_usgs_data", "schedule": crontab(minute=0, hour=0), # This task execute daily at 12 AM (UTC) }, + "import_gfd_data": { + "task": "apps.etl.etl_tasks.ext_and_transform_gfd_latest_data", + "schedule": crontab(minute=0, hour=0), # This task execute daily at 12 AM (UTC) + }, "load_data_to_stac": { "task": "apps.etl.tasks.load_data", "schedule": crontab(minute=0, hour=0), # TODO Set time to run this job diff --git a/pyproject.toml b/pyproject.toml index 6560c9c..1881e8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "sentry-sdk", "ipython", "uwsgi", + "earthengine-api>=1.5.1", ] [tool.uv.sources] diff --git a/uv.lock b/uv.lock index 055ad83..1060d82 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,9 @@ version = 1 requires-python = ">=3.12, <4" +resolution-markers = [ + "python_full_version >= '3.13'", + "python_full_version < '3.13'", +] [[package]] name = "amqp" @@ -143,6 +147,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/31/17/1776cdd6dbeaa9910f005cc8bda6c436518ba83be94a438f7b9848d49d6b/botocore-1.36.16-py3-none-any.whl", hash = "sha256:aca0348ccd730332082489b6817fdf89e1526049adcf6e9c8c11c96dd9f42c03", size = 13340384 }, ] +[[package]] +name = "cachetools" +version = "5.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/74/57df1ab0ce6bc5f6fa868e08de20df8ac58f9c44330c7671ad922d2bbeae/cachetools-5.5.1.tar.gz", hash = "sha256:70f238fbba50383ef62e55c6aff6d9673175fe59f7c6782c7a0b9e38f4a9df95", size = 28044 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/4e/de4ff18bcf55857ba18d3a4bd48c8a9fde6bb0980c9d20b263f05387fd88/cachetools-5.5.1-py3-none-any.whl", hash = "sha256:b76651fdc3b24ead3c648bbdeeb940c1b04d365b38b4af66788f9ec4a81d42bb", size = 9530 }, +] + [[package]] name = "celery" version = "5.4.0" @@ -459,6 +472,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/8f/c4d9bafc34ad7ad5d8dc16dd1347ee0e507a52c3adb6bfa8887e1c6a26ba/executing-2.2.0-py2.py3-none-any.whl", hash = "sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa", size = 26702 }, ] +[[package]] +name = "earthengine-api" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-python-client" }, + { name = "google-auth" }, + { name = "google-auth-httplib2" }, + { name = "google-cloud-storage" }, + { name = "httplib2" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/46/1132320f233675475a8422d2d136f2a6d01fe039c9ed50543b8d5d0e0c5c/earthengine_api-1.5.1.tar.gz", hash = "sha256:3bd7f6cb3fad527c4883fce7afb8875de4f4cde5e27acb89b818f3a76d7735ac", size = 413881 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/af/96d91a15df74917062c9884d3744d4c6b26cb43cf60d5fc91c42afecef0a/earthengine_api-1.5.1-py3-none-any.whl", hash = "sha256:4a0b08c25a1ba7852a854e24acda4a586e71f65f878ae105fb46473a20710ce4", size = 459594 }, +] + [[package]] name = "fiona" version = "1.10.1" @@ -547,6 +577,144 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/92/75/4bc3e242ad13f2e6c12e0b0401ab2c5e5c6f0d7da37ec69bc808e24e0ccb/humanize-4.11.0-py3-none-any.whl", hash = "sha256:b53caaec8532bcb2fff70c8826f904c35943f8cecaca29d272d9df38092736c0", size = 128055 }, ] +[[package]] +name = "google-api-core" +version = "2.24.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "googleapis-common-protos" }, + { name = "proto-plus" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/b7/481c83223d7b4f02c7651713fceca648fa3336e1571b9804713f66bca2d8/google_api_core-2.24.1.tar.gz", hash = "sha256:f8b36f5456ab0dd99a1b693a40a31d1e7757beea380ad1b38faaf8941eae9d8a", size = 163508 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/a6/8e30ddfd3d39ee6d2c76d3d4f64a83f77ac86a4cab67b286ae35ce9e4369/google_api_core-2.24.1-py3-none-any.whl", hash = "sha256:bc78d608f5a5bf853b80bd70a795f703294de656c096c0968320830a4bc280f1", size = 160059 }, +] + +[[package]] +name = "google-api-python-client" +version = "2.160.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, + { name = "google-auth-httplib2" }, + { name = "httplib2" }, + { name = "uritemplate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/42/cbf81242376c99d6e5248e62aa4376bfde5bbefbe0a69b1b06fd4b73ab25/google_api_python_client-2.160.0.tar.gz", hash = "sha256:a8ccafaecfa42d15d5b5c3134ced8de08380019717fc9fb1ed510ca58eca3b7e", size = 12304236 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/35/41623ac3b581781169eed7f5fcd24bc114c774dc491fab5c05d8eb81af36/google_api_python_client-2.160.0-py2.py3-none-any.whl", hash = "sha256:63d61fb3e4cf3fb31a70a87f45567c22f6dfe87bbfa27252317e3e2c42900db4", size = 12814302 }, +] + +[[package]] +name = "google-auth" +version = "2.38.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "pyasn1-modules" }, + { name = "rsa" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/eb/d504ba1daf190af6b204a9d4714d457462b486043744901a6eeea711f913/google_auth-2.38.0.tar.gz", hash = "sha256:8285113607d3b80a3f1543b75962447ba8a09fe85783432a784fdeef6ac094c4", size = 270866 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/47/603554949a37bca5b7f894d51896a9c534b9eab808e2520a748e081669d0/google_auth-2.38.0-py2.py3-none-any.whl", hash = "sha256:e7dae6694313f434a2727bf2906f27ad259bae090d7aa896590d86feec3d9d4a", size = 210770 }, +] + +[[package]] +name = "google-auth-httplib2" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "httplib2" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/56/be/217a598a818567b28e859ff087f347475c807a5649296fb5a817c58dacef/google-auth-httplib2-0.2.0.tar.gz", hash = "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05", size = 10842 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/8a/fe34d2f3f9470a27b01c9e76226965863f153d5fbe276f83608562e49c04/google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d", size = 9253 }, +] + +[[package]] +name = "google-cloud-core" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/1f/9d1e0ba6919668608570418a9a51e47070ac15aeff64261fb092d8be94c0/google-cloud-core-2.4.1.tar.gz", hash = "sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073", size = 35587 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/0f/2e2061e3fbcb9d535d5da3f58cc8de4947df1786fe6a1355960feb05a681/google_cloud_core-2.4.1-py2.py3-none-any.whl", hash = "sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61", size = 29233 }, +] + +[[package]] +name = "google-cloud-storage" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, + { name = "google-cloud-core" }, + { name = "google-crc32c" }, + { name = "google-resumable-media" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/d7/dfa74049c4faa3b4d68fa1a10a7eab5a76c57d0788b47c27f927bedc606d/google_cloud_storage-3.0.0.tar.gz", hash = "sha256:2accb3e828e584888beff1165e5f3ac61aa9088965eb0165794a82d8c7f95297", size = 7665253 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/ae/1a50f07161301e40a30b2e40744a7b85ffab7add16e044417925eccf9bbf/google_cloud_storage-3.0.0-py2.py3-none-any.whl", hash = "sha256:f85fd059650d2dbb0ac158a9a6b304b66143b35ed2419afec2905ca522eb2c6a", size = 173860 }, +] + +[[package]] +name = "google-crc32c" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/67/72/c3298da1a3773102359c5a78f20dae8925f5ea876e37354415f68594a6fb/google_crc32c-1.6.0.tar.gz", hash = "sha256:6eceb6ad197656a1ff49ebfbbfa870678c75be4344feb35ac1edf694309413dc", size = 14472 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/41/65a91657d6a8123c6c12f9aac72127b6ac76dda9e2ba1834026a842eb77c/google_crc32c-1.6.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:ed767bf4ba90104c1216b68111613f0d5926fb3780660ea1198fc469af410e9d", size = 30268 }, + { url = "https://files.pythonhosted.org/packages/59/d0/ee743a267c7d5c4bb8bd865f7d4c039505f1c8a4b439df047fdc17be9769/google_crc32c-1.6.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:62f6d4a29fea082ac4a3c9be5e415218255cf11684ac6ef5488eea0c9132689b", size = 30113 }, + { url = "https://files.pythonhosted.org/packages/25/53/e5e449c368dd26ade5fb2bb209e046d4309ed0623be65b13f0ce026cb520/google_crc32c-1.6.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c87d98c7c4a69066fd31701c4e10d178a648c2cac3452e62c6b24dc51f9fcc00", size = 32995 }, + { url = "https://files.pythonhosted.org/packages/52/12/9bf6042d5b0ac8c25afed562fb78e51b0641474097e4139e858b45de40a5/google_crc32c-1.6.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd5e7d2445d1a958c266bfa5d04c39932dc54093fa391736dbfdb0f1929c1fb3", size = 32614 }, + { url = "https://files.pythonhosted.org/packages/76/29/fc20f5ec36eac1eea0d0b2de4118c774c5f59c513f2a8630d4db6991f3e0/google_crc32c-1.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:7aec8e88a3583515f9e0957fe4f5f6d8d4997e36d0f61624e70469771584c760", size = 33445 }, +] + +[[package]] +name = "google-resumable-media" +version = "2.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-crc32c" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/5a/0efdc02665dca14e0837b62c8a1a93132c264bd02054a15abb2218afe0ae/google_resumable_media-2.7.2.tar.gz", hash = "sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0", size = 2163099 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/35/b8d3baf8c46695858cb9d8835a53baa1eeb9906ddaf2f728a5f5b640fd1e/google_resumable_media-2.7.2-py2.py3-none-any.whl", hash = "sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa", size = 81251 }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.66.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/a7/8e9cccdb1c49870de6faea2a2764fa23f627dd290633103540209f03524c/googleapis_common_protos-1.66.0.tar.gz", hash = "sha256:c3e7b33d15fdca5374cc0a7346dd92ffa847425cc4ea941d970f13680052ec8c", size = 114376 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/0f/c0713fb2b3d28af4b2fded3291df1c4d4f79a00d15c2374a9e010870016c/googleapis_common_protos-1.66.0-py2.py3-none-any.whl", hash = "sha256:d7abcd75fabb2e0ec9f74466401f6c119a0b498e27370e9be4c94cb7e382b8ed", size = 221682 }, +] + +[[package]] +name = "httplib2" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyparsing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/ad/2371116b22d616c194aa25ec410c9c6c37f23599dcd590502b74db197584/httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81", size = 351116 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/6c/d2fbdaaa5959339d53ba38e94c123e4e84b8fbc4b84beb0e70d7c1608486/httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc", size = 96854 }, +] + [[package]] name = "idna" version = "3.10" @@ -708,6 +876,7 @@ dependencies = [ { name = "django-environ" }, { name = "django-redis" }, { name = "django-storages", extra = ["azure", "s3"] }, + { name = "earthengine-api" }, { name = "fiona" }, { name = "flake8" }, { name = "flower" }, @@ -732,6 +901,7 @@ requires-dist = [ { name = "django-environ" }, { name = "django-redis", specifier = ">=5.4.0,<6" }, { name = "django-storages", extras = ["s3", "azure"], specifier = ">=1.14,<1.15" }, + { name = "earthengine-api", specifier = ">=1.5.1" }, { name = "fiona", specifier = ">=1.10.1" }, { name = "flake8", specifier = ">=7.1.1,<8" }, { name = "flower" }, @@ -909,6 +1079,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl", hash = "sha256:f49a827f90062e411f1ce1f854f2aedb3c23353244f8108b89283587397ac10e", size = 386595 }, ] +[[package]] +name = "proto-plus" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/79/a5c6cbb42268cfd3ddc652dc526889044a8798c688a03ff58e5e92b743c8/proto_plus-1.26.0.tar.gz", hash = "sha256:6e93d5f5ca267b54300880fff156b6a3386b3fa3f43b1da62e680fc0c586ef22", size = 56136 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/c3/59308ccc07b34980f9d532f7afc718a9f32b40e52cde7a740df8d55632fb/proto_plus-1.26.0-py3-none-any.whl", hash = "sha256:bf2dfaa3da281fc3187d12d224c707cb57214fb2c22ba854eb0c105a3fb2d4d7", size = 50166 }, +] + +[[package]] +name = "protobuf" +version = "5.29.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f7/d1/e0a911544ca9993e0f17ce6d3cc0932752356c1b0a834397f28e63479344/protobuf-5.29.3.tar.gz", hash = "sha256:5da0f41edaf117bde316404bad1a486cb4ededf8e4a54891296f648e8e076620", size = 424945 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/7a/1e38f3cafa022f477ca0f57a1f49962f21ad25850c3ca0acd3b9d0091518/protobuf-5.29.3-cp310-abi3-win32.whl", hash = "sha256:3ea51771449e1035f26069c4c7fd51fba990d07bc55ba80701c78f886bf9c888", size = 422708 }, + { url = "https://files.pythonhosted.org/packages/61/fa/aae8e10512b83de633f2646506a6d835b151edf4b30d18d73afd01447253/protobuf-5.29.3-cp310-abi3-win_amd64.whl", hash = "sha256:a4fa6f80816a9a0678429e84973f2f98cbc218cca434abe8db2ad0bffc98503a", size = 434508 }, + { url = "https://files.pythonhosted.org/packages/dd/04/3eaedc2ba17a088961d0e3bd396eac764450f431621b58a04ce898acd126/protobuf-5.29.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a8434404bbf139aa9e1300dbf989667a83d42ddda9153d8ab76e0d5dcaca484e", size = 417825 }, + { url = "https://files.pythonhosted.org/packages/4f/06/7c467744d23c3979ce250397e26d8ad8eeb2bea7b18ca12ad58313c1b8d5/protobuf-5.29.3-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:daaf63f70f25e8689c072cfad4334ca0ac1d1e05a92fc15c54eb9cf23c3efd84", size = 319573 }, + { url = "https://files.pythonhosted.org/packages/a8/45/2ebbde52ad2be18d3675b6bee50e68cd73c9e0654de77d595540b5129df8/protobuf-5.29.3-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:c027e08a08be10b67c06bf2370b99c811c466398c357e615ca88c91c07f0910f", size = 319672 }, + { url = "https://files.pythonhosted.org/packages/fd/b2/ab07b09e0f6d143dfb839693aa05765257bceaa13d03bf1a696b78323e7a/protobuf-5.29.3-py3-none-any.whl", hash = "sha256:0a18ed4a24198528f2333802eb075e59dea9d679ab7a6c5efb017a59004d849f", size = 172550 }, +] + [[package]] name = "psycopg2-binary" version = "2.9.10" @@ -958,6 +1154,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842 }, ] +[[package]] +name = "pyasn1" +version = "0.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/e9/01f1a64245b89f039897cb0130016d79f77d52669aae6ee7b159a6c4c018/pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034", size = 145322 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", size = 83135 }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/67/6afbf0d507f73c32d21084a79946bfcfca5fbc62a72057e9c23797a737c9/pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c", size = 310028 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/89/bc88a6711935ba795a679ea6ebee07e128050d6382eaa35a0a47c8032bdc/pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd", size = 181537 }, +] + [[package]] name = "pycodestyle" version = "2.12.1" @@ -1086,6 +1303,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/5d/0deb16d228362a097ee3258d0a887c9c0add4b9678bb4847b08a241e124d/pyogrio-0.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:02e54bcfb305af75f829044b0045f74de31b77c2d6546f7aaf96822066147848", size = 16158260 }, ] +[[package]] +name = "pyparsing" +version = "3.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8b/1a/3544f4f299a47911c2ab3710f534e52fea62a633c96806995da5d25be4b2/pyparsing-3.2.1.tar.gz", hash = "sha256:61980854fd66de3a90028d679a954d5f2623e83144b5afe5ee86f43d762e5f0a", size = 1067694 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/a7/c8a2d361bf89c0d9577c934ebb7421b25dc84bf3a8e3ac0a40aed9acc547/pyparsing-3.2.1-py3-none-any.whl", hash = "sha256:506ff4f4386c4cec0590ec19e6302d3aedb992fdc02c761e90416f158dacf8e1", size = 107716 }, +] + [[package]] name = "pyproj" version = "3.7.0" @@ -1273,6 +1499,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e6/0f/6f7e6cd0f4a141752caef3f79300148422fdf2b8b68b531f30b2b0c0cbda/sentry_sdk-2.20.0-py2.py3-none-any.whl", hash = "sha256:c359a1edf950eb5e80cffd7d9111f3dbeef57994cb4415df37d39fda2cf22364", size = 322576 }, ] +[[package]] +name = "rsa" +version = "4.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/65/7d973b89c4d2351d7fb232c2e452547ddfa243e93131e7cfa766da627b52/rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21", size = 29711 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/97/fa78e3d2f65c02c8e1268b9aba606569fe97f6c8f7c2d74394553347c145/rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7", size = 34315 }, +] + [[package]] name = "shapely" version = "2.0.6" @@ -1382,6 +1620,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/ab/7e5f53c3b9d14972843a647d8d7a853969a58aecc7559cb3267302c94774/tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd", size = 346586 }, ] +[[package]] +name = "uritemplate" +version = "4.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/5a/4742fdba39cd02a56226815abfa72fe0aa81c33bed16ed045647d6000eba/uritemplate-4.1.1.tar.gz", hash = "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0", size = 273898 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c0/7461b49cd25aeece13766f02ee576d1db528f1c37ce69aee300e075b485b/uritemplate-4.1.1-py2.py3-none-any.whl", hash = "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e", size = 10356 }, +] + [[package]] name = "urllib3" version = "2.3.0"