Add gfd credential in docker-compose file.

IFRCGo · Feb 13, 2025 · 15173a6 · 15173a6
1 parent f02b364
commit 15173a6
Show file tree

Hide file tree

Showing 8 changed files with 14 additions and 7 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -25,6 +25,9 @@ env:
   EMDAT_AUTHORIZATION_KEY: dummy-value
   IDMC_CLIENT_ID: dummy-value
   EOAPI_DOMAIN: https://montandon-eoapi.dummy.com
+  GFD_CREDENTIAL: dummy-value
+  GFD_SERVICE_ACCOUNT: dummy-value
+
 
 jobs:
   pre_commit_checks:

diff --git a/apps/etl/extraction/sources/gfd/extract.py b/apps/etl/extraction/sources/gfd/extract.py
@@ -51,7 +51,7 @@ def store_extraction_data(
         instance_id: int = None,
     ):
         """
-        Save extracted data into data base. Checks for duplicate conent using hashing.
+        Save extracted data into database. Checks for duplicate content using hashing.
         """
         file_extension = "json"
         file_name = f"{source}.{file_extension}"
@@ -60,7 +60,7 @@ def store_extraction_data(
         # save the additional response data after the data is fetched from api.
         extraction_instance = ExtractionData.objects.get(id=instance_id)
         extraction_instance.resp_data_type = "application/json"
-        extraction_instance.save()
+        extraction_instance.save(update_fields=["resp_data_type"])
 
         # Validate the non empty response data.
         if resp_data_content:

diff --git a/apps/etl/management/commands/extract_gfd_data.py b/apps/etl/management/commands/extract_gfd_data.py
@@ -1,11 +1,7 @@
-import logging
-
 from django.core.management.base import BaseCommand
 
 from apps.etl.etl_tasks.gfd import ext_and_transform_gfd_historical_data
 
-logger = logging.getLogger(__name__)
-
 
 class Command(BaseCommand):
     help = "Import data from gfd api"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -21,6 +21,8 @@ x-server: &base_server_setup
       # ETL Sources
       EMDAT_AUTHORIZATION_KEY: ${EMDAT_AUTHORIZATION_KEY?error}
       IDMC_CLIENT_ID: ${IDMC_CLIENT_ID?error}
+      GFD_CREDENTIAL: ${GFD_CREDENTIAL?error}
+      GFD_SERVICE_ACCOUNT: ${GFD_SERVICE_ACCOUNT?error}
       # ETL Load
       EOAPI_DOMAIN: ${EOAPI_DOMAIN?error}
   volumes:

diff --git a/helm/templates/config/secret.yaml b/helm/templates/config/secret.yaml
@@ -18,6 +18,8 @@ stringData:
   # ETL Sources
   EMDAT_AUTHORIZATION_KEY: {{ required "secrets.EMDAT_AUTHORIZATION_KEY" .Values.secrets.EMDAT_AUTHORIZATION_KEY | quote }}
   IDMC_CLIENT_ID: {{ required "secrets.IDMC_CLIENT_ID" .Values.secrets.IDMC_CLIENT_ID | quote }}
+  GFD_CREDENTIAL: {{ required "secrets.GFD_CREDENTIAL" .Values.secrets.GFD_CREDENTIAL | quote }}
+  GFD_SERVICE_ACCOUNT: {{ required "secrets.GFD_SERVICE_ACCOUNT" .Values.secrets.GFD_SERVICE_ACCOUNT | quote }}
 
   # Database
   {{- if .Values.postgresql.enabled }}

diff --git a/helm/values-test.yaml b/helm/values-test.yaml
@@ -36,6 +36,8 @@ secrets:
   # Sources
   EMDAT_AUTHORIZATION_KEY: dummy-key
   IDMC_CLIENT_ID: dummy-client-id
+  GFD_CREDENTIAL: dummy-gfd-cred
+  GFD_SERVICE_ACCOUNT: dummy-gfd-service-ac
 secretsAdditional:
   ENABLE_MAGIC_SECRET: "true"
   MAGIC_KEY: to-much-fun

diff --git a/helm/values.yaml b/helm/values.yaml
@@ -179,6 +179,8 @@ secrets:
   # ETL Sources
   EMDAT_AUTHORIZATION_KEY:
   IDMC_CLIENT_ID:
+  GFD_CREDENDIAL:
+  GFD_SERVICE_ACCOUNT:
 # NOTE: Used to pass additional secrets to api/worker containers
 # NOTE: Not used by azure vault
 secretsAdditional:

diff --git a/libs/pystac-monty b/libs/pystac-monty
+204 −0		pystac_monty/sources/gfd.py
+874 −0		tests/extensions/cassettes/test_gfd/GFDTest.test_transformer_0.yaml
+156 −0		tests/extensions/test_gfd.py