From c79a5352f1bd1588f67b458d6bf29b36309ab513 Mon Sep 17 00:00:00 2001 From: rup-narayan-rajbanshi Date: Thu, 6 Feb 2025 17:17:17 +0545 Subject: [PATCH] Manage gfd credential. --- README.md | 3 +- apps/etl/extraction/sources/gfd/extract.py | 34 +++++++++++++++------- main/settings.py | 6 ++++ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2f82e34..c119bfe 100644 --- a/README.md +++ b/README.md @@ -37,9 +37,8 @@ docker-compose exec web python manage.py extract_gidd_data ``` - Command to import Global Flood Database data. - - We need to add a json file containing credential for earthengine inside `apps/etl/extraction/sources/gfd/` ```bash - docker-compose exec web python manage.py extract_emdat_data + docker-compose exec web python manage.py extract_gfd_data ``` - To view the imported data in the admin panel you need to create yourself as a superuser: ```bash diff --git a/apps/etl/extraction/sources/gfd/extract.py b/apps/etl/extraction/sources/gfd/extract.py index b7e096a..e492675 100644 --- a/apps/etl/extraction/sources/gfd/extract.py +++ b/apps/etl/extraction/sources/gfd/extract.py @@ -1,11 +1,13 @@ +import base64 import hashlib import json import logging -import os +import tempfile from typing import Any, Callable import ee import requests +from django.conf import settings from apps.etl.extraction.sources.base.handler import BaseExtraction from apps.etl.extraction.sources.base.utils import manage_duplicate_file_content @@ -20,10 +22,24 @@ class GFDExtraction(BaseExtraction): + @classmethod + def decode_json(cls, encoded_str): + """Decodes a Base64 string back to a JSON object.""" + decoded_data = base64.urlsafe_b64decode(encoded_str.encode()).decode() + return json.loads(decoded_data) + + @classmethod + def get_json_credentials(cls, content): + with tempfile.NamedTemporaryFile(delete=False, mode="w") as temp_file: + json_string = json.dumps(content, sort_keys=True) + temp_file.write(json_string) + temp_path = temp_file.name + return temp_path + @classmethod def hash_json_content(cls, json_data): """Hashes a JSON object using SHA256.""" - json_string = json.dumps(json_data, sort_keys=True) # Ensure consistent ordering + json_string = json.dumps(json_data, sort_keys=True) return hashlib.sha256(json_string.encode()).hexdigest() @classmethod @@ -138,18 +154,14 @@ def handle_extraction(cls, url: str, source: int, start_date, end_date) -> int: @classmethod def extract_data(cls, start_date=None, end_date=None): # Set up authentication - # TODO Remove this autherntication and use official authentication - service_account = "rup-rajbanshi@gidd-1671534293682.iam.gserviceaccount.com" - - # Get the script's directory - BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + service_account = settings.GFD_SERVICE_ACCOUNT - # Construct the JSON file path - # TODO Add json key file official - json_key_path = os.path.join(BASE_DIR, "gidd-1671534293682-dab333fe24ec.json") + # # Decode the earthengine credential + decoded_json = cls.decode_json(settings.GFD_CREDENTIAL) + credential_file_path = cls.get_json_credentials(decoded_json) # Authenticate - credentials = ee.ServiceAccountCredentials(service_account, json_key_path) + credentials = ee.ServiceAccountCredentials(service_account, credential_file_path) ee.Initialize(credentials) # Load Global Flood Database (GFD) diff --git a/main/settings.py b/main/settings.py index f585079..36e8798 100644 --- a/main/settings.py +++ b/main/settings.py @@ -37,8 +37,14 @@ EMDAT_AUTHORIZATION_KEY=str, IDMC_CLIENT_ID=str, IDMC_DATA_URL=(str, "https://helix-tools-api.idmcdb.org"), + GFD_CREDENTIAL=str, + GFD_SERVICE_ACCOUNT=str, ) +GFD_SERVICE_ACCOUNT = env("GFD_SERVICE_ACCOUNT") + +GFD_CREDENTIAL = env("GFD_CREDENTIAL") + EMDAT_AUTHORIZATION_KEY = env("EMDAT_AUTHORIZATION_KEY") IDMC_CLIENT_ID = env("IDMC_CLIENT_ID")