Use git repos for gecko-strings (#1037)

mozfr · Jun 5, 2024 · d9d6028 · d9d6028
1 parent 027817f
commit d9d6028
Show file tree

Hide file tree

Showing 7 changed files with 62 additions and 161 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -49,7 +49,7 @@ jobs:
         run: composer install --prefer-dist
 
       - name: Install Node packages
-        run: sudo npm install -g eslint
+        run: sudo npm install -g eslint@8.57.0
 
       - name: Run JS linter (eslint)
         run: eslint web/js

diff --git a/app/classes/Transvision/Project.php b/app/classes/Transvision/Project.php
@@ -60,6 +60,8 @@ class Project
             'variable_patterns' => ['ios'],
         ],
         'gecko_strings'=> [
+            'git_repository'    => 'firefox-l10n',
+            'git_branch'        => 'main',
             'source_type'       => 'mixed',
             'variable_patterns' => ['dtd', 'ftl', 'l10njs', 'printf', 'properties'],
         ],
@@ -105,7 +107,8 @@ class Project
         ],
         // Products using Git
         'git'         => [
-            'android_l10n', 'firefox_ios', 'mozilla_org', 'vpn_client',
+            'android_l10n', 'firefox_ios', 'gecko_strings', 'mozilla_org',
+            'vpn_client',
         ],
         // Products using free text search on Pontoon
         'text_search' => [

diff --git a/app/classes/Transvision/VersionControl.php b/app/classes/Transvision/VersionControl.php
@@ -101,15 +101,8 @@ public static function hgPath($locale, $repo, $path)
         $exploded_path = explode('/', $path);
         $base_folder = $exploded_path[0];
 
-        if ($repo == 'comm_l10n') {
-            $url = "https://hg.mozilla.org/projects/comm-l10n/file/default/{$locale}/";
-        } else {
-            if ($locale != 'en-US') {
-                $url = "https://hg.mozilla.org/l10n-central/{$locale}/file/default/";
-            } else {
-                $url = 'https://hg.mozilla.org/l10n/gecko-strings/file/default/';
-            }
-        }
+        # comm-l10n is the last repository using Mercurial
+        $url = "https://hg.mozilla.org/projects/comm-l10n/file/default/{$locale}/";
 
         return $url . $path . '/' . $entity_file;
     }
@@ -135,6 +128,15 @@ public static function gitPath($locale, $repo, $path)
             if (isset($repo_data['git_subfolder'])) {
                 $file_path = "{$repo_data['git_subfolder']}/{$file_path}";
             }
+            if ($repo == 'gecko_strings') {
+                // Special case for gecko-strings (Firefox)
+                $file_path = explode(':', $path)[0];
+                if ($locale == 'en-US') {
+                    return "https://github.com/mozilla-l10n/firefox-l10n-source/blob/main/{$file_path}";
+                }
+
+                return "https://github.com/mozilla-l10n/{$git_repo}/blob/{$git_branch}/{$locale}/{$file_path}";
+            }
             if ($repo == 'android_l10n') {
                 // Special case for android-l10n (Android)
                 $locale_android = $locale == 'en-US'

diff --git a/app/scripts/bash_variables.sh b/app/scripts/bash_variables.sh
@@ -6,8 +6,8 @@
 folders=( $libraries )
 path_sources=${config}/sources
 
-# Path and list of locales for gecko-strings
-gecko_strings_path=${local_hg}/gecko_strings
+# Path and list of locales for Gecko locales (Firefox)
+gecko_strings_path=${local_git}/gecko_strings
 gecko_strings_locales=${path_sources}/gecko_strings.txt
 folders+=( $gecko_strings_path )
 

diff --git a/app/scripts/glossaire.sh b/app/scripts/glossaire.sh
@@ -111,147 +111,51 @@ fi
 # Create all bash variables
 source $script_path/bash_variables.sh
 
-function updateLocale() {
-    # Update this locale's repository
-    # $1: Path to l10n repository
-    # $2: Locale code
-    # $3: Repository name
-
-    # Assign input variables to variables with meaningful names
-    l10n_path="$1"
-    locale="$2"
-    repository_name="$3"
-
-    cd $l10n_path/$locale
-    # Check if there are incoming changesets
-    hg incoming -r default --bundle incoming.hg 2>&1 >/dev/null
-    incoming_changesets=$?
-    if [ $incoming_changesets -eq 0 ]
-    then
-        # Update with incoming changesets and remove the bundle
-        echogreen "Updating $repository_name"
-        hg pull --update incoming.hg
-        rm incoming.hg
-
-        # Return 1: we need to create the cache for this locale
-        return 1
-    else
-        echogreen "There are no changes to pull for $repository_name"
-
-        # Return 0: no need to create the cache
-        return 0
-    fi
-}
-
 function updateGeckoStrings() {
     function buildCache() {
         # Build the cache
-        # $1: Locale code
-        echogreen "Create cache for $repo_name/$1"
+        # $1: Path containing locale folder
+        # $2: Locale code
+        local path="$1"
+        local locale="$2"
+        echogreen "Create cache for $path/$locale"
         mkdir -p "${root}TMX/${locale}/"
-        nice -20 python $install/app/scripts/tmx/tmx_products.py --path $repo_folder/$1/ --locale $1 --ref en-US --repo $repo_name
+        nice -20 python $install/app/scripts/tmx/tmx_products.py --path $path/$locale/ --locale $locale --ref en-US --repo $repo_name
     }
 
     local repo_name="gecko_strings"
     local repo_folder="$gecko_strings_path"
     local locale_list="gecko_strings_locales"
 
-    updated_english=false
+    # Update en-US, create TMX for en-US
+    git -C $repo_folder/en-US pull
+    buildCache $repo_folder en-US
 
-    # Store md5 of the existing en-US cache before updating the repositories
-    cache_file="${root}TMX/en-US/cache_en-US_${repo_name}.php"
-    if [ -f $cache_file ]
-    then
-        existing_md5=($(md5sum $cache_file))
-    else
-        existing_md5=0
-    fi
-
-    # Update en-US, create TMX for en-US and check the updated md5
-    hg --cwd $repo_folder/en-US pull --update -r default
-    buildCache en-US
-    updated_md5=($(md5sum $cache_file))
-    if [ $existing_md5 != $updated_md5 ]
+    # Pull l10n repository if necessary
+    if [ "$checkrepo" = true ]
     then
-        echo "English strings have been updated."
-        updated_english=true
+        git -C $repo_folder/l10n pull
     fi
 
     if [ "$all_locales" = true ]
     then
-        for locale in $(cat ${!locale_list})
-        do
-            if [ $locale != "en-US" ]
-            then
-                if [ -d $repo_folder/$locale ]
-                then
-                    updated_locale=0
-                    if [ "$checkrepo" = true ]
-                    then
-                        updateLocale $repo_folder $locale $repo_name/$locale
-                        updated_locale=$?
-                    fi
-
-                    # Check if we have a cache file for this locale. If it's a brand
-                    # new locale, we'll have the folder and no updates, but we
-                    # still need to create the cache.
-                    cache_file="${root}TMX/${locale}/cache_${locale}_${repo_name}.php"
-                    if [ ! -f $cache_file ]
-                    then
-                        echored "Cache doesn't exist for ${repo_name}/${locale}"
-                        updated_locale=1
-                    else
-                        php -l $cache_file 2>&1 1>/dev/null
-                        if [ $? -ne 0 ]
-                        then
-                            # There are PHP errors, force the rebuild
-                            echored "PHP errors in $cache_file. Forcing rebuild."
-                            updated_locale=1
-                        fi
-                    fi
-
-                    if [ "$forceTMX" = true -o "$updated_english" = true -o "$updated_locale" -eq 1 ]
-                    then
-                        buildCache $locale
-                    fi
-                else
-                    echored "Folder $repo_folder/$locale does not exist. Run setup.sh to fix the issue."
-                fi
-            fi
-        done
+        locales=$(cat ${!locale_list})
     else
-        if [ -d $repo_folder/$locale_code ]
-        then
-            updated_locale=0
-            if [ "$checkrepo" = true ]
-            then
-                updateLocale $repo_folder $locale_code $repo_name/$locale_code
-                updated_locale=$?
-            fi
+        locales=($locale_code)
+    fi
 
-            cache_file="${root}TMX/${locale_code}/cache_${locale_code}_${repo_name}.php"
-            if [ ! -f $cache_file ]
+    for locale in $locales
+    do
+        if [ $locale != "en-US" ]
+        then
+            if [ -d $repo_folder/l10n/$locale ]
             then
-                echored "Cache doesn't exist for ${repo_name}/${locale_code}"
-                updated_locale=1
+                buildCache $repo_folder/l10n $locale
             else
-                php -l $cache_file 2>&1 1>/dev/null
-                if [ $? -ne 0 ]
-                then
-                    # There are PHP errors, force the rebuild
-                    echored "PHP errors in $cache_file. Forcing rebuild."
-                    updated_locale=1
-                fi
+                echored "Folder $repo_folder/l10n/$locale does not exist. Run setup.sh to fix the issue."
             fi
-
-            if [ "$forceTMX" = true -o "$updated_english" = true -o "$updated_locale" -eq 1 ]
-            then
-                buildCache $locale_code
-            fi
-        else
-            echored "Folder $repo_folder/$locale_code does not exist."
         fi
-    fi
+    done
 }
 
 function updateCommL10n() {

diff --git a/app/scripts/setup.sh b/app/scripts/setup.sh
@@ -43,31 +43,23 @@ function setupVirtualEnv() {
 function initGeckoStringsRepo() {
     local repo_folder="gecko_strings_path"
     local repo_path="https://hg.mozilla.org/l10n-central"
-    local locale_list="gecko_strings_locales"
 
     # If repo_folder="gecko_strings_path", ${!repo_folder} is equal to $gecko_strings_path
     cd ${!repo_folder}
 
-    # Checkout all locales, including en-US
-    for locale in $(cat ${!locale_list})
-        do
-            if [ ! -d $locale ]
-            then
-                mkdir $locale
-            fi
-
-            if [ ! -d $locale/.hg ]
-            then
-                echogreen "Checking out the following repo:"
-                echogreen "$repo_path/$locale/"
-                if [ "$locale" = "en-US" ]
-                then
-                    hg clone https://hg.mozilla.org/l10n/gecko-strings/ $locale
-                else
-                    hg clone $repo_path/$locale $locale
-                fi
-            fi
-    done
+    # Clone source repository as en-US
+    if [ ! -d "en-US" ];
+    then
+        echogreen "Checking out firefox-l10n-source"
+        git clone https://github.com/mozilla-l10n/firefox-l10n-source en-US
+    fi
+
+    # Clone l10n monorepo as l10n
+    if [ ! -d "l10n" ];
+    then
+        echogreen "Checking out firefox-l10n-source"
+        git clone https://github.com/mozilla-l10n/firefox-l10n l10n
+    fi
 }
 
 function initCommL10nRepo() {

diff --git a/tests/units/Transvision/VersionControl.php b/tests/units/Transvision/VersionControl.php
@@ -15,7 +15,7 @@ public function getVCSDP()
                 'mozilla_org', 'git',
             ],
             [
-                'gecko_strings', 'hg',
+                'gecko_strings', 'git',
             ],
             [
                 'firefox_ios', 'git',
@@ -63,12 +63,6 @@ public function testVCSRepoName($a, $b)
     public function hgPathDP()
     {
         return [
-            [
-                'fr',
-                'gecko_strings',
-                'browser/updater/updater.ini:TitleText',
-                'https://hg.mozilla.org/l10n-central/fr/file/default/browser/updater/updater.ini',
-            ],
             [
                 'en-US',
                 'comm_l10n',
@@ -151,11 +145,17 @@ public function testGitPath($a, $b, $c, $d)
     public function getPathDP()
     {
         return [
+            [
+                'en-US',
+                'gecko_strings',
+                'browser/updater/updater.ini:TitleText',
+                'https://github.com/mozilla-l10n/firefox-l10n-source/blob/main/browser/updater/updater.ini',
+            ],
             [
                 'fr',
                 'gecko_strings',
                 'browser/updater/updater.ini:TitleText',
-                'https://hg.mozilla.org/l10n-central/fr/file/default/browser/updater/updater.ini',
+                'https://github.com/mozilla-l10n/firefox-l10n/blob/main/fr/browser/updater/updater.ini',
             ],
             [
                 'it',