From 9aecfb81610fa7a88f76d5265a8a0164fe2de37e Mon Sep 17 00:00:00 2001 From: Eric Chen Date: Wed, 18 Dec 2024 13:22:05 -0500 Subject: [PATCH 1/4] Remove an extra _catalog api call that is made during install against an oras repo Without the _catalog call, it is already enough to do all necessary filtering for oras package installs. Doing the _catalog call is unnecessary. As _catalog is also not part of oci specs, per https://github.com/opencontainers/distribution-spec/issues/22 , the return behavior is also undefined, which can cause the install commands to fail depending on the specific implementation of the OCI registry, such as in JFrog Artifactory when combined with namespaces. Signed-off-by: Eric Chen --- src/cls/IPM/Repo/Oras/PackageService.cls | 93 +++++++++--------------- 1 file changed, 33 insertions(+), 60 deletions(-) diff --git a/src/cls/IPM/Repo/Oras/PackageService.cls b/src/cls/IPM/Repo/Oras/PackageService.cls index 2b7e1b76..a629ff6e 100644 --- a/src/cls/IPM/Repo/Oras/PackageService.cls +++ b/src/cls/IPM/Repo/Oras/PackageService.cls @@ -75,69 +75,42 @@ Method ListModules(pSearchCriteria As %IPM.Repo.SearchCriteria) As %ListOfObject #; Get all modules Set tList = ##class(%Library.ListOfObjects).%New() - Set request = ..GetHttpRequest() - - #; Make GET request - // response is a JSON structure like {"repositories":["package1", "package2", ...]} - Set tSC=request.Get(..PathPrefix _ "/v2/_catalog") - $$$ThrowOnError(tSC) - Set response=request.HttpResponse - If response.StatusCode'=200 { - // todo improve error processing - Set data = response.Data.Read() - Write !, "Error! " _ response.StatusCode _ ": " _ response.ReasonPhrase - Return "" - } - - #; Handle results - Set json = response.Data.ReadLine() - Set data = ##class(%DynamicAbstractObject).%FromJSON(json) - Set iter = data.repositories.%GetIterator() - While iter.%GetNext(.key, .package, .type) { - If (package="") { - Continue - } - #; filter by module name - If (name'="") && (package'=name) { + + #; get all versions + Set allTagsString = ..GetAllTagsPy(..Location, name, "", client) + Set allTagsList = $LISTFROMSTRING(allTagsString, ", ") + Set pointer = 0 + While $ListNext(allTagsList,pointer,tag) { + #; filter by version + Set tVersion = ##class(%IPM.General.SemanticVersion).FromString(tag) + If 'tVersion.Satisfies(tVersionExpression) { Continue } + + #; get metadata from annotations + Set metadata = ..GetPackageMetadataPy(..Location, name, "", tag, client) + set artifactMetadata = ##class(%IPM.Repo.Oras.ArtifactMetadata).%New() + Do artifactMetadata.%JSONImport(metadata) + + Set tModRef = ##class(%IPM.Storage.ModuleInfo).%New() + Set tModRef.Name = artifactMetadata.ImageTitle + Set tModRef.Repository = artifactMetadata.ImageSource + Set tModRef.VersionString = artifactMetadata.ImageVersion + Set tModRef.Description = artifactMetadata.ImageDescription + Set tModRef.Deployed = artifactMetadata.IPMDeployed + #; If $IsObject(item."platform_versions") { + #; Set tIterPVer = item."platform_versions".%GetIterator() + #; While tIterPVer.%GetNext(.tPVerKey, .platformVersion) { + #; Do tModRef.PlatformVersions.Insert(platformVersion) + #; } + #; } + Set tModRef.AllVersions = allTagsString + Set tModRef.Origin = artifactMetadata.IPMOrigin + Do tList.Insert(tModRef) - #; get all versions - Set allTagsString = ..GetAllTagsPy(..Location, package, "", client) - Set allTagsList = $LISTFROMSTRING(allTagsString, ", ") - Set pointer = 0 - While $ListNext(allTagsList,pointer,tag) { - #; filter by version - Set tVersion = ##class(%IPM.General.SemanticVersion).FromString(tag) - If 'tVersion.Satisfies(tVersionExpression) { - Continue - } - - #; get metadata from annotations - Set metadata = ..GetPackageMetadataPy(..Location, package, "", tag, client) - set artifactMetadata = ##class(%IPM.Repo.Oras.ArtifactMetadata).%New() - Do artifactMetadata.%JSONImport(metadata) - - Set tModRef = ##class(%IPM.Storage.ModuleInfo).%New() - Set tModRef.Name = artifactMetadata.ImageTitle - Set tModRef.Repository = artifactMetadata.ImageSource - Set tModRef.VersionString = artifactMetadata.ImageVersion - Set tModRef.Description = artifactMetadata.ImageDescription - Set tModRef.Deployed = artifactMetadata.IPMDeployed - #; If $IsObject(item."platform_versions") { - #; Set tIterPVer = item."platform_versions".%GetIterator() - #; While tIterPVer.%GetNext(.tPVerKey, .platformVersion) { - #; Do tModRef.PlatformVersions.Insert(platformVersion) - #; } - #; } - Set tModRef.AllVersions = allTagsString - Set tModRef.Origin = artifactMetadata.IPMOrigin - Do tList.Insert(tModRef) - - #; If not all versions are requested, return the latest one - If 'pSearchCriteria.AllVersions, name="" { - Quit - } + #; If not all versions are requested, return the latest one + If 'pSearchCriteria.AllVersions, name="" { + Quit } } Quit tList From 0d3dc3b8ec2eaee66c485f6af94f3094e8a87285 Mon Sep 17 00:00:00 2001 From: Eric Chen Date: Wed, 18 Dec 2024 15:02:42 -0500 Subject: [PATCH 2/4] Fix incorrect error handling in python Signed-off-by: Eric Chen --- src/cls/IPM/Repo/Oras/PackageService.cls | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cls/IPM/Repo/Oras/PackageService.cls b/src/cls/IPM/Repo/Oras/PackageService.cls index a629ff6e..873f5335 100644 --- a/src/cls/IPM/Repo/Oras/PackageService.cls +++ b/src/cls/IPM/Repo/Oras/PackageService.cls @@ -62,7 +62,6 @@ Method ListModules(pSearchCriteria As %IPM.Repo.SearchCriteria) As %ListOfObject Set client = ..GetClient(..Location, ..Username, ..Password, ..Token, ..TokenAuthMethod) #; Parse search criteria - // The OCI /v2/_catalog endpoint always returns all packages. We will filter by name and version on the client side. Set name = $$$lcase(pSearchCriteria.Name) Set tVersionExpression = pSearchCriteria.VersionExpression Set tSC = ##class(%IPM.General.SemanticVersionExpression).FromString(pSearchCriteria.VersionExpression, .tVersionExpression) @@ -315,7 +314,10 @@ ClassMethod GetAllTagsPy(registry As %String, package As %String, namespace As % # convert from list to comma separated string return ", ".join(str(x) for x in tags) - except e: + except ValueError: + # ValueError indicates the package is missing and has no tags. Fail silently with no tags returned + return "" + except Exception as e: print("Error: ", repr(e)) return "" } From b9bd60729303204e7dd8fbaa644197127e7858a8 Mon Sep 17 00:00:00 2001 From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com> Date: Tue, 21 Jan 2025 17:11:30 -0500 Subject: [PATCH 3/4] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2192d60e..91814ebd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - #607: Uninstall reports deletion of non-classes - #606: Don't put garbage folders in tar archive - #652: Don't create extra needless mappings (could cause deadlock with parallel installation of dependencies) +- #669: Work with a wider variety of ORAS repos (removes _catalog call) ### Deprecated - #593 CSPApplication is deprecated in favor of WebApplication. User will be warned when installing a package containing CSPApplication. From 52f1e28a5b62beeefc6cdb8154aa0c68809812f8 Mon Sep 17 00:00:00 2001 From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com> Date: Tue, 21 Jan 2025 17:12:08 -0500 Subject: [PATCH 4/4] Update CHANGELOG.md putting it in the right place... --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91814ebd..43dbc183 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - #635: When calling the "package" command, the directory is now normalized to include trailing slash (or backslash). - #696: Fix a bug that caused error status to be ignored when publishing a module. - #700: Fix a bug due to incompatible conventions between SemVer and OCI tags +- #669: Work with a wider variety of ORAS repos (removes _catalog call) ### Security - @@ -92,7 +93,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - #607: Uninstall reports deletion of non-classes - #606: Don't put garbage folders in tar archive - #652: Don't create extra needless mappings (could cause deadlock with parallel installation of dependencies) -- #669: Work with a wider variety of ORAS repos (removes _catalog call) ### Deprecated - #593 CSPApplication is deprecated in favor of WebApplication. User will be warned when installing a package containing CSPApplication.