metrics endpoints and indexer update

dbpedia · Jan 29, 2025 · 9638c06 · 9638c06
1 parent 213f17d
commit 9638c06
Show file tree

Hide file tree

Showing 58 changed files with 1,148 additions and 556 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -0,0 +1,26 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "type": "node",
+        "request": "launch",
+        "name": "Start Databus With AUTH0",
+        "program": "${workspaceFolder}/server/www",
+        "cwd": "${workspaceFolder}/server",
+        "env": {
+          "DATABUS_RESOURCE_BASE_URL": "http://localhost:3000",
+          "DATABUS_NAME": "Local Databus",
+          "DATABUS_ABSTRACT": "Go ham",
+          "DATABUS_ORG_ICON": "https://www.dbpedia.org/wp-content/uploads/2020/09/dbpedia-org-logo.png",
+          "DATABUS_BANNER_COLOR": "\\#5b798d",
+          "DATABUS_DATABASE_URL": "http://localhost:3002",
+          "DATABUS_OIDC_ISSUER_BASE_URL": "https://kilt.eu.auth0.com",
+          "DATABUS_OIDC_CLIENT_ID": "e9eOLS9IkGvuyBl7cBorWUQTNgbqejhc",
+          "DATABUS_OIDC_SECRET": "0RVo9jMnbhPnkR6Ttz0aXQRTcRuSz5DpqyUEjbbcbgRuGA4rbwCjnHM2cOlTrv9q",
+          "LOOKUP_BASE_URL": "http://localhost:3004",
+          "MAX_WORKERS": "1",
+          "METRICS_PORT": "3001"
+        }
+      }
+    ]
+  }
diff --git a/devenv/docker-compose.yml b/devenv/docker-compose.yml
@@ -1,22 +1,32 @@
 # Starts everything except the databus
 version: "3.0"
+name: databus_nodejs_devenv
 services:
   gstore:
-    image: gstore #dbpedia/gstore:dev
-    container_name: devenv_gstore
+    image: dbpedia/gstore:dev
+    container_name: databus_devenv_gstore
     environment: 
       STORAGE_USER: "dba"
       STORAGE_PASS: "everyoneknows"
       STORAGE_SPARQL_ENDPOINT_URI: http://virtuoso:8890/sparql
-      GSTORE_LOCALHOST_CONTEXT_FALLBACK_URL: http://172.17.0.1
+      DEFAULT_JSONLD_LOCALHOST_CONTEXT: http://localhost:3000/res/context.jsonld
+      DEFAULT_JSONLD_LOCALHOST_CONTEXT_LOCATION: http://host.docker.internal:3000/res/context.jsonld
     ports:
       - "3002:8080"
     volumes:
       - ./data/gstore/git:/gstore/git
       - ./data/gstore/logs:/gstore/logs
+  lookup:
+    image: lookup:dev
+    container_name: databus_devenv_lookup
+    ports:
+      - "3004:8082"
+    volumes:
+      - ./data/index/:/index
+      - ../search/servlet-config.yml:/resources/config.yml
   virtuoso:
     image: "openlink/virtuoso-opensource-7:latest"
-    container_name: devenv_virtuoso
+    container_name: databus_devenv_virtuoso
     environment:
       DBA_PASSWORD: "everyoneknows"
       SPARQL_UPDATE: "true"
@@ -29,7 +39,30 @@ services:
         echo 'grant SPARQL_LOAD_SERVICE_DATA to \"SPARQL\";' > /opt/virtuoso-opensource/initdb.d/ini.sql &&
         echo 'grant SPARQL_SPONGE to \"SPARQL\";' >> /opt/virtuoso-opensource/initdb.d/ini.sql &&
         echo 'grant SPARQL_SELECT_FED to \"SPARQL\";' > /opt/virtuoso-opensource/initdb.d/ini.sql &&
-        /virtuoso-entrypoint.sh
-      "
-      
-  
+        /virtuoso-entrypoint.sh"
+  prometheus:
+    image: prom/prometheus
+    container_name: databus_devenv_prometheus
+    ports:
+      - "3005:9090"
+    volumes:
+      - ./data/prometheus:/prometheus
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"  
+    restart: unless-stopped
+  grafana:
+    image: grafana/grafana
+    container_name: databus_devenv_grafana
+    ports:
+      - "3006:3000"
+    volumes:
+      - ./data/grafana:/var/lib/grafana  
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_USERS_ALLOW_SIGN_UP=false
+    restart: unless-stopped
+    depends_on:
+      - prometheus
diff --git a/devenv/prometheus.yml b/devenv/prometheus.yml
@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 15s 
+
+scrape_configs:
+  - job_name: 'databus-server' 
+    static_configs:
+      - targets: ['host.docker.internal:3001']
diff --git a/docker-compose-4-swarm.yml b/docker-compose-4-swarm.yml
@@ -0,0 +1,56 @@
+version: "3.8"  # Updated version for Swarm compatibility
+services:
+  ${CLIENT_ID}_databus:
+    image: "docker.io/dbpedia/databus"
+    ports:
+      - ${CLIENT_ID}_3000:3000 # HTTP: Databus web UI
+    environment:
+      DATABUS_RESOURCE_BASE_URL: ${DATABUS_RESOURCE_BASE_URL}
+      DATABUS_DATABASE_URL: "http://${CLIENT_ID}_gstore:8080"
+      DATABUS_OIDC_ISSUER_BASE_URL: ${DATABUS_OIDC_ISSUER_BASE_URL}
+      DATABUS_NAME: ${DATABUS_NAME}
+      DATABUS_ORG_ICON: ${DATABUS_ORG_ICON}
+      DATABUS_BANNER_COLOR: ${DATABUS_BANNER_COLOR}
+      DATABUS_OIDC_CLIENT_ID: ${DATABUS_OIDC_CLIENT_ID}
+      DATABUS_OIDC_SECRET: ${DATABUS_OIDC_SECRET}
+      DATABUS_PRIVATE_MODE: ${DATABUS_PRIVATE_MODE}
+      DATABUS_PROXY_SERVER_ENABLE: ${DATABUS_PROXY_SERVER_ENABLE}
+      DATABUS_PROXY_SERVER_USE_ACME: ${DATABUS_PROXY_SERVER_USE_ACME}
+      DATABUS_PROXY_SERVER_OWN_CERT: ${DATABUS_PROXY_SERVER_OWN_CERT}
+      DATABUS_PROXY_SERVER_OWN_CERT_KEY: ${DATABUS_PROXY_SERVER_OWN_CERT_KEY}
+      DATABUS_PROXY_SERVER_HOSTNAME: ${DATABUS_PROXY_SERVER_HOSTNAME}
+    volumes:
+      - ./data/${CLIENT_ID}/dav/:/databus/server/dav/
+      - ./data/${CLIENT_ID}/sqlite/:/databus/server/sqlite
+      - ./data/${CLIENT_ID}/keypair/:/databus/server/keypair
+      - ./data/${CLIENT_ID}/tls/:/tls:ro
+      - ./data/${CLIENT_ID}/tls/caddy:/root/.local/share/caddy/
+
+  ${CLIENT_ID}_gstore:
+    image: "docker.io/dbpedia/gstore"
+    environment:
+      STORAGE_USER: ${VIRTUOSO_USER}
+      STORAGE_PASS: ${VIRTUOSO_PASSWORD}
+      STORAGE_SPARQL_ENDPOINT_URI: http://${CLIENT_ID}_virtuoso:8890/sparql
+    ports:
+      - "127.0.0.1:${CLIENT_ID}_3002:8080"
+    volumes:
+      - ./data/${CLIENT_ID}/gstore/repo:/gstore/git
+      - ./data/${CLIENT_ID}/gstore/logs:/gstore/logs
+
+  ${CLIENT_ID}_virtuoso:
+    image: "docker.io/openlink/virtuoso-opensource-7"
+    environment:
+      DBA_PASSWORD: ${VIRTUOSO_PASSWORD}
+      SPARQL_UPDATE: "true"
+      DEFAULT_GRAPH: ${DATABUS_RESOURCE_BASE_URL}
+    ports:
+      - "127.0.0.1:${CLIENT_ID}_3003:8890"
+    volumes:
+      - ./data/${CLIENT_ID}/virtuoso:/database
+    entrypoint: /bin/bash -c "
+        echo 'grant SPARQL_LOAD_SERVICE_DATA to \"SPARQL\";' > /opt/virtuoso-opensource/initdb.d/ini.sql &&
+        echo 'grant SPARQL_SPONGE to \"SPARQL\";' >> /opt/virtuoso-opensource/initdb.d/ini.sql &&
+        echo 'grant SPARQL_SELECT_FED to \"SPARQL\";' > /opt/virtuoso-opensource/initdb.d/ini.sql &&
+        /virtuoso-entrypoint.sh
+      "
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,7 +1,8 @@
 version: "3.0"
+name: "databus-nodejs-dockerized"
 services:
   databus:
-    image: "docker.io/dbpedia/databus"
+    image: "databus-nodejs"
     ports:
       - 3000:3000 # HTTP:  Databus web UI
 #      - 80:80       # ** uncomment if proxy enabled only** HTTP port of included proxy (caddy) necessary for Auto-HTTPS via ACME and HTTP->HTTPS redirect                                                  

diff --git a/model/account.php b/model/account.php
@@ -102,7 +102,7 @@
     "@id": "foaf:primaryTopic",
     "@type": "@id"
   },
-  "name": {"@id": "foaf:name"},
+  "displayName": {"@id": "foaf:name"},
   "account": {
     "@id": "databus:account",
     "@type": "@id"

diff --git a/model/artifact.php b/model/artifact.php
@@ -30,8 +30,8 @@
 	sh:property [
 	  sh:path [ sh:inversePath rdf:type ] ;
 	    sh:nodekind sh:IRI ;
-      sh:pattern "/[a-zA-Z0-9\\\\-_]{4,}/[a-zA-Z0-9\\\\-_\\\\.]{3,}/[a-zA-Z0-9\\\\-_\\\\.]{3,}$" ;
-      sh:message "IRI for databus:Artifact must match /[a-zA-Z0-9\\\\-_]{4,}/[a-zA-Z0-9\\\\-_\\\\.]{3,}/[a-zA-Z0-9\\\\-_\\\\.]{3,}$"@en ;
+      sh:pattern "^[\\\\w+.-]+:\\\\/\\\\/[\\\\w+.:-]+\\\\/[\\\\w+.-]{4,}(?:\\\\/[\\\\w+.-]{3,}){2,2}$" ;
+      sh:message "IRI for databus:Artifact must be a 3-segment URI and match ^[\\\\w+.-]+:\\\\/\\\\/[\\\\w+.:-]+\\\\/[\\\\w+.-]{4,}(?:\\\\/[\\\\w+.-]{3,}){3,3}$"@en ;
 	] .';
 
 $example='"@type": "Artifact",';

diff --git a/model/generated/shacl/version.shacl b/model/generated/shacl/version.shacl
@@ -26,8 +26,8 @@
 	sh:property [
 		sh:path [ sh:inversePath rdf:type ] ;
 		  sh:nodekind sh:IRI ;
-		sh:pattern "/[a-zA-Z0-9\\-_]{4,}/[a-zA-Z0-9\\-_\\.]{1,}/[a-zA-Z0-9\\-_\\.]{1,}/[a-zA-Z0-9\\-_\\.]{1,}$" ;
-		sh:message "IRI for databus:Version must match /USER/GROUP/ARTIFACT/VERSION , |USER|>3"@en ;
+		sh:pattern "^[\\w+.-]+:\\/\\/[\\w+.:-]+\\/[\\w+.-]{4,}(?:\\/[\\w+.-]{3,}){3,3}$" ;
+		sh:message "IRI for databus:Artifact must be a 4-segment URI and match ^[\\w+.-]+:\\/\\/[\\w+.:-]+\\/[\\w+.-]{4,}(?:\\/[\\w+.-]{3,}){3,3}$)"@en ;
   ] . 
 
 <#title-version>

diff --git a/model/group.php b/model/group.php
@@ -30,8 +30,8 @@
 	sh:property [
 	  sh:path [ sh:inversePath rdf:type ] ;
 	    sh:nodekind sh:IRI ;
-      sh:pattern "/[a-zA-Z0-9\\\\-_]{4,}/[a-zA-Z0-9\\\\-_\\\\.]{3,}$" ;
-      sh:message "IRI for databus:Group must match /[a-zA-Z0-9\\\\-_]{4,}/[a-zA-Z0-9\\\\-_\\\\.]{3,}$"@en ;
+      sh:pattern "^[\\\\w+.-]+:\\\\/\\\\/[\\\\w+.:-]+\\\\/[\\\\w+.-]{4,}\\\\/[\\\\w+.-]{3,}$" ;
+      sh:message "IRI for databus:Group must be a 2-segment URI and match ^[\\\\w+.-]+:\\\\/\\\\/[\\\\w+.:-]+\\\\/[\\\\w+.-]{4,}\\\\/[\\\\w+.-]{3,}$"@en ;
 	] .';
 
 $example='"@type": "Group",';

diff --git a/model/version.php b/model/version.php
@@ -40,8 +40,8 @@
 	sh:property [
 		sh:path [ sh:inversePath rdf:type ] ;
 		  sh:nodekind sh:IRI ;
-		sh:pattern "/[a-zA-Z0-9\\\\-_]{4,}/[a-zA-Z0-9\\\\-_\\\\.]{1,}/[a-zA-Z0-9\\\\-_\\\\.]{1,}/[a-zA-Z0-9\\\\-_\\\\.]{1,}$" ;
-		sh:message "IRI for databus:Version must match /USER/GROUP/ARTIFACT/VERSION , |USER|>3"@en ;
+		sh:pattern "^[\\\\w+.-]+:\\\\/\\\\/[\\\\w+.:-]+\\\\/[\\\\w+.-]{4,}(?:\\\\/[\\\\w+.-]{3,}){3,3}$" ;
+		sh:message "IRI for databus:Version must be a 4-segment URI and match ^[\\\\w+.-]+:\\\\/\\\\/[\\\\w+.:-]+\\\\/[\\\\w+.-]{4,}(?:\\\\/[\\\\w+.-]{3,}){3,3}$"@en ;
   ] . ';
 
 

diff --git a/public/js/components/collection-hierarchy-two/collection-hierarchy.html b/public/js/components/collection-hierarchy-two/collection-hierarchy.html
@@ -333,7 +333,7 @@
                 </div>
 
                 <entity-card style="flex: 1;" label="result.label[0].highlight" type="result.typeName[0].value"
-                  uri="result.resource[0].value" absolute="true"  desc="result.comment[0].highlight">
+                  uri="result.id[0].value" absolute="true"  desc="result.comment[0].highlight">
                 </entity-card>
 
               </div>
@@ -556,7 +556,7 @@
                 </div>
 
                 <entity-card style="flex: 1;" label="result.label[0].highlight" type="result.typeName[0].value"
-                  uri="result.resource[0].value" absolute="true" desc="result.comment[0].highlight">
+                  uri="result.id[0].value" absolute="true" desc="result.comment[0].highlight">
                 </entity-card>
 
               </div>

diff --git a/public/js/components/collection-hierarchy-two/collection-hierarchy.js b/public/js/components/collection-hierarchy-two/collection-hierarchy.js
@@ -190,18 +190,18 @@ SELECT ?file WHERE {
   ctrl.addToCollection = function (source, view, result) {
 
     if (ctrl.isInCollection(result)) {
-      QueryNode.removeChildByUri(ctrl.root, result.resource[0].value);
+      QueryNode.removeChildByUri(ctrl.root, result.id[0].value);
     }
     else {
       if (result.typeName[0].value == 'Group') {
-        let node = new QueryNode(result.resource[0].value, DATAID_GROUP_PROPERTY);
+        let node = new QueryNode(result.id[0].value, DATAID_GROUP_PROPERTY);
 
         source.childNodes.push(node);
       }
 
       if (result.typeName[0].value == 'Artifact') {
 
-        var artifactUri = result.resource[0].value;
+        var artifactUri = result.id[0].value;
         let groupUri = DatabusUtils.navigateUp(artifactUri);
         let groupNode = QueryNode.findChildByUri(ctrl.root, groupUri);
 
@@ -295,7 +295,7 @@ SELECT ?file WHERE {
   }
 
   ctrl.isInCollection = function (result) {
-    let uri = result.resource[0].value;
+    let uri = result.id[0].value;
     let node = QueryNode.findChildByUri(ctrl.root, uri);
     return node != null;
   }

diff --git a/public/js/components/collection-search/collection-search.html b/public/js/components/collection-search/collection-search.html
@@ -42,7 +42,7 @@
     <div class="dataset-name" ng-repeat="result in $ctrl.results">
       <div style="display:flex; align-items:flex-end;">
         <entity-card style="flex: 1;" label="result.label[0].highlight" type="result.typeName[0].value"
-          uri="result.resource[0].value" desc="result.comment[0].highlight">
+          uri="result.id[0].value" desc="result.comment[0].highlight">
         </entity-card>
 
         <div>

diff --git a/public/js/components/collection-search/collection-search.js b/public/js/components/collection-search/collection-search.js
@@ -48,7 +48,7 @@ function CollectionSearchController(collectionManager, $http, $interval, $sce) {
 
   // TODO Fabian
   ctrl.isInCollection = function (result) {
-    let uri = result.resource[0].value;
+    let uri = result.id[0].value;
     let node = QueryNode.findChildByUri(ctrl.root, uri);
 
     return node != null;
@@ -61,19 +61,19 @@ function CollectionSearchController(collectionManager, $http, $interval, $sce) {
     var sourceNode = QueryNode.findChildByUri(ctrl.root, currentSource);
 
     if (result.inCollection) {
-      QueryNode.removeChildByUri(ctrl.root, result.resource[0].value);
+      QueryNode.removeChildByUri(ctrl.root, result.id[0].value);
     }
     else {
       if (result.typeName[0].value == 'Group') {
-        let node = new QueryNode(result.resource[0].value, 'databus:group');
+        let node = new QueryNode(result.id[0].value, 'databus:group');
 
         sourceNode.addChild(node);
         ctrl.onComponentAdded();
       }
 
       if (result.typeName[0].value == 'Artifact') {
 
-        var artifactUri = result.resource[0].value;
+        var artifactUri = result.id[0].value;
         let groupUri = DatabusUtils.navigateUp(artifactUri);
         let groupNode = QueryNode.findChildByUri(ctrl.root, groupUri);
 

diff --git a/public/js/components/nav-search/nav-search.html b/public/js/components/nav-search/nav-search.html
@@ -60,13 +60,13 @@
               resource.</i></div>
 
 
-          <div class="result-item" ng-click="$ctrl.navigateTo(result.resource[0])" ng-repeat="result in $ctrl.results">
+          <div class="result-item" ng-click="$ctrl.navigateTo(result.id[0])" ng-repeat="result in $ctrl.results">
             <type-tag width="48" height="48" ng-if="result.typeName[0]" type="result.typeName[0]"></type-tag>
             <div>
               <div class="item-header">
                 <p id="const" class="item-label" ng-bind-html="$ctrl.formatResult(result.label[0])"></p>
-                <div ng-if="result.resource[0]">
-                  <uri-breadcrumbs uri="result.resource[0]"></uri-breadcrumbs>
+                <div ng-if="result.id[0]">
+                  <uri-breadcrumbs uri="result.id[0]"></uri-breadcrumbs>
                 </div>
 
               </div>

diff --git a/public/js/components/search/search.html b/public/js/components/search/search.html
@@ -38,7 +38,7 @@
           resource.</i></div>
       <div class="dataset-name" ng-repeat="result in $ctrl.results" style="margin-bottom: 1em;">
 
-        <entity-card label="result.label[0]" type="result.typeName[0]" uri="result.resource[0]"
+        <entity-card label="result.label[0]" type="result.typeName[0]" uri="result.id[0]"
           desc="result.comment[0]">
         </entity-card>
 

diff --git a/public/js/page-controller/group-controller.js b/public/js/page-controller/group-controller.js
@@ -311,7 +311,7 @@ function GroupPageController($scope, $http, $sce, $interval, $location, collecti
         var result = response.data.docs[r];
 
         for (var artifact of $scope.artifacts) {
-          if (result.resource[0] == artifact.uri) {
+          if (result.id[0] == artifact.uri) {
             $scope.searchResult.push(artifact);
           }
         }

diff --git a/public/js/utils/databus-uris.js b/public/js/utils/databus-uris.js
@@ -18,6 +18,7 @@ class DatabusUris {
   static DATABUS_VERSION_PROPERTY = 'https://dataid.dbpedia.org/databus#version';
   static DATABUS_GROUP_PROPERTY = 'https://dataid.dbpedia.org/databus#group';
   static DATABUS_ACCOUNT_PROPERTY = 'https://dataid.dbpedia.org/databus#account';
+  static DATABUS_NAME = 'https://dataid.dbpedia.org/databus#name';
   static DATABUS_ARTIFACT_PROPERTY = 'https://dataid.dbpedia.org/databus#artifact';
   static DATABUS_FORMAT = 'https://dataid.dbpedia.org/databus#format';
   static DATABUS_FORMAT_EXTENSION = 'https://dataid.dbpedia.org/databus#formatExtension';

diff --git a/public/templates/index.ejs b/public/templates/index.ejs
@@ -50,26 +50,12 @@
               </div>
             </div>
           </div>
-
-
         </div>
       </section>
-
-
       <section class="section front">
         <div class="container">
           <div class="columns">
-
             <div class="column is-4 stats" style="padding-right: 3em;">
-
-
-
-
-
-
-
-
-
               <p class="section-label">Recent Activity</p>
               <div ng-if="activityChartData.isLoading" style="text-align:center">
                 <div class="loading-rect placeholder-400" style="min-height: 400px;">

diff --git a/search/Dockerfile b/search/Dockerfile