Skip to content

Commit

Permalink
bugfixes and dockerisation
Browse files Browse the repository at this point in the history
  • Loading branch information
holycrab13 committed Feb 12, 2025
1 parent 4289eea commit e62b9d7
Show file tree
Hide file tree
Showing 17 changed files with 266 additions and 72 deletions.
2 changes: 0 additions & 2 deletions .env
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ DATABUS_TITLE_COLOR= # Default is white
DATABUS_OIDC_ISSUER_BASE_URL=https://kilt.eu.auth0.com
DATABUS_OIDC_CLIENT_ID=K5PCEOr7OJGBGU9xN7SvBrX1RWDs4S4n
DATABUS_OIDC_SECRET=LiL_X1tzwRmReU3RO7kBlBdDopmVEGf4gj5Ve8No16kifyi3weXK7u6IS1Ttpl_q


DATABUS_PRIVATE_MODE=false

####### DATABASE SETTINGS #######
Expand Down
3 changes: 3 additions & 0 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
"name": "Start Databus With AUTH0",
"program": "${workspaceFolder}/server/www",
"cwd": "${workspaceFolder}/server",
"runtimeArgs": [
"--trace-warnings"
],
"env": {
"DATABUS_RESOURCE_BASE_URL": "http://localhost:3000",
"DATABUS_NAME": "Local Databus",
Expand Down
64 changes: 33 additions & 31 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,28 +1,34 @@
FROM node:18 AS installer
# Use a consistent Node.js version across build and release stages
FROM node:20 AS installer

COPY ./server/package.json ./server/package-lock.json /server/
COPY ./public/package.json ./public/package-lock.json /public/
# Set working directory
WORKDIR /server

# Set up the NPM projects:
RUN cd /server && \
npm install && \
cd ../public && \
npm install
# Copy package.json first to leverage Docker caching
COPY ./server/package.json ./server/package-lock.json ./
RUN npm install

WORKDIR /public
COPY ./public/package.json ./public/package-lock.json ./
RUN npm install

# Copy the rest of the application source files
WORKDIR /
COPY ./server /server
COPY ./public /public

FROM ubuntu:22.04 AS release

COPY --from=installer /server/node_modules /databus/server/node_modules
COPY --from=installer /public/node_modules /databus/public/node_modules
COPY . /databus

# Install node.js, Caddy as proxy server, and java.
# Install system dependencies, including build tools
RUN apt-get update && \
apt-get -y install curl wget systemctl debian-keyring debian-archive-keyring apt-transport-https && \
apt-get -y install curl wget systemctl build-essential python3 && \
wget https://github.com/caddyserver/caddy/releases/download/v2.7.6/caddy_2.7.6_linux_amd64.deb && \
dpkg -i ./caddy_2.7.6_linux_amd64.deb && \
caddy version && \
systemctl daemon-reload && systemctl enable --now caddy && \
curl -sL https://deb.nodesource.com/setup_16.x | bash - && \
curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
apt-get update && \
apt-get -y install \
nodejs \
Expand All @@ -34,28 +40,24 @@ RUN apt-get update && \
npm -v && \
java -version

# Disable the proxy server by default:
ENV DATABUS_PROXY_SERVER_ENABLE=false

# When using the proxy, use ACME by default:
ENV DATABUS_PROXY_SERVER_USE_ACME=true
WORKDIR /databus

# When not using ACME, what is the name of the own certificate file?
ENV DATABUS_PROXY_SERVER_OWN_CERT="cert.pem"
# Copy application source code first
COPY --from=installer /server /databus/server
COPY --from=installer /public /databus/public

# When not using ACME, what is the name of the own certificate's key file?
ENV DATABUS_PROXY_SERVER_OWN_CERT_KEY="key.pem"
# Rebuild sqlite3 in the release stage to ensure it's correctly compiled
RUN cd /databus/server && npm rebuild sqlite3 && npm install --omit=dev

# What is the hostname of this machine, when using the proxy server?
# It is necessary to know this, in order to set up ACME etc.
# Note: the host name should be identical to DATABUS_RESOURCE_BASE_URL,
# but without specifying a port, protocol i.e. HTTP(S) etc.
ENV DATABUS_PROXY_SERVER_HOSTNAME="my-databus.org"

# Define the volume for the TLS certificate:
# Define the volume for the TLS certificate
VOLUME /tls

COPY . /databus
# Set environment variables
ENV DATABUS_PROXY_SERVER_ENABLE=false
ENV DATABUS_PROXY_SERVER_USE_ACME=true
ENV DATABUS_PROXY_SERVER_OWN_CERT="cert.pem"
ENV DATABUS_PROXY_SERVER_OWN_CERT_KEY="key.pem"
ENV DATABUS_PROXY_SERVER_HOSTNAME="my-databus.org"

WORKDIR /databus
# Set up entrypoint
ENTRYPOINT [ "/bin/bash", "./setup.sh" ]
37 changes: 24 additions & 13 deletions docker-compose-4-swarm.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
version: "3.8" # Updated version for Swarm compatibility
services:
${CLIENT_ID}_databus:
image: "docker.io/dbpedia/databus"
ports:
- ${CLIENT_ID}_3000:3000 # HTTP: Databus web UI
${CLIENT_ID}_server:
image: "databus-rc12"
environment:
DATABUS_RESOURCE_BASE_URL: ${DATABUS_RESOURCE_BASE_URL}
DATABUS_DATABASE_URL: "http://${CLIENT_ID}_gstore:8080"
DATABUS_DATABASE_URL: "http://databus_gstore:8080"
LOOKUP_BASE_URL: "http://lookup:8082"
DATABUS_OIDC_ISSUER_BASE_URL: ${DATABUS_OIDC_ISSUER_BASE_URL}
DATABUS_NAME: ${DATABUS_NAME}
DATABUS_ORG_ICON: ${DATABUS_ORG_ICON}
Expand All @@ -25,20 +24,20 @@ services:
- ./data/${CLIENT_ID}/keypair/:/databus/server/keypair
- ./data/${CLIENT_ID}/tls/:/tls:ro
- ./data/${CLIENT_ID}/tls/caddy:/root/.local/share/caddy/

${CLIENT_ID}_gstore:
networks:
- ${CLIENT_ID}_network
databus_gstore:
image: "docker.io/dbpedia/gstore"
environment:
STORAGE_USER: ${VIRTUOSO_USER}
STORAGE_PASS: ${VIRTUOSO_PASSWORD}
STORAGE_SPARQL_ENDPOINT_URI: http://${CLIENT_ID}_virtuoso:8890/sparql
ports:
- "127.0.0.1:${CLIENT_ID}_3002:8080"
STORAGE_SPARQL_ENDPOINT_URI: http://databus_virtuoso:8890/sparql
volumes:
- ./data/${CLIENT_ID}/gstore/repo:/gstore/git
- ./data/${CLIENT_ID}/gstore/logs:/gstore/logs

${CLIENT_ID}_virtuoso:
networks:
- ${CLIENT_ID}_network
databus_virtuoso:
image: "docker.io/openlink/virtuoso-opensource-7"
environment:
DBA_PASSWORD: ${VIRTUOSO_PASSWORD}
Expand All @@ -53,4 +52,16 @@ services:
echo 'grant SPARQL_SPONGE to \"SPARQL\";' >> /opt/virtuoso-opensource/initdb.d/ini.sql &&
echo 'grant SPARQL_SELECT_FED to \"SPARQL\";' > /opt/virtuoso-opensource/initdb.d/ini.sql &&
/virtuoso-entrypoint.sh
"
""
networks:
- ${CLIENT_ID}_network
databus_lookup:
image: lookup:dev
volumes:
- ./data/index/:/index
- ../search/servlet-config.yml:/resources/config.yml
networks:
- ${CLIENT_ID}_network
networks:
${CLIENT_ID}_network:
driver: overlay
21 changes: 15 additions & 6 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,16 @@ version: "3.0"
name: "databus-nodejs-dockerized"
services:
databus:
image: "databus-nodejs"
image: "databus-rc12"
ports:
- 3000:3000 # HTTP: Databus web UI
# - 80:80 # ** uncomment if proxy enabled only** HTTP port of included proxy (caddy) necessary for Auto-HTTPS via ACME and HTTP->HTTPS redirect
# - 443:4000 # ** uncomment if proxy enabled only** HTTPS: Databus web UI via caddy proxy
environment:
DATABUS_RESOURCE_BASE_URL: ${DATABUS_RESOURCE_BASE_URL}
DATABUS_DATABASE_URL: "http://gstore:8080"
LOOKUP_BASE_URL: "http://lookup:8082"
METRICS_PORT: "3001"
DATABUS_OIDC_ISSUER_BASE_URL: ${DATABUS_OIDC_ISSUER_BASE_URL}
DATABUS_NAME: ${DATABUS_NAME}
DATABUS_ABSTRACT: ${DATABUS_ABSTRACT}
Expand Down Expand Up @@ -38,9 +40,9 @@ services:
# gstore now is preloading the context for localhost from a configured remote location
# only needed for dev env with context at localhost
DEFAULT_JSONLD_LOCALHOST_CONTEXT: http://localhost:3000/res/context.jsonld
DEFAULT_JSONLD_LOCALHOST_CONTEXT_LOCATION: https://databus.dbpedia.org/res/context.jsonld
DEFAULT_JSONLD_LOCALHOST_CONTEXT_LOCATION: http://databus:3000/res/context.jsonld
ports:
- "127.0.0.1:3002:8080"
- 3002:8080
volumes:
- ./data/gstore/repo:/gstore/git
- ./data/gstore/logs:/gstore/logs
Expand All @@ -51,12 +53,19 @@ services:
SPARQL_UPDATE: "true"
DEFAULT_GRAPH: ${DATABUS_RESOURCE_BASE_URL}
ports:
- "127.0.0.1:3003:8890"
- 3003:8890
volumes:
- ./data/virtuoso:/database
entrypoint: /bin/bash -c "
echo 'grant SPARQL_LOAD_SERVICE_DATA to \"SPARQL\";' > /opt/virtuoso-opensource/initdb.d/ini.sql &&
echo 'grant SPARQL_SPONGE to \"SPARQL\";' >> /opt/virtuoso-opensource/initdb.d/ini.sql &&
echo 'grant SPARQL_SELECT_FED to \"SPARQL\";' > /opt/virtuoso-opensource/initdb.d/ini.sql &&
/virtuoso-entrypoint.sh
"
/virtuoso-entrypoint.sh"
lookup:
image: lookup:dev
container_name: databus_devenv_lookup
ports:
- 3004:8082
volumes:
- ./data/index/:/index
- ../search/servlet-config.yml:/resources/config.yml
10 changes: 5 additions & 5 deletions public/dist/main.js

Large diffs are not rendered by default.

6 changes: 4 additions & 2 deletions public/js/collections/databus-collection-manager.js
Original file line number Diff line number Diff line change
Expand Up @@ -600,8 +600,10 @@ class DatabusCollectionManager {

try {

var relativeUri = new URL(collectionUri).pathname;
response = await this.http.put(relativeUri, collectionJsonLd);
// var relativeUri = new URL(collectionUri).pathname;
// response = await this.http.put(relativeUri, collectionJsonLd);

response = await this.http.post('/api/register', collectionJsonLd);

} catch (errResponse) {
console.log(errResponse);
Expand Down
64 changes: 64 additions & 0 deletions search/indexer/collection.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
version: "1.0"
indexMode: INDEX_SPARQL_ENDPOINT
sparqlEndpoint: http://virtuoso:8890/sparql
indexFields:
# COLLECTION LABEL
- fieldName: label
documentVariable: collection
query: >
SELECT DISTINCT ?collection ?label WHERE {
GRAPH ?g {
?collection a <https://dataid.dbpedia.org/databus#Collection>.
?collection <http://purl.org/dc/terms/title> ?label.
}
#VALUES#
}
# COLLECTION ABSTRACT
- fieldName: abstract
documentVariable: collection
query: >
SELECT DISTINCT ?collection ?abstract WHERE {
GRAPH ?g {
?collection a <https://dataid.dbpedia.org/databus#Collection>.
?collection <http://purl.org/dc/terms/abstract> ?abstract.
}
#VALUES#
}
# COLLECTION NAME
- fieldName: part
documentVariable: collection
query: >
SELECT ?collection ?part WHERE {
GRAPH ?g {
?collection a <https://dataid.dbpedia.org/databus#Collection>.
?collection <https://dataid.dbpedia.org/databus#name> ?part .
}
#VALUES#
}
# PUBLISHER ACCOUNT NAME
- fieldName: publisher
documentVariable: collection
query: >
SELECT ?collection ?publisher WHERE {
GRAPH ?g1 {
?collection a <https://dataid.dbpedia.org/databus#Collection> .
?collection <https://dataid.dbpedia.org/databus#account> ?account .
}
GRAPH ?g2 {
?account <http://xmlns.com/foaf/0.1/accountName> ?publisher .
}
#VALUES#
}
# COLLECTIOn TYPE NAME
- fieldName: typeName
type: string
documentVariable: collection
query: >
SELECT DISTINCT ?collection ?typeName WHERE {
GRAPH ?g {
?collection a <https://dataid.dbpedia.org/databus#Collection> .
BIND("Collection" AS ?typeName)
}
#VALUES#
}
2 changes: 1 addition & 1 deletion server/app/api/lib/account-writer.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class AccountWriter extends ResourceWriter {
throw new ApiError(400, this.uri, `Specified account name ('${accountName}') must be at least 4 characters long.`, null);
}

this.userData = await this.createUserCallback(this.userData);
this.userData = await this.createUserCallback(this.userData.sub, accountName);
}

super.onValidateUser();
Expand Down
51 changes: 51 additions & 0 deletions server/app/api/lib/collection-writer.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
const DatabusUris = require('../../../../public/js/utils/databus-uris');
const DatabusUtils = require('../../../../public/js/utils/databus-utils.js');
const JsonldUtils = require('../../../../public/js/utils/jsonld-utils.js');
const ResourceWriter = require('./resource-writer.js');

class CollectionWriter extends ResourceWriter {

constructor(logger) {
super(logger);
}

async onCreateGraphs() {

var inputCollectionGraph = JsonldUtils.getGraphById(this.inputGraphs, this.uri);

var collectionGraph = {};
collectionGraph[DatabusUris.JSONLD_ID] = this.uri;
collectionGraph[DatabusUris.JSONLD_TYPE] = DatabusUris.DATABUS_COLLECTION;
collectionGraph[DatabusUris.DATABUS_NAME] = this.resource.getArtifact();
collectionGraph[DatabusUris.DATABUS_ACCOUNT_PROPERTY] = JsonldUtils.refTo(this.resource.getAccountURI());
collectionGraph[DatabusUris.DATABUS_COLLECTION_CONTENT] = inputCollectionGraph[DatabusUris.DATABUS_COLLECTION_CONTENT];

if(inputCollectionGraph[DatabusUris.DCT_TITLE] != null) {
collectionGraph[DatabusUris.DCT_TITLE] = inputCollectionGraph[DatabusUris.DCT_TITLE];
}

if(inputCollectionGraph[DatabusUris.DCT_DESCRIPTION] != null) {
collectionGraph[DatabusUris.DCT_DESCRIPTION] = inputCollectionGraph[DatabusUris.DCT_DESCRIPTION];
}

if(inputCollectionGraph[DatabusUris.DCT_ABSTRACT] != null) {
collectionGraph[DatabusUris.DCT_ABSTRACT] = inputCollectionGraph[DatabusUris.DCT_ABSTRACT];
} else if (collectionGraph[DatabusUris.DCT_DESCRIPTION] != null) {
collectionGraph[DatabusUris.DCT_DESCRIPTION] = DatabusUtils.createAbstractFromDescription(collectionGraph[DatabusUris.DCT_DESCRIPTION]);
}

var groupGraph = {};
groupGraph[DatabusUris.JSONLD_ID] = this.resource.getGroupURI();
groupGraph[DatabusUris.JSONLD_TYPE] = DatabusUris.DATABUS_GROUP;

return [
collectionGraph
];
}

getSHACLFilePath() {
return './res/shacl/collection.shacl'
}
}

module.exports = CollectionWriter;
Loading

0 comments on commit e62b9d7

Please sign in to comment.