Skip to content

Commit

Permalink
Bump to 12.4 and fix scripts a little
Browse files Browse the repository at this point in the history
  • Loading branch information
@ccakes
ccakes committed Aug 31, 2020
1 parent 05d4a71 commit 284d977
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 188 deletions.
58 changes: 13 additions & 45 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,70 +24,33 @@ RUN mkdir -p ${GOPATH}/src/github.com/timescale/ \
&& go build -o /go/bin/timescaledb-parallel-copy

############################
# Build TimescaleDB + is_jsonb_valid
# Add Timescale, PostGIS and Patroni
############################
FROM postgres:12.3 AS build
ARG TIMESCALEDB_VERSION

# curl -L -o is_jsonb_valid.tar.gz https://github.com/furstenheim/is_jsonb_valid/tarball/master
# tar xf ../is_jsonb_valid.tar.gz --strip-components 1

RUN \
set -x \
&& buildDeps="curl build-essential ca-certificates git python gnupg libc++-dev libc++abi-dev pkg-config glib2.0 cmake libssl-dev" \
&& apt-get update -y \
&& apt-get install -y --no-install-recommends ${buildDeps} \
&& apt-get install -y --no-install-recommends postgresql-server-dev-$PG_MAJOR \
&& mkdir -p /tmp/build \
&& curl -o /tmp/build/timescaledb-${TIMESCALEDB_VERSION}.tar.lzma -SL "https://github.com/timescale/timescaledb/releases/download/${TIMESCALEDB_VERSION}/timescaledb-${TIMESCALEDB_VERSION}.tar.lzma" \
&& cd /tmp/build \
&& tar xf /tmp/build/timescaledb-${TIMESCALEDB_VERSION}.tar.lzma -C /tmp/build/ \
&& cd /tmp/build/timescaledb \
&& ./bootstrap -DPROJECT_INSTALL_METHOD="docker" -DAPACHE_ONLY=1 -DREGRESS_CHECKS=OFF \
&& cd build \
&& make \
&& make install \
&& ls -l /usr/lib/postgresql/${PG_MAJOR}/lib/ \
&& strip /usr/lib/postgresql/${PG_MAJOR}/lib/timescaledb.so \
&& strip /usr/lib/postgresql/${PG_MAJOR}/lib/timescaledb-${TIMESCALEDB_VERSION}.so \
#
# Build is_jsonb_valid
&& curl -L -o is_jsonb_valid.tar.gz https://github.com/furstenheim/is_jsonb_valid/tarball/master \
&& mkdir is_jsonb_valid && cd is_jsonb_valid \
&& tar xf ../is_jsonb_valid.tar.gz --strip-components 1 \
&& make install

############################
# Add PostGIS and Patroni
############################
FROM postgres:12.3
FROM postgres:12.4
ARG PG_MAJOR
ARG POSTGIS_MAJOR
ARG TIMESCALEDB_VERSION

COPY --from=tools /go/bin/* /usr/local/bin/
COPY --from=build /usr/lib/postgresql/${PG_MAJOR}/lib/timescale*.so /usr/lib/postgresql/${PG_MAJOR}/lib/
COPY --from=build /usr/share/postgresql/${PG_MAJOR}/extension/timescaledb.control /usr/share/postgresql/${PG_MAJOR}/extension/timescaledb.control
COPY --from=build /usr/share/postgresql/${PG_MAJOR}/extension/timescaledb--*.sql /usr/share/postgresql/${PG_MAJOR}/extension/

COPY --from=build /usr/lib/postgresql/${PG_MAJOR}/lib/bitcode /usr/lib/postgresql/${PG_MAJOR}/lib/
COPY --from=build /usr/lib/postgresql/${PG_MAJOR}/lib/is_jsonb_valid.so /usr/lib/postgresql/${PG_MAJOR}/lib/
COPY --from=build /usr/share/postgresql/${PG_MAJOR}/extension/is_jsonb_valid* /usr/share/postgresql/${PG_MAJOR}/extension/


RUN set -x \
&& apt-get update -y \
&& apt-get install -y gcc curl procps python3-dev libpython3-dev libyaml-dev apt-transport-https ca-certificates \
&& echo "deb https://packagecloud.io/timescale/timescaledb/debian/ stretch main" > /etc/apt/sources.list.d/timescaledb.list \
&& curl -L https://packagecloud.io/timescale/timescaledb/gpgkey | apt-key add - \
&& apt-get update -y \
&& apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \
&& apt-get install -y --no-install-recommends \
postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \
postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \
postgis \
postgresql-$PG_MAJOR-pgrouting \
gcc curl python3-dev libpython3-dev libyaml-dev procps \
timescaledb-postgresql-$PG_MAJOR \
\
# Install Patroni
&& apt-get install -y --no-install-recommends \
python3 python3-pip python3-setuptools \
&& pip3 install wheel zipp==1.0.0 \
&& pip3 install awscli python-consul psycopg2-binary \
&& pip3 install https://github.com/zalando/patroni/archive/v1.6.5.zip \
\
Expand All @@ -97,6 +60,11 @@ RUN set -x \
&& rm -f wal-g.linux-amd64.tar.gz \
&& mv wal-g /usr/local/bin/ \
\
# Install vaultenv
&& curl -LO https://github.com/channable/vaultenv/releases/download/v0.13.1/vaultenv-0.13.1-linux-musl \
&& install -oroot -groot -m755 vaultenv-0.13.1-linux-musl /usr/bin/vaultenv \
&& rm vaultenv-0.13.1-linux-musl \
\
# Cleanup
&& rm -rf /var/lib/apt/lists/* \
\
Expand Down
5 changes: 5 additions & 0 deletions files/002_timescaledb_tune.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ NO_TS_TUNE=${NO_TS_TUNE:-""}
TS_TUNE_MEMORY=${TS_TUNE_MEMORY:-""}
TS_TUNE_NUM_CPUS=${TS_TUNE_NUM_CPUS:-""}

if [ ! -x "/usr/local/bin/timescaledb-tune" ]; then
# timescaledb-tune isn't present, nothing we can do here
exit 0
fi

if [ ! -z "${NO_TS_TUNE:-}" ]; then
# The user has explicitly requested not to run timescaledb-tune; exit this script
exit 0
Expand Down
157 changes: 14 additions & 143 deletions files/docker-initdb.sh
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,15 @@
#!/usr/bin/env bash
set -Eeo pipefail

# This script is lifted from the official Postgres Docker image with the only real
# alteration being that we don't run `postgres` at the end.
# This script is lifted from the official Postgres Docker image with the following alterations
# so that it can be run by Patroni to initialize the database.
#
# This is run by Patroni instead of plain initdb
# - Don't run postgres at the end
# - Ignore POSTGRES_USER, POSTGRES_PASSWORD and POSTGRES_DB, they should be controlled
# by Patroni

# TODO swap to -Eeuo pipefail above (after handling all potentially-unset variables)

# usage: file_env VAR [DEFAULT]
# ie: file_env 'XYZ_DB_PASSWORD' 'example'
# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
file_env() {
local var="$1"
local fileVar="${var}_FILE"
local def="${2:-}"
if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
exit 1
fi
local val="$def"
if [ "${!var:-}" ]; then
val="${!var}"
elif [ "${!fileVar:-}" ]; then
val="$(< "${!fileVar}")"
fi
export "$var"="$val"
unset "$fileVar"
}

# check to see if this file is being run or sourced from another script
_is_sourced() {
# https://unix.stackexchange.com/a/215279
Expand Down Expand Up @@ -84,7 +64,7 @@ docker_init_database_dir() {
set -- --waldir "$POSTGRES_INITDB_WALDIR" "$@"
fi

eval 'initdb --username="$POSTGRES_USER" --pwfile=<(echo "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS"' "$@"'
eval 'initdb '"$POSTGRES_INITDB_ARGS"' "$@"'

# unset/cleanup "nss_wrapper" bits
if [ "${LD_PRELOAD:-}" = '/usr/lib/libnss_wrapper.so' ]; then
Expand All @@ -93,59 +73,6 @@ docker_init_database_dir() {
fi
}

# print large warning if POSTGRES_PASSWORD is long
# error if both POSTGRES_PASSWORD is empty and POSTGRES_HOST_AUTH_METHOD is not 'trust'
# print large warning if POSTGRES_HOST_AUTH_METHOD is set to 'trust'
# assumes database is not set up, ie: [ -z "$DATABASE_ALREADY_EXISTS" ]
docker_verify_minimum_env() {
# check password first so we can output the warning before postgres
# messes it up
if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then
cat >&2 <<-'EOWARN'
WARNING: The supplied POSTGRES_PASSWORD is 100+ characters.
This will not work if used via PGPASSWORD with "psql".
https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412)
https://github.com/docker-library/postgres/issues/507
EOWARN
fi
if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then
# The - option suppresses leading tabs but *not* spaces. :)
cat >&2 <<-'EOE'
Error: Database is uninitialized and superuser password is not specified.
You must specify POSTGRES_PASSWORD to a non-empty value for the
superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run".
You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all
connections without a password. This is *not* recommended.
See PostgreSQL documentation about "trust":
https://www.postgresql.org/docs/current/auth-trust.html
EOE
exit 1
fi
if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then
cat >&2 <<-'EOWARN'
********************************************************************************
WARNING: POSTGRES_HOST_AUTH_METHOD has been set to "trust". This will allow
anyone with access to the Postgres port to access your database without
a password, even if POSTGRES_PASSWORD is set. See PostgreSQL
documentation about "trust":
https://www.postgresql.org/docs/current/auth-trust.html
In Docker's default configuration, this is effectively any other
container on the same system.
It is not recommended to use POSTGRES_HOST_AUTH_METHOD=trust. Replace
it with "-e POSTGRES_PASSWORD=password" instead to set a password in
"docker run".
********************************************************************************
EOWARN
fi
}

# usage: docker_process_init_files [file [file [...]]]
# ie: docker_process_init_files /always-initdb.d/*
# process initializer files, based on file extensions and permissions
Expand Down Expand Up @@ -183,65 +110,17 @@ docker_process_init_files() {
# ie: docker_process_sql -f my-file.sql
# ie: docker_process_sql <my-file.sql
docker_process_sql() {
local query_runner=( psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --no-password )
if [ -n "$POSTGRES_DB" ]; then
query_runner+=( --dbname "$POSTGRES_DB" )
fi

local query_runner=( psql -v ON_ERROR_STOP=1 --username "postgres" --no-password )

"${query_runner[@]}" "$@"
}

# create initial database
# uses environment variables for input: POSTGRES_DB
docker_setup_db() {
if [ "$POSTGRES_DB" != 'postgres' ]; then
POSTGRES_DB= docker_process_sql --dbname postgres --set db="$POSTGRES_DB" <<-'EOSQL'
CREATE DATABASE :"db" ;
EOSQL
echo
fi
}

# Loads various settings that are used elsewhere in the script
# This should be called before any other functions
docker_setup_env() {
file_env 'POSTGRES_PASSWORD'

file_env 'POSTGRES_USER' 'postgres'
file_env 'POSTGRES_DB' "$POSTGRES_USER"
file_env 'POSTGRES_INITDB_ARGS'
# default authentication method is md5
: "${POSTGRES_HOST_AUTH_METHOD:=md5}"

declare -g DATABASE_ALREADY_EXISTS
# look specifically for PG_VERSION, as it is expected in the DB dir
if [ -s "$PGDATA/PG_VERSION" ]; then
DATABASE_ALREADY_EXISTS='true'
fi
}

# append POSTGRES_HOST_AUTH_METHOD to pg_hba.conf for "host" connections
pg_setup_hba_conf() {
{
echo
if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then
echo '# warning trust is enabled for all connections'
echo '# see https://www.postgresql.org/docs/12/auth-trust.html'
fi
echo "host all all all $POSTGRES_HOST_AUTH_METHOD"
} >> "$PGDATA/pg_hba.conf"
}

# start socket-only postgresql server for setting up or running scripts
# all arguments will be passed along as arguments to `postgres` (via pg_ctl)
docker_temp_server_start() {
if [ "$1" = 'postgres' ]; then
shift
fi

# internal start of server in order to allow setup using psql client
# does not listen on external TCP/IP and waits until start finishes
set -- "$@" -c listen_addresses='' -p "${PGPORT:-5432}"
set -- -c listen_addresses='' -p "${PGPORT:-5432}"

PGUSER="${PGUSER:-$POSTGRES_USER}" \
pg_ctl -D "$PGDATA" \
Expand Down Expand Up @@ -279,7 +158,6 @@ _main() {
fi

if [ "$1" = 'postgres' ] && ! _pg_want_help "$@"; then
docker_setup_env
# setup data directories and permissions (when run as root)
docker_create_db_directories
if [ "$(id -u)" = '0' ]; then
Expand All @@ -289,24 +167,17 @@ _main() {

# only run initialization on an empty data directory
if [ -z "$DATABASE_ALREADY_EXISTS" ]; then
docker_verify_minimum_env

# check dir permissions to reduce likelihood of half-initialized database
ls /docker-entrypoint-initdb.d/ > /dev/null

echo "==> A"
docker_init_database_dir
pg_setup_hba_conf

# PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless
# e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS
export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}"
echo "==> B"
docker_temp_server_start "$@"

docker_setup_db
echo "==> C"
docker_process_init_files /docker-entrypoint-initdb.d/*

echo "==> D"
docker_temp_server_stop
unset PGPASSWORD
echo "==> E"

echo
echo 'PostgreSQL init process complete; ready for start up.'
Expand Down

0 comments on commit 284d977

Please sign in to comment.