From 94547d332c2891f43af17ef26e5e5debb025eca2 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Fri, 18 Mar 2022 13:01:47 -0400 Subject: [PATCH] add dockerfile and docker-compose --- docker/Dockerfile.wallet_server | 53 ++++++++++++++++++++ docker/docker-compose.yml | 80 ++++++++++++++++++++++++++++++ docker/set_build.py | 44 ++++++++++++++++ docker/wallet_server_entrypoint.sh | 17 +++++++ 4 files changed, 194 insertions(+) create mode 100644 docker/Dockerfile.wallet_server create mode 100644 docker/docker-compose.yml create mode 100644 docker/set_build.py create mode 100755 docker/wallet_server_entrypoint.sh diff --git a/docker/Dockerfile.wallet_server b/docker/Dockerfile.wallet_server new file mode 100644 index 0000000..0be0ffc --- /dev/null +++ b/docker/Dockerfile.wallet_server @@ -0,0 +1,53 @@ +FROM debian:11-slim + +ARG user=lbry +ARG db_dir=/database +ARG projects_dir=/home/$user + +ARG DOCKER_TAG +ARG DOCKER_COMMIT=docker +ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT + +RUN apt-get update && \ + apt-get -y --no-install-recommends install \ + wget \ + tar unzip \ + build-essential \ + automake libtool \ + pkg-config \ + python3.9 \ + python3.9-dev \ + python3-pip && \ + update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \ + rm -rf /var/lib/apt/lists/* + +RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user +RUN mkdir -p $db_dir +RUN chown -R $user:$user $db_dir + +COPY . $projects_dir +RUN chown -R $user:$user $projects_dir + +USER $user +WORKDIR $projects_dir +RUN python3.9 -m pip install pip +RUN python3.9 -m pip install -e . +RUN python3.9 docker/set_build.py +RUN rm ~/.cache -rf + +# entry point +ARG host=localhost +ARG tcp_port=50001 +ARG daemon_url=http://lbry:lbry@localhost:9245/ +VOLUME $db_dir +ENV TCP_PORT=$tcp_port +ENV HOST=$host +ENV DAEMON_URL=$daemon_url +ENV DB_DIRECTORY=$db_dir +ENV MAX_SESSIONS=100000 +ENV MAX_SEND=1000000000000000000 +ENV MAX_RECEIVE=1000000000000000000 + + +COPY ./docker/wallet_server_entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..8bb764a --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,80 @@ +version: "3" + +volumes: + lbry_rocksdb: + es01: + +services: + scribe: + depends_on: + - scribe_elastic_sync + image: lbry/scribe:${SCRIBE_TAG:-latest-release} + restart: always + network_mode: host + volumes: + - "lbry_rocksdb:/database" + environment: + - HUB_COMMAND=scribe + - DAEMON_URL=http://lbry:lbry@127.0.0.1:9245 + - MAX_QUERY_WORKERS=2 + - FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8 + - BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 + scribe_elastic_sync: + depends_on: + - es01 + image: lbry/scribe:${SCRIBE_TAG:-latest-release} + restart: always + network_mode: host + ports: + - "127.0.0.1:19080:19080" # elastic notifier port + volumes: + - "lbry_rocksdb:/database" + environment: + - HUB_COMMAND=scribe-elastic-sync + - MAX_QUERY_WORKERS=2 + - ELASTIC_HOST=127.0.0.1 + - ELASTIC_PORT=9200 + - FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8 + - BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 + scribe_hub: + depends_on: + - scribe_elastic_sync + - scribe + image: lbry/scribe:${SCRIBE_TAG:-latest-release} + restart: always + network_mode: host + ports: + - "50001:50001" # electrum rpc port and udp ping port + - "2112:2112" # comment out to disable prometheus + volumes: + - "lbry_rocksdb:/database" + environment: + - HUB_COMMAND=scribe-hub + - DAEMON_URL=http://lbry:lbry@127.0.0.1:9245 # used for broadcasting transactions + - MAX_QUERY_WORKERS=4 # reader threads + - MAX_SESSIONS=100000 + - ELASTIC_HOST=127.0.0.1 + - ELASTIC_PORT=9200 + - HOST=0.0.0.0 + - PROMETHEUS_PORT=2112 + - TCP_PORT=50001 + - ALLOW_LAN_UDP=No + - FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8 + - BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 + es01: + image: docker.elastic.co/elasticsearch/elasticsearch:7.16.0 + container_name: es01 + environment: + - node.name=es01 + - discovery.type=single-node + - indices.query.bool.max_clause_count=8192 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Dlog4j2.formatMsgNoLookups=true -Xms8g -Xmx8g" # no more than 32, remember to disable swap + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - "es01:/usr/share/elasticsearch/data" + ports: + - "127.0.0.1:9200:9200" diff --git a/docker/set_build.py b/docker/set_build.py new file mode 100644 index 0000000..4cefcce --- /dev/null +++ b/docker/set_build.py @@ -0,0 +1,44 @@ +import sys +import os +import re +import logging +import scribe.build_info as build_info_mod + +log = logging.getLogger() +log.addHandler(logging.StreamHandler()) +log.setLevel(logging.DEBUG) + + +def _check_and_set(d: dict, key: str, value: str): + try: + d[key] + except KeyError: + raise Exception(f"{key} var does not exist in {build_info_mod.__file__}") + d[key] = value + + +def main(): + build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")} + + commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('GITHUB_SHA')) + if commit_hash is None: + raise ValueError("Commit hash not found in env vars") + _check_and_set(build_info, "COMMIT_HASH", commit_hash[:6]) + + docker_tag = os.getenv('DOCKER_TAG') + if docker_tag: + _check_and_set(build_info, "DOCKER_TAG", docker_tag) + _check_and_set(build_info, "BUILD", "docker") + else: + if re.match(r'refs/tags/v\d+\.\d+\.\d+$', str(os.getenv('GITHUB_REF'))): + _check_and_set(build_info, "BUILD", "release") + else: + _check_and_set(build_info, "BUILD", "qa") + + log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()])) + with open(build_info_mod.__file__, 'w') as f: + f.write("\n".join([f"{k} = \"{v}\"" for k, v in build_info.items()]) + "\n") + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/docker/wallet_server_entrypoint.sh b/docker/wallet_server_entrypoint.sh new file mode 100755 index 0000000..09b836b --- /dev/null +++ b/docker/wallet_server_entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# entrypoint for scribe Docker image + +set -euo pipefail + +if [ -z "$HUB_COMMAND" ]; then + echo "HUB_COMMAND env variable must be scribe, scribe-hub, or scribe-elastic-sync" + exit 1 +fi + +case "$HUB_COMMAND" in + scribe ) /home/lbry/.local/bin/scribe "$@" ;; + scribe-hub ) /home/lbry/.local/bin/scribe-hub "$@" ;; + scribe-elastic-sync ) /home/lbry/.local/bin/scribe-elastic-sync ;; + * ) "HUB_COMMAND env variable must be scribe, scribe-hub, or scribe-elastic-sync" && exit 1 ;; +esac