Merge pull request #158 from overleaf/em-docker-compose-v2

Use Docker Compose v2
2025-04-19 07:18:06 +02:00 · 2023-05-23 06:09:26 -04:00 · 2023-05-23 06:09:26 -04:00 · fd7891401b
commit fd7891401b
parent 63c5e1d5fd ae03cd3568
12 changed files with 211 additions and 109 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -1,5 +1,38 @@
 # Changelog

+## 2023-05-16
+### Added
+- Use Docker Compose v2 by default. Fall back to Docker Compose v1 if v2 is
+  unavailable.
+### Fixed
+- Propagate the `REDIS_PORT` variable to the sharelatex container
+
+## 2023-05-15
+### Added
+- Support listing container logs with `bin/logs` command
+- `bin/logs -n all` shows all logs for a given service
+
+## 2023-05-11
+### Added
+- Change the location of the git-bridge data directory to /data/git-bridge
+  inside the container
+
+## 2023-05-01
+### Added
+- Start Mongo in a replica set by default
+
+## 2023-04-14
+### Fixed
+- Fix openssl invocation on OS X
+
+## 2023-04-13
+### Fixed
+- Ensure git bridge is disabled by default
+
+## 2023-04-10
+### Added
+- Git bridge support in Server Pro 4.x
+
 ## 2023-03-21
 ### Added
 - Updated default [`version`](https://github.com/overleaf/toolkit/blob/master/lib/config-seed/version) to `3.5.5`.
--- a/bin/docker-compose
+++ b/bin/docker-compose
@ -18,119 +18,163 @@ fi

 source "$TOOLKIT_ROOT/lib/shared-functions.sh"

-function __main__() {
-  # Load the Overleaf version
-  read_image_version
+function build_environment() {
+  canonicalize_data_paths
+  set_base_vars

-  # Load vars from the rc file
-  read_config
-
-  # Select which docker-compose files to load
-  local compose_file_flags=("-f $TOOLKIT_ROOT/lib/docker-compose.base.yml")
-  if [[ "$REDIS_ENABLED" == "true" ]]; then
-    compose_file_flags+=("-f $TOOLKIT_ROOT/lib/docker-compose.redis.yml")
+  if [[ $REDIS_ENABLED == "true" ]]; then
+    set_redis_vars
  fi
-  if [[ "$MONGO_ENABLED" == "true" ]]; then
-    compose_file_flags+=("-f $TOOLKIT_ROOT/lib/docker-compose.mongo.yml")
-    if [[ "${IMAGE_VERSION_MAJOR}" -ge 4 ]]; then
-      MONGO_ARGS="--replSet overleaf"
-    else
-      MONGO_ARGS=""
-    fi
+  if [[ $MONGO_ENABLED == "true" ]]; then
+    set_mongo_vars
  fi
-  if [[ "$SIBLING_CONTAINERS_ENABLED" == "true" ]]; then
-    compose_file_flags+=("-f $TOOLKIT_ROOT/lib/docker-compose.sibling-containers.yml")
+  if [[ $SIBLING_CONTAINERS_ENABLED == "true" ]]; then
+    set_sibling_containers_vars
  fi
-  if [[ "${NGINX_ENABLED}" == "true" ]]; then
-    compose_file_flags+=("-f $TOOLKIT_ROOT/lib/docker-compose.nginx.yml")
+  if [[ $NGINX_ENABLED == "true" ]]; then
+    set_nginx_vars
  fi
-  if [[ $GIT_BRIDGE_ENABLED = "true" ]]; then
-    if [[ $SERVER_PRO = "true" && $IMAGE_VERSION_MAJOR -ge 4 ]]; then
-      compose_file_flags+=("-f $TOOLKIT_ROOT/lib/docker-compose.git-bridge.yml")
-      GIT_BRIDGE_IMAGE="quay.io/sharelatex/git-bridge:$IMAGE_VERSION"
-    else
-      # Git bridge is only supported in Server Pro 4+
-      GIT_BRIDGE_ENABLED=false
-    fi
+  if [[ $GIT_BRIDGE_ENABLED == "true" ]]; then
+    set_git_bridge_vars
  fi

  # Include docker-compose.override.yml if it is present
  if [[ -f "$TOOLKIT_ROOT/config/docker-compose.override.yml" ]]; then
-    compose_file_flags+=("-f $TOOLKIT_ROOT/config/docker-compose.override.yml")
+    DOCKER_COMPOSE_FLAGS+=(-f "$TOOLKIT_ROOT/config/docker-compose.override.yml")
  fi
+}

-  local image_name="sharelatex/sharelatex"
-  if [[ "${SERVER_PRO}" == "true" ]]; then
-    image_name="quay.io/sharelatex/sharelatex-pro"
-  fi
-  image_name="${SHARELATEX_IMAGE_NAME:-${image_name}}"
-
-  local full_image_spec="$image_name:$IMAGE_VERSION"
-
-  # Canonicalize data paths
+function canonicalize_data_paths() {
  SHARELATEX_DATA_PATH=$(cd "$TOOLKIT_ROOT"; realpath "$SHARELATEX_DATA_PATH")
  MONGO_DATA_PATH=$(cd "$TOOLKIT_ROOT"; realpath "$MONGO_DATA_PATH")
  REDIS_DATA_PATH=$(cd "$TOOLKIT_ROOT"; realpath "$REDIS_DATA_PATH")
  GIT_BRIDGE_DATA_PATH=$(cd "$TOOLKIT_ROOT"; realpath "$GIT_BRIDGE_DATA_PATH")
+}

-  if [ "${SHARELATEX_LISTEN_IP:-null}" == "null" ];
+# Set environment variables for docker-compose.base.yml
+function set_base_vars() {
+  DOCKER_COMPOSE_FLAGS=(-f "$TOOLKIT_ROOT/lib/docker-compose.base.yml")
+
+  local image_name
+  if [[ -n ${SHARELATEX_IMAGE_NAME:-} ]]; then
+    image_name="$SHARELATEX_IMAGE_NAME"
+  elif [[ $SERVER_PRO == "true" ]]; then
+    image_name="quay.io/sharelatex/sharelatex-pro"
+  else
+    image_name="sharelatex/sharelatex"
+  fi
+  export IMAGE="$image_name:$IMAGE_VERSION"
+
+  if [[ ${SHARELATEX_LISTEN_IP:-null} == "null" ]];
  then
    echo "WARNING: the value of SHARELATEX_LISTEN_IP is not set in config/overleaf.rc. This value must be set to the public IP address for direct container access. Defaulting to 0.0.0.0" >&2
    SHARELATEX_LISTEN_IP="0.0.0.0"
  fi
+  export SHARELATEX_LISTEN_IP

-  if [[ "$NGINX_ENABLED" == "true" ]]; then
-    if [[ -n "${TLS_PRIVATE_KEY_PATH-}" ]]; then
-      TLS_PRIVATE_KEY_PATH=$(cd "$TOOLKIT_ROOT"; realpath "$TLS_PRIVATE_KEY_PATH")
-    fi
-    if [[ -n "${TLS_CERTIFICATE_PATH-}" ]]; then
-      TLS_CERTIFICATE_PATH=$(cd "$TOOLKIT_ROOT"; realpath "$TLS_CERTIFICATE_PATH")
-    fi
-    if [[ -n "${NGINX_CONFIG_PATH-}" ]]; then
-      NGINX_CONFIG_PATH=$(cd "$TOOLKIT_ROOT"; realpath "$NGINX_CONFIG_PATH")
-    fi
+  if [[ $SERVER_PRO != "true" || $IMAGE_VERSION_MAJOR -lt 4 ]]; then
+    # Force git bridge to be disabled if not ServerPro >= 4
+    GIT_BRIDGE_ENABLED=false
  fi
-  # Print debug info
-  if [[ "${RC_DEBUG:-null}" != "null" ]]; then
+
+  export GIT_BRIDGE_ENABLED
+  export MONGO_URL
+  export REDIS_HOST
+  export REDIS_PORT
+  export SHARELATEX_DATA_PATH
+  export SHARELATEX_PORT
+}
+
+# Set environment variables for docker-compose.redis.yml
+function set_redis_vars() {
+  DOCKER_COMPOSE_FLAGS+=(-f "$TOOLKIT_ROOT/lib/docker-compose.redis.yml")
+  export REDIS_IMAGE
+  export REDIS_DATA_PATH
+}
+
+# Set environment variables for docker-compose.mongo.yml
+function set_mongo_vars() {
+  DOCKER_COMPOSE_FLAGS+=(-f "$TOOLKIT_ROOT/lib/docker-compose.mongo.yml")
+
+  if [[ $MONGO_ENABLED == "true" && $IMAGE_VERSION_MAJOR -ge 4 ]]; then
+    MONGO_ARGS="--replSet overleaf"
+  else
+    MONGO_ARGS=""
+  fi
+  export MONGO_ARGS
+
+  export MONGO_DATA_PATH
+  export MONGO_IMAGE
+}
+
+# Set environment variables for docker-compose.sibling-containers.yml
+function set_sibling_containers_vars() {
+  DOCKER_COMPOSE_FLAGS+=(-f "$TOOLKIT_ROOT/lib/docker-compose.sibling-containers.yml")
+  export DOCKER_SOCKET_PATH
+  export SHARELATEX_DATA_PATH
+}
+
+# Set environment variables for docker-compose.nginx.yml
+function set_nginx_vars() {
+  DOCKER_COMPOSE_FLAGS+=(-f "$TOOLKIT_ROOT/lib/docker-compose.nginx.yml")
+
+  if [[ -n ${TLS_PRIVATE_KEY_PATH-} ]]; then
+    TLS_PRIVATE_KEY_PATH=$(cd "$TOOLKIT_ROOT"; realpath "$TLS_PRIVATE_KEY_PATH")
+  fi
+  if [[ -n ${TLS_CERTIFICATE_PATH-} ]]; then
+    TLS_CERTIFICATE_PATH=$(cd "$TOOLKIT_ROOT"; realpath "$TLS_CERTIFICATE_PATH")
+  fi
+  if [[ -n ${NGINX_CONFIG_PATH-} ]]; then
+    NGINX_CONFIG_PATH=$(cd "$TOOLKIT_ROOT"; realpath "$NGINX_CONFIG_PATH")
+  fi
+
+  export NGINX_CONFIG_PATH
+  export NGINX_IMAGE
+  export NGINX_HTTP_PORT
+  export NGINX_HTTP_LISTEN_IP
+  export NGINX_TLS_LISTEN_IP
+  export TLS_CERTIFICATE_PATH
+  export TLS_PORT
+  export TLS_PRIVATE_KEY_PATH
+}
+
+# Set environment variables for docker-compose.git-bridge.yml
+function set_git_bridge_vars() {
+  DOCKER_COMPOSE_FLAGS+=(-f "$TOOLKIT_ROOT/lib/docker-compose.git-bridge.yml")
+  export GIT_BRIDGE_IMAGE="quay.io/sharelatex/git-bridge:$IMAGE_VERSION"
+  export GIT_BRIDGE_DATA_PATH
+}
+
+function print_debug_info() {
+  if [[ ${RC_DEBUG:-null} != "null" ]]; then
    echo ">>>>>>VARS>>>>>>"
    echo "$(set -o posix; set)" # print all vars
    echo "IMAGE_VERSION=$IMAGE_VERSION"
    echo "<<<<<<<<<<<<<<<<"
    echo ">>>>COMPOSE-ARGS>>>>"
    echo "-p $PROJECT_NAME"
-    echo "${compose_file_flags[@]}"
+    echo "${DOCKER_COMPOSE_FLAGS[@]}"
    echo "$@"
    echo "<<<<<<<<<<<<<<<<<<<<"
  fi
-
-  # Export vars for use in docker-compose files
-  export DOCKER_SOCKET_PATH
-  export GIT_BRIDGE_DATA_PATH
-  export GIT_BRIDGE_ENABLED
-  export GIT_BRIDGE_IMAGE
-  export IMAGE="$full_image_spec"
-  export MONGO_ARGS
-  export MONGO_DATA_PATH
-  export MONGO_IMAGE
-  export MONGO_URL
-  export NGINX_CONFIG_PATH
-  export NGINX_IMAGE
-  export NGINX_HTTP_PORT
-  export NGINX_HTTP_LISTEN_IP
-  export NGINX_TLS_LISTEN_IP
-  export REDIS_DATA_PATH
-  export REDIS_HOST
-  export REDIS_IMAGE
-  export REDIS_PORT
-  export SHARELATEX_DATA_PATH
-  export SHARELATEX_PORT
-  export SHARELATEX_LISTEN_IP
-  export TLS_CERTIFICATE_PATH
-  export TLS_PORT
-  export TLS_PRIVATE_KEY_PATH
-
-  # shellcheck disable=SC2068
-  exec docker-compose -p "$PROJECT_NAME" ${compose_file_flags[@]} "$@"
 }

-__main__ "$@"
+function docker_compose() {
+  local flags=(-p "$PROJECT_NAME" "${DOCKER_COMPOSE_FLAGS[@]}" "$@")
+  if docker compose version >/dev/null 2>&1; then
+    # Docker compose v2 is available
+    exec docker compose "${flags[@]}"
+  elif command -v docker-compose >/dev/null; then
+    # Fall back to docker-compose v1
+    exec docker-compose "${flags[@]}"
+  else
+    echo "ERROR: Could not find Docker Compose." >&2
+    exit 1
+  fi
+}
+
+read_image_version
+read_config
+build_environment
+print_debug_info "$@"
+docker_compose "$@"
--- a/bin/doctor
+++ b/bin/doctor
@ -113,7 +113,6 @@ function check_dependencies() {
  declare -a binaries=(
    bash
    docker
-    docker-compose
    realpath
    perl
    awk
@ -123,6 +122,16 @@ function check_dependencies() {
  for binary in "${binaries[@]}"; do
    check_for_binary "$binary"
  done
+
+  if docker compose version > /dev/null 2>&1; then
+    print_point 1 "docker compose"
+    print_point 2 "status: present"
+    print_point 2 "version info: $(docker compose version)"
+  elif command -v docker-compose > /dev/null; then
+    check_for_binary docker-compose
+  else
+    add_warning "Docker Compose not found"
+  fi
 }

 function check_docker_daemon() {
--- a/bin/logs
+++ b/bin/logs
@ -19,6 +19,8 @@ DEFAULT_TAIL_LINES=20
 ALL_SERVICES=(chat clsi contacts docstore document-updater filestore git-bridge \
    mongo notifications real-time redis spelling tags track-changes web)

+LOGS_PID_FILE="/tmp/toolkit-logs.$$.pid"
+
 function usage() {
    echo "Usage: bin/logs [OPTIONS] [SERVICES...]

@ -76,12 +78,26 @@ function docker_compose() {
    "$TOOLKIT_ROOT/bin/docker-compose" "$@" 2>/dev/null || true
 }

+function kill_background_jobs() {
+    if [[ ${#COMPOSE_LOGS_PIDS[@]} -gt 0 ]]; then
+        # Kill "docker compose logs" processes
+        kill "${COMPOSE_LOGS_PIDS[@]}" 2>/dev/null || true
+    fi
+
+    # Kill "tail -f" processes started inside the sharelatex container
+    docker_compose exec -T sharelatex bash -c "
+        [[ -f $LOGS_PID_FILE ]] && kill \$(cat $LOGS_PID_FILE) 2>/dev/null
+        rm -f $LOGS_PID_FILE"
+}
+
 function show_logs() {
-    trap 'kill -INT $(jobs -p)' INT
+    COMPOSE_LOGS_PIDS=()
+    trap kill_background_jobs EXIT

    for service in "${SERVICES[@]}"; do
        if [[ $service =~ ^(git-bridge|mongo|redis)$ ]]; then
            show_compose_logs "$service" &
+            COMPOSE_LOGS_PIDS+=($!)
        else
            show_sharelatex_logs "$service" &
        fi
@ -123,7 +139,7 @@ function show_sharelatex_logs() {
        flags+=(-n "$TAIL_LINES")
    fi

-    local logs_cmd="[[ -f $log_path ]] && tail ${flags[*]} $log_path"
+    local logs_cmd="[[ -f $log_path ]] && echo \$\$ >> $LOGS_PID_FILE && tail --pid=\$\$ ${flags[*]} $log_path"
    if [[ ${#SERVICES[@]} -gt 1 ]]; then
        # Roughly reproduce the service prefix format from docker compose
        local padded_service
--- a/bin/up
+++ b/bin/up
@ -21,9 +21,9 @@ source "$TOOLKIT_ROOT/lib/shared-functions.sh"
 function usage() {
  echo "Usage: bin/up [FLAGS...]"
  echo ""
-  echo "A wrapper around 'docker-compose up'."
+  echo "A wrapper around 'docker compose up'."
  echo ""
-  echo "This program will pass any extra flags to docker-compose,"
+  echo "This program will pass any extra flags to docker compose,"
  echo "for example: 'bin/up -d' will run in detached mode"
 }

--- a/doc/configuration.md
+++ b/doc/configuration.md
@ -11,7 +11,7 @@ This directory is excluded from the git revision control system, so it will not
 Note that changes to the configuration files will not be automatically applied
 to existing containers, even if the container is stopped and restarted (with
 `bin/stop` and `bin/start`). To apply the changes, run `bin/up`, and
-`docker-compose` will automatically apply the configuration changes to a new
+`docker compose` will automatically apply the configuration changes to a new
 container. (Or, run `bin/up -d`, if you prefer to not attach to the docker logs)


@ -53,6 +53,6 @@ The `config/version` file contains the version number of the docker images that

 ## The `docker-compose.override.yml` File

-If present, the `config/docker-compose.override.yml` file will be included in the invocation to `docker-compose`. This is useful for overriding configuration specific to docker-compose.
+If present, the `config/docker-compose.override.yml` file will be included in the invocation to `docker compose`. This is useful for overriding configuration specific to docker compose.

 See the [docker-compose documentation](https://docs.docker.com/compose/extends/#adding-and-overriding-configuration) for more details.
--- a/doc/dependencies.md
+++ b/doc/dependencies.md
@ -1,7 +1,7 @@
 # Dependencies

 This project requires a modern unix system as a base (such as Ubuntu Linux).
-It also requires `bash`, `docker`, and `docker-compose`. 
+It also requires `bash` and `docker`. 

 The `bin/doctor` script can be used to check for missing dependencies.

--- a/doc/docker-compose.md
+++ b/doc/docker-compose.md
@ -1,8 +1,8 @@
-# Working with Docker-Compose Services
+# Working with Docker Compose Services

 The Overleaf Toolkit runs Overleaf inside a docker container, plus the
 supporting databases (MongoDB and Redis), in their own containers. All of this
-is orchestrated with `docker-compose`.
+is orchestrated with `docker compose`.

 Note: for legacy reasons, the main Overleaf container is called `sharelatex`,
 and is based on the `sharelatex/sharelatex` docker image. This is because the
@ -15,12 +15,12 @@ Overleaf naming scheme.

 ## The `bin/docker-compose` Wrapper

-The `bin/docker-compose` script is a wrapper around `docker-compose`. It
+The `bin/docker-compose` script is a wrapper around `docker compose`. It
 loads configuration from the `config/` directory, before invoking
-`docker-compose` with whatever arguments were passed to the script.
+`docker compose` with whatever arguments were passed to the script.

 You can treat `bin/docker-compose` as a transparent wrapper for the
-`docker-compose` program installed on your machine.
+`docker compose` program installed on your machine.

 For example, we can check which containers are running with the following:

--- a/doc/overview.md
+++ b/doc/overview.md
@ -18,9 +18,9 @@ Community Edition is the free version of Overleaf, while Server Pro is our enter
 When you set up Overleaf using the toolkit, you will start with Community Edition, and can easily switch to Server Pro by changing just one setting.


-## Docker, Docker-Compose, and Overleaf
+## Docker, Docker Compose, and Overleaf

-The toolkit uses [Docker](https://www.docker.com) and [Docker-Compose](https://docs.docker.com/compose/) to run the Overleaf software in an isolated sandbox. While we do recommend becoming familiar with both Docker and Docker-Compose, we also aim to make it as easy as possible to run Overleaf on your own computer.
+The toolkit uses [Docker](https://www.docker.com) and [Docker Compose](https://docs.docker.com/compose/) to run the Overleaf software in an isolated sandbox. While we do recommend becoming familiar with both Docker and Docker Compose, we also aim to make it as easy as possible to run Overleaf on your own computer.


 ## How do I get the Toolkit?
--- a/doc/quick-start-guide.md
+++ b/doc/quick-start-guide.md
@ -6,10 +6,9 @@ The Overleaf Toolkit depends on the following programs:

 - bash
 - docker
- docker-compose

-We recommend that you install the most recent version of docker and docker-compose that 
-are available on your system.
+We recommend that you install the most recent version of docker that is
+available on your system.


 ## Install
@ -78,7 +77,7 @@ These are the three configuration files you will interact with:

 ## Starting Up

-The Overleaf Toolkit uses `docker-compose` to manage the overleaf docker containers. The toolkit provides a set of scripts which wrap `docker-compose`, and take care of most of the details for you.
+The Overleaf Toolkit uses `docker compose` to manage the overleaf docker containers. The toolkit provides a set of scripts which wrap `docker compose`, and take care of most of the details for you.

 Let's start the docker services:

@ -87,7 +86,7 @@ $ bin/up
 ```

 You should see some log output from the docker containers, indicating that the containers are running. 
-If you press `CTRL-C` at the terminal, the services will shut down. You can start them up again (without attaching to the log output) by running `bin/start`. More generally, you can run `bin/docker-compose` to control the `docker-compose` system directly, if you find that the convenience scripts don't cover your use-case.
+If you press `CTRL-C` at the terminal, the services will shut down. You can start them up again (without attaching to the log output) by running `bin/start`. More generally, you can run `bin/docker-compose` to control the `docker compose` system directly, if you find that the convenience scripts don't cover your use-case.


 ## Create the first admin account
@ -161,10 +160,10 @@ We should see some output similar to this:
        - version info: 5.0.17(1)-release
    - docker
        - status: present
-        - version info: Docker version 19.03.6, build 369ce74a3c
-    - docker-compose
+        - version info: Docker version 23.06.6, build 369ce74a3c
+    - docker compose
        - status: present
-        - version info: docker-compose version 1.24.0, build 0aa59064
+        - version info: docker compose version v2.17.3
    ...
 ====== Configuration ======
    ...
--- a/doc/the-doctor.md
+++ b/doc/the-doctor.md
@ -38,10 +38,10 @@ You will see some output like this:
        - version info: 5.0.17(1)-release
    - docker
        - status: present
-        - version info: Docker version 19.03.6, build 369ce74a3c
-    - docker-compose
+        - version info: Docker version 23.0.6, build 369ce74a3c
+    - docker compose
        - status: present
-        - version info: docker-compose version 1.24.0, build 0aa59064
+        - version info: Docker Compose version v2.17.3
    - realpath
        - status: present
        - version info: realpath (GNU coreutils) 8.30
--- a/lib/docker-compose.base.yml
+++ b/lib/docker-compose.base.yml
@ -15,6 +15,7 @@ services:
          GIT_BRIDGE_HOST: "git-bridge"
          GIT_BRIDGE_PORT: "8000"
          REDIS_HOST: "${REDIS_HOST}"
+          REDIS_PORT: "${REDIS_PORT}"
          SHARELATEX_MONGO_URL: "${MONGO_URL}"
          SHARELATEX_REDIS_HOST: "${REDIS_HOST}"
          V1_HISTORY_URL: "http://sharelatex:3100/api"