Move VGRID_FILES_WRITABLE default into env for simplicity.

Expose openid_store through the new OPENID_STORE env similarly to the existing MIG_SYSTEM_RUN, so that it can also easily be pointed to a fast tmpfs mount for improved OpenID 2.0 scalability and performance. Both new env options are documented, too. Updates to the inline docs in the env files for consistency.
ucphhpc · Mar 27, 2024 · 0c75174 · 0c75174
1 parent da209f8
commit 0c75174
Show file tree

Hide file tree

Showing 7 changed files with 80 additions and 34 deletions.
diff --git a/development.env b/development.env
@@ -147,12 +147,24 @@ EXTRA_USERPAGE_STYLES=""
 # be used by default.
 # NOTE: a shared mig_system_run scratch space on tmpfs can be made with
 #       something like:                                                                             
-# tmpfs   /storage-mem/mig_system_run  tmpfs   nosuid,nodev,noatime,noexec,uid=1000,gid=1000,mode=0770,size=128m   0 0
+# tmpfs   /storage/tmpfs/mig_system_run  tmpfs   nosuid,nodev,noatime,noexec,uid=1000,gid=1000,mode=0770,size=128m   0 0
 # in /etc/fstab. Manual mount can be done with:                                                     
-# sudo mount /storage-mem/mig_system_run
+# sudo mount /storage/tmpfs/mig_system_run
 # NOTE: toggle commenting on  next two lines if you have such a tmpfs set up in the given path
-#MIG_SYSTEM_RUN=/storage-mem/mig_system_run
+#MIG_SYSTEM_RUN=/storage/tmpfs/mig_system_run
 MIG_SYSTEM_RUN=${DOCKER_MIGRID_ROOT}/state/mig_system_run
+# The apache auth openid module performs and scales better if the associated
+# internal openid store directory runs from fast storage. It's a volatile data
+# store, which allows more concurrent OpenID 2.0 clients if it e.g. uses tmpfs.
+# If you have migoid or extoid in LOGIN_METHODS you likely want to look into
+# that. The instructions for mig_system_run can be mostly reused in that case.
+# Otherwise you can safely ignore the OPENID_STORE setting.
+# NOTE: toggle commenting on next two lines if you have such a tmpfs set up in the given path
+#OPENID_STORE=/storage/tmpfs/openid_store
+OPENID_STORE=${DOCKER_MIGRID_ROOT}/state/openid_store
+# We need a read-only bind mounted version of the vgrid_files_writable
+# directory and the underlying location can be configured here.
+VGRID_FILES_WRITABLE=${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable
 
 # Which svn repo and version of migrid should be used
 #MIG_SVN_REPO=https://svn.code.sf.net/p/migrid/code/trunk

diff --git a/development_gdp.env b/development_gdp.env
@@ -147,12 +147,24 @@ EXTRA_USERPAGE_STYLES=""
 # be used by default.
 # NOTE: a shared mig_system_run scratch space on tmpfs can be made with
 #       something like:                                                                             
-# tmpfs   /storage-mem/mig_system_run  tmpfs   nosuid,nodev,noatime,noexec,uid=1000,gid=1000,mode=0770,size=128m   0 0
+# tmpfs   /storage/tmpfs/mig_system_run  tmpfs   nosuid,nodev,noatime,noexec,uid=1000,gid=1000,mode=0770,size=128m   0 0
 # in /etc/fstab. Manual mount can be done with:                                                     
-# sudo mount /storage-mem/mig_system_run
+# sudo mount /storage/tmpfs/mig_system_run
 # NOTE: toggle commenting on  next two lines if you have such a tmpfs set up in the given path
-#MIG_SYSTEM_RUN=/storage-mem/mig_system_run
+#MIG_SYSTEM_RUN=/storage/tmpfs/mig_system_run
 MIG_SYSTEM_RUN=${DOCKER_MIGRID_ROOT}/state/mig_system_run
+# The apache auth openid module performs and scales better if the associated
+# internal openid store directory runs from fast storage. It's a volatile data
+# store, which allows more concurrent OpenID 2.0 clients if it e.g. uses tmpfs.
+# If you have migoid or extoid in LOGIN_METHODS you likely want to look into
+# that. The instructions for mig_system_run can be mostly reused in that case.
+# Otherwise you can safely ignore the OPENID_STORE setting.
+# NOTE: toggle commenting on next two lines if you have such a tmpfs set up in the given path
+#OPENID_STORE=/storage/tmpfs/openid_store
+OPENID_STORE=${DOCKER_MIGRID_ROOT}/state/openid_store
+# We need a read-only bind mounted version of the vgrid_files_writable
+# directory and the underlying location can be configured here.
+VGRID_FILES_WRITABLE=${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable
 
 # Which svn repo and version of migrid should be used
 #MIG_SVN_REPO=https://svn.code.sf.net/p/migrid/code/trunk

diff --git a/doc/source/sections/configuration/variables.rst b/doc/source/sections/configuration/variables.rst
@@ -409,6 +409,12 @@ Variables
    * - MIG_SYSTEM_RUN
      - "state/mig_system_run"
      - A preferably fast (e.g. tmpfs-backed) scratch folder path for various internal cache and helper files. It must have read/write access by the same USER:GROUP running in the containers and will be shared among all containers for cache and state coherence.
+   * - OPENID_STORE
+     - "state/openid_store"
+     - A preferably fast (e.g. tmpfs-backed) scratch folder path for the optional OpenID 2.0 authentication of users in the apache web server. It must have read/write access by the same USER:GROUP running in the containers and will only be exposed in the migrid container for mod auth openid cache and session state.
+   * - VGRID_FILES_WRITABLE
+     - "state/vgrid_files_writable"
+     - The path where the vgrid_files_writable directory is available. It is used as a source for a read-only bind mount of the data there onto the vgrid_files_readonly directory in order to support users write-protecting VGrids/Workgroups shared folders on the site.
    * - GDP_EMAIL_NOTIFY
      - True
      - Whether to send project administration emails to address(es) configured in state/gdp_home/notifyemails.txt when in GDP mode

diff --git a/docker-compose_development.yml b/docker-compose_development.yml
@@ -121,17 +121,18 @@ services:
         target: /home/mig/state
       # NOTE: for read-only vgrids/workgroups we need a local 'ro' bind-mount
       - type: bind
-        source: ${VGRID_FILES_WRITABLE:-${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable}
+        source: ${VGRID_FILES_WRITABLE}
         target: /home/mig/state/vgrid_files_readonly
         read_only: true
       # NOTE: mig_system_run is a shared volatile cache which gains from using host tmpfs
       - type: bind
         source: ${MIG_SYSTEM_RUN}
         target: /home/mig/state/mig_system_run
-    # NOTE: openid_store is a volatile cache and should use fast local tmpfs
-    # TODO: enable the tmpfs or a similar structure as MIG_SYSTEM_RUN here?
-    #tmpfs:
-    #  - /home/mig/state/openid_store:mode=770,uid=${UID:-1000},gid=${GID:-1000}
+      # NOTE: openid_store is a volatile state for apache mod auth openid and
+      #       gains scalability and performance from using a fast local tmpfs
+      - type: bind
+        source: ${OPENID_STORE}
+        target: /home/mig/state/openid_store
     env_file:
       - migrid-httpd.env
     # IMPORTANT: please ONLY run with this test@ user for non-public hosts
@@ -220,7 +221,7 @@ services:
         target: /home/mig/state
       # NOTE: for read-only vgrids/workgroups we need a local 'ro' bind-mount
       - type: bind
-        source: ${VGRID_FILES_WRITABLE:-${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable}
+        source: ${VGRID_FILES_WRITABLE}
         target: /home/mig/state/vgrid_files_readonly
         read_only: true
       # NOTE: mig_system_run is a shared volatile cache which gains from using host tmpfs
@@ -267,7 +268,7 @@ services:
         target: /home/mig/state
       # NOTE: for read-only vgrids/workgroups we need a local 'ro' bind-mount
       - type: bind
-        source: ${VGRID_FILES_WRITABLE:-${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable}
+        source: ${VGRID_FILES_WRITABLE}
         target: /home/mig/state/vgrid_files_readonly
         read_only: true
       # NOTE: mig_system_run is a shared volatile cache which gains from using host tmpfs
@@ -313,7 +314,7 @@ services:
         target: /home/mig/state
       # NOTE: for read-only vgrids/workgroups we need a local 'ro' bind-mount
       - type: bind
-        source: ${VGRID_FILES_WRITABLE:-${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable}
+        source: ${VGRID_FILES_WRITABLE}
         target: /home/mig/state/vgrid_files_readonly
         read_only: true
       # NOTE: mig_system_run is a shared volatile cache which gains from using host tmpfs

diff --git a/docker-compose_development_gdp.yml b/docker-compose_development_gdp.yml
@@ -121,20 +121,22 @@ services:
         target: /home/mig/state
       # NOTE: for read-only vgrids/workgroups we need a local 'ro' bind-mount
       - type: bind
-        source: ${VGRID_FILES_WRITABLE:-${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable}
+        source: ${VGRID_FILES_WRITABLE}
         target: /home/mig/state/vgrid_files_readonly
         read_only: true
       # NOTE: mig_system_run is a shared volatile cache which gains from using host tmpfs
       - type: bind
         source: ${MIG_SYSTEM_RUN}
         target: /home/mig/state/mig_system_run
+      # NOTE: openid_store is a volatile state for apache mod auth openid and
+      #       gains scalability and performance from using a fast local tmpfs
+      - type: bind
+        source: ${OPENID_STORE}
+        target: /home/mig/state/openid_store
+      # To avoid a bit of docker dbus noise
       - type: bind
         source: /run/dbus/system_bus_socket
         target: /run/dbus/system_bus_socket
-    # NOTE: openid_store is a volatile cache and should use fast local tmpfs
-    # TODO: enable the tmpfs or a similar structure as MIG_SYSTEM_RUN here?
-    #tmpfs:
-    #  - /home/mig/state/openid_store:mode=770,uid=${UID:-1000},gid=${GID:-1000}
     env_file:
       - migrid-httpd.env
     # IMPORTANT: please ONLY run with this test@ user for non-public hosts
@@ -223,7 +225,7 @@ services:
         target: /home/mig/state
       # NOTE: for read-only vgrids/workgroups we need a local 'ro' bind-mount
       - type: bind
-        source: ${VGRID_FILES_WRITABLE:-${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable}
+        source: ${VGRID_FILES_WRITABLE}
         target: /home/mig/state/vgrid_files_readonly
         read_only: true
       # NOTE: mig_system_run is a shared volatile cache which gains from using host tmpfs
@@ -270,7 +272,7 @@ services:
         target: /home/mig/state
       # NOTE: for read-only vgrids/workgroups we need a local 'ro' bind-mount
       - type: bind
-        source: ${VGRID_FILES_WRITABLE:-${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable}
+        source: ${VGRID_FILES_WRITABLE}
         target: /home/mig/state/vgrid_files_readonly
         read_only: true
       # NOTE: mig_system_run is a shared volatile cache which gains from using host tmpfs
@@ -316,7 +318,7 @@ services:
         target: /home/mig/state
       # NOTE: for read-only vgrids/workgroups we need a local 'ro' bind-mount
       - type: bind
-        source: ${VGRID_FILES_WRITABLE:-${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable}
+        source: ${VGRID_FILES_WRITABLE}
         target: /home/mig/state/vgrid_files_readonly
         read_only: true
       # NOTE: mig_system_run is a shared volatile cache which gains from using host tmpfs

diff --git a/docker-compose_production_bind.yml b/docker-compose_production_bind.yml
@@ -98,17 +98,18 @@ services:
         target: /home/mig/state
       # NOTE: for read-only vgrids/workgroups we need a local 'ro' bind-mount
       - type: bind
-        source: ${VGRID_FILES_WRITABLE:-${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable}
+        source: ${VGRID_FILES_WRITABLE}
         target: /home/mig/state/vgrid_files_readonly
         read_only: true
       # NOTE: mig_system_run is a shared volatile cache which gains from using host tmpfs
       - type: bind
         source: ${MIG_SYSTEM_RUN}
         target: /home/mig/state/mig_system_run
-    # NOTE: openid_store is a volatile cache and should use fast local tmpfs
-    # TODO: enable the tmpfs or a similar structure as MIG_SYSTEM_RUN here?
-    #tmpfs:
-    #  - /home/mig/state/openid_store:mode=770,uid=${UID:-1000},gid=${GID:-1000}
+      # NOTE: openid_store is a volatile state for apache mod auth openid and
+      #       gains scalability and performance from using a fast local tmpfs
+      - type: bind
+        source: ${OPENID_STORE}
+        target: /home/mig/state/openid_store
     env_file:
       - migrid-httpd.env
     command: /app/docker-entry.sh -k -V
@@ -188,7 +189,7 @@ services:
         target: /home/mig/state
       # NOTE: for read-only vgrids/workgroups we need a local 'ro' bind-mount
       - type: bind
-        source: ${VGRID_FILES_WRITABLE:-${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable}
+        source: ${VGRID_FILES_WRITABLE}
         target: /home/mig/state/vgrid_files_readonly
         read_only: true
       # NOTE: mig_system_run is a shared volatile cache which gains from using host tmpfs
@@ -231,7 +232,7 @@ services:
         target: /home/mig/state
       # NOTE: for read-only vgrids/workgroups we need a local 'ro' bind-mount
       - type: bind
-        source: ${VGRID_FILES_WRITABLE:-${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable}
+        source: ${VGRID_FILES_WRITABLE}
         target: /home/mig/state/vgrid_files_readonly
         read_only: true
       # NOTE: mig_system_run is a shared volatile cache which gains from using host tmpfs
@@ -274,7 +275,7 @@ services:
         target: /home/mig/state
       # NOTE: for read-only vgrids/workgroups we need a local 'ro' bind-mount
       - type: bind
-        source: ${VGRID_FILES_WRITABLE:-${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable}
+        source: ${VGRID_FILES_WRITABLE}
         target: /home/mig/state/vgrid_files_readonly
         read_only: true
       # NOTE: mig_system_run is a shared volatile cache which gains from using host tmpfs

diff --git a/production.env b/production.env
@@ -151,16 +151,28 @@ EXTRA_USERPAGE_STYLES=""
 # be used by default.
 # NOTE: a shared mig_system_run scratch space on tmpfs can be made with
 #       something like:                                                                             
-# tmpfs   /storage-mem/mig_system_run  tmpfs   nosuid,nodev,noatime,noexec,uid=1000,gid=1000,mode=0770,size=128m   0 0
+# tmpfs   /storage/tmpfs/mig_system_run  tmpfs   nosuid,nodev,noatime,noexec,uid=1000,gid=1000,mode=0770,size=128m   0 0
 # in /etc/fstab. Manual mount can be done with:                                                     
-# sudo mount /storage-mem/mig_system_run
+# sudo mount /storage/tmpfs/mig_system_run
 # NOTE: toggle commenting on  next two lines if you have such a tmpfs set up in the given path
-#MIG_SYSTEM_RUN=/storage-mem/mig_system_run
+#MIG_SYSTEM_RUN=/storage/tmpfs/mig_system_run
 MIG_SYSTEM_RUN=${DOCKER_MIGRID_ROOT}/state/mig_system_run
+# The apache auth openid module performs and scales better if the associated
+# internal openid store directory runs from fast storage. It's a volatile data
+# store, which allows more concurrent OpenID 2.0 clients if it e.g. uses tmpfs.
+# If you have migoid or extoid in LOGIN_METHODS you likely want to look into
+# that. The instructions for mig_system_run can be mostly reused in that case.
+# Otherwise you can safely ignore the OPENID_STORE setting.
+# NOTE: toggle commenting on next two lines if you have such a tmpfs set up in the given path
+#OPENID_STORE=/storage/tmpfs/openid_store
+OPENID_STORE=${DOCKER_MIGRID_ROOT}/state/openid_store
+# We need a read-only bind mounted version of the vgrid_files_writable
+# directory and the underlying location can be configured here.
+VGRID_FILES_WRITABLE=${DOCKER_MIGRID_ROOT}/state/vgrid_files_writable
 
 # Which svn repo and version of migrid should be used
 #MIG_SVN_REPO=https://svn.code.sf.net/p/migrid/code/trunk
-#MIG_SVN_REV=5787
+#MIG_SVN_REV=HEAD
 
 # NOTE: use 'git experimental' here for future python3 version
 # Which git repo and version of migrid should be used