Update Sahana + Sahana Demo for spoc

lxc-apps/sahana-demo/install.sh

@@ -1,43 +1,67 @@
 #!/bin/sh
 set -ev
 
+# Create containers
+spoc-container create \
+    -m sahana-demo/postgres_data:var/lib/postgresql \
+    sahana-demo-postgres postgis_3.0.0-200207
+spoc-container create \
+    -d sahana-demo-postgres \
+    -m sahana-demo/sahana_conf:srv/web2py/applications/eden/models \
+    -m sahana-demo/sahana_data/default:srv/web2py/applications/eden/modules/templates/default \
+    -m sahana-demo/sahana_data/databases:srv/web2py/applications/eden/databases \
+    -m sahana-demo/sahana_data/uploads:srv/web2py/applications/eden/uploads \
+    sahana-demo sahana_0.0.1-200207
+
+# TODO: Populate from spoc
+LAYER_DIR="/var/lib/spoc/layers"
+VOLUME_DIR="/var/lib/spoc/volumes/sahana-demo"
+POSTGRES_UID=70
+POSTGRES_GID=70
+POSTGRES_NSUID=100070
+POSTGRES_NSGID=100070
+SAHANA_UID=103
+SAHANA_GID=104
+SAHANA_NSUID=100103
+SAHANA_NSGID=100104
+
 # Create Postgres instance
-mkdir -p /srv/sahana-demo/postgres_data
-chown -R 105432:105432 /srv/sahana-demo/postgres_data
-chmod 700 /srv/sahana-demo/postgres_data
-lxc-execute -n sahana-demo-postgres -- initdb -D /var/lib/postgresql
+mkdir -p ${VOLUME_DIR}/postgres_data
+chown -R ${POSTGRES_NSUID}:${POSTGRES_NSGID} ${VOLUME_DIR}/postgres_data
+chmod 700 ${VOLUME_DIR}/postgres_data
+spoc-container exec sahana-demo-postgres -- initdb -D /var/lib/postgresql
 
 # Configure Postgres
-cp postgres_data/postgresql.conf /srv/sahana-demo/postgres_data/postgresql.conf
-cp postgres_data/pg_hba.conf /srv/sahana-demo/postgres_data/pg_hba.conf
+cp postgres_data/postgresql.conf ${VOLUME_DIR}/postgres_data/postgresql.conf
+cp postgres_data/pg_hba.conf ${VOLUME_DIR}/postgres_data/pg_hba.conf
 
 # Create PostgreSQL user and database
 export SAHANADEMO_PWD=$(head -c 18 /dev/urandom | base64 | tr -d '+/=')
-service lxc-sahana-demo-postgres start
-envsubst <createdb.sql | lxc-attach -u 5432 -g 5432 sahana-demo-postgres -- psql
+spoc-container start sahana-demo-postgres
+envsubst <createdb.sql | spoc-container exec sahana-demo-postgres -- psql
 
 # Prepare persistent directory structure
-mkdir -p /srv/sahana-demo/sahana_data/databases /srv/sahana-demo/sahana_data/uploads
-chown -R 108080:108080 /srv/sahana-demo/sahana_data
-lxchelper extract sahana-demo /srv/web2py/applications/eden/models /srv/sahana-demo/sahana_conf
-lxchelper extract sahana-demo /srv/web2py/applications/eden/modules/templates/default /srv/sahana-demo/sahana_data/default
+mkdir -p ${VOLUME_DIR}/sahana_data/databases ${VOLUME_DIR}/sahana_data/uploads
+chown -R ${SAHANA_NSUID}:${SAHANA_NSGID} ${VOLUME_DIR}/sahana_data
+cp -rp ${LAYER_DIR}/sahana_0.0.1-200207/srv/web2py/applications/eden/models ${VOLUME_DIR}/sahana_conf
+cp -rp ${LAYER_DIR}/sahana_0.0.1-200207/srv/web2py/applications/eden/modules/templates/default ${VOLUME_DIR}/sahana_data/default
 
 # Configure Sahana
 export SAHANADEMO_HMAC=$(head -c 18 /dev/urandom | base64 | tr -d '+/=')
 export SAHANADEMO_ADMIN_USER=admin@example.com
 export SAHANADEMO_ADMIN_PWD=$(head -c 12 /dev/urandom | base64 | tr -d '+/=')
-envsubst <sahana_conf/000_config.py >/srv/sahana-demo/sahana_conf/000_config.py
-envsubst <masterUsers.csv >/srv/sahana-demo/sahana_data/default/users/masterUsers.csv
-chown -R 108080:108080 /srv/sahana/sahana_conf /srv/sahana/sahana_data
+envsubst <sahana_conf/000_config.py >${VOLUME_DIR}/sahana_conf/000_config.py
+envsubst <masterUsers.csv >${VOLUME_DIR}/sahana_data/default/users/masterUsers.csv
+chown -R ${SAHANA_NSUID}:${SAHANA_NSGID} ${VOLUME_DIR}/sahana_conf ${VOLUME_DIR}/sahana_data
 
 # Populate database
-lxc-execute -u 8080 -g 8080 sahana-demo -- sh -c 'cd /srv/web2py; ./web2py.py -S eden -M -R applications/eden/static/scripts/tools/noop.py'
+spoc-container exec -u sahana sahana-demo -- sh -c 'cd /srv/web2py; ./web2py.py -S eden -M -R applications/eden/static/scripts/tools/noop.py'
 
 # Install config update script
-cp update-conf.sh /srv/sahana-demo/update-conf.sh
+cp update-conf.sh ${VOLUME_DIR}/update-conf.sh
 
 # Stop services required for setup
-service lxc-sahana-demo-postgres stop
+spoc-container stop sahana-postgres
 
 # Register application
-vmmgr register-app sahana-demo sahana-demo "${SAHANADEMO_ADMIN_USER}" "${SAHANADEMO_ADMIN_PWD}"
+#vmmgr register-app sahana-demo sahana-demo "${SAHANADEMO_ADMIN_USER}" "${SAHANADEMO_ADMIN_PWD}"

lxc-apps/sahana/image

@@ -3,14 +3,14 @@ FROM alpine3.11-python3.8_3.8.1-200207
 
 RUN EOF
     # Install runtime dependencies
-    apk --no-cache add geos nginx py3-gdal py3-dateutil py3-lxml py3-numpy py3-pillow py3-psycopg2 py3-requests uwsgi-python3
+    apk --no-cache add geos nginx py3-gdal py3-dateutil py3-lxml py3-numpy py3-pillow py3-psycopg2 py3-requests py3-sgmllib3k uwsgi-python3
 
     # Install build dependencies
-    apk --no-cache add --virtual .deps build-base git freetype-dev libpng-dev py3-numpy-dev py3-pip python3-dev ttf-dejavu
+    apk --no-cache add --virtual .deps build-base freetype-dev git py3-numpy-dev py3-pip python3-dev ttf-dejavu
 
     # Install web2py
     git clone --recursive https://github.com/web2py/web2py.git /srv/web2py
-    git -C /srv/web2py checkout 6128d03
+    git -C /srv/web2py checkout 59700b8
     git -C /srv/web2py submodule update
 
     # Symlink WSGI handler
@@ -36,15 +36,20 @@ COPY image.d
 
 RUN EOF
     # Create OS user
-    addgroup sahana
-    adduser -h /srv/web2py -s /bin/false -g sahana -G sahana sahana
+    addgroup -S sahana
+    adduser -S -h /srv/web2py -s /bin/false -g sahana -G sahana sahana
     chown -R sahana:sahana /srv/web2py
 
+    # Patch web2py for python 3.8
+    cd /srv/web2py
+    patch -p0 </tmp/python38.patch
+
     # Cleanup
     apk --no-cache del .deps
     find /srv/web2py -name '.git*' -exec rm -rf {} +
-    rm -r /srv/web2py/applications/admin /srv/web2py/applications/examples /srv/web2py/applications/welcome
+    # rm -r /srv/web2py/applications/admin /srv/web2py/applications/examples /srv/web2py/applications/welcome
     rm -r /root/.cache
+    rm /tmp/python38.patch
 EOF
 
-CMD s6-svscan /etc/services.d
+CMD /bin/s6-svscan /etc/services.d

lxc-apps/sahana/image.d/srv/web2py/routes.py

@@ -5,7 +5,6 @@ default_function = 'index'
 routes_onerror = [
     ('eden/400', '!'),
     ('eden/401', '!'),
-    ('eden/509', '!'),
     ('eden/*', '/eden/errors/index'),
     ('*/*', '/eden/errors/index'),
 ]

lxc-apps/sahana/image.d/tmp/python38.patch

@@ -0,0 +1,115 @@
+--- gluon/compileapp.py
++++ gluon/compileapp.py
+@@ -61,7 +61,7 @@
+ TEST_CODE = \
+     r"""
+ def _TEST():
+-    import doctest, sys, cStringIO, types, cgi, gluon.fileutils
++    import doctest, sys, cStringIO, types, gluon.fileutils
+     if not gluon.fileutils.check_credentials(request):
+         raise HTTP(401, web2py_error='invalid credentials')
+     stdout = sys.stdout
+--- gluon/languages.py
++++ gluon/languages.py
+@@ -16,7 +16,6 @@
+ import sys
+ import pkgutil
+ import logging
+-from cgi import escape
+ from threading import RLock
+ 
+ from pydal._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin
+--- gluon/packages/dal/pydal/_compat.py
++++ gluon/packages/dal/pydal/_compat.py
+@@ -32,7 +32,6 @@
+     from urllib2 import urlopen
+     from string import maketrans
+     from types import ClassType
+-    import cgi
+     import cookielib
+     from xmlrpclib import ProtocolError
+     BytesIO = StringIO
+Only in web2py: gluon/packages/dal/pydal/_compat.py.orig
+--- gluon/packages/dal/pydal/adapters/oracle.py
++++ gluon/packages/dal/pydal/adapters/oracle.py
+@@ -96,7 +96,7 @@
+         return self.dialect.quote(tablename)
+ 
+     def _build_value_for_insert(self, field, value, r_values):
+-        if field.type is 'text':
++        if field.type == 'text':
+             r_values[':' + field._rname] = self.expand(value, field.type)
+             return ':' + field._rname
+         return self.expand(value, field.type)
+Only in web2py: gluon/packages/dal/pydal/adapters/oracle.py.orig
+--- gluon/packages/dal/pydal/contrib/portalocker.py
++++ gluon/packages/dal/pydal/contrib/portalocker.py
+@@ -193,6 +193,9 @@
+     def read(self, size=None):
+         return self.file.read() if size is None else self.file.read(size)
+ 
++    def readinto(self, b):
++        b[:] = self.file.read()
++
+     def readline(self):
+         return self.file.readline()
+ 
+--- gluon/packages/dal/pydal/helpers/classes.py
++++ gluon/packages/dal/pydal/helpers/classes.py
+@@ -532,6 +532,9 @@
+         self.p += len(data)
+         return data
+ 
++    def readinto(self, bytes):
++        return self.read(bytes)
++
+     def readline(self):
+         i = self.data.find('\n', self.p)+1
+         if i > 0:
+--- gluon/packages/dal/pydal/validators.py
++++ gluon/packages/dal/pydal/validators.py
+@@ -148,7 +148,7 @@
+ 
+ 
+ def validator_caller(func, value):
+-    if getattr(func, 'validate', None) is Validator.validate:
++    if getattr(func, 'validate', None) is not Validator.validate:
+         return func.validate(value)
+     value, error = func(value)
+     if error is not None:
+@@ -3154,7 +3154,7 @@
+                 if not all_special.count(True) >= self.special:
+                     failures.append(self.translator("Must include at least %s of the following: %s")
+                                     % (self.special, self.specials))
+-            elif self.special is 0:
++            elif self.special == 0 and self.special is not False:
+                 if len(all_special) > 0:
+                     failures.append(self.translator("May not contain any of the following: %s")
+                                     % self.specials)
+@@ -3169,7 +3169,7 @@
+                 if not len(all_upper) >= self.upper:
+                     failures.append(self.translator("Must include at least %s uppercase")
+                                     % str(self.upper))
+-            elif self.upper is 0:
++            elif self.upper == 0 and self.upper is not False:
+                 if len(all_upper) > 0:
+                     failures.append(
+                         self.translator("May not include any uppercase letters"))
+@@ -3179,7 +3179,7 @@
+                 if not len(all_lower) >= self.lower:
+                     failures.append(self.translator("Must include at least %s lowercase")
+                                     % str(self.lower))
+-            elif self.lower is 0:
++            elif self.lower == 0 and self.lower is not False:
+                 if len(all_lower) > 0:
+                     failures.append(
+                         self.translator("May not include any lowercase letters"))
+@@ -3192,7 +3192,7 @@
+                 if not len(all_number) >= self.number:
+                     failures.append(self.translator("Must include at least %s %s")
+                                     % (str(self.number), numbers))
+-            elif self.number is 0:
++            elif self.number == 0 and self.number is not False:
+                 if len(all_number) > 0:
+                     failures.append(self.translator("May not include any numbers"))
+         if len(failures) == 0:

lxc-apps/sahana/install.sh

@@ -1,54 +1,78 @@
 #!/bin/sh
 set -ev
 
+# Create containers
+spoc-container create \
+    -m sahana/postgres_data:var/lib/postgresql \
+    sahana-postgres postgis_3.0.0-200207
+spoc-container create \
+    -d sahana-postgres \
+    -m sahana/sahana_conf:srv/web2py/applications/eden/models \
+    -m sahana/sahana_data/Spotter:srv/web2py/applications/eden/modules/templates/Spotter \
+    -m sahana/sahana_data/databases:srv/web2py/applications/eden/databases \
+    -m sahana/sahana_data/uploads:srv/web2py/applications/eden/uploads \
+    sahana sahana_0.0.1-200207
+
+# TODO: Populate from spoc
+LAYER_DIR="/var/lib/spoc/layers"
+VOLUME_DIR="/var/lib/spoc/volumes/sahana"
+POSTGRES_UID=70
+POSTGRES_GID=70
+POSTGRES_NSUID=100070
+POSTGRES_NSGID=100070
+SAHANA_UID=103
+SAHANA_GID=104
+SAHANA_NSUID=100103
+SAHANA_NSGID=100104
+
 # Create Postgres instance
-mkdir -p /srv/sahana/postgres_data
-chown -R 105432:105432 /srv/sahana/postgres_data
-chmod 700 /srv/sahana/postgres_data
-lxc-execute -n sahana-postgres -- initdb -D /var/lib/postgresql
+mkdir -p ${VOLUME_DIR}/postgres_data
+chown -R ${POSTGRES_NSUID}:${POSTGRES_NSGID} ${VOLUME_DIR}/postgres_data
+chmod 700 ${VOLUME_DIR}/postgres_data
+spoc-container exec sahana-postgres -- initdb -D /var/lib/postgresql
 
 # Configure Postgres
-cp postgres_data/postgresql.conf /srv/sahana/postgres_data/postgresql.conf
-cp postgres_data/pg_hba.conf /srv/sahana/postgres_data/pg_hba.conf
+cp postgres_data/postgresql.conf ${VOLUME_DIR}/postgres_data/postgresql.conf
+cp postgres_data/pg_hba.conf ${VOLUME_DIR}/postgres_data/pg_hba.conf
 
 # Create PostgreSQL user and database
 export SAHANA_PWD=$(head -c 18 /dev/urandom | base64 | tr -d '+/=')
-service lxc-sahana-postgres start
-envsubst <createdb.sql | lxc-attach -u 5432 -g 5432 sahana-postgres -- psql
+spoc-container start sahana-postgres
+envsubst <createdb.sql | spoc-container exec sahana-postgres -- psql
 
 # Prepare persistent directory structure
-mkdir -p /srv/sahana/sahana_data/databases /srv/sahana/sahana_data/uploads
-chown -R 108080:108080 /srv/sahana/sahana_data
-lxchelper extract sahana /srv/web2py/applications/eden/models /srv/sahana/sahana_conf
+mkdir -p ${VOLUME_DIR}/sahana_data/databases ${VOLUME_DIR}/sahana_data/uploads
+chown -R ${SAHANA_NSUID}:${SAHANA_NSGID} ${VOLUME_DIR}/sahana_data
+cp -rp ${LAYER_DIR}/sahana_0.0.1-200207/srv/web2py/applications/eden/models ${VOLUME_DIR}/sahana_conf
 
 # Copy Spotter template
-cp -r sahana_data/Spotter /srv/sahana/sahana_data/
-chown -R 108080:108080 /srv/sahana/sahana_data
+cp -r sahana_data/Spotter ${VOLUME_DIR}/sahana_data/
+chown -R ${SAHANA_NSUID}:${SAHANA_NSGID} ${VOLUME_DIR}/sahana_data
 
 # Configure Sahana
 export SAHANA_HMAC=$(head -c 18 /dev/urandom | base64 | tr -d '+/=')
 export SAHANA_ADMIN_USER=admin@example.com
 export SAHANA_ADMIN_PWD=$(head -c 12 /dev/urandom | base64 | tr -d '+/=')
-envsubst <sahana_conf/000_config.py >/srv/sahana/sahana_conf/000_config.py
-envsubst <sahana_data/Spotter/masterUsers.csv >/srv/sahana/sahana_data/Spotter/masterUsers.csv
-cp sahana_conf/00_settings.py /srv/sahana/sahana_conf/00_settings.py
-chown -R 108080:108080 /srv/sahana/sahana_conf /srv/sahana/sahana_data
+envsubst <sahana_conf/000_config.py >${VOLUME_DIR}/sahana_conf/000_config.py
+envsubst <sahana_data/Spotter/masterUsers.csv >${VOLUME_DIR}/sahana_data/Spotter/masterUsers.csv
+cp sahana_conf/00_settings.py ${VOLUME_DIR}/sahana_conf/00_settings.py
+chown -R ${SAHANA_NSUID}:${SAHANA_NSGID} ${VOLUME_DIR}/sahana_conf ${VOLUME_DIR}/sahana_data
 
 # Populate database
-lxc-execute -u 8080 -g 8080 sahana -- sh -c 'cd /srv/web2py; ./web2py.py -S eden -M -R applications/eden/static/scripts/tools/noop.py'
+spoc-container exec -u sahana sahana -- sh -c 'cd /srv/web2py; ./web2py.py -S eden -M -R applications/eden/static/scripts/tools/noop.py'
 
 # Set "production values" (increases performance) only if the DEBUG environment variable is not set
 if [ ${DEBUG:-0} -eq 0 ]; then
-    sed -i 's/settings.base.migrate = True/settings.base.migrate = False/' /srv/sahana/sahana_conf/000_config.py
-    sed -i 's/settings.base.debug = True/settings.base.debug = False/' /srv/sahana/sahana_conf/000_config.py
-    sed -i 's/#settings.base.prepopulate = 0/settings.base.prepopulate = 0/' /srv/sahana/sahana_conf/000_config.py
+    sed -i 's/settings.base.migrate = True/settings.base.migrate = False/' ${VOLUME_DIR}/sahana_conf/000_config.py
+    sed -i 's/settings.base.debug = True/settings.base.debug = False/' ${VOLUME_DIR}/sahana_conf/000_config.py
+    sed -i 's/#settings.base.prepopulate = 0/settings.base.prepopulate = 0/' ${VOLUME_DIR}/sahana_conf/000_config.py
 fi
 
 # Install config update script
-cp update-conf.sh /srv/sahana/update-conf.sh
+cp update-conf.sh ${VOLUME_DIR}/update-conf.sh
 
 # Stop services required for setup
-service lxc-sahana-postgres stop
+spoc-container stop sahana-postgres
 
 # Register application
-vmmgr register-app sahana sahana "${SAHANA_ADMIN_USER}" "${SAHANA_ADMIN_PWD}"
+#vmmgr register-app sahana sahana "${SAHANA_ADMIN_USER}" "${SAHANA_ADMIN_PWD}"