Installation Guidelines: Linux (Production: Cherokee/PostgreSQL)

Installs that we maintain are usually done using Cherokee & PostgreSQL

• Cherokee is a high-performance web server with a graphical admin UI
• PostgreSQL is a database with powerful geospatial support

The simplest way to install is using our attached installation scripts.

This is split into 2 halves, as the 1st script can create a base image which is then customised per-instance using the configure script.

Note: We install using the latest Trunk versions of both Web2Py & Eden. This can occasionally cause problems.

Before Imaging

install-eden-cherokee-postgis.sh

#!/bin/sh

# Script to turn a generic Debian Squeeze box into an Eden server
# with Cherokee & PostgreSQL
# - tunes PostgreSQL for 1Gb RAM

# Update system
apt-get update
apt-get upgrade -y

# Install Admin Tools
apt-get install -y unzip psmisc mlocate telnet lrzsz vim elinks-lite rcconf htop sudo
# Email
apt-get -y install exim4-config exim4-daemon-light

#########
# Python
#########
# Install Libraries
apt-get -y install libgeos-c1

# Install Python 2.6
apt-get -y install python2.6 python-dev ipython
apt-get -y install python-lxml python-setuptools python-shapely python-dateutil
apt-get -y install python-serial
apt-get -y install python-imaging python-reportlab
apt-get -y install python-xlwt python-xlrd
apt-get -y install bzr

#########
# Web2Py
#########
# Install Web2Py
adduser --system --disabled-password web2py
addgroup web2py
cd /home
bzr branch lp:~mdipierro/web2py/devel web2py --use-existing-dir
ln -s /home/web2py ~
cat << EOF > "/home/web2py/routes.py"
#!/usr/bin/python
default_application = 'eden'
default_controller = 'default'
default_function = 'index'
routes_onerror = [
        ('eden/400', '!'),
        ('eden/401', '!'),
        ('eden/*', '/eden/errors/index'),
        ('*/*', '/eden/errors/index'),
    ]
EOF


##############
# Sahana Eden
##############
# Install Sahana Eden
cd web2py
cd applications
bzr branch lp:sahana-eden eden
# Fix permissions
chown web2py ~web2py
chown web2py ~web2py/applications/admin/cache
chown web2py ~web2py/applications/admin/cron
chown web2py ~web2py/applications/admin/databases
chown web2py ~web2py/applications/admin/errors
chown web2py ~web2py/applications/admin/sessions
chown web2py ~web2py/applications/eden
chown web2py ~web2py/applications/eden/cache
chown web2py ~web2py/applications/eden/cron
chown web2py ~web2py/applications/eden/databases
chown web2py ~web2py/applications/eden/errors
chown web2py ~web2py/applications/eden/models
chown web2py ~web2py/applications/eden/sessions
chown web2py ~web2py/applications/eden/static/img/markers
chown web2py ~web2py/applications/eden/uploads
mkdir -p ~web2py/applications/eden/uploads/gis_cache
mkdir -p ~web2py/applications/eden/uploads/images
mkdir -p ~web2py/applications/eden/uploads/tracks
chown web2py ~web2py/applications/eden/uploads/gis_cache
chown web2py ~web2py/applications/eden/uploads/images
chown web2py ~web2py/applications/eden/uploads/tracks
ln -s /home/web2py/applications/eden ~

##########
# Cherokee
##########
# Debian current version
#apt-get -y install libcherokee-mod-libssl libcherokee-mod-rrd
# Source version
CHEROKEE_CONF="/usr/local/etc/cherokee/cherokee.conf"
apt-get install -y gettext libgeoip-dev libssl-dev rrdtool make
cd /tmp
wget http://www.cherokee-project.com/download/1.2/1.2.99/cherokee-1.2.99.tar.gz
tar zxvf cherokee-1.2.99.tar.gz
cd cherokee-1.2.99
./configure --datarootdir=/var/www
make
make install
mkdir /var/log/cherokee
chown www-data /var/log/cherokee
wget http://eden.sahanafoundation.org/raw-attachment/wiki/InstallationGuidelinesCherokee/cherokee -O /etc/init.d/cherokee
chmod +x /etc/init.d/cherokee
ldconfig
rcconf --on cherokee

# Install uWSGI
apt-get install -y libxml2-dev
cd /tmp
wget http://projects.unbit.it/downloads/uwsgi-0.9.9.2.tar.gz
tar zxvf uwsgi-0.9.9.2.tar.gz
cd uwsgi-0.9.9.2
make
cp uwsgi /usr/local/bin
cd ..

# Configure Cherokee
cat << EOF > "/home/web2py/uwsgi.xml"
<uwsgi>
    <pythonpath>/home/web2py/</pythonpath>
    <app mountpoint="/">
        <script>wsgihandler</script>
    </app>
    <workers>4</workers>
</uwsgi>
EOF

mv "$CHEROKEE_CONF" /tmp
cat << EOF > "$CHEROKEE_CONF"
config!version = 001002002
server!bind!1!port = 80
server!collector = rrd
server!fdlimit = 10240
server!group = www-data
server!ipv6 = 0
server!keepalive = 1
server!keepalive_max_requests = 500
server!panic_action = /usr/share/cherokee/cherokee-panic
server!pid_file = /var/run/cherokee.pid
server!server_tokens = product
server!timeout = 300
server!user = www-data
vserver!10!collector!enabled = 1
vserver!10!directory_index = index.html
vserver!10!document_root = /var/www
vserver!10!error_writer!filename = /var/log/cherokee/cherokee.error
vserver!10!error_writer!type = file
vserver!10!logger = combined
vserver!10!logger!access!buffsize = 16384
vserver!10!logger!access!filename = /var/log/cherokee/cherokee.access
vserver!10!logger!access!type = file
vserver!10!nick = default
vserver!10!rule!10!handler = common
vserver!10!rule!10!handler!iocache = 1
vserver!10!rule!10!match = default
vserver!20!collector!enabled = 1
vserver!20!directory_index = index.html
vserver!20!document_root = /var/www
vserver!20!error_writer!filename = /var/log/cherokee/cherokee.error
vserver!20!error_writer!type = file
vserver!20!logger = combined
vserver!20!logger!access!buffsize = 16384
vserver!20!logger!access!filename = /var/log/cherokee/cherokee.access
vserver!20!logger!access!type = file
vserver!20!match = wildcard
vserver!20!match!domain!1 = *
vserver!20!match!nick = 0
vserver!20!nick = maintenance
vserver!20!rule!210!handler = file
vserver!20!rule!210!match = fullpath
vserver!20!rule!210!match!fullpath!1 = /maintenance.html
vserver!20!rule!110!handler = redir
vserver!20!rule!110!handler!rewrite!10!regex = ^/*
vserver!20!rule!110!handler!rewrite!10!show = 1
vserver!20!rule!110!handler!rewrite!10!substring = /maintenance.html
vserver!20!rule!110!match = directory
vserver!20!rule!110!match!directory = /
vserver!20!rule!10!handler = common
vserver!20!rule!10!handler!iocache = 1
vserver!20!rule!10!match = default
vserver!30!collector!enabled = 1
vserver!30!directory_index = index.html
vserver!30!document_root = /var/www
vserver!30!error_writer!filename = /var/log/cherokee/cherokee.error
vserver!30!error_writer!type = file
vserver!30!logger = combined
vserver!30!logger!access!buffsize = 16384
vserver!30!logger!access!filename = /var/log/cherokee/cherokee.access
vserver!30!logger!access!type = file
vserver!30!match = wildcard
vserver!30!match!domain!1 = *
vserver!30!match!nick = 0
vserver!30!nick = Production
vserver!30!rule!400!document_root = /home/web2py/applications/eden/static
vserver!30!rule!400!encoder!deflate = allow
vserver!30!rule!400!encoder!gzip = allow
vserver!30!rule!400!expiration = time
vserver!30!rule!400!expiration!time = 7d
vserver!30!rule!400!handler = file
vserver!30!rule!400!match = fullpath
vserver!30!rule!400!match!fullpath!1 = /favicon.ico
vserver!30!rule!400!match!fullpath!2 = /robots.txt
vserver!30!rule!400!match!fullpath!3 = /crossdomain.xml
vserver!30!rule!300!document_root = /home/web2py/applications/eden/static
vserver!30!rule!300!encoder!deflate = allow
vserver!30!rule!300!encoder!gzip = allow
vserver!30!rule!300!expiration = time
vserver!30!rule!300!expiration!time = 7d
vserver!30!rule!300!handler = file
vserver!30!rule!300!match = directory
vserver!30!rule!300!match!directory = /eden/static/
vserver!30!rule!300!match!final = 1
vserver!30!rule!200!encoder!deflate = allow
vserver!30!rule!200!encoder!gzip = allow
vserver!30!rule!200!handler = uwsgi
vserver!30!rule!200!handler!balancer = round_robin
vserver!30!rule!200!handler!balancer!source!10 = 1
vserver!30!rule!200!handler!check_file = 0
vserver!30!rule!200!handler!error_handler = 1
vserver!30!rule!200!handler!modifier1 = 0
vserver!30!rule!200!handler!modifier2 = 0
vserver!30!rule!200!handler!pass_req_headers = 1
vserver!30!rule!200!match = directory
vserver!30!rule!200!match!directory = /
vserver!30!rule!100!handler = common
vserver!30!rule!100!handler!iocache = 1
vserver!30!rule!100!match = default
source!1!env_inherited = 1
source!1!group = web2py
source!1!host = 127.0.0.1:59025
source!1!interpreter = /usr/local/bin/uwsgi -s 127.0.0.1:59025 -x /home/web2py/uwsgi.xml
source!1!nick = uWSGI 1
source!1!timeout = 300
source!1!type = interpreter
source!1!user = web2py
EOF

grep 'icons!' /tmp/cherokee.conf >> "$CHEROKEE_CONF"
grep 'mime!' /tmp/cherokee.conf >> "$CHEROKEE_CONF"

cat << EOF >> "$CHEROKEE_CONF"
admin!ows!enabled = 0
EOF

# For a static home page, push 400->500 & 300->400 & insert this
#vserver!30!rule!300!document_root = /home/web2py/applications/eden/static
#vserver!30!rule!300!handler = redir
#vserver!30!rule!300!handler!rewrite!10!regex = ^.*$
#vserver!30!rule!300!handler!rewrite!10!show = 1
#vserver!30!rule!300!handler!rewrite!10!substring = /eden/static/index.html
#vserver!30!rule!300!match = and
#vserver!30!rule!300!match!final = 1
#vserver!30!rule!300!match!left = fullpath
#vserver!30!rule!300!match!left!fullpath!1 = /
#vserver!30!rule!300!match!right = not
#vserver!30!rule!300!match!right!right = header
#vserver!30!rule!300!match!right!right!complete = 0
#vserver!30!rule!300!match!right!right!header = Cookie
#vserver!30!rule!300!match!right!right!match = re
#vserver!30!rule!300!match!right!right!type = regex


cat << EOF > "/var/www/maintenance.html"
<html><body><h1>Site Maintenance</h1>Please try again later...</body></html>
EOF

chown -R www-data /usr/local/var/lib/cherokee
/etc/init.d/cherokee restart

# Logfile rotation
cat << EOF > "/etc/logrotate.d/cherokee"
/var/log/cherokee/*.error /var/log/cherokee/*.access /var/log/cherokee.access /var/log/cherokee.error {
  rotate 7
  weekly
  compress
  copytruncate
  missingok
  notifempty
  copytruncate
}
EOF

############
# PostgreSQL
############
apt-get -y install postgresql-8.4 python-psycopg2 postgresql-8.4-postgis ptop

# Tune PostgreSQL
cat << EOF >> "/etc/sysctl.conf"
## Increase Shared Memory available for PostgreSQL
# 512Mb
#kernel.shmmax = 279134208
# 1024Mb (may need more)
kernel.shmmax = 552992768
kernel.shmall = 2097152
EOF
sysctl -w kernel.shmmax=552992768
sysctl -w kernel.shmall=2097152

sed -i 's|#track_counts = on|track_counts = on|' /etc/postgresql/8.4/main/postgresql.conf
sed -i 's|#autovacuum = on|autovacuum = on|' /etc/postgresql/8.4/main/postgresql.conf
sed -i 's|shared_buffers = 28MB|shared_buffers = 160MB|' /etc/postgresql/8.4/main/postgresql.conf
sed -i 's|#effective_cache_size = 128MB|effective_cache_size = 512MB|' /etc/postgresql/8.4/main/postgresql.conf
sed -i 's|#work_mem = 1MB|work_mem = 4MB|' /etc/postgresql/8.4/main/postgresql.conf

#####################
# Management scripts
#####################
cat << EOF > "/usr/local/bin/backup"
#!/bin/sh
mkdir /var/backups/eden
chown postgres /var/backups/eden
NOW=\$(date +"%Y-%m-%d")
su -c - postgres "pg_dump -c sahana > /var/backups/eden/sahana-\$NOW.sql"
su -c - postgres "pg_dump -Fc gis > /var/backups/eden/gis.dmp"
OLD=\$(date --date='7 day ago' +"%Y-%m-%d")
rm -f /var/backups/eden/sahana-\$OLD.sql
EOF
chmod +x /usr/local/bin/backup

cat << EOF > "/usr/local/bin/compile"
#!/bin/sh
cd ~web2py
python web2py.py -S eden -R applications/eden/static/scripts/tools/compile.py
killall cherokee
sleep 2
/etc/init.d/cherokee start
EOF
chmod +x /usr/local/bin/compile

#@ToDo: Complete Maintenance Site
# (currently just disables Scheduler)
cat << EOF > "/usr/local/bin/maintenance"
#!/bin/sh
if [ "" != "off" ]
then
    #a2dissite maintenance
    #a2ensite production
    cd ~web2py && sudo -H -u web2py python web2py.py -K eden -Q >/dev/null 2>&1 &
else
    killall -u web2py python
    #a2ensite maintenance
    #a2dissite production
fi
killall cherokee
sleep 2
/etc/init.d/cherokee start
EOF
chmod +x /usr/local/bin/maintenance

cat << EOF > "/usr/local/bin/pull"
#!/bin/sh
cd ~web2py/applications/eden
sed -i 's/deployment_settings.base.migrate = False/deployment_settings.base.migrate = True/g' models/000_config.py
bzr pull
/usr/local/bin/maintenance
rm -rf compiled
cd ~web2py
sudo -H -u web2py python web2py.py -S eden -M -R applications/eden/static/scripts/tools/noop.py
cd ~web2py/applications/eden
sed -i 's/deployment_settings.base.migrate = True/deployment_settings.base.migrate = False/g' models/000_config.py
/usr/local/bin/maintenance off
/usr/local/bin/compile
EOF
chmod +x /usr/local/bin/pull

# Change the value of prepopulate, if-necessary
cat << EOF > "/usr/local/bin/clean"
#!/bin/sh
cd ~web2py/applications/eden
rm -f databases/*
rm -f errors/*
rm -f sessions/*
rm -f uploads/*
pkill -f 'postgres: sahana sahana'
sudo -H -u postgres dropdb sahana
sed -i 's/deployment_settings.base.migrate = False/deployment_settings.base.migrate = True/g' models/000_config.py
sed -i 's/deployment_settings.base.prepopulate = 0/deployment_settings.base.prepopulate = 1/g' models/000_config.py
rm -rf compiled
su -c - postgres "createdb -O sahana -E UTF8 sahana -T template0"
su -c - postgres "createlang plpgsql -d sahana"
su -c - postgres "psql -q -d sahana -f /usr/share/postgresql/8.4/contrib/postgis-1.5/postgis.sql"
su -c - postgres "psql -q -d sahana -f /usr/share/postgresql/8.4/contrib/postgis-1.5/spatial_ref_sys.sql"
cd ~web2py
sudo -H -u web2py python web2py.py -S eden -M -R applications/eden/static/scripts/tools/noop.py
cd ~web2py/applications/eden
sed -i 's/deployment_settings.base.migrate = True/deployment_settings.base.migrate = False/g' models/000_config.py
sed -i 's/deployment_settings.base.prepopulate = 1/deployment_settings.base.prepopulate = 0/g' models/000_config.py
/usr/local/bin/maintenance off
/usr/local/bin/compile
EOF
chmod +x /usr/local/bin/clean

cat << EOF > "/usr/local/bin/w2p"
#!/bin/sh
cd ~web2py
python web2py.py -S eden -M
EOF
chmod +x /usr/local/bin/w2p

# END

After Imaging

configure-eden-cherokee-postgis.sh

#!/bin/sh

# Script to configure an Eden server
# - assumes that install-eden-cherokee-postgis.sh has been run

BRANCH="~flavour/sahana-eden/trunk"

if [ "$BRANCH" != "~flavour/sahana-eden/trunk" ]; then
    echo "parent_location = http://bazaar.launchpad.net/$BRANCH" > /home/web2py/applications/eden/.bzr/branch/branch.conf
fi

#RAM=1
#if [ $RAM != 1 ]; then
    # @ToDo: Option to tune PostgreSQL from the default 1Gb RAM
    # 512
    #/etc/sysctl.conf
    #552992768->279134208
    #/etc/postgresql/8.4/main/postgresql.conf
#fi

echo -e "What domain name should we use? : \c "
read DOMAIN

echo -e "What host name should we use? : \c "
read hostname
sitename=$hostname".$DOMAIN"

echo -e "What is the new PostgreSQL password: \c "
read password

echo "Now reconfiguring system to use the hostname: $hostname"

cd /etc
filename="hosts"
sed -i "s|localdomain localhost|localdomain localhost $hostname|" $filename

cd /etc
filename="hostname"
echo $hostname > $filename

cd /etc
filename="mailname"
echo $sitename >  $filename

# Update system (in case run at a much later time than the install script)
apt-get update
apt-get upgrade -y
cd ~web2py
bzr pull
cd ~web2py/applications/eden
bzr pull
# -----------------------------------------------------------------------------
# Email
# -----------------------------------------------------------------------------
echo configure for Internet mail delivery
dpkg-reconfigure exim4-config

# -----------------------------------------------------------------------------
# Sahana Eden
# -----------------------------------------------------------------------------
echo "Setting up Sahana"

# Copy Templates
cp ~web2py/applications/eden/deployment-templates/cron/crontab ~web2py/applications/eden/cron
cp ~web2py/applications/eden/deployment-templates/models/000_config.py ~web2py/applications/eden/models

sed -i 's|EDITING_CONFIG_FILE = False|EDITING_CONFIG_FILE = True|' ~web2py/applications/eden/models/000_config.py
sed -i "s|akeytochange|$sitename$password|" ~web2py/applications/eden/models/000_config.py
sed -i "s|127.0.0.1:8000|$sitename|" ~web2py/applications/eden/models/000_config.py
sed -i 's|base.cdn = False|base.cdn = True|' ~web2py/applications/eden/models/000_config.py

# PostgreSQL
echo "CREATE USER sahana WITH PASSWORD '$password';" > /tmp/pgpass.sql
su -c - postgres "psql -q -d template1 -f /tmp/pgpass.sql"
rm -f /tmp/pgpass.sql
su -c - postgres "createdb -O sahana -E UTF8 sahana -T template0"
su -c - postgres "createlang plpgsql -d sahana"

# PostGIS
su -c - postgres "psql -q -d sahana -f /usr/share/postgresql/8.4/contrib/postgis-1.5/postgis.sql"
su -c - postgres "psql -q -d sahana -f /usr/share/postgresql/8.4/contrib/postgis-1.5/spatial_ref_sys.sql"

# Configure Database
sed -i 's|deployment_settings.database.db_type = "sqlite"|deployment_settings.database.db_type = "postgres"|' ~web2py/applications/eden/models/000_config.py
sed -i "s|deployment_settings.database.password = \"password\"|deployment_settings.database.password = \"$password\"|" ~web2py/applications/eden/models/000_config.py
sed -i 's|deployment_settings.gis.spatialdb = False|deployment_settings.gis.spatialdb = True|' ~web2py/applications/eden/models/000_config.py

# Create the Tables & Populate with base data
sed -i 's|deployment_settings.base.prepopulate = 0|deployment_settings.base.prepopulate = 1|' ~web2py/applications/eden/models/000_config.py
sed -i 's|deployment_settings.base.migrate = False|deployment_settings.base.migrate = True|' ~web2py/applications/eden/models/000_config.py
cd ~web2py
sudo -H -u web2py python web2py.py -S eden -M -R applications/eden/static/scripts/tools/noop.py

# PostGIS the Sahana DB
cat << EOF > "/tmp/geometry.sql"
UPDATE public.gis_location SET wkt = 'POINT (' || lon || ' ' || lat || ')' WHERE gis_feature_type = 1;
SELECT AddGeometryColumn( 'public', 'gis_location', 'the_geom', 4326, 'GEOMETRY', 2 );
UPDATE public.gis_location SET the_geom = ST_SetSRID(ST_GeomFromText(wkt), 4326);
EOF
su -c - postgres "psql -q -d sahana -f /tmp/geometry.sql"
cat << EOF > "/tmp/autopopulate.sql"
CREATE OR REPLACE FUNCTION s3_update_geometry()
  RETURNS "trigger" AS \$$
  DECLARE
  BEGIN
    if (NEW.wkt != '') then
        NEW.the_geom = SetSRID(GeomFromText(NEW.wkt), 4326);
        end if;

    RETURN NEW;
  END;
\$$  LANGUAGE 'plpgsql' VOLATILE;
ALTER FUNCTION s3_update_geometry() OWNER TO sahana;
CREATE TRIGGER s3_locations_update
  BEFORE INSERT
  ON gis_location
  FOR EACH ROW
  EXECUTE PROCEDURE s3_update_geometry();

EOF
su -c - postgres "psql -q -d sahana -f /tmp/autopopulate.sql"

# Configure for Production
sed -i 's|deployment_settings.base.prepopulate = 1|deployment_settings.base.prepopulate = 0|' ~web2py/applications/eden/models/000_config.py
sed -i 's|deployment_settings.base.migrate = True|deployment_settings.base.migrate = False|' ~web2py/applications/eden/models/000_config.py
cd ~web2py
sudo -H -u web2py python web2py.py -S eden -R applications/eden/static/scripts/tools/compile.py

# Add Scheduler
sed -i 's|exit 0|cd ~web2py \&\& python web2py.py -K eden -Q >/dev/null 2>\&1 \&|' /etc/rc.local
echo "exit 0" >> /etc/rc.local

#read -p "Press any key to Reboot..."
echo "Now rebooting.."
reboot

# END