norrs · November 23, 2012 12:05 · norrs · Mar 7, 2013
diff --git a/nav-dev.sh b/nav-dev.sh
 #!/bin/sh
 # Script depends on gnu sed and perl.
 # Rest of the dependencies gets installed by apt-get. 
 set -e

 YOUR_USER=
 SRCDIR=
 DEPLOYDIR=
 PIP_REQUIREMENTS=
 RAPID_DEVELOPMENT=
 DISPLAY_HELP=
 SKIP_MCC_RUN=

 die_with_message() {
  echo "$@"
  exit 1
 }

 display_help() {
  echo "nav-dev.sh (Roy Sindre Norangshol aka Rockj)"
  echo "https://gist.github.com/norrs/4135334"
  echo ""
  echo "Required settings:"
  echo "-u username to chown deploy directory to"
  echo "-s source dir where to find NAV sources"
  echo "-d deploy dir where to deploy NAV virtualenv and install"
  echo "Optional settings:"
  echo "-m Skip mcc.py execution that builds cricket file, faster «redeployments»"
  echo "-p use provided pip resources file to deploy a virtualenv with specific/testing libraries"
  echo "-r rapid development, symlink media/*, templates and python code from source dir into deploydir"
  echo "-h display this help"
  echo ""
  echo "Important: flags who has no arguments need to come before flags with arguments!"
  echo "Example: $0 -m -r -u norangshol -s /source/nav-mirror -d /deploy/nav-mirror"
  [ -z "$1" ]
  exit $?
 }

 set -- $(getopt hmru:s:d:p: "$@")
 while [ $# -gt 0 ]
 do
    case "$1" in
    (-u) YOUR_USER="$2";shift;;
    (-s) SRCDIR=$2;shift;;
    (-d) DEPLOYDIR=$2;shift;;
    (-p) PIP_REQUIREMENTS=$2;shift;;
    (-r) RAPID_DEVELOPMENT=1;;
    (-h) DISPLAY_HELP=1;;
    (-m) SKIP_MCC_RUN=1;;
    (--) shift; break;;
    (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;;
    (*)  break;;
    esac
    shift
 done

 [ -n "$DISPLAY_HELP" ] && display_help
 [ -z "$YOUR_USER" ] && display_help 1
 [ -z "$SRCDIR" ] && display_help 1
 [ -z "$DEPLOYDIR" ] && display 1

 echo "Using settings:"
 echo "User             : $YOUR_USER"
 echo "Src              : $SRCDIR"
 echo "Deploy dir       : $DEPLOYDIR"
 echo "requirements.txt : $PIP_REQUIREMENTS (blank means default)"
 echo "rapid_development: $RAPID_DEVELOPMENT"


 setup_virtualenv() {
  test -z "$1"          && echo dir missing        && exit 1
  
  VIRTENV="$1/.env"

  
  # Create virtualenv for some required python packages, and keep it around to
  # avoid hitting the network each time a job runs.
  if [ -d "$VIRTENV" ]; then
      echo "**> virtualenv exists"
  else
      echo "**> creating virtualenv"
      opt=
      test -n "$PYTHON_VERSION" && opt="-p $PYTHON_VERSION"
      virtualenv --no-site-packages $opt "$VIRTENV"
  fi
 }
 install_dependencies_and_build() {
  test -z "$PGDATABASE" && echo PGDATABASE missing && exit 1
  test -z "$PGUSER"     && echo PGUSER missing     && exit 1
  test -z "$1"          && echo source dir missing && exit 1
  test -z "$2"          && echo «deploy» dir missing && exit 1


  SRCDIR="$1"
  BUILDDIR="$2/build"
  VIRTENV="$2/.env"

  # Clear build directory
  test -d "$BUILDDIR" && rm -rf "$BUILDDIR"
  
  . "$VIRTENV/bin/activate"
  easy_install pip || exit 1
  # hacky hacky fall back to always included cheetah and psycopg2 :-p
  pip install cheetah
  pip install psycopg2 

  [ -z "$PIP_REQUIREMENTS"] && PIP_REQUIREMENTS="$SRCDIR/tests/requirements.txt"

  pip install -r "$PIP_REQUIREMENTS" || exit 1
  
  
  # Make install code into given directory
  ./autogen.sh || exit 1
  ./configure --prefix "$BUILDDIR" || exit 1
  make || exit 1
  make install || exit 1
  
  export PYTHONPATH="$BUILDDIR/lib/python"
  
  # Update db config
  sed -i'' -e "s,^db_nav\s*=\s*nav,db_nav=$PGDATABASE," "$BUILDDIR/etc/db.conf"
  sed -i'' -e "s/^script_default\s*=\s*nav/script_default=$PGUSER/" "$BUILDDIR/etc/db.conf"
  sed -i'' -e "s/^userpw_nav\s*=.*/userpw_$PGUSER=$PGPASSWORD/" "$BUILDDIR/etc/db.conf"
  if [ -n "$PGHOST" ]; then sed -i'' -e "s/^dbhost\s*=\s*localhost/dbhost=$PGHOST/" "$BUILDDIR/etc/db.conf"; fi
  if [ -n "$PGPORT" ]; then sed -i'' -e "s/^dbport\s*=.*/dbport=$PGPORT/" "$BUILDDIR/etc/db.conf"; fi
  
  # Remove existing DB, if any, and create new one
  #dropdb $PGDATABASE || true
  #sql/syncdb.py -c || exit 1
  
  if [ -n "$ADMINPASSWORD" ]; then psql -c "UPDATE account SET password = '$ADMINPASSWORD' WHERE login = 'admin'"; fi
  
  # Add non-ASCII chars to the admin user's login name to test encoding
  # compliance for all Cheetah based web pages.
  psql -c "UPDATE account SET name = 'Administrator ÆØÅ' WHERE login = 'admin'"
  
 }

 check_python_version() {
  [ $# -eq 0 ] && die_with_message "No arguments given for check_python_version, bailing"
  VERSION="$1"
  [ -z "$VERSION" ] && die_with_message "No arguments given for check_python_version, bailing"
  TEST_PYTHON=$($DEPLOYDIR/.env/bin/python -V 2>&1 | grep -e "^Python $VERSION")
  [ -n "$TEST_PYTHON" ]
  return $?
 }


 export DEPLOYDIR="$DEPLOYDIR"

 if [ ! -f "$SRCDIR/tests/bootstrap-test-environment.sh" ]; then
 	die_with_message "Are you sure you have NAV sources deployed in $SRCDIR? \$SRCDIR need to be set of root of checkout"
 fi

 DISTRO_CODENAME=$(lsb_release -c | tr '\t' ' ' | cut -d ' ' -f 2)

 if [ $DISTRO_CODENAME = "squeeze" ]; then
 	apt-get install -t squeeze-backports -y mercurial
 else
 	apt-get install -y mercurial
 fi

 apt-get install -y subversion

 apt-get install -y python-virtualenv build-essential autoconf
 apt-get install -y postgresql-client
 apt-get build-dep -y psycopg2

 if [ ! -d /etc/nav ]; then
 	mkdir /etc/nav
 	echo "Creating /etc/nav"
 fi

 cd "$SRCDIR"

 cat <<DATABASECONFIG > /etc/nav/database
 #!/bin/sh
 export PGDATABASE=nav
 export PGUSER=nav
 export PGPASSWORD=nav
 export PGHOST=127.0.0.1
 export PGPORT=5432
 DATABASECONFIG

 . /etc/nav/database

 setup_virtualenv "$DEPLOYDIR" 

 . "$DEPLOYDIR/.env/bin/activate"
 check_python_version 2.6 && [ $? -eq 0 ] && export PYTHON_VERSION=python2.6
 check_python_version 2.7 && [ $? -eq 0 ] && export PYTHON_VERSION=python2.7

 [ -z "$PYTHON_VERSION" ] && die_with_message "Could not detect python version in virtualenv in $DEPLOYDIR"


 # Build env should be fine now.
 install_dependencies_and_build "$SRCDIR" "$DEPLOYDIR" || exit 1

 getent group nav || addgroup --system nav 
 getent passwd navcron 
 if [ $? -ne 0 ]; then
 	adduser --system --no-create-home --home "$DEPLOYDIR/build" --shell /bin/sh --ingroup nav navcron
 else 
 	usermod -d "$DEPLOYDIR/build" navcron
 fi
 chown -R navcron:nav "$DEPLOYDIR/build/var"

 apt-get install -y cricket

 # not needed?
 #cp -r "$DEPLOYDIR/build/doc/cricket/cricket-config" "$DEPLOYDIR/build/etc/"
 chown -R navcron:nav "$DEPLOYDIR/build/etc/cricket-config"


 # should do a sed here.  but /etc/cricket/cricket-conf.pl should have:
 # $gConfigRoot = "/deploy/nav-mirror/build/etc/cricket-config";
 #sed 's/$gConfigRoot.*/$gConfigRoot = \"'"$DEPLOYDIR"'\/build\/etc\/cricket-config\";/g' -i /etc/cricket/cricket-conf.pl
 perl -pi -e 's/\$gConfigRoot.*/\$gConfigRoot = \"$ENV{DEPLOYDIR}\/build\/etc\/cricket-config\";/' /etc/cricket/cricket-conf.pl
 # confirm with:  grep gConfigRoot /etc/cricket/cricket-conf.pl 
 (grep gConfigRoot /etc/cricket/cricket-conf.pl | grep -o $DEPLOYDIR)


 [ ! -d "$DEPLOYDIR/build/var/cricket-data" ] && mkdir "$DEPLOYDIR/build/var/cricket-data"
 chown navcron "$DEPLOYDIR/build/var/cricket-data"

 su navcron -c cricket-compile


 # need to do something tricky here FIRST! :]
 # like rm rf python lib and symlink it . not delete whole lbi
 rm -rf "$DEPLOYDIR/build/lib/python/nav"
 ln -sf "$SRCDIR/python/nav" "$DEPLOYDIR/build/lib/python/nav"


 ln -sf "$DEPLOYDIR/build/lib/python/nav" "$DEPLOYDIR/.env/lib/$PYTHON_VERSION/site-packages/"


 # IN THE SOURCE ENVIRONMENT!
 pip install django==1.2.3
 pip install ipy
 [ -z "$SKIP_MCC_RUN" ] && (echo "Running mcc.py, might take some time if your have a few devices already added in DB"; su navcron -s /bin/sh -c ". $DEPLOYDIR/.env/bin/activate && $DEPLOYDIR/build/bin/mcc.py")

 mkdir "$DEPLOYDIR/build/share/htdocs/cricket"
 cd /usr/share/cricket
 ln -sf $PWD/grapher.cgi "$DEPLOYDIR/build/share/htdocs/cricket/"
 ln -sf $PWD/mini-graph.cgi  "$DEPLOYDIR/build/share/htdocs/cricket/"
 cd "$DEPLOYDIR/build/share/htdocs/cricket"
 ln -sf grapher.cgi index.cgi
 #todo update! 
 #cp "$DEPLOYDIR/build/doc/cricket/public_html/cricket.css" .

 ln -sf /usr/share/cricket/images "$DEPLOYDIR/build/share/htdocs/cricket/"


 #Apache2 vhost:

 cat <<APACHEVHOST > /etc/apache2/sites-available/navdeploy
 <VirtualHost *:80>
        ServerAdmin webmaster@localhost

        Include /deploy/nav-mirror/build/etc/apache/apache.conf

        ErrorLog \${APACHE_LOG_DIR}/error.log

        # Possible values include: debug, info, notice, warn, error, crit,
        # alert, emerg.
        LogLevel warn

        CustomLog \${APACHE_LOG_DIR}/access.log combined
 </VirtualHost>
 APACHEVHOST

 apt-get install -y libapache2-mod-wsgi
 apt-get install -y libapache2-mod-python

 #vim $DEPLOYDIR/build/etc/apache/apache.conf
 #Insert:

 # requires gnu sed. 
 sed -i '/PythonPath/d' "$DEPLOYDIR/build/etc/apache/apache.conf"
 perl -pi -e "s{^((\s*)PythonHandler django.core.handlers.modpython)}{\1\n\2PythonPath \"\(lambda oldpath:\(__import__\('site'\)\.addsitedir\('$DEPLOYDIR/.env/lib/$PYTHON_VERSION/site-packages'\),filter\( \(lambda x: x not in oldpath\),sys\.path\)\+oldpath\)\)\(list\(sys\.path\)\)\[\-1\]\"}" "$DEPLOYDIR/build/etc/apache/apache.conf"

 a2ensite navdeploy
 /etc/init.d/apache2 restart

 # nav.asyncdns.py
 # need to install from tarball due to pip not accepting names with spaces in its repository
 # blame pip-tool or twisted guys!
 # https://github.com/pypa/pip/issues/307
 pip install twisted
 pip install http://twistedmatrix.com/Releases/Names/12.2/TwistedNames-12.2.0.tar.bz2

 #  freebsd doesnt have newer ... or yeah meh.
 pip install simplejson==2.0.6


 #pip install pysnmp
 pip install http://downloads.sourceforge.net/project/twistedsnmp/pysnmp-se/3.5.2/pysnmp-se-3.5.2.tar.gz

 pip install networkx


 apt-get build-dep -y librrd-dev
 apt-get install -y librrd-dev
 #pip install rrdtool
 pip install python-rrdtool


 pip install xmpppy

 #PRØV libsnmp15 først! Skal ikke være nødvendig med de 2 neste for pynetsnmp
 #apt-get install libsnmp-base
 #apt-get install libsnmp-dev

 apt-get install -y libsnmp15

 pip install svn+http://dev.zenoss.org/svn/tags/pynetsnmp-0.28.14

 #------------------------------------
 #FINAL LIST
 #-----------------------------------
 #pip freeze -l
 #Cheetah==2.4.4
 #Django==1.2.3
 #IPy==0.75
 #Jinja2==2.6
 #Markdown==2.2.0
 #Pygments==1.5
 #Sphinx==1.1.3
 #Twisted==12.2.0
 #Twisted-Names==12.2.0
 #distribute==0.6.10
 #docutils==0.9.1
 #logilab-astng==0.24.0
 #logilab-common==0.58.1
 #mock==1.0.0
 #networkx==1.7
 #psycopg2==2.4.5
 #py==1.4.9
 #pyasn1==0.1.4
 #pycrypto==2.6
 #pylint==0.25.2
 #pysnmp-se==3.5.2
 #pytest==2.2.4
 #python-rrdtool==1.4.7
 #simplejson==2.0.6
 #unittest2==0.5.1
 #xmpppy==0.5.0rc1
 #zope.interface==4.0.1
 #

 #--------

 #ln -sf /nav-mirror/media/ /deploy/nav-mirror/build/lib/ ??
 if [ -n "$RAPID_DEVELOPMENT" ]; then
  echo "Checking for making RAPID DEVELOPEMENT symlinks..."
  [ ! -L "$DEPLOYDIR/build/share/htdocs/images" ] && rm -rf "$DEPLOYDIR/build/share/htdocs/images"
  ln -sf "$SRCDIR/media/images" "$DEPLOYDIR/build/share/htdocs/images" 
  [ ! -L "$DEPLOYDIR/build/share/htdocs/js" ] && rm -rf "$DEPLOYDIR/build/share/htdocs/js"
  ln -sf "$SRCDIR/media/js" "$DEPLOYDIR/build/share/htdocs/js"
  [ ! -L "$DEPLOYDIR/build/share/htdocs/style" ] && rm -rf "$DEPLOYDIR/build/share/htdocs/style"
  ln -sf "$SRCDIR/media/style" "$DEPLOYDIR/build/share/htdocs/style"
  [ ! -L "$DEPLOYDIR/build/lib/templates" ] && rm -rf "$DEPLOYDIR/build/lib/templates"
  ln -sf "$SRCDIR/templates" "$DEPLOYDIR/build/lib/templates"
 fi


 #-------------


 #------
 #FOR RUNNING TESTS

 pip install minimock
 apt-get build-dep -y python-lxml
 pip install lxml

 # Bør migrere til pytidylib .. og noke magi for å håndtere tidypakken i debian.
 #pip install pytidylib 

 echo "We should be done, navdev"