Make the slave system-installable and configurable

* Move configurable constants from const.py to config.py
  - These values are now overidden by /etc/autotua/slave.cfg
    * config.VERBOSE is overidden in slave.cfg
  - Fix all packages to import config.py instead
  - Change USE_PROXY to IGNORE_PROXY
* bai bai test_modules.py. Testing is now done by executing modules
* Install git-proxy-cmd.sh into /usr/bin

Advantages:
* python setup.py sdist
* python setup.py install --root ~/autotua-slave
* python setup.py help # ;-)

13 files changed, 194 insertions, 114 deletions

diff --git a/scripts/init-autotua-tmpdirs.sh b/scripts/init-autotua-tmpdirs.sh
deleted file mode 100755
index 8cb60c5..0000000
--- a/scripts/init-autotua-tmpdirs.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-mkdir -p /var/tmp/autotua/{chroots/{pristine,updated},jobtage,tarballs/{jobfiles,stages},work}
-: > /var/log/autotua-slave.log
diff --git a/slave/autotua/__init__.py b/slave/autotua/__init__.py
index 601937d..7d5ef6c 100644
--- a/slave/autotua/__init__.py
+++ b/slave/autotua/__init__.py
@@ -9,7 +9,7 @@
 import os, shutil, urllib2, atexit
 import os.path as osp
 import cPickle as pickle
-from autotua import fetch, const, sync, chroot, jobuild
+from autotua import fetch, config, sync, chroot, jobuild
 
 class Jobs:
     """Interface to jobs on the master server that we can do"""
@@ -23,7 +23,7 @@ class Jobs:
         (skeleton code atm)
         """
         jobs = []
-        job_list = pickle.load(urllib2.urlopen(const.AUTOTUA_MASTER+'/slave_api/jobs'))
+        job_list = pickle.load(urllib2.urlopen(config.AUTOTUA_MASTER+'/slave_api/jobs'))
         for job_data in job_list:
             jobs.append(Job(job_data))
         return jobs
@@ -36,7 +36,7 @@ class Job:
         self.maint = job_data['maintainer']['username']
         self.name = job_data['name']
         self.stage = fetch.Fetchable(uri=[job_data['stage']])
-        self.jobdir = osp.join(const.WORKDIR, self.maint, self.name)
+        self.jobdir = osp.join(config.WORKDIR, self.maint, self.name)
         self.jobtagedir = osp.join(self.jobdir, 'jobtage')
         self.jobtagerev = job_data['jobtagerev']
         self.atoms = job_data['atoms']
@@ -55,7 +55,7 @@ class Job:
     def _setup_jobfiles(self):
         job_src = []
         action = 'link'
-        fetcher = fetch.Fetcher(const.JOBFILE_DIR)
+        fetcher = fetch.Fetcher(config.JOBFILE_DIR)
         processor = jobuild.Processor(None, self.chroot)
         for jbld in self.jobuilds:
             processor.jobuild = jbld
@@ -63,7 +63,7 @@ class Job:
             for i in src_uri:
                 job_src.append(fetch.Fetchable(uri=[i]))
         src = fetcher.tarballdir+'/%s'
-        dest = self.chroot.chrootdir+const.CHAUTOTUA_DIR+'/jobfiles/%s'
+        dest = self.chroot.chrootdir+config.CHAUTOTUA_DIR+'/jobfiles/%s'
         for fetchable in job_src:
             fetcher.fetch(fetchable)
             src = src % fetchable.filename
@@ -81,14 +81,14 @@ class Job:
         # Job metadata stuff
         ## Get stage3 (if required)
         print 'Fetching stage...'
-        fetcher = fetch.Fetcher(const.STAGE_DIR)
+        fetcher = fetch.Fetcher(config.STAGE_DIR)
         fetcher.fetch(self.stage)
         # Sync jobtage tree
         print 'Syncing jobtage tree...'
         sync.Syncer().sync()
         # Export from local jobtage tree
         print 'Exporting jobtage tree...'
-        sync.Syncer(uri=const.JOBTAGE_DIR, destdir=self.jobtagedir,
+        sync.Syncer(uri=config.JOBTAGE_DIR, destdir=self.jobtagedir,
                     rev=self.jobtagerev, scheme="git-export").sync()
         ## Read config, get portage snapshot if required
         #self._fetch_portage_snapshot()
@@ -119,4 +119,14 @@ class Job:
         # Tidy up before cleaning
         self.tidy()
         shutil.rmtree(self.jobdir)
-        os.removedirs(osp.join(const.WORKDIR, self.maint))
+        os.removedirs(osp.join(config.WORKDIR, self.maint))
+
+if __name__ == "__main__":
+    job = Jobs().getjobs()[0]
+    job.fetch()
+    if os.getuid() == 0:
+        job.prepare()
+        job.run()
+        job.tidy()
+    else:
+        print 'You need to be root to run job.prepare(), job.run() and job.tidy()'
diff --git a/slave/autotua/chroot/__init__.py b/slave/autotua/chroot/__init__.py
index 042b44d..07638b2 100644
--- a/slave/autotua/chroot/__init__.py
+++ b/slave/autotua/chroot/__init__.py
@@ -8,7 +8,7 @@
 
 import os, shutil, subprocess, re
 import os.path as osp
-from .. import const, sync
+from .. import config, sync
 from time import strftime
 
 class PristineChroot(object):
@@ -21,9 +21,9 @@ class PristineChroot(object):
         @type stage: string
         """
         # stage-my.tar.bz2 -> stage-my
-        self.dir = osp.join(const.CHROOT_DIR, stage_filename[:stage_filename.rfind('.tar')])
+        self.dir = osp.join(config.CHROOT_DIR, stage_filename[:stage_filename.rfind('.tar')])
         self.stage = stage_filename
-        self.tar_args = (osp.join(const.STAGE_DIR, self.stage), self.dir)
+        self.tar_args = (osp.join(config.STAGE_DIR, self.stage), self.dir)
 
     def _extract_stage(self):
         os.makedirs(self.dir)
@@ -33,17 +33,17 @@ class PristineChroot(object):
 
     def check(self, fix=False):
         """Verify that the pristine is intact"""
-        if const.VERBOSE:
+        if config.VERBOSE:
             print "Cross-checking with stage tarball \"%s\"" % self.stage
         process = subprocess.Popen('tar jfd "%s" -C "%s"' % self.tar_args, stderr=subprocess.PIPE, shell=True)
-        if const.VERBOSE:
+        if config.VERBOSE:
             print "PID: %s" % process.pid
         output = process.communicate()[1]
         if output.count('No such file or directory') > 0:
             print "Chroot is no longer pristine."
             regex = re.compile(r'tar: (.*?):.*')
             output = regex.sub(r'"\1"', output)
-            if const.VERBOSE:
+            if config.VERBOSE:
                 print "The following files are missing:"
                 print output
             if not fix:
@@ -59,7 +59,7 @@ class PristineChroot(object):
         if osp.exists(self.dir):
             print "Pristine ready."
             return True
-        if const.VERBOSE:
+        if config.VERBOSE:
             print "Extracting from stage...",
         self._extract_stage()
         print "Pristine ready."
@@ -67,10 +67,10 @@ class PristineChroot(object):
 
     def reset(self):
         """Re-extract the chroot from the tarball"""
-        if const.VERBOSE:
+        if config.VERBOSE:
             print "Removing existing chroot..."
         shutil.rmtree(self.dir)
-        if const.VERBOSE:
+        if config.VERBOSE:
             print "Extracting %s" % self.stage
         self._extract_stage()
         print "Pristine ready."
@@ -115,18 +115,18 @@ class WorkChroot(object):
         for dir in ['/dev', '/sys', '/proc']:
             self._bind(dir, dir)
         # FIXME: else: Fetch portage tarball and use.
-        if const.PORTAGE_DIR:
-            if not osp.isdir(const.PORTAGE_DIR):
-                print "\"%s\" is not a directory, cannot mount" % const.PORTAGE_DIR
+        if config.PORTAGE_DIR:
+            if not osp.isdir(config.PORTAGE_DIR):
+                print "\"%s\" is not a directory, cannot mount" % config.PORTAGE_DIR
             else:
-                self._bind(const.PORTAGE_DIR, '/usr/portage')
-        if const.DISTFILES_DIR and not os.path.samefile(const.DISTFILES_DIR, const.PORTAGE_DIR+'/distfiles'):
-            if not osp.isdir(const.DISTFILES_DIR):
-                print "\"%s\" is not a directory, cannot mount" % const.DISTFILES_DIR
+                self._bind(config.PORTAGE_DIR, '/usr/portage')
+        if config.DISTFILES_DIR and not os.path.samefile(config.DISTFILES_DIR, config.PORTAGE_DIR+'/distfiles'):
+            if not osp.isdir(config.DISTFILES_DIR):
+                print "\"%s\" is not a directory, cannot mount" % config.DISTFILES_DIR
             else:
-                self._bind(const.DISTFILES_DIR, '/usr/portage/distfiles', ro=False)
-        self._bind(const.AUTOTUA_DIR+'/bin', const.CHAUTOTUA_DIR+'/bin')
-        self._bind(self.jobdir+'/jobtage', const.CHAUTOTUA_DIR+'/jobtage')
+                self._bind(config.DISTFILES_DIR, '/usr/portage/distfiles', ro=False)
+        self._bind(config.AUTOTUA_DIR+'/bin', config.CHAUTOTUA_DIR+'/bin')
+        self._bind(self.jobdir+'/jobtage', config.CHAUTOTUA_DIR+'/jobtage')
 
     def setup(self):
         """
@@ -142,7 +142,7 @@ class WorkChroot(object):
         print "Preparing Work Chroot..."
         sync.Syncer(uri=self.pristine.dir+"/", destdir=self.chrootdir, scheme='rsync-nc').sync()
         for dir in ['bin', 'jobfiles', 'jobtage', 'src']:
-            os.makedirs('%s/%s/%s' % (self.chrootdir, const.CHAUTOTUA_DIR, dir))
+            os.makedirs('%s/%s/%s' % (self.chrootdir, config.CHAUTOTUA_DIR, dir))
         self._setup_mounts()
         print "Work Chroot ready."
 
@@ -151,8 +151,8 @@ class WorkChroot(object):
         Cleanup when done.
         """
         self._clean_mounts()
-        if osp.isdir(self.chrootdir+const.CHAUTOTUA_DIR):
-            if not osp.isdir(self.chrootdir+const.CHAUTOTUA_DIR+'-old'):
-                os.makedirs(self.chrootdir+const.CHAUTOTUA_DIR+'-old')
-            shutil.move(self.chrootdir+const.CHAUTOTUA_DIR,
-                        self.chrootdir+const.CHAUTOTUA_DIR+'-old/autotua-'+strftime('%Y%m%d%H%M%S'))
+        if osp.isdir(self.chrootdir+config.CHAUTOTUA_DIR):
+            if not osp.isdir(self.chrootdir+config.CHAUTOTUA_DIR+'-old'):
+                os.makedirs(self.chrootdir+config.CHAUTOTUA_DIR+'-old')
+            shutil.move(self.chrootdir+config.CHAUTOTUA_DIR,
+                        self.chrootdir+config.CHAUTOTUA_DIR+'-old/autotua-'+strftime('%Y%m%d%H%M%S'))
diff --git a/slave/autotua/config.py b/slave/autotua/config.py
new file mode 100644
index 0000000..63a5209
--- /dev/null
+++ b/slave/autotua/config.py
@@ -0,0 +1,71 @@
+# vim: set sw=4 sts=4 et :
+# Copyright: 2008 Gentoo Foundation
+# Author(s): Nirbheek Chauhan <nirbheek.chauhan@gmail.com>
+# License: GPL-2
+#
+# Immortal lh!
+#
+
+"""
+Default configuration, overriden by /etc/autotua/slave.cfg
+"""
+
+import os, ConfigParser, const
+
+VERBOSE             = False
+FULL_CLEAN          = False
+IGNORE_PROXY        = False
+
+LOGFILE             = '/var/log/autotua/slave.log'
+TMPDIR              = '/var/tmp/autotua'
+
+AUTOTUA_MASTER      = 'http://www.autotua.org:8000'
+JOBTAGE_URI         = 'git://git.overlays.gentoo.org/proj/jobtage.git'
+
+# Bind mounted inside the chroots for use if defined
+PORTAGE_DIR         = '/usr/portage'
+DISTFILES_DIR       = '/usr/portage/distfiles'
+
+# Read settings from slave.cfg which override the above
+options = locals().copy()
+cfg = ConfigParser.ConfigParser()
+cfg.readfp(open('%s/slave.cfg' % const.CONFIG_PATH))
+for option, value in options.iteritems():
+    if not isinstance(value, (str, int, bool)):
+        continue
+    if cfg.has_option('global', option.lower()):
+        if isinstance(value, str):
+            if not option.startswith('__'):
+                exec('%s = %s' % (option, cfg.get('global', option.lower())))
+        elif isinstance(value, bool):
+            exec('%s = %s' % (option, cfg.getboolean('global', option.lower())))
+        elif isinstance(value, int):
+            exec('%s = %s' % (option, cfg.getint('global', option.lower())))
+
+# Import all the constants now
+from const import *
+
+# Derivative variables
+TARBALL_DIR         = TMPDIR+'/tarballs'
+STAGE_DIR           = TARBALL_DIR+'/stages'
+JOBFILE_DIR         = TARBALL_DIR+'/jobfiles'
+CHROOT_DIR          = TMPDIR+'/chroots/pristine'
+SAVED_CHROOT_DIR    = TMPDIR+'/chroots/saved'
+WORKDIR             = TMPDIR+'/work'
+JOBTAGE_DIR         = TMPDIR+'/jobtage'
+
+# If using http_proxy, enable git proxy support
+if not IGNORE_PROXY and os.environ.has_key('http_proxy'):
+    prefix = AUTOTUA_DIR+'/../scripts/'
+    # If git-proxy-cmd.sh is in PATH, use that
+    for path in os.environ['PATH'].split(':'):
+        if os.access(path+"/git-proxy-cmd.sh", os.X_OK):
+            prefix = ''
+    os.environ['GIT_PROXY_COMMAND'] = prefix+'git-proxy-cmd.sh'
+
+# Create autotua directories if they don't exist
+if not os.path.isdir(os.path.dirname(LOGFILE)):
+    os.mkdir(os.path.dirname(LOGFILE))
+for dir in [TMPDIR, TARBALL_DIR, STAGE_DIR, JOBFILE_DIR, CHROOT_DIR, SAVED_CHROOT_DIR, WORKDIR, JOBTAGE_DIR]:
+    if not os.path.isdir(dir):
+        os.mkdir(dir)
diff --git a/slave/autotua/const.py b/slave/autotua/const.py
index 7d23e1f..dd7b969 100644
--- a/slave/autotua/const.py
+++ b/slave/autotua/const.py
@@ -12,35 +12,8 @@ Internal Constants
 
 import os
 
-VERBOSE             = True
-FULL_CLEAN          = False
-USE_PROXY           = True
-
+CONFIG_PATH         = '/etc/autotua'
 AUTOTUA_DIR         = os.path.abspath(os.path.dirname(__file__))
-LOGFILE             = '/var/log/autotua-slave.log'
-TMPDIR              = '/var/tmp/autotua'
-TARBALL_DIR         = TMPDIR+'/tarballs'
-STAGE_DIR           = TARBALL_DIR+'/stages'
-JOBFILE_DIR         = TARBALL_DIR+'/jobfiles'
-CHROOT_DIR          = TMPDIR+'/chroots/pristine'
-WORKDIR             = TMPDIR+'/work'
-JOBTAGE_DIR         = TMPDIR+'/jobtage'
-
-AUTOTUA_MASTER      = 'http://www.autotua.org:8000'
-JOBTAGE_URI         = 'git://git.overlays.gentoo.org/proj/jobtage.git'
 
 # Autotua variables inside the chroot
 CHAUTOTUA_DIR       = '/tmp/autotua'
-
-# Bind mounted inside the chroots for use if defined
-PORTAGE_DIR         = ''
-DISTFILES_DIR       = ''
-
-# If using http_proxy, enable git proxy support
-if os.environ.has_key('http_proxy'):
-    prefix = AUTOTUA_DIR+'/../../scripts/'
-    # If git-proxy-cmd.sh is in PATH, use that
-    for path in os.environ['PATH'].split(':'):
-        if os.access(path+"/git-proxy-cmd.sh", os.X_OK):
-            prefix = ''
-    os.environ['GIT_PROXY_COMMAND'] = prefix+'git-proxy-cmd.sh'
diff --git a/slave/autotua/daemon/__init__.py b/slave/autotua/daemon/__init__.py
index 6d42817..a5e4f86 100644
--- a/slave/autotua/daemon/__init__.py
+++ b/slave/autotua/daemon/__init__.py
@@ -7,7 +7,7 @@
 #
 
 import subprocess
-from .. import const
+from .. import config
 
 class Spawn(object):
     """
@@ -16,7 +16,7 @@ class Spawn(object):
     sent to stdin with a 'pong\n'
     """
 
-    def __init__(self, chroot, jobtagedir, workdir, command, logfile=const.LOGFILE):
+    def __init__(self, chroot, jobtagedir, workdir, command, logfile=config.LOGFILE):
         """
         @param chroot: The path to the chroot in which 
                        the command will be run
diff --git a/slave/autotua/fetch/__init__.py b/slave/autotua/fetch/__init__.py
index 36588d6..224f056 100644
--- a/slave/autotua/fetch/__init__.py
+++ b/slave/autotua/fetch/__init__.py
@@ -65,3 +65,9 @@ class Fetcher(object):
         while index >= 0:
             subprocess.check_call(self.command % { 'file': filename, 'uri': target.uri[index], 'tries': 5 }, shell=True)
             index -= 1
+
+if __name__ == "__main__":
+    import autotua
+    job = autotua.Jobs().getjobs()[0]
+    fetcher = Fetcher(tmpdir)
+    fetcher.fetch(job.stage)
diff --git a/slave/autotua/jobuild/__init__.py b/slave/autotua/jobuild/__init__.py
index a250b66..54d3ca0 100644
--- a/slave/autotua/jobuild/__init__.py
+++ b/slave/autotua/jobuild/__init__.py
@@ -11,7 +11,7 @@
 
 import re, subprocess, sys, os
 import os.path as osp
-from .. import const, daemon
+from .. import config, daemon
 
 class Jobuild(object):
     """A jobuild"""
@@ -129,10 +129,10 @@ class Processor(object):
 
     def _msg(self, msg):
         chroot = None
-        workdir = const.AUTOTUA_DIR
+        workdir = config.AUTOTUA_DIR
         if self.chroot:
             chroot = self.chroot.chrootdir
-            workdir = const.CHAUTOTUA_DIR
+            workdir = config.CHAUTOTUA_DIR
         proc = daemon.Spawn(chroot, self.jobuild.jobtagedir, workdir, '\"%s\"/bin/jobuild.sh %s' % (workdir, msg))
         # XXX: This waits for process to close stderr.
         # Replace with a loop during daemonisation
@@ -177,6 +177,10 @@ class Resolver(object):
         return newlist
 
     def _unique(self, deplist):
+        """
+        Makes deplist unique and reverses order
+        This gives us the required merge order
+        """
         newlist = []
         index = len(deplist)
         while index >= 0:
diff --git a/slave/autotua/sync/__init__.py b/slave/autotua/sync/__init__.py
index 97ca095..7ad356f 100644
--- a/slave/autotua/sync/__init__.py
+++ b/slave/autotua/sync/__init__.py
@@ -8,7 +8,7 @@
 
 import os, subprocess, shutil
 import os.path as osp
-from .. import const
+from .. import config
 
 # init == sync (example: rsync)
 ONLY_SYNC = 1
@@ -23,7 +23,7 @@ class Syncer(object):
     Sync stuff
     """
 
-    def __init__(self, scheme='git', uri=const.JOBTAGE_URI, rev=None, destdir=const.JOBTAGE_DIR):
+    def __init__(self, scheme='git', uri=config.JOBTAGE_URI, rev=None, destdir=config.JOBTAGE_DIR):
         """
         Default is to sync the jobtage tree from the default location.
 
@@ -143,3 +143,8 @@ class Command(object):
         """Run a sync command"""
         args = self._get_args(action=action)
         subprocess.check_call(args, shell=True)
+
+if __name__ == "__main__":
+    syncer = Syncer(destdir='/tmp/jobtage')
+    syncer.sync()
+
diff --git a/slave/config/slave.cfg b/slave/config/slave.cfg
new file mode 100644
index 0000000..ca17132
--- /dev/null
+++ b/slave/config/slave.cfg
@@ -0,0 +1,14 @@
+# AutotuA Slave configuration
+# Default values are commented
+# No trailing slashes please
+
+[global]
+verbose             = True
+;full_clean         = False
+;ignore_proxy       = False
+;logfile            = '/var/log/autotua/slave.log'
+;tmpdir             = '/var/tmp/autotua'
+;autotua_master     = 'http://www.autotua.org:8000'
+;jobtage_uri        = 'git://git.overlays.gentoo.org/proj/jobtage.git'
+;portage_dir        = '/usr/portage'
+;distfiles_dir      = '/usr/portage/distfiles'
diff --git a/scripts/git-proxy-cmd.sh b/slave/scripts/git-proxy-cmd.sh
index aa9fc61..aa9fc61 100755
--- a/scripts/git-proxy-cmd.sh
+++ b/slave/scripts/git-proxy-cmd.sh
diff --git a/slave/setup.py b/slave/setup.py
new file mode 100644
index 0000000..a9c0bd5
--- /dev/null
+++ b/slave/setup.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# vim: set sw=4 sts=4 et :
+# Copyright: 2008 Gentoo Foundation
+# Author(s): Nirbheek Chauhan <nirbheek.chauhan@gmail.com>
+# License: GPL-2
+#
+# Immortal lh!
+#
+
+import os
+from distutils.core import Command, setup
+
+class print_help(Command):
+    description = "Print help"
+    user_options = [('help','h',"Help help")]
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        print \
+        """
+        setup.py: Run this file as root to install the AutotuA slave
+        * Edit /etc/autotua/slave.cfg for the settings
+        * Set the http_proxy variable if you want to use an http proxy (will tunnel everything)
+          - You can force autotua to not use the http_proxy for tunnelling by setting ignore_proxy = true
+        """
+
+setup(name='autotua-slave',
+      version='0.0.1',
+      description='Build slave for AutotuA',
+      author='Nirbheek Chauhan',
+      author_email='nirbheek.chauhan@gmail.com',
+      license='GPL-2',
+      url='http://www.autotua.org/',
+      cmdclass={'help': print_help},
+      data_files=[('/etc/autotua', ['config/slave.cfg']), ('/usr/bin', ['scripts/git-proxy-cmd.sh'])],
+      packages=['autotua', 'autotua.chroot', 'autotua.daemon', 'autotua.fetch', 'autotua.jobuild', 'autotua.sync'],
+      package_data={'autotua': ['bin/*.sh']},
+     )
diff --git a/slave/test_modules.py b/slave/test_modules.py
deleted file mode 100755
index 34013a1..0000000
--- a/slave/test_modules.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env python
-# vim: set sw=4 sts=4 et :
-# Copyright: 2008 Gentoo Foundation
-# Author(s): Nirbheek Chauhan <nirbheek.chauhan@gmail.com>
-# License: GPL-2
-#
-# Immortal lh!
-#
-
-import autotua, shutil, os, sys
-import os.path as osp
-from autotua import chroot, fetch, sync
-
-tmpdir = '/tmp'
-# Default test (tests everything)
-modules = ['job']
-# All tests:
-#modules = ['job', 'fetch', 'sync', 'chroot']
-
-if 'fetch' in modules:
-    job = autotua.Jobs().getjobs()[0]
-    fetcher = autotua.fetch.Fetcher(tmpdir)
-    fetcher.fetch(job.stage)
-
-if 'sync' in modules:
-    destdir = tmpdir+'/jobtage'
-    syncer = autotua.sync.Syncer(destdir=destdir)
-    syncer.sync()
-
-if 'chroot' in modules:
-    job = autotua.Jobs().getjobs()[0]
-    chroot = autotua.chroot.WorkChroot(job.jobdir, job.stage.filename)
-    chroot.setup()
-    chroot.tidy()
-
-if 'job' in modules:
-    job = autotua.Jobs().getjobs()[0]
-    job.fetch()
-    if os.getuid() == 0:
-        job.prepare()
-        job.run()
-        job.tidy()
-    else:
-        print 'You need to be root to run job.prepare(), job.run() and job.tidy()'