setup-master.py now does db initialization

- After the previous models.py mega-change, the database needed the info
  that was in master.process.const initialised before it was usable.
- setup-master.py + custom.db_defaults can now do all this
- If you do not already have autotua-slave installed as a module, you
  will have to export the PYTHONPATH variable to allow setup-master.py
  and manage.py to import autotua

- Update TODOs
- Fix default to copy in setup-master.py

4 files changed, 156 insertions, 31 deletions

diff --git a/master/TODO b/master/TODO
index 82ecb5f..4868bd8 100644
--- a/master/TODO
+++ b/master/TODO
@@ -1,10 +1,13 @@
 TODO:
  * Implement input via the webinterface (manual via command line atm)
-   - Also, use/implement input validation (master.process.validation)
  * Better jobuild dependency resolution for atom list (basic right now)
  * Job status tracking and management (none right now)
  * Asymmetrical key authentication for accepting jobs
 
+setup-master.py:
+ * Should prompt for super-user
+ * Should initialize with a jobtage tree and a sample job for testing
+
 Low-priority TODO:
  * setup-master.py should do fastcgi settings as well
 
diff --git a/master/custom/db_defaults.py b/master/custom/db_defaults.py
new file mode 100644
index 0000000..f49826d
--- /dev/null
+++ b/master/custom/db_defaults.py
@@ -0,0 +1,47 @@
+# vim: set sw=4 sts=4 et :
+# Copyright: 2008 Gentoo Foundation
+# Author(s): Nirbheek Chauhan <nirbheek.chauhan@gmail.com>
+# License: GPL-2
+#
+# Immortal lh!
+#
+
+# Database initialization values
+
+providers = ('gentoo', 'funtoo',)
+# No 'arm' in here because the dir structure is weird
+# and it hasn't been updated in forever anyway.
+# Use a custom stage url if you want to use arm
+# 
+# 'Mirror dir': (<archs that reside there>)
+archs = {   'gentoo':   (('x86',        ('i686', 'x86',)),
+                        ('amd64',       ('amd64',)),
+                        ('ppc',         ('970-32ul', '970-64ul', 'g4',
+                                         'power5', 'ppc', 'ppc64',)),
+                        ('sparc',       ('sparc64',)),
+                        ('alpha',       ('alpha',)),
+                        ('ia64',        ('ia64',)),
+                        ('hppa',        ('hppa1.1', 'hppa2.0',)),
+                        ('mips',        ('cobalt', 'mips3', 'mips4', 
+                                         'n32',)),
+                        ('sh',          ('sh4',)),
+                        ('s390',        ('s390', 's390x',)),),
+            'funtoo':   ('amd64', 'athlon-xp', 'core32', 'core64',
+                         'i686', 'openvz', 'pentium4', 'x86', '~amd64',
+                         '~x86'), }
+
+stages = {  'gentoo':   ('stage1', 'stage2', 'stage3',),
+            'funtoo':   ('stage1', 'stage2', 'stage3',), }
+
+# Ask drobbins to have a 'latest' symlink.
+releases = {'gentoo':   ('2007.0', '2008.0'),
+            'funtoo':   ('latest',)}
+
+mirrors = { 'gentoo':   {'servers': (('http://gentoo.osuosl.org/', 'releases/'),
+                                    ('http://ftp.jaist.ac.jp/', 'pub/Linux/Gentoo/releases/'),),
+                         'structure': '%(mirror)s/releases/%(gen_arch)s/%(release)s/stages/%(stage)s-%(arch)s-%(release)s.tar.bz2',},
+                            
+            'funtoo':   {'servers': ('http://www.funtoo.org/', 'linux/'),
+                         'structure': '%(arch)s/%(owner)s-%(arch)s-%(release)/%(stage)s-%(arch)s-%(release)s.tar.bz2'} }
+
+#stage_types = ('', 'hardened', 'hardened+nomultilib')
diff --git a/master/setup-master.py b/master/setup-master.py
index 448933d..896f680 100755
--- a/master/setup-master.py
+++ b/master/setup-master.py
@@ -8,61 +8,135 @@
 #
 
 import sys, os, shutil
+try:
+    from django.core import management
+except ImportError:
+    print "You need to install django-1.0 first."
+    sys.exit(1)
+
+# XXX: Document this in the help
+if os.environ.has_key('PYTHONPATH'):
+    sys.path.insert(0, os.environ['PYTHONPATH'])
 
 DESTDIR = 'autotua_master'
-SYMLINKS = True
+SYMLINKS = False
 
 def print_help():
     print \
     """\
 %(file)s: Setup the django webapp for autotua
 Usage:
-    %(file)s <where>
+    %(file)s install <where>
+    <edit db settings>
+    %(file)s syncdb <where>
     
 Example:
-    %(file)s /home/me/projects/
-will create `/home/me/projects/autotua` with all the necessary files.
+    %(file)s install /home/me/projects/
+will create `/home/me/projects/autotua_master` with all the necessary files.
+
+You then need to edit the database settings in
+    /home/me/projects/autotua_master/settings.py
+
+After this, running
+    %(file)s syncdb /home/me/projects/autotua_master
+will initialize the db
 
 By default, the directory will have the files copied. 
 Toggle `SYMLINKS` to symlink the files instead.
     """ % {'file': sys.argv[0]}
 
-if len(sys.argv) == 1:
+def install_master():
+    """Start the new project"""
+
+    management.call_command('startproject', DESTDIR)
+    if SYMLINKS:
+        os.symlink(cwd+'/autotua', DESTDIR+'/master')
+        for file in ['autotua_settings.py', 'urls.py']:
+            dest_file = DESTDIR+'/'+file
+            if os.path.isfile(dest_file):
+                os.remove(dest_file)
+            os.symlink(cwd+'/custom/'+file, dest_file)
+    else:
+        shutil.copytree(cwd+'/autotua', DESTDIR+'/master')
+        for file in ['autotua_settings.py', 'urls.py']:
+            shutil.copy(cwd+'/custom/'+file, DESTDIR)
+    settings = open(DESTDIR+'/settings.py', 'a')
+    settings.write('\nfrom autotua_settings import *\n')
+    settings.close()
+
+def syncdb_master():
+    """Initialize the database"""
+    import settings
+    from django.core.management import setup_environ
+    setup_environ(settings)
+    from db_defaults import providers, archs, stages, releases, mirrors
+    from master.models import StageProvider, Arch, Stage, Release, Mirror
+    import copy
+
+    management.call_command('syncdb')
+    for provider in providers:
+        provobj = StageProvider(name=provider)
+        provobj.save()
+        # Populate arch list
+        for generic_arch in archs[provider]:
+            gen_archobj = Arch(provider=provobj)
+            # If (generic, (specific1, specific2))
+            if isinstance(generic_arch, tuple):
+                gen_archobj.generic = generic_arch[0]
+                for arch in generic_arch[1]:
+                    archobj = copy.copy(gen_archobj)
+                    archobj.specific = arch
+                    archobj.save()
+            # If (specific1, specific2, specific3)
+            else:
+                gen_archobj.generic = generic_arch
+                gen_archobj.specific = generic_arch
+                gen_archobj.save()
+        # Populate stage list
+        for stage in stages[provider]:
+            stageobj = Stage(provider=provobj)
+            stageobj.name = stage
+            stageobj.save()
+        # Populate release list
+        for release in releases[provider]:
+            releaseobj = Release(provider=provobj)
+            releaseobj.name = release
+            releaseobj.save()
+        # Populate mirror list
+        obj = Mirror(owner=provobj)
+        obj.structure = mirrors[provider]['structure']
+        for server in mirrors[provider]['servers']:
+            serverobj = copy.copy(obj)
+            serverobj.server = server[0]
+            serverobj.prefix = server[1]
+            serverobj.save()
+
+if len(sys.argv) < 3:
     print_help()
     sys.exit(1)
 
 os.chdir(os.path.dirname(sys.argv[0]))
 cwd = os.getcwd()
-if not os.path.isdir(sys.argv[1]):
-    os.makedirs(sys.argv[1])
-os.chdir(sys.argv[1])
-
-try:
-    from django.core import management
-except ImportError:
-    print "You need to install django-1.0 first."
-    sys.exit(1)
+if not os.path.isdir(sys.argv[2]):
+    os.makedirs(sys.argv[2])
+os.chdir(sys.argv[2])
 
 if management.get_version() < '0.99':
     print "You need django-1.0 to use this."
     sys.exit(1)
 
-management.call_command('startproject', DESTDIR)
+sys.path.append(os.getcwd())
+sys.path.append(cwd+'/custom')
 
-if SYMLINKS:
-    os.symlink(cwd+'/autotua', DESTDIR+'/master')
-    for file in ['autotua_settings.py', 'urls.py']:
-        dest_file = DESTDIR+'/'+file
-        if os.path.isfile(dest_file):
-            os.remove(dest_file)
-        os.symlink(cwd+'/custom/'+file, dest_file)
+if sys.argv[1] == 'install':
+    install_master()
+    print """Setup done. 
+Now you need to edit the database settings in %(dest)s/settings.py
+and run `./setup-master.py syncdb %(dest)s`""" % { 'dest': os.path.join(sys.argv[2], DESTDIR) }
+elif sys.argv[1] == 'syncdb':
+    syncdb_master()
+    print "All done! Now you can start the master with `python manage.py runserver`"
 else:
-    shutil.copytree(cwd+'/autotua', DESTDIR+'/master')
-    for file in ['autotua_settings.py', 'urls.py']:
-        shutil.copy(cwd+'/custom/'+file, DESTDIR)
-
-settings = open(DESTDIR+'/settings.py', 'a')
-settings.write('\nfrom autotua_settings import *\n')
-settings.close()
+    print_help()
+    sys.exit(1)
 
-print "All done. Now you need to run `python manage.py syncdb` after editing the database settings in %s/settings.py" % DESTDIR
diff --git a/slave/TODO b/slave/TODO
index 550ba6a..e469100 100644
--- a/slave/TODO
+++ b/slave/TODO
@@ -1,5 +1,6 @@
 - Structure and API are not final. Expect changes.
 - Implement locking for the workdir/chrootdir
+- Define test jobs so that a master-server is not required to run the module tests
 
 Long-term:
 - Custom exceptions plz, kthx.