diff options
| -rw-r--r-- | slave/autotua/__init__.py | 5 | ||||
| -rw-r--r-- | slave/autotua/bin/jobuild-functions.sh | 4 | ||||
| -rwxr-xr-x | slave/autotua/bin/jobuild.sh | 6 | ||||
| -rw-r--r-- | slave/autotua/chroot/__init__.py | 17 | ||||
| -rw-r--r-- | slave/autotua/const.py | 2 | ||||
| -rw-r--r-- | slave/autotua/daemon/__init__.py | 12 | ||||
| -rw-r--r-- | slave/autotua/jobuild/__init__.py | 84 |
7 files changed, 96 insertions, 34 deletions
diff --git a/slave/autotua/__init__.py b/slave/autotua/__init__.py index f390c7a..5ddc53c 100644 --- a/slave/autotua/__init__.py +++ b/slave/autotua/__init__.py @@ -37,6 +37,7 @@ class Job: self.name = job_data['name'] self.stage = fetch.Fetchable(uri=[job_data['stage']]) self.jobdir = osp.join(const.WORKDIR, self.maint, self.name) + self.jobtagedir = osp.join(self.jobdir, 'jobtage') self.jobtagerev = job_data['jobtagerev'] self.atoms = job_data['atoms'] self.jobuilds = [] @@ -86,7 +87,7 @@ class Job: sync.Syncer().sync() # Export from local jobtage tree print 'Exporting jobtage tree...' - sync.Syncer(uri=const.JOBTAGE_DIR, destdir=osp.join(self.jobdir, 'jobtage'), + sync.Syncer(uri=const.JOBTAGE_DIR, destdir=self.jobtagedir, rev=self.jobtagerev, scheme="git-export").sync() ## Read config, get portage snapshot if required #self._fetch_portage_snapshot() @@ -98,7 +99,7 @@ class Job: self.chroot.setup() # Create jobuild objects for parsing for atom in self.atoms: - self.jobuilds.append(jobuild.Jobuild(self.jobdir, atom)) + self.jobuilds.append(jobuild.Jobuild(self.jobtagedir, atom)) print 'Fetch jobuild SRC_URI and hardlink/copy into chroot' self._setup_jobfiles() diff --git a/slave/autotua/bin/jobuild-functions.sh b/slave/autotua/bin/jobuild-functions.sh index a86a372..0ec5dfa 100644 --- a/slave/autotua/bin/jobuild-functions.sh +++ b/slave/autotua/bin/jobuild-functions.sh @@ -176,10 +176,6 @@ get_param() { eval echo \$${param} } -get_jobuild_path() { - echo "${AUTOTUA_DIR}/jobtage/${1}" -} - run_all_phases() { local jobuild="${1}" initialise diff --git a/slave/autotua/bin/jobuild.sh b/slave/autotua/bin/jobuild.sh index 81a657a..ce0f3ea 100755 --- a/slave/autotua/bin/jobuild.sh +++ b/slave/autotua/bin/jobuild.sh @@ -31,6 +31,8 @@ speak "pong" listen AUTOTUA_DIR [ -z "${AUTOTUA_DIR}" ] && die "empty AUTOTUA_DIR variable, bai bai" +listen JOBTAGE_DIR +[ -z "${JOBTAGE_DIR}" ] && die "empty JOBTAGE_DIR variable, bai bai" get_var() { speak $(get_param "${1}" "${2}") @@ -53,12 +55,12 @@ run_phase() { case "${1}" in GET_VAR) var="${2}" - jobuild="$(get_jobuild_path ${3})" + jobuild="${JOBTAGE_DIR}/${3}" get_var "${var}" "${jobuild}" ;; RUN_PHASE) phase="${2}" - jobuild="$(get_jobuild_path ${3})" + jobuild="${JOBTAGE_DIR}/${3}" run_phase "${phase}" "${jobuild}" ;; *) diff --git a/slave/autotua/chroot/__init__.py b/slave/autotua/chroot/__init__.py index cdc02c2..4ed0cec 100644 --- a/slave/autotua/chroot/__init__.py +++ b/slave/autotua/chroot/__init__.py @@ -114,6 +114,7 @@ class WorkChroot(object): def _setup_mounts(self): for dir in ['/dev', '/sys', '/proc']: self._bind(dir, dir) + # FIXME: else: Fetch portage tarball and use. if const.PORTAGE_DIR: if not osp.isdir(const.PORTAGE_DIR): print "\"%s\" is not a directory, cannot mount" % const.PORTAGE_DIR @@ -122,8 +123,8 @@ class WorkChroot(object): if not osp.isdir(const.DISTFILES_DIR): print "\"%s\" is not a directory, cannot mount" % const.DISTFILES_DIR self._bind(const.DISTFILES_DIR, '/usr/portage/distfiles', ro=False) - self._bind(const.AUTOTUA_DIR+'/bin', '/tmp/autotua/bin') - self._bind(self.jobdir+'/jobtage', '/tmp/autotua/jobtage') + self._bind(const.AUTOTUA_DIR+'/bin', const.CHAUTOTUA_DIR+'/bin') + self._bind(self.jobdir+'/jobtage', const.CHAUTOTUA_DIR+'/jobtage') def setup(self): """ @@ -139,7 +140,7 @@ class WorkChroot(object): print "Preparing Work Chroot..." sync.Syncer(uri=self.pristine.dir+"/", destdir=self.chrootdir, scheme='rsync-nc').sync() for dir in ['bin', 'jobfiles', 'jobtage', 'src']: - os.makedirs('%s/tmp/autotua/%s' % (self.chrootdir, dir)) + os.makedirs('%s/%s/%s' % (self.chrootdir, const.CHAUTOTUA_DIR, dir)) self._setup_mounts() print "Work Chroot ready." @@ -148,8 +149,8 @@ class WorkChroot(object): Cleanup when done. """ self._clean_mounts() - if osp.isdir('%s/tmp/autotua' % self.chrootdir): - if not osp.isdir('%s/tmp/autotua-old' % self.chrootdir): - os.makedirs('%s/tmp/autotua-old' % self.chrootdir) - shutil.move('%s/tmp/autotua' % (self.chrootdir), - '%s/tmp/autotua-old/autotua-%s' % (self.chrootdir, strftime('%Y%m%d%H%M%S'))) + if osp.isdir(self.chrootdir+const.CHAUTOTUA_DIR): + if not osp.isdir(self.chrootdir+const.CHAUTOTUA_DIR+'-old'): + os.makedirs(self.chrootdir+const.CHAUTOTUA_DIR+'-old') + shutil.move(self.chrootdir+const.CHAUTOTUA_DIR, + self.chrootdir+const.CHAUTOTUA_DIR+'-old/autotua-'+strftime('%Y%m%d%H%M%S')) diff --git a/slave/autotua/const.py b/slave/autotua/const.py index 8719570..7d23e1f 100644 --- a/slave/autotua/const.py +++ b/slave/autotua/const.py @@ -16,7 +16,7 @@ VERBOSE = True FULL_CLEAN = False USE_PROXY = True -AUTOTUA_DIR = os.path.dirname(__file__) +AUTOTUA_DIR = os.path.abspath(os.path.dirname(__file__)) LOGFILE = '/var/log/autotua-slave.log' TMPDIR = '/var/tmp/autotua' TARBALL_DIR = TMPDIR+'/tarballs' diff --git a/slave/autotua/daemon/__init__.py b/slave/autotua/daemon/__init__.py index 35111a5..6d42817 100644 --- a/slave/autotua/daemon/__init__.py +++ b/slave/autotua/daemon/__init__.py @@ -16,7 +16,7 @@ class Spawn(object): sent to stdin with a 'pong\n' """ - def __init__(self, chroot, command, logfile=const.LOGFILE): + def __init__(self, chroot, jobtagedir, workdir, command, logfile=const.LOGFILE): """ @param chroot: The path to the chroot in which the command will be run @@ -27,10 +27,15 @@ class Spawn(object): @type command: string """ self.logfile = logfile + self.jobtagedir = jobtagedir + self.workdir = workdir # Messages goto 3 and then to /dev/tty1 # stderr goes to stdout # stdout goes to self.logfile - self.command = 'chroot \"%s\" /bin/bash -c \'%s\' 3>&2 2>&1 | tee -a \"%s\"' % (chroot, command, self.logfile) + if chroot: + self.command = 'chroot \"%s\" /bin/bash -c \'%s\' 3>&2 2>&1 | tee -a \"%s\"' % (chroot, command, self.logfile) + else: + self.command = '/bin/bash -c \'%s\' 3>&2 2>&1 | tee -a \"%s\"' % (command, self.logfile) self.process = self._init() def _init(self): @@ -40,5 +45,6 @@ class Spawn(object): if not response == 'pong\n': # FIXME: Custom exceptions raise 'Communication error: received %s when expecting "pong"' % response - process.stdin.write(const.CHAUTOTUA_DIR+'\n') + process.stdin.write(self.workdir+'\n') + process.stdin.write(self.jobtagedir+'\n') return process diff --git a/slave/autotua/jobuild/__init__.py b/slave/autotua/jobuild/__init__.py index b2d254f..a250b66 100644 --- a/slave/autotua/jobuild/__init__.py +++ b/slave/autotua/jobuild/__init__.py @@ -16,20 +16,20 @@ from .. import const, daemon class Jobuild(object): """A jobuild""" - def __init__(self, jobdir, atom): + def __init__(self, jobtagedir, atom): """ - @param jobdir: Job directory (of the form tmpdir/maint/job_name) - @type jobdir: string + @param jobtagedir: Jobtage directory + @type jobtagdir: string @param atom: Atom for finding the corresponding jobuild - @type jobdir: string + @type atom: string """ - self.name = atom - self.jobdir = jobdir - self.where = self._best_jobuild(atom) + self.atom = atom + self.jobtagedir = jobtagedir + self.which = self._best_jobuild(atom) def __str__(self): - return '%s jobuild object' % self.name + return '%s jobuild object' % self.atom def _split_atom(self, atom): regex = re.compile(r'^([<>]?=)?([a-zA-Z0-9_+-]+)/([a-zA-Z0-9_+-]+)(-|$)([0-9]+\.[0-9]+)?') # Fancy regex aye? @@ -69,7 +69,7 @@ class Jobuild(object): 'maint': parts[1], 'pn': parts[2], 'pv': parts[3], - 'jobtage': '%s/jobtage' % self.jobdir} + 'jobtage': self.jobtagedir} if data['op'] == '=': pass # Nothing to be done elif not data['op']: @@ -115,9 +115,8 @@ class Processor(object): cleanup: cleanup() """ args = {'phase': phase, - 'jobuild': self.jobuild.where, - 'chroot': self.chroot.chrootdir,} - self._msg('RUN_PHASE "%(phase)s" "%(jobuild)s" "%(chroot)s"' % args) + 'jobuild': self.jobuild.which,} + self._msg('RUN_PHASE "%(phase)s" "%(jobuild)s"' % args) def get_var(self, var): """ @@ -125,11 +124,16 @@ class Processor(object): (yay-something-works function) """ args = {'var': var, - 'jobuild': self.jobuild.where,} + 'jobuild': self.jobuild.which,} return self._msg('GET_VAR "%(var)s" "%(jobuild)s"' % args) def _msg(self, msg): - proc = daemon.Spawn(self.chroot.chrootdir, '\"%s\"/bin/jobuild.sh %s' % (const.CHAUTOTUA_DIR, msg)) + chroot = None + workdir = const.AUTOTUA_DIR + if self.chroot: + chroot = self.chroot.chrootdir + workdir = const.CHAUTOTUA_DIR + proc = daemon.Spawn(chroot, self.jobuild.jobtagedir, workdir, '\"%s\"/bin/jobuild.sh %s' % (workdir, msg)) # XXX: This waits for process to close stderr. # Replace with a loop during daemonisation response = proc.process.stderr.read()[:-1] # Remove trailing newline @@ -138,3 +142,55 @@ class Processor(object): # FIXME: Custom exceptions raise 'Daemon died with exit code \'%s\'' % -returncode return response + +class Resolver(object): + """Jobuild dependency resolver""" + + def __init__(self, jobuild): + """ + @param jobuild: Jobuild to resolve dependencies of + @type jobuild: L{autotua.jobuild.Jobuild} + """ + self.jobuild = jobuild + self.processor = Processor(self.jobuild, None) + + def _get_deplist(self, deplist): + retlist = deplist + index = 0 + while index < len(deplist): + if not isinstance(deplist[index], list): + jobj = Jobuild(self.jobuild.jobtagedir, deplist[index]) + self.processor.jobuild = jobj + deps = self.processor.get_var('DEPEND').split() + if deps: + retlist.append(self._get_deplist(deps)) + index += 1 + return retlist + + def _flatten(self, deplist): + newlist = [] + for item in deplist: + if isinstance(item, list): + newlist.extend(self._flatten(item)) + else: + newlist.append(item) + return newlist + + def _unique(self, deplist): + newlist = [] + index = len(deplist) + while index >= 0: + index -= 1 + if deplist[index] not in newlist: + newlist.append(deplist[index]) + return newlist + + def resolve(self): + """ + Resolve all the dependencies of the Jobuild and + return a list of jobs in running order + """ + deps = [self.jobuild.atom] + deps = self._get_deplist(deps) + deps = self._flatten(deps) + return self._unique(deps) |