diff options
author | 2009-08-07 17:46:04 -0400 | |
---|---|---|
committer | 2009-08-07 17:46:04 -0400 | |
commit | 6bf140ca63493397ce7aefea89ca6e3d4ba9cf52 (patch) | |
tree | c9eac3c699cd4aa3431226e7948e24a47e773474 | |
parent | Delete work dir for failed builds also (diff) | |
download | ingenue-6bf140ca63493397ce7aefea89ca6e3d4ba9cf52.tar.gz ingenue-6bf140ca63493397ce7aefea89ca6e3d4ba9cf52.tar.bz2 ingenue-6bf140ca63493397ce7aefea89ca6e3d4ba9cf52.zip |
Major backend reorganization to split build into steps and allow automatic resuming after failure
30 files changed, 319 insertions, 237 deletions
diff --git a/backend/backend.php b/backend/backend.php index d5c989e..2874b1a 100755 --- a/backend/backend.php +++ b/backend/backend.php @@ -43,56 +43,56 @@ if (posix_geteuid() !== 0) fputs(STDERR, "Not running as root... this is not going to accomplish much.\n"); if (@file_put_contents($pidfile, posix_getpid())) $unlinkpidfile=true; -require_once(dirname(__FILE__).'/../shared/include/includes.php'); // USE __DIR__ once 5.3.0 is out (and 2 lines down) +require_once(dirname(__FILE__).'/../shared/include/includes.php'); // USE __DIR__ once 5.3.0 is out require_once(BACKEND.'/include/signals.php'); declare(ticks=1); require_once(SHARED.'/include/dbinit.php'); while (true) { // TODO check first for builds that need to be resumed (and figure out how to resume things) - query('LOCK TABLES `builds` WRITE'); - $r=query('SELECT * FROM `builds` WHERE `status`=-128 ORDER BY `ctime` ASC LIMIT 1'); - if ($r->rowCount()) { - $build=new sql_build($r->fetch(PDO::FETCH_ASSOC)); + while (true) { + $r=query('SELECT * FROM `builds` WHERE `backend`="'.$S['conf']['backend_id'].'" AND `status` IN ("queued","cancel","uploading","upload_failed","building","got_signal") ORDER BY `ctime` ASC LIMIT 1'); + if ($r->rowCount()) + break; + else { + $r=query('UPDATE `builds` SET `backend`="'.$S['conf']['backend_id'].'" WHERE `status`="queued" AND `backend` IS NULL ORDER BY `ctime` ASC LIMIT 1'); + if ($r->rowCount() == 0) + sleep(5); + } + } + $build=new sql_build($r->fetch(PDO::FETCH_ASSOC)); + if (!isset($build->start)) { $build->start=time(); - $build->status=-1; $build->write(); - query('UNLOCK TABLES'); - debug('Starting build id='.$build->id); - $owner=$build->get_owner(); - $file=null; + } + debug('Starting build id='.$build->id); + $file=null; + $owner=$build->get_owner(); + $workdir=WORK."/build-$build->id"; + if (($image=$build->build($workdir)) !== false) { try { - if ($S['conf']['split_setup']) { - $opt=new sql_buildopt($build->id, 'backend', $S['conf']['backend_id']); - $opt->write(); - unset($opt); - } + $bundler=$build->get_opt('bundler'); + $bundle_proc="bundle_$bundler"; + if (!function_exists($bundle_proc)) + throw_exception("No bundler function defined for bundler $bundler"); $opts=$build->get_opts(); - $build_proc=$build->module.'_build'; - require_once(BACKEND."/modules/$build->module/build.php"); - // TODO check that build_proc exists - $workdir=WORK.'/build-'.$build->id; - log_status('Creating work directory '.$workdir, mkdir($workdir, 0700)); - $image=$build_proc($build, $opts, $workdir); - require_once(BACKEND."/bundlers/{$opts['bundler']}.php"); - $proc='bundle_'.$opts['bundler']; - $file=$proc($image, $workdir, $opts); + $file=$bundle_proc($image, $workdir, $opts); end_internal_task(0); // Just in case } catch (Exception $e) { + log_msg('Caught exception: '.$e->getMessage()); end_internal_task(1); -// log_msg('Caught exception: '.$e->getMessage()); - $build->status=INGENUE_BUILD_FAILED; - xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build failed', 'Your build has failed. You can find more information at <a href="'.url("build/$build->id").'">'.url("build/$build->id").'</a>'); + $build->status='failed'; + $build->write(); + xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build failed', 'Your build has failed in bundling stage. You can find more information at <a href="'.url("build/$build->id").'">'.url("build/$build->id").'</a>'); } $build->finish=time(); debug('Finished with build id='.$build->id); if (isset($file)) { debug("Completed build successfully"); if ($S['conf']['split_setup']) { - $build->status=INGENUE_BUILD_UPLOADING; + $build->status='uploading'; $build->write(); $key=randstring(30); - $opt=new sql_buildopt($build->id, 'uploadkey', $key); - $opt->write(); + $build->set_opt('uploadkey', $key); $c=curl_init(url('backend/upload_image')); curl_setopt($c, CURLOPT_POST, 1); curl_setopt($c, CURLOPT_POSTFIELDS, array( @@ -106,30 +106,30 @@ while (true) { debug($result); } if ($result === false || strpos($result, 'Upload successful') === false) { - $build->status=INGENUE_BUILD_UPLOAD_FAILED; + $build->status='upload_failed'; + $build->write(); } else { debug("Transferred $file... unlinking it"); unlink($file); - $build->status=INGENUE_BUILD_COMPLETE; + $build->status='complete'; + $build->write(); + shell_exec('rm -rf '.escapeshellarg($workdir)); } } else { - $build->status=INGENUE_BUILD_COMPLETE; + $build->status='complete'; $base=basename($file); $ext=substr($base, strpos($base, '.')); rename($file, COMPLETED.'/build-'.$build->id.$ext); + $build->write(); + shell_exec('rm -rf '.escapeshellarg($workdir)); } xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build finished', 'Your build has completed successfully. You can find more information and download the completed image at <a href="'.url("build/$build->id").'">'.url("build/$build->id").'</a>'); } - $build->write(); - if (!$S['conf']['debug']) { - shell_exec('rm -rf "'.$workdir.'"'); - } - unset($build); } else { - query('UNLOCK TABLES'); + $build->status='failed'; + $build->write(); + xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build failed', 'Your build has failed. You can find more information at <a href="'.url("build/$build->id").'">'.url("build/$build->id").'</a>'); } - // log_msg('Sleeping...', false); - sleep(5); - // log_msg("done"); + unset($build); } ?> diff --git a/backend/bundlers/cd.inc b/backend/bundlers/cd.inc new file mode 100644 index 0000000..6fb5aac --- /dev/null +++ b/backend/bundlers/cd.inc @@ -0,0 +1,12 @@ +<?php +$profile=new sql_gentoo_profile($opts['profile']); +$headers=$profile->get_headers(); +if (strpos($headers['chost'], 'x86_64') === false) + $minimaliso=CACHE.'/cd/install-x86-minimal-20090623.iso'; +else + $minimaliso=CACHE.'/cd/install-amd64-minimal-20090625.iso'; +makedir("$I/boot"); +execute_command('Extract kernel, initrd, and squashfs from CD image ', LIB."/bkisofs-cli '$minimaliso' extract /isolinux/gentoo '$I/boot/kernel' extract /isolinux/gentoo.igz '$I/boot/initrd' extract /image.squashfs '$W/'"); +//file_put_contents("$W/unsquashfs-files", "/lib64/modules\n/lib/modules\n"); +execute_command('Extract kernel modules from SquashFS to image', "unsquashfs -i -d '$W/modules' '$W/image.squashfs' /lib/modules /lib64/modules; cp -avT '$W/modules' '$I'; rm -rf '$W/modules'"); +?> diff --git a/backend/bundlers/installcd.php b/backend/bundlers/installcd.php index 9a4f857..63ffb5a 100644 --- a/backend/bundlers/installcd.php +++ b/backend/bundlers/installcd.php @@ -1,17 +1,8 @@ <?php function bundle_installcd($I, $W, &$opts) { - $profile=new sql_gentoo_profile($opts['profile']); - $headers=$profile->get_headers(); - if (strpos($headers['chost'], 'x86_64') === false) - $minimaliso=CACHE.'/cd/install-x86-minimal-20090623.iso'; - else - $minimaliso=CACHE.'/cd/install-amd64-minimal-20090625.iso'; - makedir("$I/boot"); - execute_command('Extract kernel, initrd, and squashfs from CD image ', LIB."/bkisofs-cli '$minimaliso' extract /isolinux/gentoo '$I/boot/kernel' extract /isolinux/gentoo.igz '$I/boot/initrd' extract /image.squashfs '$W/'"); - file_put_contents("$W/unsquashfs-files", "/lib64/modules\n/lib/modules\n"); - execute_command('Extract kernel modules from SquashFS', "unsquashfs -i -d '$W/modules' -e '$W/unsquashfs-files' '$W/image.squashfs'"); - execute_command('Copy extracted modules to image', "cp -av '$W/modules/*' '$I/'"); + require(dirname(__FILE__).'/cd.inc'); execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/image.tar.bz2' -C '$I' ."); execute_command('Create ISO image', LIB."/bkisofs-cli '$minimaliso' add / '$W/image.tar.bz2' write '$W/image.iso'"); return "$W/image.iso"; } +?> diff --git a/backend/bundlers/livecd.php b/backend/bundlers/livecd.php index d8a40fd..740c72b 100644 --- a/backend/bundlers/livecd.php +++ b/backend/bundlers/livecd.php @@ -1,17 +1,8 @@ <?php function bundle_livecd($I, $W, &$opts) { - $profile=new sql_gentoo_profile($opts['profile']); - $headers=$profile->get_headers(); - if (strpos($headers['chost'], 'x86_64') === false) - $minimaliso=CACHE.'/cd/install-x86-minimal-20090623.iso'; - else - $minimaliso=CACHE.'/cd/install-amd64-minimal-20090625.iso'; - makedir("$I/boot"); - execute_command('Extract kernel, initrd, and squashfs from CD image ', LIB."/bkisofs-cli '$minimaliso' extract /isolinux/gentoo '$I/boot/kernel' extract /isolinux/gentoo.igz '$I/boot/initrd' extract /image.squashfs '$W/'"); - file_put_contents("$W/unsquashfs-files", "/lib64/modules\n/lib/modules\n"); - execute_command('Extract kernel modules from SquashFS', "unsquashfs -i -d '$W/modules' -e '$W/unsquashfs-files' '$W/image.squashfs'"); - execute_command('Copy extracted modules to image', "cp -av '$W/modules/*' '$I/'"); -execute_command('Compress finished image to squashfs', "mksquashfs '$I' '$W/image.squashfs' -noappend -info"); + emerge('app-misc/livecd-toold', 'Install LiveCD utilities'); + require(dirname(__FILE__).'/cd.inc'); + execute_command('Compress finished image to squashfs', "mksquashfs '$I' '$W/image.squashfs' -noappend -info"); execute_command('Create ISO image', LIB."/bkisofs-cli '$minimaliso' replace /image.squashfs '$W/image.squashfs' write '$W/image.iso'"); return "$W/image.iso"; } diff --git a/backend/functions/api.php b/backend/functions/api.php index 1a0cc43..d8d648c 100644 --- a/backend/functions/api.php +++ b/backend/functions/api.php @@ -97,4 +97,8 @@ function makedir($dir) { if (!is_dir($dir)) log_status('Create '.$dir, mkdir($dir, 0700, true)); } +function add_step($step) { + global $S; + $S['build_steps'][]=$step; +} ?> diff --git a/backend/functions/order_management.php b/backend/functions/order_management.php index 306e42d..b7609f6 100644 --- a/backend/functions/order_management.php +++ b/backend/functions/order_management.php @@ -7,7 +7,11 @@ function task_get_order() { $order++; } else { $buildid=$build->id; - $order=0; + $order=query('SELECT MAX(`order`) FROM `tasks` WHERE `build`="'.$buildid.'"')->fetch(PDO::FETCH_COLUMN); + if ($order === null) + $order=0; + else + $order++; } return $order; } diff --git a/backend/functions/signals.php b/backend/functions/signals.php index 2a895e5..70231c7 100644 --- a/backend/functions/signals.php +++ b/backend/functions/signals.php @@ -13,7 +13,8 @@ function handle_signal($sig=null) { debug('$task not set'); } $build->finish=time(); - $build->status=$sig; + $build->status='got_signal'; + $build->details=$sig; $build->write(); debug("build $build->id given status $build->status"); } diff --git a/backend/include/includes.php b/backend/include/includes.php new file mode 100644 index 0000000..d20c3ac --- /dev/null +++ b/backend/include/includes.php @@ -0,0 +1,4 @@ +<?php +foreach (glob(BACKEND.'/bundlers/*.php') as $file) + require_once($file); +?> diff --git a/backend/modules/gentoo_portage/base-system.php b/backend/modules/gentoo_portage/base-system.php index 6cf2194..c99447e 100644 --- a/backend/modules/gentoo_portage/base-system.php +++ b/backend/modules/gentoo_portage/base-system.php @@ -1,10 +1,10 @@ <?php $file=CACHE.'/stage3/'.$profile->stage3; execute_command('Unpack base system', "tar -xvjpf '$file' -C '$I'"); -if ($opts['basesystem'] == 'manual' && $opts['prunepkgs']) { +if ($opts['basesystem'] == 'user_prune' && $opts['prunepkgs']) { emerge($opts['prunepkgs'], 'Prune base system packages', '-C'); -} elseif ($opts['basesystem'] == 'autoprune') { - throw_exception('Base system autoprune not implemented - need package list'); +} elseif ($opts['basesystem'] == 'auto_prune') { + throw_exception('Base system auto-prune not implemented - need package list'); $keep=explode(' ', $keep_pkgs); $remove=array(); $r=query('SELECT * FROM `gentoo_basepkgs` WHERE `profile`='.$profile->id); @@ -16,5 +16,7 @@ if ($opts['basesystem'] == 'manual' && $opts['prunepkgs']) { unset($keep[$i]); } emerge($remove, 'Automatically prune base system packages', '-C'); +} elseif ($opts['basesystem'] == 'emerge') { + emerge('system', 'Emerge base system'); } ?> diff --git a/backend/modules/gentoo_portage/build.php b/backend/modules/gentoo_portage/build.php index 631cf0f..781aaef 100644 --- a/backend/modules/gentoo_portage/build.php +++ b/backend/modules/gentoo_portage/build.php @@ -1,37 +1,26 @@ <?php -require_once(dirname(__FILE__).'/packages.php'); // __DIR__ 5.3.0 -// This is the main function that carries out a build from start to finish -function gentoo_portage_build(&$build, &$opts, &$W) { - global $S; - $profile=new sql_gentoo_profile($opts['profile']); - $headers=$profile->get_headers(); - $I="$W/image"; - require(dirname(__FILE__).'/setup.php'); // __DIR__ in 5.3.0 - if ($S['conf']['debug']) - execute_command_with_env('Log portage setup', 'emerge --info', $prtg_cfgrt); - require(dirname(__FILE__).'/base-system.php'); // __DIR__ 5.3.0 - $extra=explode(' ', $opts['options']); - if (in_array('portage', $extra)) - require(dirname(__FILE__).'/portage.php'); // __DIR__ 5.3.0 - if (in_array('pruneinit', $extra)) - require(dirname(__FILE__).'/init.d.php'); // __DIR__ 5.3.0 - if (in_array('timezone', $extra)) - require(dirname(__FILE__).'/timezone.php'); // __DIR__ 5.3.0 - if (in_array('hostname', $extra)) - require(dirname(__FILE__).'/hostname.php'); // __DIR__ 5.3.0 - if (in_array('dev-manager', $extra)) - require(dirname(__FILE__).'/dev-manager.php'); // __DIR__ 5.3.0 - if ($opts['bundler'] == 'livecd') - emerge('app-misc/livecd-tools', 'Install LiveCD utilities'); - if (strlen($opts['pkgsets'])) { - foreach (explode(' ', $opts['pkgsets']) as $pkgset) { - if (strlen($opts['pkgset-'.$pkgset])) { - emerge($opts['pkgset-'.$pkgset]); - } - } - } - if (strlen($opts['install_packages'])) - emerge($opts['install_packages'], 'Install selected packages'); - return $I; -} +require_once(dirname(__FILE__).'/packages.php'); // __DIR__ 5.3.0 +$profile=new sql_gentoo_profile($opts['profile']); +$headers=$profile->get_headers(); +$I="$workdir/image"; +$C="$workdir/config_root"; +$S['prtg_cfgrt']=array('PORTAGE_CONFIGROOT' => $C); +add_step('setup'); +add_step('base-system'); +$extra=explode(' ', $opts['options']); +if (in_array('portage', $extra)) + add_step('portage'); +if (in_array('prune_init', $extra)) + add_step('init.d.php'); +if (in_array('timezone', $extra)) + add_step('timezone'); +if (in_array('hostname', $extra)) + add_step('hostname'); +if (in_array('dev-manager', $extra)) + add_step('dev-manager'); +if (strlen($opts['pkgsets'])) + add_step('pkgsets'); +if (strlen($opts['install_packages'])) + add_step('misc-pkgs'); +return $I; ?> diff --git a/backend/modules/gentoo_portage/misc-pkgs.php b/backend/modules/gentoo_portage/misc-pkgs.php new file mode 100644 index 0000000..2c5185f --- /dev/null +++ b/backend/modules/gentoo_portage/misc-pkgs.php @@ -0,0 +1,3 @@ +<?php +emerge($opts['install_packages'], 'Install selected packages'); +?> diff --git a/backend/modules/gentoo_portage/packages.php b/backend/modules/gentoo_portage/packages.php index 167ca73..0274f35 100644 --- a/backend/modules/gentoo_portage/packages.php +++ b/backend/modules/gentoo_portage/packages.php @@ -1,6 +1,6 @@ <?php function emerge($pkgs, $desc=null, $opts=null, $use=null) { - global $prtg_cfgrt, $S; + global $S; $opts=$S['conf']['emerge_default_opts'].' '.($opts === null?'-K -n --root-deps=rdeps':$opts); if (!$pkgs) return; if (!is_array($pkgs)) @@ -9,6 +9,6 @@ function emerge($pkgs, $desc=null, $opts=null, $use=null) { $desc='Install '.(count($pkgs) > 1?'packages':$pkgs[0]); foreach ($pkgs as $i => &$pkg) $pkg=escapeshellarg($pkg); - execute_command_with_env($desc, ($use?'env USE="'.(is_array($use)?implode(' ', $use):$use).'" ':'').'emerge '.($opts?$opts.' ':'').implode(' ', $pkgs), $prtg_cfgrt); + execute_command_with_env($desc, ($use?'env USE="'.(is_array($use)?implode(' ', $use):$use).'" ':'').'emerge '.($opts?$opts.' ':'').implode(' ', $pkgs), $S['prtg_cfgrt']); } ?> diff --git a/backend/modules/gentoo_portage/pkgsets.php b/backend/modules/gentoo_portage/pkgsets.php new file mode 100644 index 0000000..c0ed4e7 --- /dev/null +++ b/backend/modules/gentoo_portage/pkgsets.php @@ -0,0 +1,7 @@ +<?php +foreach (explode(' ', $opts['pkgsets']) as $pkgset) { + if (strlen($opts['pkgset-'.$pkgset])) { + emerge($opts['pkgset-'.$pkgset]); + } +} +?> diff --git a/backend/modules/gentoo_portage/portage.php b/backend/modules/gentoo_portage/portage.php index 291a858..a1eb3ed 100644 --- a/backend/modules/gentoo_portage/portage.php +++ b/backend/modules/gentoo_portage/portage.php @@ -18,16 +18,21 @@ if ($file) { end_internal_task(0); } start_internal_task('Set up portage in image'); +$use=explode(' ', $headers['use']); +foreach(explode(' ', shell_exec(dirname(__FILE__).'/profile_use.py '.escapeshellarg("{$S['conf']['portdir']}/profiles/{$headers['profile']}"))) as $flag) + if (($i=array_search($flag, $use)) !== false) + unset($use[$i]); $makeconf=array( 'chost' => $headers['chost'], 'accept_keywords' => $headers['accept_keywords'], 'gentoo_mirrors' => $headers['gentoo_mirrors'], - 'use' => $headers['use'] + 'use' => implode(' ', $use) ); $contents=''; foreach ($makeconf as $name => $val) - $contents.=strtoupper($name).'='.escapeshellarg($val)."\n"; + $contents.=strtoupper($name).'="'.str_replace('"', '\"', $val)."\"\n"; unset($makeconf); +log_msg("/etc/make.conf:\n$contents"); log_status('Writing /etc/make.conf', file_put_contents("$I/etc/make.conf", $contents)); log_status('Remove previous make.profile', unlink("$I/etc/make.profile"), false); log_status("Symlink make.profile -> /usr/portage/profiles/{$headers['profile']}", symlink("/usr/portage/profiles/{$headers['profile']}", "$I/etc/make.profile")); diff --git a/backend/modules/gentoo_portage/setup.php b/backend/modules/gentoo_portage/setup.php index 5738bee..d96eb51 100644 --- a/backend/modules/gentoo_portage/setup.php +++ b/backend/modules/gentoo_portage/setup.php @@ -1,25 +1,22 @@ <?php start_internal_task('Create portage target environment'); -$C=$W.'/config_root'; -makedirs($I, $C, "$W/log", "$W/tmp"); -log_status("Making symlink $C/etc -> .", symlink('.', "$C/etc")); +makedirs($I, $C, "$workdir/log", "$workdir/tmp"); +log_status("Make symlink $C/etc -> .", symlink('.', "$C/etc")); $makeconf=array( 'pkgdir' => $S['conf']['pkgdir_root'].'/'.$profile->pkgdir, 'chost' => $headers['chost'], 'accept_keywords' => $headers['accept_keywords'], 'root' => $I, - 'port_logdir' => "$W/log", - 'emerge_log_dir' => "$W/log", - 'portage_tmpdir' => "$W/tmp" + 'port_logdir' => "$workdir/log", + 'emerge_log_dir' => "$workdir/log", + 'portage_tmpdir' => "$workdir/tmp" ); $contents=''; foreach ($makeconf as $name => $val) - $contents.=strtoupper($name).'='.escapeshellarg($val)."\n"; + $contents.=strtoupper($name).'="'.str_replace('"', '\"', $val)."\"\n"; unset($makeconf); -log_status("Writing $C/etc/make.conf", file_put_contents("$C/etc/make.conf", $contents)); +log_status("Write $C/etc/make.conf", file_put_contents("$C/etc/make.conf", $contents)); unset($contents); -log_status('Making make.profile symlink to '.$S['conf']['portdir'].'/profiles/'.$headers['profile'], symlink($S['conf']['portdir'].'/profiles/'.$headers['profile'], $C.'/etc/make.profile')); -global $prtg_cfgrt; -$prtg_cfgrt=array('PORTAGE_CONFIGROOT' => $C); +log_status('Make make.profile symlink to '.$S['conf']['portdir'].'/profiles/'.$headers['profile'], symlink($S['conf']['portdir'].'/profiles/'.$headers['profile'], $C.'/etc/make.profile')); end_internal_task(0); ?> diff --git a/frontend/modules/gentoo/step2.php b/frontend/modules/gentoo/step2.php index 7fa5456..c2d169c 100644 --- a/frontend/modules/gentoo/step2.php +++ b/frontend/modules/gentoo/step2.php @@ -1,8 +1,8 @@ <?php global $S; $profile=new sql_gentoo_profile($this->get_opt('profile')); -$this->checkbox_array('options', 'options', 'Configuration options', array('timezone' => 'Select timezone', 'hostname' => 'Choose hostname', 'dev-manager' => 'Select /dev manager', 'pruneinit' => 'Remove enabled-by-default init scripts', 'portage' => 'Install portage snapshot and configure portage')); -$this->select('basesystem', 'basesystem', 'Base system', array('stage3' => 'Stage3 Tarball', /*'autoprune' => 'Remove all non-vital packages',*/ 'manual' => 'Manually select packages to remove from stage3')); +$this->checkbox_array('options', 'options', 'Configuration options', array('timezone' => 'Select timezone', 'hostname' => 'Choose hostname', 'dev-manager' => 'Select /dev manager', 'prune_init' => 'Remove enabled-by-default init scripts', 'portage' => 'Install portage snapshot and configure portage')); +$this->select('basesystem', 'basesystem', 'Base system', array('emerge' => 'emerge @system', 'stage3' => 'Stage3 Tarball', /*'auto_prune' => 'Remove all non-vital packages',*/ 'user_prune' => 'Manually select packages to remove from stage3')); $pkgsets=array(); $r=query('SELECT * FROM `gentoo_pkgsets` WHERE `profile`='.$profile->id); while ($pkgset=$r->fetch(PDO::FETCH_ASSOC)) { diff --git a/frontend/pages/builds/delete.php b/frontend/pages/builds/delete.php index 5ff1d9a..6fc6ae3 100644 --- a/frontend/pages/builds/delete.php +++ b/frontend/pages/builds/delete.php @@ -9,14 +9,27 @@ function init_builds_delete(&$S) { return array('title' => 'Delete Build'); } function body_builds_delete(&$S) { - if ($S['build']->status >= 0 || $S['build']->status == INGENUE_BUILD_QUEUED) { + switch($S['build']->status) { + case 'queued': + if (isset($S['build']->backend)) + die(print_warning('Oops', 'You tried to delete this build just as it was about to start being built. Please try to cancel it in a moment.')); + case 'upload_failed': + case 'canceled': + case 'failed': + case 'complete': + case 'got_signal': $S['build']->delete(); echo print_success('Build deleted.'); - } elseif ($S['build']->status != INGENUE_BUILD_CANCEL) { - $S['build']->status=INGENUE_BUILD_CANCEL; + break; + case 'cancel': + echo print_error('This build is already queued for cancellation.'); + break; + case 'uploading': + case 'building': + default: + $S['build']->status='cancel'; $S['build']->write(); echo print_success('Build queued for cancellation.'); - } else - echo print_error('This build is already queued for cancellation.'); + } } ?> diff --git a/frontend/pages/builds/index.php b/frontend/pages/builds/index.php index adc6566..bb976b9 100644 --- a/frontend/pages/builds/index.php +++ b/frontend/pages/builds/index.php @@ -4,7 +4,7 @@ function init_builds_index(&$S) { return array('title' => 'My Builds'); } function body_builds_index(&$S) { - $r=query('SELECT * FROM `builds` WHERE `owner`='.$S['user']->id.' ORDER BY `ctime` IS NULL ASC, `ctime` ASC, `status` DESC'); + $r=query('SELECT * FROM `builds` WHERE `owner`='.$S['user']->id.' ORDER BY `ctime` IS NULL ASC, `ctime` ASC'); if ($r->rowCount() == 0) { echo print_warning('No builds found.'); } diff --git a/frontend/pages/configurations/manager.php b/frontend/pages/configurations/manager.php index 6396aaa..928ad67 100644 --- a/frontend/pages/configurations/manager.php +++ b/frontend/pages/configurations/manager.php @@ -47,13 +47,14 @@ function body_configurations_manager(&$S) { } else { echo $c->status; } - echo ' (<a href="'.url("config/$c->id/status").'">view</a>)'; - echo '</td><td>'.$c->summary().'</td><td>'; + echo '</td><td><a href="'.url("config/$c->id/status").'">View</a></td><td>'; $builds=$c->get_builds(); if ($builds) { + $buildlinks=array(); foreach ($builds as $build) { - echo '<a href="'.url("build/$build").'">'.$build.'</a> '; + $buildlinks[]='<a href="'.url("build/$build->id").'">'.($build->name?htmlentities($build->name):$build->id).'</a>'; } + echo implode(', ', $buildlinks); } else { echo '<i>None</i>'; } diff --git a/frontend/pages/welcome.php b/frontend/pages/welcome.php index 2de99a1..91efd14 100644 --- a/frontend/pages/welcome.php +++ b/frontend/pages/welcome.php @@ -27,7 +27,7 @@ function body_welcome(&$S) { } echo '</div>'; echo '<div class="box"><h3>Recently Built</h3>'; - $r=query('SELECT * FROM `builds` WHERE `status`="finished/success" ORDER BY `finish` DESC LIMIT 3'); + $r=query('SELECT * FROM `builds` WHERE `status`="complete" ORDER BY `finish` DESC LIMIT 3'); if ($r->rowCount()) { while ($build=$r->fetch(PDO::FETCH_ASSOC)) { $build=new sql_build($build); diff --git a/shared/classes/0sql_row_obj.php b/shared/classes/0sql_row_obj.php index 30fe479..d625071 100644 --- a/shared/classes/0sql_row_obj.php +++ b/shared/classes/0sql_row_obj.php @@ -564,78 +564,82 @@ class sql_col { if (isset($array['refers_to'])) $this->refers_to=$array['refers_to']; } elseif (is_string($array)) { - $line=$array; - list($type, $line)=explode(' ', $line, 2); + $type=$array; + if (strpos($type, ' ')) + list($type, $opts)=explode(' ', $type, 2); if (strpos($type, '(') !== false) { + // TODO differentiate between things with length and ENUM $length=substr($type, strpos($type, '(')); $type=substr($type, 0, strlen($type)-strlen($length)); $length=substr($length, 1, strlen($length)-2); $this->length=$length; } $this->type=strtoupper($type); - $opts=explode(' ', $line); - for ($i=0; $i<count($opts); $i++) { - $word=$opts[$i]; - switch (strtoupper($word)) { - case 'NOT': - if (strtoupper($opts[++$i]) == 'NULL') { - $this->not_null=true; - } else { - $i--; // We assume it's NULL and backtrack otherwise - } - break; - case 'NULL': - $this->not_null=false; - break; - case 'UNSIGNED': - $this->unsigned=true; - break; - case 'AUTO_INCREMENT': - $this->auto_increment=true; - break; - case 'CHARACTER': - if (strtoupper($opts[$i+1]) == 'SET') { - $word.=' '.$opts[++$i]; - } else { - break; - } - case 'COLLATE': - case 'DEFAULT': - case 'COMMENT': - $string=$opts[++$i]; - if ($string == "''") { - $string=''; - } elseif (substr($string, 0, 1) == "'") { - // An odd number of ' at the end means an unquoted ' - // The $i<count($opts) is just to avoid infinite loops that shouldn't happen - //echo strtoupper($word).' '.$string; - while ( ( strlen($string) - strlen(rtrim($string, '\'')) ) % 2 == 0 && $i<count($opts)) { - $string.=' '.$opts[++$i]; - } - $string=str_replace("''", "'", $string); - $string=substr($string, 1, strlen($string)-2); - } elseif (strtoupper($string) == 'NULL') { - $string=null; - } + if (isset($opts)) { + $opts=explode(' ', $opts); + for ($i=0; $i<count($opts); $i++) { + $word=$opts[$i]; switch (strtoupper($word)) { - case 'CHARACTER SET': - $this->charset=$string; + case 'NOT': + if (strtoupper($opts[++$i]) == 'NULL') { + $this->not_null=true; + } else { + $i--; // We assume it's NULL and backtrack otherwise + } break; - case 'COLLATE': - $this->collate=$string; + case 'NULL': + $this->not_null=false; break; - case 'DEFAULT': - $this->default=$string; + case 'UNSIGNED': + $this->unsigned=true; break; - case 'COMMENT': - if (preg_match('/^refers to(?::| |: )([a-zA-Z0-9_$]+\.[a-zA-Z0-9_$]+)$/', $string, $match)) { - $this->refers_to=$match[1]; + case 'AUTO_INCREMENT': + $this->auto_increment=true; + break; + case 'CHARACTER': + if (strtoupper($opts[$i+1]) == 'SET') { + $word.=' '.$opts[++$i]; } else { - $this->comment=$string; + break; + } + case 'COLLATE': + case 'DEFAULT': + case 'COMMENT': + $string=$opts[++$i]; + if ($string == "''") { + $string=''; + } elseif (substr($string, 0, 1) == "'") { + // An odd number of ' at the end means an unquoted ' + // The $i<count($opts) is just to avoid infinite loops that shouldn't happen + //echo strtoupper($word).' '.$string; + while ( ( strlen($string) - strlen(rtrim($string, '\'')) ) % 2 == 0 && $i<count($opts)) { + $string.=' '.$opts[++$i]; + } + $string=str_replace("''", "'", $string); + $string=substr($string, 1, strlen($string)-2); + } elseif (strtoupper($string) == 'NULL') { + $string=null; + } + switch (strtoupper($word)) { + case 'CHARACTER SET': + $this->charset=$string; + break; + case 'COLLATE': + $this->collate=$string; + break; + case 'DEFAULT': + $this->default=$string; + break; + case 'COMMENT': + if (preg_match('/^refers to(?::| |: )([a-zA-Z0-9_$]+\.[a-zA-Z0-9_$]+)$/', $string, $match)) { + $this->refers_to=$match[1]; + } else { + $this->comment=$string; + } + break; } break; } - break; } } } diff --git a/shared/classes/1conf_build_common.php b/shared/classes/1conf_build_common.php index af20eba..27c2198 100644 --- a/shared/classes/1conf_build_common.php +++ b/shared/classes/1conf_build_common.php @@ -54,7 +54,10 @@ abstract class conf_build_common extends sql_row_obj { public function init() { global $S; $this->owner=$S['user']->id; - $this->status=1; + if ($this->table == 'configurations') + $this->status=1; + else + $this->status='queued'; $fails=0; while (true) { $id=randstring(6); diff --git a/shared/classes/build.php b/shared/classes/build.php index c863f1f..64730fe 100644 --- a/shared/classes/build.php +++ b/shared/classes/build.php @@ -30,11 +30,23 @@ class sql_build extends conf_build_common { 'length' => '\'public\',\'private\'', 'not_null' => true ), + 'backend' => array ( + 'type' => 'VARCHAR', + 'length' => 255 + ), 'status' => array ( + 'type' => 'ENUM', + 'length' => '\'queued\',\'uploading\',\'cancel\',\'complete\',\'upload_failed\',\'canceled\',\'failed\',\'got_signal\',\'building\',\'bundling\'', + 'not_null' => true + ), + 'details' => array ( 'type' => 'TINYINT', - 'length' => 4, - 'not_null' => true, - 'default' => 0 + 'length' => 4 + ), + 'build_step' => array ( + 'type' => 'TINYINT', + 'length' => 3, + 'unsigned' => true ), 'ctime' => array ( 'type' => 'INT', @@ -59,46 +71,48 @@ class sql_build extends conf_build_common { $perms=$this->visibility == 'public' || owner_or_admin($this->id); $html='<div class="build"><span class="name">'.(isset($this->name) && strlen($this->name)?htmlentities($this->name):'Unnamed Build').'</span> '; $links=array(); - if ($this->status == INGENUE_BUILD_QUEUED) { - $total=query('SELECT COUNT(*) FROM `builds` WHERE `status`=-128')->fetch(PDO::FETCH_COLUMN); - $num=query('SELECT COUNT(*) FROM `builds` WHERE `status`=-128 AND `ctime` <= '.$this->ctime)->fetch(PDO::FETCH_COLUMN); + if ($this->status == 'queued') { + $total=query('SELECT COUNT(*) FROM `builds` WHERE `status`="queued"')->fetch(PDO::FETCH_COLUMN); + $num=query('SELECT COUNT(*) FROM `builds` WHERE `status`="queued" AND `ctime` <= '.$this->ctime)->fetch(PDO::FETCH_COLUMN); $html.="<span class=\"status queued\">[queued ($num/$total)]</span>"; - } elseif ($this->status == INGENUE_BUILD_UPLOADING) { + } elseif ($this->status == 'uploading') { $html.='<span class="status successful">[uploading]</span>'; if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status == INGENUE_BUILD_CANCEL) { + } elseif ($this->status == 'cancel') { $html.='<span class="status queued">[pending cancellation]</span>'; if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status < 0) { - // TODO Build stage X - $html.='<span class="status building">[building]</span>'; + } elseif ($this->status == 'building') { + // TODO stage x/y + $html.='<span class="status building">[building ('.$this->build_step.'/'.$this->details.')]</span>'; if ($perms) { //$links['Watch']="build/$this->id/live"; $links['Build Log']="build/$this->id"; } - } elseif ($this->status == INGENUE_BUILD_COMPLETE) { + } elseif ($this->status == 'complete') { $r=query('SELECT COUNT(*) as `count`, MAX(`time`) as `time` FROM `downloads` WHERE `build`="'.$this->id.'"')->fetch(PDO::FETCH_ASSOC); $d=($perms && $r['count']?'<a href="'.url("build/$this->id/history").'">':'').$r['count'].' download'.($r['count'] != 1?'s':'').($r['count']?($perms?'</a>':'').'<br/><span class="time">(last at '.date($format, $r['time']).')</span>':''); $html.='<span class="downloads">'.$d.'</span><span class="status successful">[successful]</span>'; $links['Download image']="build/$this->id/download"; if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status == INGENUE_BUILD_UPLOAD_FAILED) { + } elseif ($this->status == 'upload_failed') { $html.='<span class="status failed">[upload failed]</span>'; if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status == INGENUE_BUILD_FAILED) { - $html.='<span class="status failed">[failed]</span>'; + } elseif ($this->status == 'failed') { + $html.='<span class="status failed">[failed after step '.$this->build_step.']</span>'; if ($perms) { //$links['View output of failed command']="build/$this->id/failure"; $links['Build log']="build/$this->id"; } - } elseif ($this->status == INGENUE_BUILD_CANCELED) { + } elseif ($this->status == 'canceled') { $html.='<span class="status failed">[canceled]</span>'; if ($perms) $links['Build log']="build/$this->id"; - } else { - $html.='<span class="status failed">[failed: got signal '.$this->status.']</span>'; + } elseif ($this->status == 'got_signal') { + $html.='<span class="status failed">[failed: got signal '.$this->details.' after step '.$this->build_step.']</span>'; if ($perms) $links['Build log']="build/$this->id"; + } else { + $html.='<span class="status failed">[UNKNOWN STATUS: '.$this->status.']</span>'; } - if ($this->status >= 0 || $this->status == INGENUE_BUILD_QUEUED) // Finished or queued + if ($perms && ($this->status == 'upload_failed' || $this->status == 'failed' || $this->status == 'canceled' || $this->status == 'queued' || $this->status == 'complete' || $this->status == 'got_signal')) $links['Delete']="build/$this->id/delete"; if ($links) { foreach ($links as $label => $url) { @@ -144,5 +158,46 @@ class sql_build extends conf_build_common { unlink($file); parent::delete(); } + public function build($workdir) { + global $S; + try { + if (!is_dir($workdir)) + log_status('Create work directory '.$workdir, mkdir($workdir, 0700)); + $opts=$this->get_opts(); + $S['build_steps']=array(); + if (!is_readable(BACKEND."/modules/$this->module/build.php")) + throw_exception("No build script for module $this->module"); + $dir=require(BACKEND."/modules/$this->module/build.php"); + switch ($this->status) { + case 'queued': + $this->build_step=0; + case 'got_signal': + case 'failed': + $this->status='building'; + $this->details=count($S['build_steps']); + $this->write(); + case 'building': + $step=$this->build_step; + break; + case 'uploading': + case 'upload_failed': + case 'cancel': + case 'bundling': + default: + $step=count($S['build_steps']); + } + while ($step < count($S['build_steps'])) { + require(BACKEND."/modules/$this->module/{$S['build_steps'][$step]}.php"); + $step++; + $this->build_step=$step; + $this->write(); + } + return $dir; + } catch(Exception $e) { + log_msg('Caught exception: '.$e->getMessage()); + end_internal_task(1); + return false; + } + } } ?> diff --git a/shared/classes/configuration.php b/shared/classes/configuration.php index c7795f4..8780b9d 100644 --- a/shared/classes/configuration.php +++ b/shared/classes/configuration.php @@ -59,18 +59,18 @@ class sql_configuration extends conf_build_common { $opt->write(); } $build->ctime=time(); - $build->status=-128; + $build->status='queued'; $build->write(); return $build; } // Returns an array of the IDs of all the builds that report this configuration as their source public function get_builds() { global $S; - $r=query('SELECT `build` FROM `buildopts` WHERE `name`="configuration" AND `value`="'.$this->id.'"'); + $r=query('SELECT `builds`.* FROM `buildopts` INNER JOIN `builds` WHERE `buildopts`.`name`="configuration" AND `buildopts`.`value`="'.$this->id.'" AND `builds`.`id`=`buildopts`.`build`'); if ($r->rowCount()) { $builds=array(); - while ($b=$r->fetch(PDO::FETCH_COLUMN)) { - $builds[]=$b; + while ($b=$r->fetch(PDO::FETCH_ASSOC)) { + $builds[]=new sql_build($b); } return $builds; } else { diff --git a/shared/config.php b/shared/config.php index 62a1c4d..9270d2d 100644 --- a/shared/config.php +++ b/shared/config.php @@ -15,7 +15,7 @@ $debug=true; // Whether to print debugging information // $timezone_root='/usr/share/zoneinfo'; // Directory to search for timezone data (sys-libs/timezone-data) $emailfrom='noreply@gentoo.org'; // Used as the From: field in emails $check_email_dns=true; // Use DNS to check the domain of submitted emails for validity -$split_setup=true; // Whether the frontend and backend are running on different hosts +// $split_setup=true; // Whether the frontend and backend are running on different hosts // Frontend options: // $registration=false; // Whether users can create new accounts without an invite // $invite='admin'; // Who can use the invite function: true or 'user'=users; admin=admins; false=nobody @@ -25,5 +25,5 @@ $split_setup=true; // Whether the frontend and backend are running on different $pkgdir_root='/home/eitan/soc/tinderbox'; // The directory to recursively search for pkgdirs (Backend only) // $emerge_default_opts='-v --color=y'; // Options to add to all emerge commands // $portdir='/usr/portage'; // The directory conatining the portage tree to use (/usr/portage unless you have a reason to think otherwise) -$backend_id='red'; // A name or other way of identifying this backend as opposed to other backends working for the same frontend TODO use gethostname() by default in 5.3.0 +$backend_id='red'; // A name or other way of identifying this backend as opposed to other backends working for the same frontend ?> diff --git a/shared/functions/query.php b/shared/functions/query.php index d92ee0f..88f177a 100644 --- a/shared/functions/query.php +++ b/shared/functions/query.php @@ -1,8 +1,9 @@ <?php -function query($q) { - debug('query', $q); +function &query($q) { // 5.3.0: func_get_args() can be used directly in call_user_func_array() $args=func_get_args(); - return call_user_func_array(array(&$GLOBALS['S']['pdo'], 'query'), $args); + $r=call_user_func_array(array(&$GLOBALS['S']['pdo'], 'query'), $args); + debug('query', "$q (".$r->rowCount()." rows affected)"); + return $r; } ?> diff --git a/shared/include/defaults.php b/shared/include/defaults.php index 4e93a7a..ced0e01 100644 --- a/shared/include/defaults.php +++ b/shared/include/defaults.php @@ -25,7 +25,7 @@ $mod_rewrite=true; $timezone_root='/usr/share/zoneinfo'; $emailfrom='noreply@noreply.net'; $check_email_dns=false; -$split_setup=false; +$split_setup=true; $registration=false; $invite='admin'; $logview_max=1000; diff --git a/shared/include/definitions.php b/shared/include/definitions.php deleted file mode 100644 index b568959..0000000 --- a/shared/include/definitions.php +++ /dev/null @@ -1,9 +0,0 @@ -<?php -define('INGENUE_BUILD_QUEUED', -128); -define('INGENUE_BUILD_UPLOADING', -127); -define('INGENUE_BUILD_CANCEL', -126); -define('INGENUE_BUILD_COMPLETE', 0); -define('INGENUE_BUILD_UPLOAD_FAILED', 127); -define('INGENUE_BUILD_FAILED', 126); -define('INGENUE_BUILD_CANCELED', 125); -?> diff --git a/shared/include/includes.php b/shared/include/includes.php index 1394049..410177f 100644 --- a/shared/include/includes.php +++ b/shared/include/includes.php @@ -1,7 +1,6 @@ <?php date_default_timezone_set('UTC'); require_once(dirname(__FILE__).'/paths.php'); // USE __dir__ in 5.3.0 -require_once(SHARED.'/include/definitions.php'); // Load functions and classes from the shared directory and either foreground or background foreach (array('functions', 'classes') as $type) { foreach (array(SHARED, ($_SERVER['DOCUMENT_ROOT']?FRONTEND:BACKEND)) as $dir) { @@ -10,6 +9,9 @@ foreach (array('functions', 'classes') as $type) { } } } -unset($dir, $file, $type); +$includes=($_SERVER['DOCUMENT_ROOT']?FRONTEND:BACKEND).'/include/includes.php'; +if (is_readable($includes)) + require_once($includes); +unset($dir, $file, $type, $includes); load_config(); ?> @@ -5,11 +5,11 @@ Add cleanup functions to the frontend and backend (tasks dir in backend containi Only offer bundlers that will work - profiles without CD in map file can't make livecd, installcd, etc. Allow config viewing for builds, not just configurations Add `flags` column to configurations, builds, use it to implement public and private things -Break backend into steps defined by include files - add an API function to require a file, add them up, store the total in the database, track status and provide resumability Add safe shutdown to backend so it will stop once it gets to the end of a step that can be resumed Add 'cancel', option to builds, allow deletion of currently running builds (have backend check if canceled before and after each task) Add build->configuration and configuration duplication -Add map file for liveCD, load it into DB, etc. +Consider adding `configuration` col to builds +Add map file for liveCD, load it into DB, etc. (currently hardcoded = evil) Add gentoo_profileopts column for liveCD, stage3, etc. Write script for fetching latest stage3's from the desired FTP dirs Add ability to trim the stage3 to a minimum set of packages automatically (cached) @@ -19,11 +19,13 @@ Add option to upload an arbitrary tar.gz/bz2 to be unzipped over the finished im *** Implement selected items from gentoo-steps *** *** Documentation *** Ponder whether to remove the split-setup option and always upload to the frontend -Improve the upload step so it tries more than once +Make backend not retry indefinitely on failure (`failures` column?) Offer FTP upload Offer SCP upload? -Use zmedico's deafult USE flags script in backend/modules/gentoo_portage to trim the USE we put in the make.conf generated Ask someone to add the necessary USE flags to php on tinderbox -Fix module copying from CD to image for Install/LiveCD (See build log) -Find out why 'backend' isn't making it into buildopts -Find out why vserv with debug off didn't delete the work dir +Add rollback to backend so it can resume after a partial task +Use 'bundling' status on builds, consider adding 'bundling_failed' +Handle 'cancel' status in backend, offer it in frontend +Change builds->display() to use a switch($this->status) +Confirm that backend stops after success +*** Tidy up backend, move the proper parts into build->build() |