Wrote a script for automatically keeping cache files (stage3, install ISO, etc.) up to date by ftp, updated frontend and backend to reflect storing stage3 and iso filenames in the 'cache' table instead of 'gentoo_profiles'

19 files changed, 251 insertions, 98 deletions

diff --git a/backend/bundlers/cd-head.php b/backend/bundlers/cd-head.php
index 2e96bba..385b261 100644
--- a/backend/bundlers/cd-head.php
+++ b/backend/bundlers/cd-head.php
@@ -1,8 +1,8 @@
 <?php
 $profile=new sql_gentoo_profile($opts['profile']);
-$headers=$profile->get_headers();
-if (strpos($headers['chost'], 'x86_64') === false)
-	$minimaliso=CACHE.'/cd/install-x86-minimal-20090623.iso';
-else
-	$minimaliso=CACHE.'/cd/install-amd64-minimal-20090625.iso';
+$r=query('SELECT * FROM `cache` WHERE `type`="cd" AND `key`="'.$profile->get_arch().'"');
+if ($r->rowCount() == 0)
+	throw_exception('No CD in cache for arch '.$profile->get_arch());
+$entry=new sql_cache_entry($r->fetch(PDO::FETCH_ASSOC));
+$minimaliso=CACHE."/$entry->file";
 ?>
diff --git a/backend/functions/read_stage3.php b/backend/functions/read_stage3.php
new file mode 100644
index 0000000..21cfd01
--- /dev/null
+++ b/backend/functions/read_stage3.php
@@ -0,0 +1,26 @@
+<?php
+function read_stage3($entry, $update=false) {
+	global $S;
+	if ($update) {
+		query('DELETE FROM `gentoo_basepkgs` WHERE `key`="'.$entry->key.'"');
+		query('DELETE FROM `gentoo_baseinit` WHERE `key`="'.$entry->key.'"');
+	}
+	$file=CACHE."/$entry->file";
+	if (!is_readable($file)) return false;
+	$opt='-tv'.(substr($file, -3) == 'bz2'?'j':'z').'f';
+	$prefix='./var/db/pkg/';
+	$files=explode("\n", is_readable("$file.CONTENTS")?file_get_contents("$file.CONTENTS"):shell_exec('tar '.$opt.' '.escapeshellarg($file)));
+	if (!is_file("$file.CONTENTS"))
+		file_put_contents("$file.CONTENTS", implode("\n", $files));
+	foreach ($files as $file) {
+		if (preg_match('#^[^.]+\./var/db/pkg/(.*/.*)/$#', $file, $match)) {
+			$pkg=new sql_gentoo_basepkg($entry->key, $match[1]);
+			$pkg->write();
+		} elseif (preg_match('#^[^.]+\./etc/runlevels/([^/]+)/([^/]+) -> /etc/init\.d/#', $file, $match)) {
+			$init=new sql_gentoo_baseinit($entry->key, $match[2], $match[1]);
+			$init->write();
+		}
+	}
+	return true;
+}
+?>
diff --git a/backend/modules/gentoo_portage/base-system.php b/backend/modules/gentoo_portage/base-system.php
index 0c730e9..c578ce5 100644
--- a/backend/modules/gentoo_portage/base-system.php
+++ b/backend/modules/gentoo_portage/base-system.php
@@ -1,22 +1,28 @@
 <?php
-$file=CACHE.'/stage3/'.$profile->stage3;
-execute_command('Unpack base system', "tar -xvjpf '$file' -C '$imagedir'");
-if ($opts['basesystem'] == 'user_prune' && $opts['prunepkgs']) {
-	emerge($opts['prunepkgs'], 'Prune base system packages', '-C');
-} elseif ($opts['basesystem'] == 'auto_prune') {
-	throw_exception('Base system auto-prune not implemented - need package list');
-	$keep=explode(' ', $keep_pkgs);
-	$remove=array();
-	$r=query('SELECT * FROM `gentoo_basepkgs` WHERE `profile`='.$profile->id);
-	while ($pkg=$r->fetch(PDO::FETCH_ASSOC)) {
-		$pkg=$pkg['pkg'];
-		if (($i=array_search($pkg, $keep)) === false)
-			$remove[]=$pkg;
-		else
-			unset($keep[$i]);
-	}
-	emerge($remove, 'Automatically prune base system packages', '-C');
-} elseif ($opts['basesystem'] == 'emerge') {
+if ($opts['basesystem'] == 'emerge') {
 	emerge('system', 'Emerge base system');
+} else {
+	$r=query('SELECT * FROM `cache` WHERE `type`="stage3" AND `key`="'.$profile->get_identifier().'"');
+	if ($r->rowCount() == 0)
+		throw_exception('No stage3 in cache for '.$profile->get_identifier());
+	$entry=new sql_cache_entry($r->fetch(PDO::FETCH_ASSOC));
+	$file=CACHE."/$entry->file";
+	execute_command('Unpack base system', "tar -xvjpf '$file' -C '$imagedir'");
+	if ($opts['basesystem'] == 'user_prune' && $opts['prunepkgs']) {
+		emerge($opts['prunepkgs'], 'Prune base system packages', '-C');
+	} elseif ($opts['basesystem'] == 'auto_prune') {
+		throw_exception('Base system auto-prune not implemented - need package list');
+		$keep=explode(' ', $keep_pkgs);
+		$remove=array();
+		$r=query('SELECT * FROM `gentoo_basepkgs` WHERE `profile`='.$profile->id);
+		while ($pkg=$r->fetch(PDO::FETCH_ASSOC)) {
+			$pkg=$pkg['pkg'];
+			if (($i=array_search($pkg, $keep)) === false)
+				$remove[]=$pkg;
+			else
+				unset($keep[$i]);
+		}
+		emerge($remove, 'Automatically prune base system packages', '-C');
+	}
 }
 ?>
diff --git a/backend/modules/gentoo_portage/portage.php b/backend/modules/gentoo_portage/portage.php
index bcfa7d7..324a370 100644
--- a/backend/modules/gentoo_portage/portage.php
+++ b/backend/modules/gentoo_portage/portage.php
@@ -1,8 +1,8 @@
 <?php
-$file=glob(CACHE.'/portage/*.tar.bz2');
-if ($file) {
-	$file=array_reverse($file);
-	$file=$file[0];
+$r=query('SELECT * FROM `cache` WHERE `type`="portage"');
+if ($r->rowCount() > 0) {
+	$entry=new sql_cache_entry($r->fetch(PDO::FETCH_ASSOC));
+	$file=CACHE."/$entry->file";
 	execute_command('Unpack portage snapshot', "tar -xvjpf '$file' -C '$imagedir/usr'");
 } else {
 	start_internal_task('Copy local portage tree to image');
@@ -13,6 +13,7 @@ if ($file) {
 		$cmd="cp -av -t '$imagedir/usr/portage/' '$from'";
 		error_get_last();
 		@shell_exec($cmd);
+		// TODO something wrong with this end_internal task being in the foreach
 		end_internal_task((int)(bool)error_get_last());
 	}
 	end_internal_task(0);
diff --git a/cache/conf b/cache/conf
new file mode 100644
index 0000000..cdce10b
--- /dev/null
+++ b/cache/conf
@@ -0,0 +1,10 @@
+# ftp-regex	type	key
+ftp://mirrors.tds.net/pub/gentoo/releases/amd64/current-stage3/stage3-amd64-([0-9]+)\.tar\.bz2	stage3	amd64
+ftp://mirrors.tds.net/pub/gentoo/releases/amd64/current-stage3/stage3-amd64-([0-9]+)\.tar\.bz2.CONTENTS
+ftp://mirrors.tds.net/pub/gentoo/releases/x86/current-stage3/hardened/stage3-i686-hardened-([0-9]+)\.tar\.bz2	stage3	hardened-x86
+ftp://mirrors.tds.net/pub/gentoo/releases/x86/current-stage3/hardened/stage3-i686-hardened-([0-9]+)\.tar\.bz2.CONTENTS
+ftp://mirrors.tds.net/pub/gentoo/releases/arm/current-stage3/armv5tel-softfloat-linux-gnueabi/stage3-armv5tel-([0-9]+)\.tar\.bz2	stage3	arm
+ftp://mirrors.tds.net/pub/gentoo/releases/arm/current-stage3/armv5tel-softfloat-linux-gnueabi/stage3-armv5tel-([0-9]+)\.tar\.bz2.CONTENTS
+ftp://mirrors.tds.net/pub/gentoo/releases/amd64/current-iso/install-amd64-minimal-([0-9]+)\.iso	cd	amd64
+ftp://mirrors.tds.net/pub/gentoo/releases/x86/current-iso/install-x86-minimal-([0-9]+)\.iso	cd	x86
+ftp://mirrors.tds.net/pub/gentoo/snapshots/portage-([0-9]+)\.tar\.bz2	portage
diff --git a/depend b/depend
index f20dbac..bf8514e 100644
--- a/depend
+++ b/depend
@@ -1,4 +1,4 @@
->=dev-lang/php-5.2.2 USE=pdo hash pcntl pcre cli mysql apache2 curl ctype reflection posix
+>=dev-lang/php-5.2.2 USE=pdo hash pcntl pcre cli mysql apache2 curl ctype reflection posix ftp
 >=virtual/mysql-5
 sys-apps/portage # In case you use paludis
 sys-apps/sed
diff --git a/frontend/modules/gentoo/step3.php b/frontend/modules/gentoo/step3.php
index 6af5b2e..83b101c 100644
--- a/frontend/modules/gentoo/step3.php
+++ b/frontend/modules/gentoo/step3.php
@@ -21,8 +21,8 @@ if (strlen($pkgsets=$this->get_opt('pkgsets'))) {
 		$this->checkbox_array('pkgset-'.$pkgset->id, 'pkgset-'.$pkgset->id, $pkgset->name, $pkgs);
 	}
 }
-if (in_array('pruneinit', $opts)) {
-	$r=query('SELECT * FROM `gentoo_baseinit` WHERE `profile`='.$profile->id.' ORDER BY `name`, `runlevel`');
+if (in_array('prune_init', $opts)) {
+	$r=query('SELECT * FROM `gentoo_baseinit` WHERE `key`="'.$profile->get_identifier().'" ORDER BY `name`, `runlevel`');
 	$scripts=array();
 	while ($script=$r->fetch(PDO::FETCH_ASSOC)) {
 		$script=new sql_gentoo_baseinit($script);
@@ -30,8 +30,8 @@ if (in_array('pruneinit', $opts)) {
 	}
 	$this->checkbox_array('pruneinit', 'pruneinit', 'Remove the following init scripts', $scripts);
 }
-if ($this->get_opt('basesystem') == 'manual') {
-	$r=query('SELECT * FROM `gentoo_basepkgs` WHERE `profile`='.$profile->id.' ORDER BY `pkg`');
+if ($this->get_opt('basesystem') == 'user_prune') {
+	$r=query('SELECT * FROM `gentoo_basepkgs` WHERE `key`="'.$profile->get_identifier().'" ORDER BY `pkg`');
 	$pkgs=array();
 	while ($pkg=$r->fetch(PDO::FETCH_ASSOC)) {
 		$pkg=$pkg['pkg'];
diff --git a/gentoo_setup.php b/gentoo_setup.php
index 29f56d6..c37059f 100644
--- a/gentoo_setup.php
+++ b/gentoo_setup.php
@@ -11,14 +11,11 @@ foreach (get_pkgdirs() as $dir) {
 	$profile->read_pkgsets();
 	echo "done\n";
 }
-foreach (explode("\n", trim(file_get_contents(CACHE.'/stage3/map'))) as $line) {
-	list($profile, $file)=explode(' ', $line);
-	if (!is_numeric($profile)) continue;
-	$profile=new sql_gentoo_profile($profile);
-	$profile->stage3=$file;
-	$profile->write();
-	echo "Loading $profile->pkgdir stage3 data from $file...";
-	$profile->read_stage3();
+$r=query('SELECT * FROM `cache` WHERE `type`="stage3"');
+while ($row=$r->fetch(PDO::FETCH_ASSOC)) {
+	$entry=new sql_cache_entry($row);
+	echo "Reading contents of stage3 file $entry->file...";
+	read_stage3($entry);
 	echo "done\n";
 }
 ?>
diff --git a/shared/classes/cache_entry.php b/shared/classes/cache_entry.php
new file mode 100644
index 0000000..4395632
--- /dev/null
+++ b/shared/classes/cache_entry.php
@@ -0,0 +1,21 @@
+<?php
+class sql_cache_entry extends sql_row_obj {
+	protected $table='cache', $primary_key=array('type', 'key'), $columns=array(
+		'type' => array (
+			'type' => 'ENUM',
+			'length' => '\'cd\',\'portage\',\'stage3\'',
+			'not_null' => true
+		),
+		'key' => array (
+			'type' => 'VARCHAR',
+			'length' => 255,
+			'not_null' => true
+		),
+		'file' => array (
+			'type' => 'VARCHAR',
+			'length' => 255
+		)
+
+	);
+}
+?>
diff --git a/shared/classes/gentoo_baseinit.php b/shared/classes/gentoo_baseinit.php
index 03c9569..c9acc4e 100644
--- a/shared/classes/gentoo_baseinit.php
+++ b/shared/classes/gentoo_baseinit.php
@@ -1,12 +1,10 @@
 <?php
 class sql_gentoo_baseinit extends sql_row_obj {
 	protected $table='gentoo_baseinit', $columns=array(
-		'profile' => array (
-			'type' => 'TINYINT',
-			'length' => 3,
-			'unsigned' => true,
-			'not_null' => true,
-			'default' => 0
+		'key' => array (
+			'type' => 'VARCHAR',
+			'length' => 255,
+			'not_null' => true
 		),
 		'name' => array (
 			'type' => 'VARCHAR',
diff --git a/shared/classes/gentoo_basepkg.php b/shared/classes/gentoo_basepkg.php
index d61b3dc..c140ee9 100644
--- a/shared/classes/gentoo_basepkg.php
+++ b/shared/classes/gentoo_basepkg.php
@@ -1,13 +1,10 @@
 <?php
 class sql_gentoo_basepkg extends sql_row_obj {
 	protected $table='gentoo_basepkgs', $columns=array(
-		'profile' => array (
-			'type' => 'TINYINT',
-			'length' => 3,
-			'unsigned' => true,
-			'not_null' => true,
-			'default' => 0,
-			'refers_to' => 'gentoo_profiles.id'
+		'key' => array (
+			'type' => 'VARCHAR',
+			'length' => 255,
+			'not_null' => true
 		),
 		'pkg' => array (
 			'type' => 'VARCHAR',
diff --git a/shared/classes/gentoo_profile.php b/shared/classes/gentoo_profile.php
index d7726f8..d41497b 100644
--- a/shared/classes/gentoo_profile.php
+++ b/shared/classes/gentoo_profile.php
@@ -15,12 +15,6 @@ class sql_gentoo_profile extends sql_row_obj {
 			'default' => '',
 			'unique' => true
 		),
-		'stage3' => array (
-			'type' => 'VARCHAR',
-			'length' => 255,
-			'not_null' => true,
-			'default' => ''
-		),
 		'name' => array (
 			'type' => 'VARCHAR',
 			'length' => 255,
@@ -171,30 +165,6 @@ class sql_gentoo_profile extends sql_row_obj {
 		if ($update)
 			query('DELETE FROM `gentoo_pkgsets` WHERE `profile`='.$this->id.($exists?' AND `id` NOT IN ('.implode(',', $exists).')':''));
 	}
-	public function read_stage3($update=false) {
-		global $S;
-		if ($update) {
-			query('DELETE FROM `gentoo_basepkgs` WHERE `profile`='.$this->id);
-			query('DELETE FROM `gentoo_baseinit` WHERE `profile`='.$this->id);
-		}
-		$file=realpath(CACHE.'/stage3/'.$this->stage3);
-		if (!is_readable($file)) return false;
-		$opt='-tv'.(substr($file, -3) == 'bz2'?'j':'z').'f';
-		$prefix='./var/db/pkg/';
-		$files=explode("\n", is_readable("$file.CONTENTS")?file_get_contents("$file.CONTENTS"):shell_exec('tar '.$opt.' '.escapeshellarg($file)));
-		if (!is_file("$file.CONTENTS"))
-			file_put_contents("$file.CONTENTS", implode("\n", $files));
-		foreach ($files as $file) {
-			if (preg_match('#^[^.]+\./var/db/pkg/(.*/.*)/$#', $file, $match)) {
-				$pkg=new sql_gentoo_basepkg($this->id, $match[1]);
-				$pkg->write();
-			} elseif (preg_match('#^[^.]+\./etc/runlevels/([^/]+)/([^/]+) -> /etc/init\.d/#', $file, $match)) {
-				$init=new sql_gentoo_baseinit($this->id, $match[2], $match[1]);
-				$init->write();
-			}
-		}
-		return true;
-	}
 	public function &get_packages($omit_masked=false, $trim=null) {
 		global $S;
 		$skip_masked=!in_array('masked', $trim);
@@ -214,5 +184,13 @@ class sql_gentoo_profile extends sql_row_obj {
 		$arch=ltrim($arch[0], '~');
 		return $arch;
 	}
+	function is_hardened() {
+		$profile=$this->get_headers();
+		$profile=$profile['profile'];
+		return (strpos(strtolower($profile), 'hardened') !== false);
+	}
+	function get_identifier() {
+		return ($this->is_hardened()?'hardened-':'').$this->get_arch();
+	}
 }
 ?>
diff --git a/shared/config.php b/shared/config.php
index 8a93454..e8ed1e2 100644
--- a/shared/config.php
+++ b/shared/config.php
@@ -25,4 +25,5 @@ $pkgdir_root='/home/eitan/soc/tinderbox'; // The directory to recursively search
 // $emerge_default_opts='-v --color=y'; // Options to add to all emerge commands
 // $portdir='/usr/portage'; // The directory conatining the portage tree to use (/usr/portage unless you have a reason to think otherwise)
 $backend_id='red'; // A name or other way of identifying this backend as opposed to other backends working for the same frontend
+// $tmpdir=isset($_ENV['TMPDIR'])?$_ENV['TMPDIR'].'/ingenue':'/var/tmp/ingenue'; // The directory to use for temporary files
 ?>
diff --git a/shared/functions/load_config.php b/shared/functions/load_config.php
index d719fb9..4560f5d 100644
--- a/shared/functions/load_config.php
+++ b/shared/functions/load_config.php
@@ -3,7 +3,7 @@ function load_config() {
 	require(SHARED.'/include/defaults.php');
 	require(SHARED.'/config.php');	       	
 	$modules=explode(' ', $modules);
-	foreach (explode(' ', 'title url sqlhost sqluser sqlpass sqldb debug modules bundlers cookiename sessionlength mod_rewrite timezone_root emailfrom check_email_dns registration invite logview_max progressbar_width pkgdir_root emerge_default_opts portdir backend_id') as $var) {
+	foreach (explode(' ', 'title url sqlhost sqluser sqlpass sqldb debug modules bundlers cookiename sessionlength mod_rewrite timezone_root emailfrom check_email_dns registration invite logview_max progressbar_width pkgdir_root emerge_default_opts portdir backend_id tmpdir') as $var) {
 		if (isset($$var)) {
 			$GLOBALS['S']['conf'][$var]=$$var;
 		}
diff --git a/shared/include/defaults.php b/shared/include/defaults.php
index 95cbfc2..057e5fd 100644
--- a/shared/include/defaults.php
+++ b/shared/include/defaults.php
@@ -30,4 +30,5 @@ $progressbar_width=350;
 $emerge_default_opts='-v --color=y';
 $portdir='/usr/portage';
 //$backend_id=gethostname(); // TODO Uncomment in 5.3.0
+$tmpdir=isset($_ENV['TMPDIR'])?$_ENV['TMPDIR'].'/ingenue':'/var/tmp/ingenue'; 
 ?>
diff --git a/shared/include/includes.php b/shared/include/includes.php
index 8a75b58..c7491ae 100644
--- a/shared/include/includes.php
+++ b/shared/include/includes.php
@@ -11,4 +11,7 @@ foreach (array('functions', 'classes') as $type) {
 }
 unset($dir, $file, $type);
 load_config();
+define('TMPDIR', $S['conf']['tmpdir']);
+if (!is_dir(TMPDIR))
+	mkdir(TMPDIR, 0700);
 ?>
diff --git a/todo b/todo
index 1158b17..9d17f1e 100644
--- a/todo
+++ b/todo
@@ -8,14 +8,11 @@ Allow config viewing for builds, not just configurations
 Change `visibility` columns to `flags` in configurations, builds for expandability
 Add build->configuration and configuration duplication
 Consider adding `configuration` col to builds
-Add map file for liveCD, load it into DB, etc. (currently hardcoded = evil)
-Add gentoo_profileopts column for liveCD, stage3, etc.
-Write script for fetching latest stage3's from the desired FTP dirs
 Add ability to trim the stage3 to a minimum set of packages automatically (cached)
 Add option to add arbitrary runscripts
 Add option to upload a kernel
 Add option to upload an arbitrary tar.gz/bz2 to be unzipped over the finished image
-*** Implement selected items from gentoo-steps ***
+Implement locale selection? (only item in gentoo-steps still marked as undone)
 *** Documentation ***
 Offer FTP upload
 Offer SCP upload?
@@ -23,4 +20,4 @@ Ask someone to add the necessary USE flags to php on tinderbox
 Add rollback to backend so it can resume after a partial task
 Offer option in frontend to submit a failed build for resume
 Change builds->display() to handle `failed` column
-Find out what on earth is wrong with error reporting
+Find out what on earth is wrong with error reporting in the frontend
diff --git a/update_cache.php b/update_cache.php
new file mode 100755
index 0000000..b6e1ac1
--- /dev/null
+++ b/update_cache.php
@@ -0,0 +1,120 @@
+#!/usr/bin/php
+<?php
+function update_cache($url, $type=null, $key=null) {
+	if (strtolower(substr($url, 0, 6)) == 'ftp://') {
+		$result=update_cache_ftp(substr($url, 6));
+		if ($result) {
+			list($command, $file, $mtime, $size)=$result;
+		}
+	}
+	if (isset($command)) {
+		if (!is_file(CACHE."/$file") || filemtime(CACHE."/$file") < $mtime) {
+			echo "$command\n";
+			passthru($command);
+			if (filesize(TMPDIR."/$file") != $size) return false;
+			touch(TMPDIR."/$file", $mtime);
+			rename(TMPDIR."/$file", CACHE."/$file");
+		}
+		if (isset($type)) {
+			if (!isset($key)) $key='';
+			$r=query('SELECT * FROM `cache` WHERE `type`="'.$type.'" AND `key`="'.$key.'"');
+			if ($r->rowCount()) {
+				$obj=new sql_cache_entry($r->fetch(PDO::FETCH_ASSOC));
+				if ($obj->file != $file)
+					unlink(CACHE."/$obj->file");
+			} else {
+				$obj=new sql_cache_entry();
+				$obj->type=$type;
+				$obj->key=$key;
+			}
+			$obj->file=$file;
+			$obj->write();
+		}
+		return true;
+	}
+	return false;
+}
+function update_cache_ftp($url) {
+	$args=array('-c', '--timeout=20');
+	$pattern=substr($url, strrpos($url, '/')+1);
+	$url=substr($url, 0, strlen($url)-strlen($pattern)-1);
+	if (strpos($url, '/') === false) {
+		$server=$url;
+		$sdir=null;
+	} else {
+		list($server, $sdir)=explode('/', $url, 2);
+	}
+	if (strpos($server, '@') === false) {
+		$user='anonymous';
+		$pass='ingenue@soc.dev.gentoo.org';
+	} else {
+		list($user, $server)=explode('@', $server);
+		if (strpos($user, ':') === false) {
+			$pass='ingenue@soc.dev.gentoo.org';
+			$args[]=escapeshellarg("--password=$pass");
+		} else
+			list($user, $pass)=explode(':', $user, 2);
+		$args[]=escapeshellarg("--user=$user");
+	}
+	if (strpos($server, ':') === false)
+		$port=21;
+	else
+		list($server, $port)=explode(':', $server);
+	echo "Connecting to $server:$port...";
+	$ftp=ftp_connect($server, $port, 20); if (!$ftp) return false; // Timeout=20
+	echo "done\n";
+	echo "Logging in as $user...";
+	$r=ftp_login($ftp, $user, $pass); if (!$r) return false;
+	echo "done\n";
+	echo "Changing dir to /$sdir...";
+	$r=ftp_chdir($ftp, '/'.$sdir); if (!$r) return false;
+	echo "done\n";
+	echo "Getting list of files...";
+	$list=ftp_nlist($ftp, '.');
+	echo "done\n";
+	$newest=null;
+	$newest_id=null;
+	foreach ($list as $file) {
+		if (!preg_match("/^$pattern$/", $file, $match)) continue;
+		if (isset($match[1])) {
+			if ($match[1] > $newest_id) {
+				$newest_id=$match[1];
+				$newest=$file;
+			}
+		} else {
+			$mtime=ftp_mdtm($ftp, $file);
+			if ($mtime > $newest_id) {
+				$newest=$file;
+				$newest_id=$mtime;
+			}
+		}
+	}
+	if ($newest === null) {
+		echo "No files matched pattern\n";
+		return;
+	}
+	$mtime=ftp_mdtm($ftp, $newest);
+	$size=ftp_size($ftp, $newest);
+	ftp_close($ftp);
+	$args[]=escapeshellarg('--directory-prefix='.TMPDIR);
+	$args[]="ftp://$server/$sdir/$newest";
+	$cmd='wget '.implode(' ', $args);
+	return array($cmd, $newest, $mtime, $size);
+}
+require_once(dirname(__FILE__).'/shared/include/includes.php'); // __DIR__ 5.3.0
+require_once(SHARED.'/include/dbinit.php');
+$file=fopen(CACHE.'/conf', 'r');
+for ($line=fgets($file, 10240); !feof($file); $line=fgets($file, 10240)) {
+	$line=trim($line);
+	if (!$line || substr($line, 0, 1) == '#') continue;
+	$array=explode("\t", $line);
+	foreach ($array as &$val) {
+		if (strlen($val) == 0 || strtolower($val) == 'null') $val=null;
+	}
+	for ($i=0; $i < 5; $i++) {
+		if (call_user_func_array('update_cache', $array)) break;
+		echo "Failed... sleep 5\n";
+		sleep(5);
+	}
+}
+?>
diff --git a/update_gentoo_profiles.php b/update_gentoo_profiles.php
index e15e895..8da7443 100755
--- a/update_gentoo_profiles.php
+++ b/update_gentoo_profiles.php
@@ -15,14 +15,11 @@ while ($p=$r->fetch(PDO::FETCH_ASSOC)) {
 	$profile->read_pkgsets(true);
 	echo "done\n";
 }
-foreach (explode("\n", trim(file_get_contents(CACHE.'/stage3/map'))) as $line) {
-	list($p, $file)=explode(' ', $line, 2);
-	if (!is_numeric($p)) continue;
-	$profile=&$profiles[$p];
-	if ($profile->stage3 == $file) continue;
-	$profile->stage3=$file;
-	$profile->write();
-	echo "Loading info from profile $profile->pkgdir stage3 at $file...";
-	echo ($profile->read_stage3(true)?'done':color('failed', 'red'))."\n";
+$r=query('SELECT * FROM `cache` WHERE `type`="stage3"');
+while ($row=$r->fetch(PDO::FETCH_ASSOC)) {
+	$entry=new sql_cache_entry($row);
+	echo "Reading contents of stage3 file $entry->file...";	
+	read_stage3($entry, true);
+	echo "done\n";
 }
 ?>