diff options
-rw-r--r-- | portage/pym/portage/package/ebuild/fetch.py | 8 | ||||
-rwxr-xr-x | twrapper/premerge.sh | 2 | ||||
-rwxr-xr-x | twrapper/twrapper.py | 418 |
3 files changed, 151 insertions, 277 deletions
diff --git a/portage/pym/portage/package/ebuild/fetch.py b/portage/pym/portage/package/ebuild/fetch.py index 30b3eec..bb0f56b 100644 --- a/portage/pym/portage/package/ebuild/fetch.py +++ b/portage/pym/portage/package/ebuild/fetch.py @@ -16,6 +16,7 @@ import shutil import stat import sys import tempfile +import json #import idfetch_settings import portage @@ -242,11 +243,14 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", # create lock file idfetch_pkg_list_file_name=mysettings['USE_TWRAPPER']+"/need_to_download/pkg_list.list" print("Using TWRAPPER. Saving pkg_list.list file to ",idfetch_pkg_list_file_name) + + try: idfetch_pkg_list_file = open(idfetch_pkg_list_file_name,"r") - idfetch_pkg_list=pickle.load(idfetch_pkg_list_file) + idfetch_pkg_list=json.loads(idfetch_pkg_list_file.read()) idfetch_pkg_list_file.close() except: + print("EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE") idfetch_pkg_list=list() idfetch_pkg=dict() @@ -1251,7 +1255,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if use_twrapper_to_download: try: idfetch_pkg_list_file = open(idfetch_pkg_list_file_name,"w") - pickle.dump(idfetch_pkg_list,idfetch_pkg_list_file) + idfetch_pkg_list_file.write(json.dumps(idfetch_pkg_list, sort_keys=1, indent=4)) idfetch_pkg_list_file.close() # print("---------------->>>>>>>>>>> idfetch_pkg_list_file saved <<<<<<<<<<<<<---------------------------") except: diff --git a/twrapper/premerge.sh b/twrapper/premerge.sh index ca65ded..1c21d0c 100755 --- a/twrapper/premerge.sh +++ b/twrapper/premerge.sh @@ -2,7 +2,7 @@ if [[ -n "${1}" ]]; then emerge -f $@ ./twrapper.py - emerge $@ +# emerge $@ else echo echo "Usage:" diff --git a/twrapper/twrapper.py b/twrapper/twrapper.py index 9cd3d2a..bd3bd2f 100755 --- a/twrapper/twrapper.py +++ b/twrapper/twrapper.py @@ -16,17 +16,19 @@ import signal from portage.checksum import perform_md5, verify_all from portage.process import spawn import idfetch_settings -import pickle +#import pickle import curses import time import subprocess -import shlex +#import shlex +import json from threading import Thread CLEAN_LINE=" " MAX_ACTIVE_DOWNLOADS=8 TASK_SPACE=3 USE_CURSES_FLAG=1 DEBUG_ON=0 +INSTALL_ON=1 def fit_string(text): text=text+CLEAN_LINE @@ -61,10 +63,22 @@ def download_msg(index,msg_text): def status_msg(index,msg_text): msg(TASK_SPACE*index+1," ["+msg_text+"]") -def total_msg(finished_downloads,index,downloaded_size,total_size): - msg(2,"Downloaded "+str(finished_downloads)+" of "+str(index)+" distfiles ("\ - +str(downloaded_size)+" of "+str(total_size)+" bytes = "\ - +str(downloaded_size*100/total_size)+"% )") +def total_msg(): + msg(2,"-> Pkgs: "\ + +"I:"+str(pkgs_installed_count)\ + +"/D:"+str(pkgs_downloaded_count)\ + +"/T:"+str(pkgs_count)\ + +" - Distfiles D:"+str(distfiles_downloaded_count)\ + +"/T:"+str(distfiles_count)\ + +" [Bytes D:"+str(downloaded_size)\ + +" of T:"+str(total_size)\ + +" bytes = "+str(downloaded_size*100/total_size)+"%] <-") + if install_pkgs_is_running: + msg(26,"EMERGE: [INSTALLING...]") + else: + msg(26,"EMERGE: [IDLE]") + + def progress_msg(index,msg_text1): msg(TASK_SPACE*index+1,msg_text1) @@ -72,6 +86,9 @@ def progress_msg(index,msg_text1): def error_msg(msg_text): msg(max_y-4,"WARNING: "+msg_text) +def emerge_msg(msg_text): + msg(max_y-7,"Emerging: "+msg_text) + def open_task_list(): debug(" function open_task_list start") idfetch_pkg_list_file_name=idfetch_settings.TASK_DIR+"/pkg_list.list" @@ -85,20 +102,21 @@ def open_task_list(): error_msg("open_task_list(): Can't open file: "+idfetch_pkg_list_file_lock_name) try: idfetch_pkg_list_file = open(idfetch_pkg_list_file_name,"r") - idfetch_pkg_list=pickle.load(idfetch_pkg_list_file) + idfetch_pkg_list=json.loads(idfetch_pkg_list_file.read()) idfetch_pkg_list_file.close() except: idfetch_pkg_list=[] error_msg("open_task_list(): Can't open file: "+idfetch_pkg_list_file_name+" >>> Nothing to do.") time.sleep(3) - #delete task list file - try: - os.unlink(idfetch_pkg_list_file_name) - debug("deleted:"+idfetch_pkg_list_file_name) - except: - error_msg("open_task_list(): Can't delete file: "+idfetch_pkg_list_file_name) - time.sleep(3) + if not(DEBUG_ON): + #delete task list file + try: +# os.unlink(idfetch_pkg_list_file_name) + debug("deleted:"+idfetch_pkg_list_file_name) + except: + error_msg("open_task_list(): Can't delete file: "+idfetch_pkg_list_file_name) + time.sleep(3) #unlock file os.unlink(idfetch_pkg_list_file_lock_name) @@ -107,12 +125,56 @@ def open_task_list(): return idfetch_pkg_list +class install_pkgs(Thread): + def __init__ (self,idfetch_pkg_list): + Thread.__init__(self) + self.idfetch_pkg_list=idfetch_pkg_list + self.status = 2 + def run(self): + global pkgs_installed_count + global install_pkgs_is_running + while pkgs_installed_count < pkgs_count: + self.atom_name=self.idfetch_pkg_list[pkgs_installed_count]['pkg_name'][0:self.idfetch_pkg_list[pkgs_installed_count]['pkg_name'].find("-")] + + if self.idfetch_pkg_list[pkgs_installed_count]['fetched_distfiles_count'] >= self.idfetch_pkg_list[pkgs_installed_count]['distfiles_count']: + self.emerge_subprocess = subprocess.Popen([\ +# 'echo', + '/usr/bin/emerge',\ + self.atom_name],\ + bufsize=1024,stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + mypids.append(self.emerge_subprocess.pid) + while 1: + try: + self.line=self.emerge_subprocess.stdout.readline() + if self.line: + emerge_msg("pkg "+str(pkgs_installed_count)+" :"+self.atom_name+" "+self.line.rstrip()) +# self.igot = re.findall('aksldjfklsjd;lfkj',self.line) +# if self.igot: +# self.status=0 +# self.emerge_subprocess.wait() +# break + else: + self.emerge_subprocess.stdout.close() + pkgs_installed_count+=1 + emerge_msg("emerge ended") + total_msg() + break + except: + pass + error_msg("ERROR!!! CODE:108") + self.emerge_subprocess.stdout.close() + break + else: + break + install_pkgs_is_running=0 + class fetchit(Thread): - def __init__ (self,distfile,index): + def __init__ (self,parent_pkg, distfile, distfiles_count): Thread.__init__(self) -# sms("_init_ in fetchit") + debug("_init_ in fetchit") + self.parent_pkg=parent_pkg self.distfile = distfile - self.index=index + self.index=distfiles_count self.status = 2 self.place_in_the_list = 1 # self.wget_log_file_name = idfetch_settings.TASK_DIR+'/logs/log_wget/'+self.download_file+".log" @@ -187,191 +249,15 @@ class fetchit(Thread): def run(self): sms("started "+str(self.index)) global exit_flag -# self.download_file_url_list_file=open(idfetch_settings.TASK_DIR+'/urls/'+self.download_file+".urllist") -# self.download_file_url_list=pickle.load(self.download_file_url_list_file) -# self.download_file_url_list_file.close() self.myfile_path=idfetch_settings.DIST_DIR+"/"+self.distfile['name'] -# for myfile in filedict: # """ # fetched status # 0 nonexistent # 1 partially downloaded # 2 completely downloaded # """ -# fetched = 0 - -# orig_digests = mydigests.get(myfile, {}) -# size = orig_digests.get("size") - -# if size == 0: - # Zero-byte distfiles are always invalid, so discard their digests. -# del mydigests[myfile] -# orig_digests.clear() -# size = None -# pruned_digests = orig_digests -# if parallel_fetchonly: -# pruned_digests = {} -# if size is not None: -# pruned_digests["size"] = size - -# myfile_path = os.path.join(mysettings["DISTDIR"], myfile) -# has_space = True -# has_space_superuser = True -# file_lock = None -# if listonly: -# writemsg_stdout("\n", noiselevel=-1) -# else: -# # check if there is enough space in DISTDIR to completely store myfile - # overestimate the filesize so we aren't bitten by FS overhead -# if size is not None and hasattr(os, "statvfs"): -# vfs_stat = os.statvfs(mysettings["DISTDIR"]) -# try: -# mysize = os.stat(myfile_path).st_size -# except OSError as e: -# if e.errno not in (errno.ENOENT, errno.ESTALE): -# raise -# del e -# mysize = 0 -# if (size - mysize + vfs_stat.f_bsize) >= \ -# (vfs_stat.f_bsize * vfs_stat.f_bavail): - -# if (size - mysize + vfs_stat.f_bsize) >= \ -# (vfs_stat.f_bsize * vfs_stat.f_bfree): -# has_space_superuser = False - -# if not has_space_superuser: -# has_space = False -# elif secpass < 2: -# has_space = False -# elif userfetch: -# has_space = False - -# if not has_space: -# writemsg(_("!!! Insufficient space to store %s in %s\n") % \ -# (myfile, mysettings["DISTDIR"]), noiselevel=-1) - -# if has_space_superuser: -# writemsg(_("!!! Insufficient privileges to use " -# "remaining space.\n"), noiselevel=-1) -# if userfetch: -# writemsg(_("!!! You may set FEATURES=\"-userfetch\"" -# " in /etc/make.conf in order to fetch with\n" -# "!!! superuser privileges.\n"), noiselevel=-1) - -# if distdir_writable and use_locks: - -# lock_kwargs = {} -# if fetchonly: -# lock_kwargs["flags"] = os.O_NONBLOCK - -# try: -# file_lock = lockfile(myfile_path, -# wantnewlockfile=1, **lock_kwargs) -# except TryAgain: -# writemsg(_(">>> File '%s' is already locked by " -# "another fetcher. Continuing...\n") % myfile, -# noiselevel=-1) -# continue -# try: -# if not listonly: -# -# eout = EOutput() -# eout.quiet = mysettings.get("PORTAGE_QUIET") == "1" -# match, mystat = _check_distfile( -# myfile_path, pruned_digests, eout) -# if match: -# if distdir_writable: -# try: -# apply_secpass_permissions(myfile_path, -# gid=portage_gid, mode=0o664, mask=0o2, -# stat_cached=mystat) -# except PortageException as e: -# if not os.access(myfile_path, os.R_OK): -# writemsg(_("!!! Failed to adjust permissions:" -# " %s\n") % str(e), noiselevel=-1) -# del e -# continue - -# if distdir_writable and mystat is None: -# # Remove broken symlinks if necessary. -# try: -# os.unlink(myfile_path) -# except OSError: -# pass - -# if mystat is not None: -# if stat.S_ISDIR(mystat.st_mode): -# writemsg_level( -# _("!!! Unable to fetch file since " -# "a directory is in the way: \n" -# "!!! %s\n") % myfile_path, -# level=logging.ERROR, noiselevel=-1) -# return 0 - -# if mystat.st_size == 0: -# if distdir_writable: -# try: -# os.unlink(myfile_path) -# except OSError: -# pass -# elif distdir_writable: -# if mystat.st_size < fetch_resume_size and \ -# mystat.st_size < size: -# # If the file already exists and the size does not -# # match the existing digests, it may be that the -# # user is attempting to update the digest. In this -# # case, the digestgen() function will advise the -# # user to use `ebuild --force foo.ebuild manifest` -# # in order to force the old digests to be replaced. -# # Since the user may want to keep this file, rename -# # it instead of deleting it. -# writemsg(_(">>> Renaming distfile with size " -# "%d (smaller than " "PORTAGE_FETCH_RESU" -# "ME_MIN_SIZE)\n") % mystat.st_size) -# temp_filename = \ -# _checksum_failure_temp_file( -# mysettings["DISTDIR"], myfile) -# writemsg_stdout(_("Refetching... " -# "File renamed to '%s'\n\n") % \ -# temp_filename, noiselevel=-1) -# elif mystat.st_size >= size: -# temp_filename = \ -# _checksum_failure_temp_file( -# mysettings["DISTDIR"], myfile) -# writemsg_stdout(_("Refetching... " -# "File renamed to '%s'\n\n") % \ -# temp_filename, noiselevel=-1) - -# if distdir_writable and ro_distdirs: -# readonly_file = None -# for x in ro_distdirs: -# filename = os.path.join(x, myfile) -# match, mystat = _check_distfile( -# filename, pruned_digests, eout) -# if match: -# readonly_file = filename -# break -# if readonly_file is not None: -# try: -# os.unlink(myfile_path) -# except OSError as e: -# if e.errno not in (errno.ENOENT, errno.ESTALE): -# raise -# del e -# os.symlink(readonly_file, myfile_path) -# continue - -# if fsmirrors and not os.path.exists(myfile_path) and has_space: -# for mydir in fsmirrors: -# mirror_file = os.path.join(mydir, myfile) -# try: -# shutil.copyfile(mirror_file, myfile_path) -# writemsg(_("Local mirror has file: %s\n") % myfile) -# break -# except (IOError, OSError) as e: -# if e.errno not in (errno.ENOENT, errno.ESTALE): -# raise -# del e + self.fetched = 0 + if 1==1: try: self.mystat = os.stat(self.myfile_path) @@ -382,42 +268,20 @@ class fetchit(Thread): del e self.fetch() else: -# try: -# apply_secpass_permissions( -# self.myfile_path, gid=portage_gid, mode=0o664, mask=0o2, -# stat_cached=self.mystat) -# except PortageException as e: -# if not os.access(self.myfile_path, os.R_OK): -# writemsg(_("!!! Failed to adjust permissions:" -# " %s\n") % str(e), noiselevel=-1) - if 1==1: # If the file is empty then it's obviously invalid. Remove # the empty file and try to download if possible. -# if mystat.st_size == 0: -# if distdir_writable: -# try: -# os.unlink(myfile_path) -# except EnvironmentError: -# pass -# elif myfile not in mydigests: - # We don't have a digest, but the file exists. We must - # assume that it is fully downloaded. -# continue -# else: + if self.mystat.st_size == 0: + try: + os.unlink(myfile_path) + self.fetch() + except EnvironmentError: + debug("Can\'t delete the file") + else: if self.mystat.st_size < self.distfile["size"]: # and \ # not restrict_fetch: self.fetched = 1 # Try to resume this download. -# elif parallel_fetchonly and \ -# mystat.st_size == mydigests[myfile]["size"]: -# eout = EOutput() -# eout.quiet = \ -# mysettings.get("PORTAGE_QUIET") == "1" -# eout.ebegin( -# "%s size ;-)" % (myfile, )) -# eout.eend(0) -# continue - debug("size too small, continue fetch") + debug("Size is small, continue fetching...") self.fetch() else: self.verified_ok, self.reason = verify_all( @@ -427,17 +291,7 @@ class fetchit(Thread): os.unlink(self.myfile_path) self.fetch() else: -# eout = EOutput() -# eout.quiet = \ -# mysettings.get("PORTAGE_QUIET", None) == "1" -# digests = mydigests.get(myfile) -# if digests: -# digests = list(digests) -# digests.sort() -# eout.ebegin( -# "%s %s ;-)" % (myfile, " ".join(digests))) -# eout.eend(0) -# continue # fetch any remaining files + # Already fetched and with correct hashes self.status=0 @@ -448,7 +302,6 @@ def do_tasks(idfetch_pkg_list): msg(1,"TASK DIR: "+idfetch_settings.TASK_DIR) msg(0,"DOWNLOADING with twrapper...") - # Tidfetch_pkg_list = list of Tidfetch_pkg; # Tidfetch_pkg = dict @@ -459,26 +312,38 @@ def do_tasks(idfetch_pkg_list): # ['name'] of string; # ['url_list'] of list of strings; # ['size'] of int; -# ['hash see fetch.py for more details -# ['hash2 -# ['hash3 +# ['RMD160'] see fetch.py for more details +# ['SHA1'] +# ['SHA256'] fetch_distfile_thread_list = [] - index=0 + global pkgs_count + global pkgs_downloaded_count + global pkgs_installed_count + global distfiles_count + global distfiles_downloaded_count global total_size - total_size=0 global downloaded_size + global install_pkgs_is_running + + pkgs_count=len(idfetch_pkg_list) + pkgs_downloaded_count=0 + pkgs_installed_count=0 + distfiles_count=0 + distfiles_downloaded_count=0 + total_size=0 downloaded_size=0 -# print(idfetch_pkg_list) - while idfetch_pkg_list: - current_pkg=idfetch_pkg_list.pop() -# print(current_pkg['pkg_name']) -# distfiles_list_file = open(idfetch_settings.TASK_DIR+'/pkgs/'+current_pkg) -# distfiles_list=pickle.load(distfiles_list_file) -# distfiles_list_file.close() + install_pkgs_is_running=0 + +# idfetch_pkg_list.reverse() + prev_pkg=0 + for current_pkg in idfetch_pkg_list: + current_pkg['installed']=0 + current_pkg['prev_pkg']=prev_pkg + current_pkg['distfiles_count']=0 + current_pkg['fetched_distfiles_count']=0 + for current_distfile in current_pkg['distfile_list']: -# print("distfile",current_distfile['name']) - #some files may have duplicates, let's skip them no_duplicate=1 for cur_fetch_distfile_thread in fetch_distfile_thread_list: @@ -487,16 +352,19 @@ def do_tasks(idfetch_pkg_list): break if no_duplicate: #create new thread because there is no duplicate - index+=1 +# pkg_contains_distfiles_for_download=1 + distfiles_count+=1 + current_pkg['distfiles_count']+=1 debug("============================================") debug("pkg: "+current_pkg['pkg_name']) debug("name: "+current_distfile['name']) debug("size: "+str(current_distfile['size']/1000)) total_size+=current_distfile['size'] - current_fetch_distfile_thread = fetchit(current_distfile, index) + current_fetch_distfile_thread = fetchit(current_pkg, current_distfile, distfiles_count) fetch_distfile_thread_list.append(current_fetch_distfile_thread) + prev_pkg=current_pkg - finished_downloads=0 +# pkgs_downloaded_count place_in_the_list=0 running_fetch_distfile_thread_list=[] fetch_distfile_thread_list.reverse() @@ -509,7 +377,7 @@ def do_tasks(idfetch_pkg_list): if place_in_the_list>=MAX_ACTIVE_DOWNLOADS: break - total_msg(finished_downloads,index,downloaded_size,total_size) + total_msg() key='x' if USE_CURSES_FLAG: @@ -518,15 +386,29 @@ def do_tasks(idfetch_pkg_list): for current_fetch_distfile_thread in running_fetch_distfile_thread_list: if current_fetch_distfile_thread.status==0: - finished_downloads+=1 + distfiles_downloaded_count+=1 downloaded_size+=current_fetch_distfile_thread.distfile['size'] - total_msg(finished_downloads,index,downloaded_size,total_size) running_fetch_distfile_thread_list.remove(current_fetch_distfile_thread) if to_start_fetch_distfile_thread_list: #start next one on the same place in the list starting_fetch_distfile_thread=to_start_fetch_distfile_thread_list.pop() running_fetch_distfile_thread_list.append(starting_fetch_distfile_thread) starting_fetch_distfile_thread.start(current_fetch_distfile_thread.get_place_in_the_list()) + #install if package has all the files and previous pkgs are installed + current_fetch_distfile_thread.parent_pkg['fetched_distfiles_count']+=1 + if current_fetch_distfile_thread.parent_pkg['fetched_distfiles_count']>=current_fetch_distfile_thread.parent_pkg['distfiles_count']: + pkgs_downloaded_count+=1 + if not(install_pkgs_is_running): + if idfetch_pkg_list[pkgs_installed_count]['fetched_distfiles_count'] >= idfetch_pkg_list[pkgs_installed_count]['distfiles_count']: + install_pkgs_is_running=1 + cur_install_pkg_thread=install_pkgs(idfetch_pkg_list) + cur_install_pkg_thread.start() + else: + msg(28,"Next package to install:"+str(pkgs_installed_count+1)\ + +" pkg_name: "+idfetch_pkg_list[pkgs_installed_count]['pkg_name']\ + +" fetched:"+str(idfetch_pkg_list[pkgs_installed_count]['fetched_distfiles_count'])\ + +" of "+str(idfetch_pkg_list[pkgs_installed_count]['distfiles_count'])) + total_msg() time_msg() # Don't make it too hard for your cpu. Not much changes in 0.5 secs. Let's wait and give cpu some rest. time.sleep(0.5) @@ -534,8 +416,8 @@ def do_tasks(idfetch_pkg_list): key = stdscr.getch() if (key == ord('q')) or (key == ord('Q')): msg(28,">>>>>>>>>>>>>>>>> EXITING <<<<<<<<<<<<<<<<<<<<<<<<<<") - if finished_downloads == index: - msg(28,"All files have been downloaded. Exiting in 3 secs ...") + if (distfiles_downloaded_count == distfiles_count) and not(install_pkgs_is_running): + msg(28,"All files have been downloaded and installed. Exiting in 3 secs ...") time.sleep(3) exit_flag=1 break @@ -543,6 +425,7 @@ def do_tasks(idfetch_pkg_list): def main(idfetch_pkg_list): + try: idfetch_pkg_list=open_task_list() do_tasks(idfetch_pkg_list) @@ -563,22 +446,9 @@ def main(idfetch_pkg_list): curses.endwin() print("twrapper exited") -def cleanup(): - while spawned_pids: - pid = spawned_pids.pop() - try: - if os.waitpid(pid, os.WNOHANG) == (0, 0): - os.kill(pid, signal.SIGTERM) - os.waitpid(pid, 0) - except OSError: - # This pid has been cleaned up outside - # of spawn(). - pass - - if __name__ == '__main__': exit_flag=0 - task_list=[] + idfetch_pkg_list=[] # mypids will hold the pids of all processes created. mypids = [] if USE_CURSES_FLAG: @@ -593,6 +463,6 @@ if __name__ == '__main__': max_y=100 max_x=100 try: - main(task_list) + main(idfetch_pkg_list) finally: sys.exit() |