diff options
author | Kostyantyn Ovechko <fastinetserver@gmail.com> | 2010-07-04 03:27:52 +0300 |
---|---|---|
committer | Kostyantyn Ovechko <fastinetserver@gmail.com> | 2010-07-04 03:27:52 +0300 |
commit | 4e55a32db0798b897798e65fe7ab5ead5e9945b7 (patch) | |
tree | dd091946688177336b2fd6e998003ab30a1b8516 | |
parent | Add competitions among networks with equal priority levels. (diff) | |
download | idfetch-4e55a32db0798b897798e65fe7ab5ead5e9945b7.tar.gz idfetch-4e55a32db0798b897798e65fe7ab5ead5e9945b7.tar.bz2 idfetch-4e55a32db0798b897798e65fe7ab5ead5e9945b7.zip |
Add support for local mirrors
network#.conf :: [network_mirrors]
SYNOPSIS: NETWORK_USES_OWN_MIRROR_LIST_ONLY_ON=0 | 1
- If set to 1, segget will replace mirror list provided by portage system with
the list from network1_mirrors.conf
- If set to 0, segget will use ONLY mirror list provided by portage system,
and will NOT use the list from network1_mirrors.conf file
Default:
use_own_mirror_list_only_on=0
-rw-r--r-- | segget/checksum.cpp | 2 | ||||
-rw-r--r-- | segget/config.cpp | 4 | ||||
-rw-r--r-- | segget/connection.cpp | 57 | ||||
-rw-r--r-- | segget/connection.h | 6 | ||||
-rw-r--r-- | segget/distfile.cpp | 81 | ||||
-rw-r--r-- | segget/distfile.h | 7 | ||||
-rw-r--r-- | segget/mirror.cpp | 33 | ||||
-rw-r--r-- | segget/mirror.h | 61 | ||||
-rw-r--r-- | segget/mirror_local.cpp | 124 | ||||
-rw-r--r-- | segget/network.cpp | 35 | ||||
-rw-r--r-- | segget/network.h | 12 | ||||
-rw-r--r-- | segget/network0.conf | 10 | ||||
-rw-r--r-- | segget/network1.conf | 10 | ||||
-rw-r--r-- | segget/network1_mirrors.conf | 4 | ||||
-rw-r--r-- | segget/networkbroker.cpp | 26 | ||||
-rw-r--r-- | segget/networkbroker.h | 16 | ||||
-rw-r--r-- | segget/segget.conf | 2 | ||||
-rw-r--r-- | segget/segget.cpp | 41 | ||||
-rw-r--r-- | segget/segment.cpp | 21 | ||||
-rw-r--r-- | segget/segment.h | 12 | ||||
-rw-r--r-- | segget/settings.h | 2 | ||||
-rw-r--r-- | segget/tui.h | 2 |
22 files changed, 443 insertions, 125 deletions
diff --git a/segget/checksum.cpp b/segget/checksum.cpp index ed7d3fa..60a42f2 100644 --- a/segget/checksum.cpp +++ b/segget/checksum.cpp @@ -36,7 +36,7 @@ #include "cryptopp/md5.h" #include "cryptopp/files.h" #include "cryptopp/hex.h" -#include "tui.cpp" +#include "tui.h" #include "str.h" bool rmd160_ok(string distfile_filename, string portage_hash){ diff --git a/segget/config.cpp b/segget/config.cpp index 2a7f5cc..e75418b 100644 --- a/segget/config.cpp +++ b/segget/config.cpp @@ -29,7 +29,7 @@ #include <string> #include <map> -#include "str.cpp" +#include "str.h" #include "tui.h" using namespace std; @@ -66,7 +66,7 @@ string trim(std::string const& source, char const* delims = " \t\r\n") { } void ConfigFile::load_settings_from_config_file() { - ; + ifstream file; file.exceptions (ifstream::failbit | ifstream::badbit); try{ diff --git a/segget/connection.cpp b/segget/connection.cpp index 1cbbc44..6cfca24 100644 --- a/segget/connection.cpp +++ b/segget/connection.cpp @@ -25,24 +25,62 @@ */ #include "connection.h" -void Tconnection::start(uint network_number){ +void Tconnection::start(CURLM *cm, uint network_number, uint distfile_num, Tsegment *started_segment, uint best_mirror_num){ try{ + segment=started_segment; + mirror_num=best_mirror_num; network_num=network_number; total_dld_bytes=0; bytes_per_last_interval=0; gettimeofday(&start_time,NULL); active=true; network_array[network_num].connect(); + segment->prepare_for_connection(cm, connection_num, network_num, distfile_num, mirror_num); }catch(...){ error_log("Error in connection.cpp: start()"); } } -void Tconnection::stop(){ +void Tconnection::stop(uint connection_result){ try{ msg_clean_connection(connection_num); active=false; network_array[network_num].disconnect(); + segment->segment_file.close(); + + Tdistfile* prnt_distfile=segment->parent_distfile; + prnt_distfile->active_connections_num--; + + Tmirror *Pcurr_mirror; + if (network_array[network_num].use_own_mirror_list_only_on){ + Pcurr_mirror=&network_array[network_num].benchmarked_mirror_list[mirror_num]; +// find_mirror(strip_mirror_name(segment->url)); + }else{ + Pcurr_mirror=find_mirror(strip_mirror_name(segment->url)); + } + + timeval now_time; + gettimeofday(&now_time,NULL); + + if (connection_result!=0){ + // error -> start downloading again + msg_status2(segment->connection_num, toString(connection_result)+"]- Failed download "+segment->file_name); + debug(toString(connection_result)+"]- Failed download "+segment->url); + Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),0); + if (segment->try_num>=settings.max_tries){ + segment->status=FAILED; + error_log("Segment:"+segment->file_name+" has reached max_tries limit - segment.status set to FAILED"); + } + else segment->status=WAITING; + }else{ + // no error => count this one and start new + log("Succesfully downloaded "+segment->file_name+" on connection#"+toString(connection_num)); + debug(" Successful download "+segment->url); + Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),segment->segment_size); + segment->status=DOWNLOADED; + prnt_distfile->inc_dld_segments_count(segment); + }; + }catch(...){ error_log("Error in connection.cpp: stop()"); } @@ -61,12 +99,15 @@ void Tconnection::show_connection_progress(ulong time_diff){ try{ if (active){ stats.total_bytes_per_last_interval+=bytes_per_last_interval; - msg_segment_progress(segment->connection_num, network_num, - segment->segment_num, segment->try_num, - segment->downloaded_bytes, - segment->segment_size, - (bytes_per_last_interval*1000)/time_diff, - (total_dld_bytes*1000)/time_left_from(start_time)); + msg_segment_progress( + segment->connection_num, + network_num, + segment->segment_num, + segment->try_num, + segment->downloaded_bytes, + segment->segment_size, + (bytes_per_last_interval*1000)/time_diff, + (total_dld_bytes*1000)/time_left_from(start_time)); bytes_per_last_interval=0; } }catch(...){ diff --git a/segget/connection.h b/segget/connection.h index 2648775..f189b98 100644 --- a/segget/connection.h +++ b/segget/connection.h @@ -39,6 +39,7 @@ class Tconnection{ private: uint connection_num; uint network_num; + uint mirror_num; bool active; ulong total_dld_bytes; ulong bytes_per_last_interval; @@ -48,13 +49,14 @@ class Tconnection{ Tconnection(): connection_num(total_connections), network_num(0), + mirror_num(0), active(0), total_dld_bytes(0), bytes_per_last_interval(0), start_time(), segment(0){total_connections++;}; - void start(uint network_number); - void stop(); + void start(CURLM *cm, uint network_number, uint distfile_num, Tsegment *started_segment, uint best_mirror_num); + void stop(uint connection_result); void inc_bytes_per_last_interval(ulong new_bytes_count); void show_connection_progress(ulong time_diff); }; diff --git a/segget/distfile.cpp b/segget/distfile.cpp index 4852108..df972bc 100644 --- a/segget/distfile.cpp +++ b/segget/distfile.cpp @@ -28,7 +28,9 @@ void Tdistfile::init(){ for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){ - networkbrokers_array[network_num].network_num=network_num; + network_distfile_brokers_array[network_num].network_num=network_num; + Tmirror cur_mirror; + network_array[network_num].benchmarked_mirror_list.push_back(cur_mirror); } } void Tdistfile::load_url_list(json_object* json_array_distfile_urllist){ @@ -156,8 +158,7 @@ bool Tdistfile::choose_best_mirror(CURLM* cm, uint connection_num, uint network_ if (Pbest_mirror){ debug("Downloading from BEST_MIRROR:"+url_list[best_mirror_num]); Pbest_mirror->start(); - dn_segments[seg_num].prepare_for_connection(cm, connection_num, network_num, num, url_list[best_mirror_num]); - connection_array[connection_num].segment=&dn_segments[seg_num]; + connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num); return 0; } else{ @@ -165,14 +166,42 @@ bool Tdistfile::choose_best_mirror(CURLM* cm, uint connection_num, uint network_ return 1; } } +bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num){ + long best_mirror_num=-1; // the best isn't set let's find it -int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ + ulong best_mirror_self_rating=-1; + ulong curr_mirror_self_rating; + + for (ulong mirror_num=0; mirror_num<network_array[network_num].mirror_list.size(); mirror_num++){ + if (network_array[network_num].benchmarked_mirror_list[mirror_num].get_active_num()<settings.max_connections_num_per_mirror){ + curr_mirror_self_rating=network_array[network_num].benchmarked_mirror_list[mirror_num].mirror_on_the_wall(); + if (curr_mirror_self_rating<best_mirror_self_rating){ + best_mirror_num=mirror_num; + best_mirror_self_rating=curr_mirror_self_rating; + } + if (best_mirror_self_rating==0) + // 0 can not be improved - it's one of the best + break; + } + } + if (best_mirror_num!=-1){ + debug("Downloading from BEST_LOCAL_MIRROR:"+network_array[network_num].mirror_list[best_mirror_num]); + network_array[network_num].benchmarked_mirror_list[best_mirror_num].start(); + connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num); + return 0; + } + else{ + error_log("Can't choose LOCAL mirror for segment:"+dn_segments[seg_num].file_name); + return 1; + } +} +int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ try{ active_connections_num++; - //choose network for (uint cur_network_priority=10; cur_network_priority>0; cur_network_priority--){ debug("cur_network_priority="+toString(cur_network_priority)); + //choose network //---------------------------------------------------------------------------------------------------------- // // Several criterions can be used here to choose among networks with equal priority: @@ -180,31 +209,49 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ // add these options to segget.conf file // //---------------------------------------------------------------------------------------------------------- + int best_local_network_num=-1; int best_network_num=-1; for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){ - debug(" network_num="+toString(network_num)); //if network priority set then it's active if (network_array[network_num].priority){ if (network_array[network_num].priority==cur_network_priority){ debug(" network_priority="+toString(network_array[network_num].priority)); - if (networkbrokers_array[network_num].get_allowed_status()){ + if (network_distfile_brokers_array[network_num].get_allowed_status()){ debug(" Allowed network#:"+toString(network_num)); - if - ((best_network_num==-1) - or - (network_array[best_network_num].active_connections_num>network_array[network_num].active_connections_num)){ - best_network_num=network_num; - debug(" Replace best network to network#:"+toString(network_num)); - } + if (network_array[network_num].use_own_mirror_list_only_on){ + if ((best_local_network_num==-1) + or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){ + best_local_network_num=network_num; + debug(" Replace best LOCAL network to network#:"+toString(network_num)); + } + }else{ + if + ((best_network_num==-1) + or + (network_array[best_network_num].active_connections_num>network_array[network_num].active_connections_num)){ + best_network_num=network_num; + debug(" Replace best network to network to network#:"+toString(network_num)); + } + } //work with network } } } } - if (best_network_num!=-1){ + if (best_local_network_num!=-1){ //best network has been found - debug(" So best network is network#:"+toString(best_network_num)); - return choose_best_mirror(cm, connection_num, best_network_num, seg_num); + //work with network + debug(" So best LOCAL network is network#:"+toString(best_local_network_num)); + int resultik=choose_best_local_mirror(cm, connection_num, best_local_network_num, seg_num); + return resultik; + }else{ + // remote_mirrors_go_second + if (best_network_num!=-1){ + //best network has been found + //work with network + debug(" So best network is network#:"+toString(best_network_num)); + return choose_best_mirror(cm, connection_num, best_network_num, seg_num); + } } } }catch(...){ diff --git a/segget/distfile.h b/segget/distfile.h index c1e38c2..b13ef9b 100644 --- a/segget/distfile.h +++ b/segget/distfile.h @@ -39,8 +39,8 @@ #include <fstream> #include <iostream> #include <json/json.h> -#include "segment.cpp" -#include "mirror.cpp" +#include "segment.h" +#include "mirror.h" #include "checksum.cpp" #include "network.h" #include "networkbroker.h" @@ -52,9 +52,10 @@ typedef unsigned int uint; class Tdistfile{ private: uint dld_segments_count; - Tnetworkbroker networkbrokers_array[MAX_NETWORKS]; + bool choose_best_local_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num); bool choose_best_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num); public: + Tnetwork_distfile_broker network_distfile_brokers_array[MAX_NETWORKS]; bool downloaded; uint active_connections_num; string *url_list; diff --git a/segget/mirror.cpp b/segget/mirror.cpp index b9cbd2d..b0e2002 100644 --- a/segget/mirror.cpp +++ b/segget/mirror.cpp @@ -23,36 +23,8 @@ * License along with Segget; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -#ifndef __MIRROR_BENCHMARKER_H__ -#define __MIRROR_BENCHMARKER_H__ -#include <string> -#include <map> -#include "str.h" -#include "tui.h" -#include "settings.cpp" -using namespace std; - -extern Tsettings settings; -class Tmirror{ - private: - uint active_num; - public: - ulong dld_time; - ulong dld_size; - double honesty; - Tmirror(): - active_num(0), - dld_time(0), - dld_size(1), - honesty(1){}; - void start(); - void stop(ulong time, uint size); - double mirror_on_the_wall(); - uint get_active_num(){return active_num;}; -}; - -map<string,Tmirror *> mirror_list; +#include "mirror.h" double Tmirror::mirror_on_the_wall(){ try{ @@ -120,5 +92,4 @@ string strip_mirror_name(string path){ error_log("Error in mirror.cpp: strip_mirror_name()"); return ""; } -} -#endif
\ No newline at end of file +}
\ No newline at end of file diff --git a/segget/mirror.h b/segget/mirror.h new file mode 100644 index 0000000..0dff255 --- /dev/null +++ b/segget/mirror.h @@ -0,0 +1,61 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ +#ifndef __MIRROR_BENCHMARKER_H__ +#define __MIRROR_BENCHMARKER_H__ + +#include <string> +#include <map> +#include "str.h" +#include "tui.h" +#include "settings.h" +using namespace std; + +//extern Tsettings settings; + +class Tmirror{ + private: + string url; + uint active_num; + public: + ulong dld_time; + ulong dld_size; + double honesty; + Tmirror(): + url(""), + active_num(0), + dld_time(0), + dld_size(1), + honesty(1){}; + void start(); + void stop(ulong time, uint size); + double mirror_on_the_wall(); + uint get_active_num(){return active_num;}; +}; + +map<string,Tmirror *> mirror_list; +string strip_mirror_name(string path); +Tmirror* find_mirror(string mirror_url); +#endif
\ No newline at end of file diff --git a/segget/mirror_local.cpp b/segget/mirror_local.cpp new file mode 100644 index 0000000..e29217f --- /dev/null +++ b/segget/mirror_local.cpp @@ -0,0 +1,124 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ +#ifndef __MIRROR_BENCHMARKER_H__ +#define __MIRROR_BENCHMARKER_H__ + +#include <string> +#include <map> +#include "str.h" +#include "tui.h" +#include "settings.cpp" +using namespace std; + +extern Tsettings settings; +class Tmirror_local{ + private: + uint active_num; + public: + ulong dld_time; + ulong dld_size; + double honesty; + Tmirror(): + active_num(0), + dld_time(0), + dld_size(1), + honesty(1){}; + void start(); + void stop(ulong time, uint size); + double mirror_on_the_wall(); + uint get_active_num(){return active_num;}; +}; + +map<string,Tmirror *> mirror_local_list; + +double Tmirror_local::mirror_on_the_wall(){ + try{ + double criterion=honesty*1000000000*dld_time/dld_size; + honesty=honesty*100/(100+settings.benchmark_oblivion); + return criterion; + }catch(...){ + error_log("Error in mirror.cpp: mirror_on_the_wall()"); + return 0; + } +} +void Tmirror::start(){ + try{ + if (!dld_time) + dld_time=1; + active_num++; + honesty=1; + }catch(...){ + error_log("Error in mirror.cpp: start()"); + } +} + +void Tmirror::stop(ulong time, uint size){ + try{ + dld_time+=time/1000; + dld_size+=size; + honesty=1; + debug(toString(time)+"---"+toString(size)); + active_num--; + }catch(...){ + error_log("Error in mirror.cpp: stop()"); + } +} + +Tmirror* find_mirror(string mirror_url){ + try{ + map<string,Tmirror*>::const_iterator mirror_iterator = mirror_list.find(mirror_url); + if (mirror_iterator==mirror_list.end()){ + Tmirror * Pnew_mirror=new Tmirror; + debug("Cant find mirror:"+mirror_url+" - creating new record"); + mirror_list[mirror_url]=Pnew_mirror; + return Pnew_mirror; + } + else{ + debug("Found mirror:"+mirror_url); + debug("==================>>"); + debug(" time:"+toString(mirror_iterator->second->dld_time)); + debug(" size:"+toString(mirror_iterator->second->dld_size)); + debug(" honesty:"+toString(mirror_iterator->second->honesty)); + debug(" criterion:"+toString(mirror_iterator->second->mirror_on_the_wall())); + return mirror_iterator->second; + } + }catch(...){ + error_log("Error in mirror.cpp: find_mirror()"); + return 0; + } +} + +string strip_mirror_name(string path){ + try{ + string mirror_name; + mirror_name=path.substr(0,path.find("/",(path.find("://",0)+3))); + return mirror_name; + }catch(...){ + error_log("Error in mirror.cpp: strip_mirror_name()"); + return ""; + } +} +#endif
\ No newline at end of file diff --git a/segget/network.cpp b/segget/network.cpp index c975c94..e2c288b 100644 --- a/segget/network.cpp +++ b/segget/network.cpp @@ -24,6 +24,34 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "network.h" +void Tnetwork::load_mirror_list(){ + ifstream file; + string mirror_list_file_name="network"+toString(network_num)+"_mirrors.conf"; + file.exceptions (ifstream::failbit | ifstream::badbit); + try{ + file.open(mirror_list_file_name.c_str()); + } + catch(...){ + error_log("Can NOT open mirror list file: "+mirror_list_file_name+". Network will be disabled"); + priority=0; + return; + } + try{ + //processing file + string mirror_line; + while (not(file.eof())) { + getline(file,mirror_line); + if (! mirror_line.length()) continue; + if (mirror_line[0] == '#') continue; + if (mirror_line[0] == ';') continue; + mirror_list.push_back(mirror_line); + debug("LOCAL_MIRROR_ADDED:"+mirror_line); + } + } + catch(...){ + error_log("Mirror list file: "+mirror_list_file_name+" was opened, but an error occured while reading from it."); + } +} void Tnetwork::init(uint priority_value){ try{ @@ -45,6 +73,13 @@ void Tnetwork::init(uint priority_value){ conf.set(proxy_off, "network_proxy", "proxy_off"); conf.set(proxy_user, "network_proxy", "proxy_user"); conf.set(proxy_password, "network_proxy", "proxy_password"); + + conf.set(use_own_mirror_list_only_on, "network_mirrors", "use_own_mirror_list_only_on"); + + if (use_own_mirror_list_only_on){ + load_mirror_list(); + log("Network#"+toString(network_num)+" LOCAL_MIRROR_LIST size:"+toString(mirror_list.size())); + } } catch(...) { diff --git a/segget/network.h b/segget/network.h index b37642e..6653196 100644 --- a/segget/network.h +++ b/segget/network.h @@ -29,6 +29,8 @@ //#include <string> //#include <map> #include "str.h" +#include <vector> +#include "mirror.h" //#include "tui.h" //#include "settings.cpp" using namespace std; @@ -38,7 +40,10 @@ using namespace std; class Tnetwork{ static uint network_count; private: + void load_mirror_list(); public: + vector<string> mirror_list; + vector<Tmirror> benchmarked_mirror_list; uint active_connections_num; //network uint network_num; @@ -64,8 +69,10 @@ class Tnetwork{ bool proxy_off; string proxy_user; string proxy_password; + bool use_own_mirror_list_only_on; Tnetwork(): - //private: + mirror_list(), + benchmarked_mirror_list(), active_connections_num(0), //network network_num(network_count), @@ -90,7 +97,8 @@ class Tnetwork{ proxy_port(3128), proxy_off(1), proxy_user("none"), - proxy_password("none") + proxy_password("none"), + use_own_mirror_list_only_on(0) {network_count++;}; void init(uint priority_value); bool get_busy_status(); diff --git a/segget/network0.conf b/segget/network0.conf index 764ecca..215c3aa 100644 --- a/segget/network0.conf +++ b/segget/network0.conf @@ -171,9 +171,11 @@ proxy_password=none proxy_off=1 [network_mirrors] -# SYNOPSIS: NETWORK_USES_OWN_MIRROR_LIST_ON=0 | 1 +# SYNOPSIS: NETWORK_USES_OWN_MIRROR_LIST_ONLY_ON=0 | 1 # - If set to 1, segget will replace mirror list provided by portage system with -# the list from network1_mirros.list +# the list from network0_mirrors.conf file +# - If set to 0, segget will use ONLY mirror list provided by portage system, +# and will NOT use the list from network0_mirrors.conf file # Default: -# use_own_mirror_list_on=0 -use_own_mirror_list_on=0
\ No newline at end of file +# use_own_mirror_list_only_on=0 +use_own_mirror_list_only_on=0
\ No newline at end of file diff --git a/segget/network1.conf b/segget/network1.conf index de08c95..dc7b80b 100644 --- a/segget/network1.conf +++ b/segget/network1.conf @@ -171,9 +171,11 @@ proxy_password=none proxy_off=1 [network_mirrors] -# SYNOPSIS: NETWORK_USES_OWN_MIRROR_LIST_ON=0 | 1 +# SYNOPSIS: NETWORK_USES_OWN_MIRROR_LIST_ONLY_ON=0 | 1 # - If set to 1, segget will replace mirror list provided by portage system with -# the list from network1_mirros.list +# the list from network1_mirrors.conf +# - If set to 0, segget will use ONLY mirror list provided by portage system, +# and will NOT use the list from network1_mirrors.conf file # Default: -# use_own_mirror_list_on=0 -use_own_mirror_list_on=0
\ No newline at end of file +# use_own_mirror_list_only_on=0 +use_own_mirror_list_only_on=1
\ No newline at end of file diff --git a/segget/network1_mirrors.conf b/segget/network1_mirrors.conf new file mode 100644 index 0000000..4164689 --- /dev/null +++ b/segget/network1_mirrors.conf @@ -0,0 +1,4 @@ +ftp://192.168.56.11/ +ftp://192.168.56.12/ +ftp://192.168.56.13/ +ftp://192.168.56.14/
\ No newline at end of file diff --git a/segget/networkbroker.cpp b/segget/networkbroker.cpp index 920fc56..8b7212f 100644 --- a/segget/networkbroker.cpp +++ b/segget/networkbroker.cpp @@ -26,7 +26,31 @@ #include "networkbroker.h" -bool Tnetworkbroker::get_allowed_status(){ +bool Tnetwork_distfile_broker::get_allowed_status(){ debug(" get_allowed_status_for_network_num="+toString(network_num)); return network_array[network_num].get_busy_status(); +} + +Tmirror* Tnetwork_distfile_broker::find_mirror(string mirror_url){ + try{ + map<string,Tmirror*>::const_iterator mirror_iterator = mirror_list.find(mirror_url); + if (mirror_iterator==mirror_list.end()){ + Tmirror * Pnew_mirror=new Tmirror; + debug("Cant find mirror:"+mirror_url+" - creating new record"); + mirror_list[mirror_url]=Pnew_mirror; + return Pnew_mirror; + } + else{ + debug("Found mirror:"+mirror_url); + debug("==================>>"); + debug(" time:"+toString(mirror_iterator->second->dld_time)); + debug(" size:"+toString(mirror_iterator->second->dld_size)); + debug(" honesty:"+toString(mirror_iterator->second->honesty)); + debug(" criterion:"+toString(mirror_iterator->second->mirror_on_the_wall())); + return mirror_iterator->second; + } + }catch(...){ + error_log("Error in mirror.cpp: find_mirror()"); + return 0; + } }
\ No newline at end of file diff --git a/segget/networkbroker.h b/segget/networkbroker.h index ba78e08..79781f2 100644 --- a/segget/networkbroker.h +++ b/segget/networkbroker.h @@ -23,18 +23,20 @@ * License along with Segget; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -#ifndef __NETWORKBROKER_H__ -#define __NETWORKBROKER_H__ +#ifndef __NETWORK_DISTFILE_BROKER_H__ +#define __NETWORK_DISTFILE_BROKER_H__ -class Tnetworkbroker{ +class Tnetwork_distfile_broker{ public: +// map<string,Tmirror *> benchmarked_mirror_list; ulong network_num; - bool local_mirrors; - bool get_allowed_status(); - Tnetworkbroker(): + bool local; + Tnetwork_distfile_broker(): network_num(0), - local_mirrors(0) + local(0) {}; + bool get_allowed_status(); + Tmirror* find_mirror(string mirror_url); }; #endif diff --git a/segget/segget.conf b/segget/segget.conf index 326ca91..eab2763 100644 --- a/segget/segget.conf +++ b/segget/segget.conf @@ -184,7 +184,7 @@ network0_priority=10 # Define network specific settings in network1.conf # Default: # network1_priority=0 -network1_priority=0 +network1_priority=10 # NETWORK2_PRIORITY # Description: same as for NETWORK0_PRIORITY diff --git a/segget/segget.cpp b/segget/segget.cpp index e2e42a3..e4b13f1 100644 --- a/segget/segget.cpp +++ b/segget/segget.cpp @@ -32,10 +32,15 @@ #include <ncurses.h> #include "pkg.cpp" #include "distfile.cpp" +#include "mirror.cpp" +#include "segment.cpp" #include "connection.cpp" #include "utils.cpp" #include "network.cpp" #include "networkbroker.cpp" +#include "str.cpp" +#include "tui.cpp" +#include "settings.cpp" //#include "settings.cpp" using namespace std; @@ -219,39 +224,13 @@ int download_pkgs(){ Tsegment *current_segment; CURL *e = msg->easy_handle; curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, ¤t_segment); - uint result=msg->data.result; - string result_msg_text="RESULT:"+toString(result)+" "+curl_easy_strerror(msg->data.result)+" while downloading segment"; + uint connection_result=msg->data.result; + string result_msg_text="RESULT:"+toString(connection_result)+" "+curl_easy_strerror(msg->data.result)+" while downloading segment"; msg_status1(current_segment->connection_num,current_segment->segment_num,result_msg_text); curl_multi_remove_handle(cm, e); - current_segment->segment_file.close(); - Tmirror *Pcurr_mirror=find_mirror(strip_mirror_name(current_segment->url)); - timeval now_time; - gettimeofday(&now_time,NULL); - Tdistfile* prnt_distfile; - prnt_distfile=(Tdistfile*)current_segment->parent_distfile; - connection_array[current_segment->connection_num].stop(); - prnt_distfile->active_connections_num--; - if (result!=0){ - // error -> start downloading again - msg_status2(current_segment->connection_num, toString(result)+"]- Failed download "+current_segment->file_name); - debug(toString(result)+"]- Failed download "+current_segment->url); - Pcurr_mirror->stop(time_left_from(connection_array[current_segment->connection_num].start_time),0); - if (current_segment->try_num>=settings.max_tries){ - current_segment->status=FAILED; - error_log("Segment:"+current_segment->file_name+" has reached max_tries limit - segment.status set to FAILED"); - } - else current_segment->status=WAITING; - //log("Restarting "+current_segment->file_name+" on connection#"+toString(current_segment->connection_num)); - //prnt_distfile->provide_segment(cm,current_segment->connection_num,current_segment->segment_num); - //U++; - }else{ - // no error => count this one and start new - log("Succesfully downloaded "+current_segment->file_name+" on connection#"+toString(current_segment->connection_num)); - debug(" Successful download "+current_segment->url); - Pcurr_mirror->stop(time_left_from(connection_array[current_segment->connection_num].start_time),current_segment->segment_size); - current_segment->status=DOWNLOADED; - prnt_distfile->inc_dld_segments_count(current_segment); - }; + + connection_array[current_segment->connection_num].stop(connection_result); + if (not choose_segment(current_segment->connection_num)) { U++; // just to prevent it from remaining at 0 if there are more URLs to get }; diff --git a/segget/segment.cpp b/segget/segment.cpp index 93c5098..56c5542 100644 --- a/segget/segment.cpp +++ b/segget/segment.cpp @@ -26,7 +26,7 @@ #include "segment.h" -void Tsegment::set_segment(void *prnt_distfile, uint seg_num, string distfile_name, ulong default_seg_size, ulong range_end){ +void Tsegment::set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfile_name, ulong default_seg_size, ulong range_end){ try{ parent_distfile=prnt_distfile; segment_num=seg_num; @@ -56,15 +56,26 @@ void Tsegment::set_segment(void *prnt_distfile, uint seg_num, string distfile_na error_log("Error in segment.cpp: prepare_for_connection()"); } } -void Tsegment::prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, string segment_url){ +void Tsegment::prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, uint mirror_num){ try{ - msg_connecting(con_num,distfile_num, segment_num,"Downloading from "+segment_url); + string url__; + debug("==========NETWORK:"+toString(network_num)); + debug("==========local_on:"+toString(network_array[network_num].use_own_mirror_list_only_on)); + if (network_array[network_num].use_own_mirror_list_only_on){ + url=network_array[network_num].mirror_list[mirror_num]+parent_distfile->name; + url__=network_array[network_num].mirror_list[mirror_num]+parent_distfile->name; + debug("URL:"+url); + debug("URL___________________:"+url__); + }else{ + url=parent_distfile->url_list[mirror_num]; + } + msg_connecting(con_num,distfile_num, segment_num,"Downloading from "+url); + msg_connecting(con_num,distfile_num, segment_num,"Downloading from "+url); segments_in_progress[con_num]=this; status=DOWNLOADING; downloaded_bytes=0; connection_num=con_num; - connection_array[con_num].start(network_num); - url=segment_url; +// connection_array[con_num].start(network_num); try_num++; add_easy_handle_to_multi(cm, network_num); }catch(...){ diff --git a/segget/segment.h b/segget/segment.h index 3151999..81e9a82 100644 --- a/segget/segment.h +++ b/segget/segment.h @@ -26,6 +26,8 @@ #ifndef __SEGMENT_H__ #define __SEGMENT_H__ +using namespace std; + #include <sstream> #include <fstream> #include <cstring> @@ -36,9 +38,11 @@ #include <ncurses.h> #include <curl/curl.h> #include "settings.h" + +#include "distfile.h" +class Tdistfile; #include "stats.cpp" -using namespace std; extern Tsettings settings; unsigned long downloaded_bytes=0; @@ -54,7 +58,7 @@ class Tsegment{ string file_name; Tstatus status; uint try_num; - void* parent_distfile; + Tdistfile* parent_distfile; uint connection_num; uint segment_num; uint segment_size; @@ -80,8 +84,8 @@ class Tsegment{ Tsegment(const Tsegment &L); // copy constructor Tsegment & operator=(const Tsegment &L); ~Tsegment(); - void set_segment(void *prnt_distfile, uint seg_num, string distfile_name, ulong default_seg_size, ulong range_end); - void prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, string segment_url); + void set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfile_name, ulong default_seg_size, ulong range_end); + void prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, uint mirror_num); string get_file_name(){return file_name;}; int add_easy_handle_to_multi(CURLM *cm, uint network_num); }; diff --git a/segget/settings.h b/segget/settings.h index c633a2c..a9d048b 100644 --- a/segget/settings.h +++ b/segget/settings.h @@ -123,4 +123,4 @@ class Tsettings{ }; Tsettings settings; -#endif +#endif
\ No newline at end of file diff --git a/segget/tui.h b/segget/tui.h index f20b413..c90ffc5 100644 --- a/segget/tui.h +++ b/segget/tui.h @@ -33,7 +33,7 @@ void msg_total(string msg_text); void msg(uint y, uint x, string msg_text); void msg_connecting(uint connection_num, uint distfile_num, uint segment_num, string msg_text); -void msg_segment_progress(uint connection_num, uint network_num, uint segment_num, uint try_num, ulong dld_bytes, ulong total_bytes, ulong speed); +void msg_segment_progress(uint connection_num, uint network_num, uint segment_num, uint try_num, ulong dld_bytes, ulong total_bytes, ulong speed, ulong avg_speed); void msg_status1(uint connection_num, uint segment_num, string msg_text); void msg_status2(uint connection_num, string msg_text); void msg_clean_connection(uint connection_num); |