diff options
author | 2010-07-21 22:07:05 +0300 | |
---|---|---|
committer | 2010-07-21 22:07:05 +0300 | |
commit | 2106d95831eaa50bcfbd77d56d313808fab7d024 (patch) | |
tree | 30a615e49d05d29e06eb59d8799ae52d05215ae6 | |
parent | Consider distfile failed if one of its segments is failed (diff) | |
download | idfetch-2106d95831eaa50bcfbd77d56d313808fab7d024.tar.gz idfetch-2106d95831eaa50bcfbd77d56d313808fab7d024.tar.bz2 idfetch-2106d95831eaa50bcfbd77d56d313808fab7d024.zip |
Add to tui: ETA, AVG speed and active/total connections
-rw-r--r-- | segget/connection.cpp | 27 | ||||
-rw-r--r-- | segget/connection.h | 2 | ||||
-rw-r--r-- | segget/distfile.cpp | 61 | ||||
-rw-r--r-- | segget/segget.cpp | 51 | ||||
-rw-r--r-- | segget/segment.cpp | 8 | ||||
-rw-r--r-- | segget/stats.cpp | 37 | ||||
-rw-r--r-- | segget/stats.h | 6 | ||||
-rw-r--r-- | segget/str.cpp | 3 | ||||
-rw-r--r-- | segget/tui.cpp | 25 | ||||
-rw-r--r-- | segget/tui.h | 1 | ||||
-rw-r--r-- | segget/utils.cpp | 82 | ||||
-rw-r--r-- | segget/utils.h | 3 |
12 files changed, 229 insertions, 77 deletions
diff --git a/segget/connection.cpp b/segget/connection.cpp index 89bfa4c..affa21f 100644 --- a/segget/connection.cpp +++ b/segget/connection.cpp @@ -37,7 +37,7 @@ void init_connections(){ void Tconnection::start(CURLM *cm, uint network_number, uint distfile_num, Tsegment *started_segment, uint best_mirror_num){ try{ - + stats.active_connections_counter++; segment=started_segment; debug("Starting connection for distfile: "+segment->parent_distfile->name); mirror_num=best_mirror_num; @@ -79,6 +79,7 @@ void Tconnection::start(CURLM *cm, uint network_number, uint distfile_num, Tsegm void Tconnection::stop(int connection_result){ try{ + stats.active_connections_counter--; debug("Finished connection for distfile: "+segment->parent_distfile->name+" Segment#:"+toString(segment->segment_num)+" Network#"+toString(network_num)+" Status: "+toString(connection_result)); error_log("Finished connection for distfile: "+segment->parent_distfile->name+" Segment#:"+toString(segment->segment_num)+" Network#"+toString(network_num)+" Status: "+toString(connection_result)); @@ -174,19 +175,17 @@ void Tconnection::inc_bytes_per_last_interval(ulong new_bytes_count){ void Tconnection::show_connection_progress(ulong time_diff){ try{ - if (active){ - stats.total_bytes_per_last_interval+=bytes_per_last_interval; - msg_segment_progress( - segment->connection_num, - network_num, - segment->segment_num, - segment->try_num, - segment->downloaded_bytes, - segment->segment_size, - (bytes_per_last_interval*1000)/time_diff, - (total_dld_bytes*1000)/time_left_from(start_time)); - bytes_per_last_interval=0; - } + stats.total_bytes_per_last_interval+=bytes_per_last_interval; + msg_segment_progress( + segment->connection_num, + network_num, + segment->segment_num, + segment->try_num, + segment->downloaded_bytes, + segment->segment_size, + (bytes_per_last_interval*1000)/time_diff, + (total_dld_bytes*1000)/time_left_from(start_time)); + bytes_per_last_interval=0; }catch(...){ error_log("Error in connection.cpp: show_connection_progress()"); } diff --git a/segget/connection.h b/segget/connection.h index 03d502f..972792f 100644 --- a/segget/connection.h +++ b/segget/connection.h @@ -42,9 +42,9 @@ class Tconnection{ Tnetwork_distfile_broker_phases connection_start_time_network_phase_for_pf_networks; uint network_num; uint mirror_num; + public: ulong total_dld_bytes; ulong bytes_per_last_interval; - public: uint connection_num; bool active; timeval start_time; diff --git a/segget/distfile.cpp b/segget/distfile.cpp index 72dc158..8e85d7a 100644 --- a/segget/distfile.cpp +++ b/segget/distfile.cpp @@ -121,7 +121,26 @@ bool Tdistfile::allows_new_actions(){ // if (downloaded) return false; // else return true; // int time_left=0; - if (status==DDOWNLOADED) return false; + if (status==DDOWNLOADED){ + debug("No new connection for distfile:"+name+". Distfile has DDOWNLOADED status"); + return false; + } + if (status==DFAILED){ + debug("No new connection for distfile:"+name+". Distfile has DFAILED status"); + return false; + } +// debug("Distfile "+Ppkg_array[pkg_num]->distfile_vector[distfile_num]->name+" allows new connections"); +// debug(" distfile_num:"+toString(distfile_num)); + if (active_connections_num<settings.max_connection_num_per_distfile){ + debug("Allow new connection for ============================================= Distfile "+name); + debug("max_connection limit has not been reached"); + return true; + }else{ + debug("No new connection for distfile: "+name+". It already has " + +toString(active_connections_num) + +" connections => choose another distfile."); + return false; + } // if (((status==DPROXY_QUEUED) || (status==DPROXY_DOWNLOADING)) && (time_left<100)) return false; //oterwise allow connections // DNEW, @@ -131,9 +150,6 @@ bool Tdistfile::allows_new_actions(){ // DPROXY_DOWNLOADED, // DWAITING, // DDOWNLOADING, -// DDOWNLOADED, -// DFAILED - return true; } void Tdistfile::init(){ @@ -293,31 +309,34 @@ bool Tdistfile::choose_best_mirror(CURLM* cm, uint connection_num, uint network_ bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num){ try{ long best_mirror_num=-1; // the best isn't set let's find it + bool all_mirrors_failed=true; ulong best_mirror_self_rating=-1; ulong curr_mirror_self_rating; debug("Choosing mirror for network"+toString(network_num)); for (ulong mirror_num=0; mirror_num<network_array[network_num].benchmarked_mirror_list.size(); mirror_num++){ debug("Evaluating mirror:"+network_array[network_num].benchmarked_mirror_list[mirror_num].url); - if (network_array[network_num].benchmarked_mirror_list[mirror_num].get_active_num()<settings.max_connections_num_per_mirror){ - curr_mirror_self_rating=network_array[network_num].benchmarked_mirror_list[mirror_num].mirror_on_the_wall(); - if (curr_mirror_self_rating<best_mirror_self_rating){ - best_mirror_num=mirror_num; - best_mirror_self_rating=curr_mirror_self_rating; - } - if (best_mirror_self_rating==0) - // 0 can not be improved - it's one of the best - break; + if (network_distfile_brokers_array[network_num].mirror_fails_vector[mirror_num]){ + debug("mirror:"+network_array[network_num].benchmarked_mirror_list[mirror_num].url+" has FAILED status - will be skipped"); }else{ - debug("Mirror already has:" - +toString(network_array[network_num].benchmarked_mirror_list[mirror_num].get_active_num()) - +" connections which doesn't meet limit:"+toString(settings.max_connections_num_per_mirror)); + all_mirrors_failed=false; + if (network_array[network_num].benchmarked_mirror_list[mirror_num].get_active_num()<settings.max_connections_num_per_mirror){ + curr_mirror_self_rating=network_array[network_num].benchmarked_mirror_list[mirror_num].mirror_on_the_wall(); + if (curr_mirror_self_rating<best_mirror_self_rating){ + best_mirror_num=mirror_num; + best_mirror_self_rating=curr_mirror_self_rating; + } + if (best_mirror_self_rating==0) + // 0 can not be improved - it's one of the best + break; + }else{ + debug("Mirror already has:" + +toString(network_array[network_num].benchmarked_mirror_list[mirror_num].get_active_num()) + +" connections which doesn't meet limit:"+toString(settings.max_connections_num_per_mirror)); + } } } if (best_mirror_num!=-1){ debug("Downloading from BEST_LOCAL_MIRROR:"+network_array[network_num].benchmarked_mirror_list[best_mirror_num].url); - - - // active_connections_num++; connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num); return R_R_DOWNLOAD_STARTED; @@ -325,6 +344,10 @@ bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint ne else{ debug("Can't choose LOCAL mirror for segment:"+dn_segments[seg_num].file_name); error_log("Can't choose LOCAL mirror for segment:"+dn_segments[seg_num].file_name); + if (all_mirrors_failed){ + debug("All local mirrors failed in network#"+toString(network_num)); + error_log("All local mirrors failed in network#"+toString(network_num)); + } return 1; } }catch(...){ diff --git a/segget/segget.cpp b/segget/segget.cpp index 4cf5f9d..8829d6d 100644 --- a/segget/segget.cpp +++ b/segget/segget.cpp @@ -103,12 +103,6 @@ int pkg_choose_segment(Tpkg * cur_pkg, uint connection_num){ while(distfile_num<cur_pkg->distfile_count){ // if (Ppkg_array[pkg_num]->distfile_vector[distfile_num].allows_new_actions()){ if (cur_pkg->Pdistfile_list[distfile_num]->allows_new_actions()){ - debug("============================================= Distfile " - +cur_pkg->Pdistfile_list[distfile_num]->name+" allows new connections"); -// debug("Distfile "+Ppkg_array[pkg_num]->distfile_vector[distfile_num]->name+" allows new connections"); -// debug(" distfile_num:"+toString(distfile_num)); - if (cur_pkg->Pdistfile_list[distfile_num]->active_connections_num<settings.max_connection_num_per_distfile){ - debug("max_connection limit has not been reached"); debug("segment_num:"+toString(segment_num)); debug("segment_count:"+toString(cur_pkg->Pdistfile_list[distfile_num]->segments_count)); while (segment_num<cur_pkg->Pdistfile_list[distfile_num]->segments_count){ @@ -132,11 +126,6 @@ int pkg_choose_segment(Tpkg * cur_pkg, uint connection_num){ segment_num++; } } - }else{ - debug(" distfile "+cur_pkg->Pdistfile_list[distfile_num]->name+" has " - +toString(cur_pkg->Pdistfile_list[distfile_num]->active_connections_num) - +" connections => choosing another distfile."); - } } distfile_num++; segment_num=0; @@ -195,26 +184,39 @@ int download_pkgs(){ // } // }; bool keep_running_flag=true; - while (keep_running_flag){ - U=1; - while (U) { - // Use free connections to download segments connections + struct timeval prev_connection_activation_cycle_time; + while (keep_running_flag) { + // Use free connections to download segments connections + if (1000>time_left_from(prev_connection_activation_cycle_time)){ + debug("Not enough time left to start connection activation cycle"); + sleep(1); + }else{ debug("Entering connection activation cycle"); + gettimeofday(&prev_connection_activation_cycle_time,NULL); for (uint connection_num = 0; connection_num < settings.max_connections; ++connection_num) { debug("connection_num:"+toString(connection_num)); if ( ! connection_array[connection_num].active){ debug("connection is not active - choosing segment"); - choose_segment(connection_num); -// // activate only one connection -// break; + if (choose_segment(connection_num)){ + // if no success don't try the same for other connections + debug("No segment found for connection:"+toString(connection_num)+" => no reason to look for a segment for other connections"); + break; + }else{ +// U++; + } } else{ debug("connection is active"); } }; debug("Exit connection activation sycle"); - while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(cm, &U)){}; - if (U) { + } + U=stats.active_connections_counter; + debug("before multi_perform"); +// while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(cm, &U)){}; + if (CURLM_CALL_MULTI_PERFORM != curl_multi_perform(cm, &U)){ + debug("after multi_perform"); +// if (U) { FD_ZERO(&R); FD_ZERO(&W); FD_ZERO(&E); @@ -245,8 +247,10 @@ int download_pkgs(){ return EXIT_FAILURE; } } - } +// } + debug("before multi_info_read"); while ((msg = curl_multi_info_read(cm, &Q))) { + debug("inside multi_info_read"); if (msg->msg == CURLMSG_DONE) { Tsegment *current_segment; CURL *e = msg->easy_handle; @@ -268,11 +272,6 @@ int download_pkgs(){ } } } - debug("bbbbbblllllllllllllaaaaaaaaaaaaa"); - // nothing to download - wait 5 secs and check upper_proxy_fetchers again - // later will be replaced by server_waiting_for_incoming connections and new tasks - // with timeout to check downloads from upper proxy fetcher - sleep(1); } curl_multi_cleanup(cm); curl_global_cleanup(); diff --git a/segget/segment.cpp b/segget/segment.cpp index cab6343..d5c2ea3 100644 --- a/segget/segment.cpp +++ b/segget/segment.cpp @@ -169,10 +169,16 @@ int Tsegment::add_easy_handle_to_multi(CURLM *cm, uint network_num){ void show_progress(double time_diff){ try{ stats.total_bytes_per_last_interval=0; + stats.avg_total_speed=0; for (uint con_num=0; con_num<MAX_CONNECTS; con_num++){ // ulong speed=bytes_written*1000/(diff_sec+diff_milli); //if connection is not NULL - connection_array[con_num].show_connection_progress(time_diff); + if (connection_array[con_num].active){ + connection_array[con_num].show_connection_progress(time_diff); + if (time_left_from(connection_array[con_num].start_time)>0){ + stats.avg_total_speed+=(connection_array[con_num].total_dld_bytes*1000)/time_left_from(connection_array[con_num].start_time); + } + } } stats.last_time_interval=time_diff; stats.show_totals(); diff --git a/segget/stats.cpp b/segget/stats.cpp index 1a0a271..0dbc4c1 100644 --- a/segget/stats.cpp +++ b/segget/stats.cpp @@ -53,17 +53,32 @@ void Tstats::show_totals(){ show_total_size=total_size; struct timeval now_timee; gettimeofday(&now_timee,NULL); - msg_total("Total" - +field(" PKGs:", pkg_count,4) - +field(" = DFs:", dld_distfiles_count,4) - +field(" / ", distfiles_count,4) - +field(" = Size:", dld_size/1000,7) - +field(" / ", total_size/1000,7)+" Kb " - +field(" = ", dld_size*100/(1+show_total_size),3)+"%" - +field(" Total speed: ", (total_bytes_per_last_interval/(show_last_time_interval)),7) - +" Kb/s" -// +" Secs:"+toString(now_timee.tv_sec) -// +" usecs:"+toString(now_timee.tv_usec) + string eta_str; + if (avg_total_speed==0){ + eta_str=" ETA: inf"; + }else{ + eta_str=" ETA: "+secsToString((total_size-dld_size)/avg_total_speed); + } + + string avg_speed_str=" AVG speed: "+speedToString(avg_total_speed); + + msg_total("Total CON:" + +field("", active_connections_counter,2)+"/" + +field("", settings.max_connections,2) +// +field(" PKGs:", pkg_count,4) +// +field(" = DFs:", dld_distfiles_count,4) + +" = DF:"+toString(dld_distfiles_count) + +field("/", distfiles_count,4) +// +field(" = Size:", dld_size/1000,7) + +" = Size:"+toString(dld_size/1000) + +field("(", ((double)dld_size/show_total_size)*100,3)+"%)" +// +field("/", total_size/1000,7)+" Kb " + +"/"+toString(total_size/1000)+"Kb" + +" Total spd: "+speedToString(total_bytes_per_last_interval*1000/show_last_time_interval) + +avg_speed_str + +eta_str + // +" Secs:"+toString(now_timee.tv_sec) + // +" usecs:"+toString(now_timee.tv_usec) ); reset_previous_time(); }catch(...){ diff --git a/segget/stats.h b/segget/stats.h index 2cb78a3..6402cc8 100644 --- a/segget/stats.h +++ b/segget/stats.h @@ -42,8 +42,10 @@ class Tstats{ ulong total_bytes_per_last_interval; struct timeval previous_time; double last_time_interval; + ulong avg_total_speed; uint pkg_count; uint distfiles_count; + ulong active_connections_counter; Tstats(): dld_size(0), dld_distfiles_count(0), @@ -51,8 +53,10 @@ class Tstats{ total_bytes_per_last_interval(0), previous_time(), last_time_interval(1), + avg_total_speed(0), pkg_count(0), - distfiles_count(0) + distfiles_count(0), + active_connections_counter(0) {}; void inc_dld_size(ulong more_bytes){ dld_size+=more_bytes;}; ulong get_dld_size(){return dld_size;}; diff --git a/segget/str.cpp b/segget/str.cpp index c45787a..4ffbe2f 100644 --- a/segget/str.cpp +++ b/segget/str.cpp @@ -58,6 +58,7 @@ template<typename T> string toString(T t){ return s.str(); } */ +/* template<typename T> string field(string prefix,T t, int width){ try{ stringstream s1,s2; @@ -71,7 +72,7 @@ template<typename T> string field(string prefix,T t, int width){ return ""; } } - +*/ string field(string prefix,ulong t, int width){ try{ stringstream s1,s2; diff --git a/segget/tui.cpp b/segget/tui.cpp index d84e17e..fa194b6 100644 --- a/segget/tui.cpp +++ b/segget/tui.cpp @@ -58,17 +58,35 @@ void msg_connecting(uint connection_num, uint distfile_num, uint segment_num, st void msg_segment_progress(uint connection_num, uint network_num, uint segment_num, uint try_num, ulong dld_bytes, ulong total_bytes, ulong speed, ulong avg_speed){ try{ + + string eta_string; + if (speed==0){ + eta_string=" ETA: inf"; + }else{ + eta_string=" ETA: "+secsToString((total_bytes-dld_bytes)/speed); + } + string speed_str; string avg_speed_str; +/* if (speed<1000) speed_str=field(" Speed:",speed,7)+" b/s"; else speed_str=field(" Speed:",speed/1000,7)+" Kb/s"; +*/ + + speed_str=" Speed: "+speedToString(speed); +/* if (avg_speed<1000) - avg_speed_str=field(" AVG speed:",avg_speed,7)+" b/s"; + avg_speed_str=field(" AVG",avg_speed,7)+" b/s"; else - avg_speed_str=field(" AVG speed:",avg_speed/1000,7)+" Kb/s"; + avg_speed_str=field(" AVG",avg_speed/1000,7)+" Kb/s"; +*/ + avg_speed_str=" AVG speed: "+speedToString(avg_speed); + int percent=dld_bytes*100/total_bytes; + + msg(connection_num*CONNECTION_LINES,0, field("[",connection_num,2)+"]" +field(" Net",network_num,1) @@ -78,7 +96,8 @@ void msg_segment_progress(uint connection_num, uint network_num, uint segment_nu +field(" / ",total_bytes,7) +field(" = ",percent,3)+"%" +speed_str - +avg_speed_str); + +avg_speed_str + +eta_string); }catch(...){ error_log_no_msg("Error in tui.cpp: msg_segment_progress()"); } diff --git a/segget/tui.h b/segget/tui.h index baf718f..8f24b47 100644 --- a/segget/tui.h +++ b/segget/tui.h @@ -29,6 +29,7 @@ #include <ncurses.h> #include "settings.h" #include "ui_server.h" +#include "utils.h" using namespace std; diff --git a/segget/utils.cpp b/segget/utils.cpp index 1179363..9d80cb4 100644 --- a/segget/utils.cpp +++ b/segget/utils.cpp @@ -40,3 +40,85 @@ ulong time_left_from(timeval from_time){ } } +string secsToString(ulong secs){ + string result=""; + ulong mins=0; + ulong hours=0; + ulong days=0; + ldiv_t time_div_spliter; + if (secs>60){ + time_div_spliter = ldiv (secs,60); + secs=time_div_spliter.rem; + if (time_div_spliter.quot>60){ + time_div_spliter = ldiv (time_div_spliter.quot,60); + mins=time_div_spliter.rem; + if (time_div_spliter.quot>24){ + time_div_spliter = ldiv (time_div_spliter.quot,24); + hours=time_div_spliter.rem; + days=time_div_spliter.quot; + if (days>999) return ("inf"); + result=result+field(" ", days,3)+"d"; + }else{ + hours=time_div_spliter.quot; + } + result=result+field(" ", hours,2)+"h"; + }else{ + mins=time_div_spliter.quot; + } + result=result+field(" ", mins,2)+"m"; + } + result=result+field(" ", secs,2)+"s"; + + return result; +} + +string speedToString(ulong dld_bytes, ulong time_left){ + try{ + string speed_str; + if (time_left==0){ + speed_str="N/a"; + }else{ + ulong speed=(dld_bytes*1000/time_left); + string suffix; + if (speed>1500){ + if (speed>1500000){ + suffix=" MB/s"; + speed=speed/1000000; + }else{ + suffix=" KB/s"; + speed=speed/1000; + } + }else{ + suffix=" B/s"; + } + speed_str=field("",speed,4)+suffix; + } + return speed_str; + }catch(...){ + error_log("Error in utils.cpp: speedToString()"); + } + return ""; +} + +string speedToString(ulong speed){ + try{ + string speed_str; + string suffix; + if (speed>1500){ + if (speed>1500000){ + suffix=" MB/s"; + speed=speed/1000000; + }else{ + suffix=" KB/s"; + speed=speed/1000; + } + }else{ + suffix=" B/s"; + } + speed_str=field("",speed,4)+suffix; + return speed_str; + }catch(...){ + error_log("Error in utils.cpp: speedToString()"); + } + return ""; +}
\ No newline at end of file diff --git a/segget/utils.h b/segget/utils.h index b897402..dfd5eff 100644 --- a/segget/utils.h +++ b/segget/utils.h @@ -34,5 +34,8 @@ using namespace std; ulong time_left_from(timeval from_time); +string secsToString(ulong secs); +string speedToString(ulong dld_bytes, ulong time_left); +string speedToString(ulong speed); #endif
\ No newline at end of file |