diff options
author | 2010-07-24 15:31:12 +0300 | |
---|---|---|
committer | 2010-07-24 15:31:12 +0300 | |
commit | 3410b7deba33302c2e3f2d4c3b5bafc15f357f5a (patch) | |
tree | 810dff13ee7264e281e47997994319248298ab9a | |
parent | Add /etc/init.d/seggetd script to start|stop|restart|status segget daemon (diff) | |
download | idfetch-3410b7deba33302c2e3f2d4c3b5bafc15f357f5a.tar.gz idfetch-3410b7deba33302c2e3f2d4c3b5bafc15f357f5a.tar.bz2 idfetch-3410b7deba33302c2e3f2d4c3b5bafc15f357f5a.zip |
Add request tool.
Request tool reads list of distfiles from ./pkg.list file and requests
seggetd daemon to download distfiles from the list.
-rw-r--r-- | request/Makefile | 29 | ||||
-rw-r--r-- | request/config.cpp | 177 | ||||
-rw-r--r-- | request/config.h | 54 | ||||
-rw-r--r-- | request/distfile.cpp | 742 | ||||
-rw-r--r-- | request/distfile.h | 155 | ||||
-rw-r--r-- | request/phase.cpp | 37 | ||||
-rw-r--r-- | request/phase.h | 43 | ||||
-rw-r--r-- | request/pkg.h | 67 | ||||
-rw-r--r-- | request/request.conf | 288 | ||||
-rw-r--r-- | request/request.cpp | 247 | ||||
-rw-r--r-- | request/request.h | 77 | ||||
-rw-r--r-- | request/response.cpp | 65 | ||||
-rw-r--r-- | request/response.h | 61 | ||||
-rw-r--r-- | request/restrict.conf | 2 | ||||
-rw-r--r-- | request/settings.cpp | 44 | ||||
-rw-r--r-- | request/settings.h | 130 | ||||
-rw-r--r-- | request/shorttypes.h | 32 | ||||
-rw-r--r-- | request/str.cpp | 126 | ||||
-rw-r--r-- | request/str.h | 49 | ||||
-rw-r--r-- | request/utils.cpp | 44 | ||||
-rw-r--r-- | request/utils.h | 39 | ||||
-rwxr-xr-x | segget/1 | 4 | ||||
-rw-r--r-- | segget/Makefile | 2 | ||||
-rw-r--r-- | segget/pkg.cpp | 30 | ||||
-rw-r--r-- | segget/pkg.h | 2 | ||||
-rw-r--r-- | segget/proxyfetcher.cpp | 160 | ||||
-rw-r--r-- | segget/requestserver.cpp | 121 | ||||
-rw-r--r-- | segget/requestserver.h | 45 | ||||
-rw-r--r-- | segget/segget.cpp | 88 | ||||
-rw-r--r-- | segget/segget.h | 7 | ||||
-rw-r--r-- | segget/settings.cpp | 3 | ||||
-rw-r--r-- | segget/settings.h | 6 |
32 files changed, 2860 insertions, 116 deletions
diff --git a/request/Makefile b/request/Makefile new file mode 100644 index 0000000..d7d45c7 --- /dev/null +++ b/request/Makefile @@ -0,0 +1,29 @@ +BINS = request +OBJS = $(addsuffix .o,$(BINS)) +PKGCONFIG_MODULES = json +MISSING_PLUGINS_LIBS = +LIBS_PKGCONFIG := $(foreach mod,$(PKGCONFIG_MODULES),$(shell pkg-config --libs $(mod))) +CXXFLAGS_PKGCONFIG := $(foreach mod,$(PKGCONFIG_MODULES),$(shell pkg-config --libs $(mod))) + +CXXFLAGS_WARNINGS = -pedantic -Wall -Wextra -Wformat -Weffc++ +#CXXFLAGS_WARNINGS = -pedantic -Wextra -Wformat -Weffc++ +CXXFLAGS_OPT = -O0 -g3 + +LIBS = $(LIBS_PKGCONFIG) +CXXFLAGS = $(CXXFLAGS_PKGCONFIG) $(CXXFLAGS_WARNINGS) $(CXXFLAGS_OPT) + +all: $(BINS) + +%.o: %.cxx + $(CXX) -c -o $@ $(CXXFLAGS) $^ + +request: request.o config.o phase.o response.o settings.o str.o utils.o + $(CXX) -o $@ $(LIBS) $(CIBS) $^ + +clean: + -rm -f $(BINS) + -rm -f $(OBJS) + -rm -f *~ + -rm -f *.o + +.PRECIOUS: %.o diff --git a/request/config.cpp b/request/config.cpp new file mode 100644 index 0000000..0cc5395 --- /dev/null +++ b/request/config.cpp @@ -0,0 +1,177 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "config.h" + +void Tconfig::load_settings_from_config_file(){ + try{ + ifstream file; + file.exceptions (ifstream::failbit | ifstream::badbit); + try{ + file.open((settings.conf_dir+"/"+config_file_name).c_str()); + } + catch(...){ + merror("Error opening settings file: "+settings.conf_dir+"/"+config_file_name+". Default settings will be used. Check if config file: "+config_file_name+" exists and segget has rights to access it."); + return; + } + try{ + //processing file + string line; + string name; + string value; + string inSection; + int posEqual; + while (not(file.eof())) { + getline(file,line); + if (! line.length()) continue; + if (line[0] == '#') continue; + if (line[0] == ';') continue; + line=noupper(line); + if (line[0] == '[') { + inSection=trim(line.substr(1,line.find(']')-1)); + continue; + } + posEqual=line.find('='); + name = trim(line.substr(0,posEqual)); + value = trim(line.substr(posEqual+1)); + content_[inSection+'/'+name]=noupper(value); + } + }catch(ifstream::failure e){ + if (!file.eof()){ + merror("Settings file: "+config_file_name+" was opened, but an error occured while reading it."); + return; + } + }catch(...){ + merror("Settings file: "+config_file_name+" was opened, but an error occured while reading settings from it."); + } + }catch(...){ + merror("Error in config.cpp: load_settings_from_config_file()"); + } +} + +int Tconfig::set(string const& section, string const& entry, string &dst) const { + try{ + map<string,string>::const_iterator ci = content_.find(section + '/' + entry); + if (ci == content_.end()){ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+" has not been set."); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+dst+". Default value forced."); + return 1; + } + else{ + dst=ci->second; + merror("Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+dst); + return 0; + } + }catch(...){ + merror("Error in config.cpp: set(string &dst, string const& section, string const& entry)"); + return 1; + } +} +int Tconfig::set(string const& section, string const& entry, ulong &dst, uint const& min_limit, uint const& max_limit) const { + try{ + uint return_value; + map<string,string>::const_iterator ci = content_.find(section + '/' + entry); + if (ci == content_.end()){ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+" has not been set."); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(dst)+". Default value forced."); + return 1; + } + else{ + return_value=atoi(ci->second.c_str()); + if (return_value==0) + if (toString(return_value)!=ci->second){ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry + +" must have an integer value in range from "+toString(min_limit) + +" to "+toString(max_limit) + +". Can't convert "+ci->second + +" to integer. "); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(dst)+". Default value forced."); + return 1; + } + if ((return_value>=min_limit) and (return_value<=max_limit)){ + //everything is ok + merror("Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(return_value)); + dst=return_value; + return 0; + }else{ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry + +" must have an integer value in range from "+toString(min_limit) + +" to "+toString(max_limit)+"."); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(dst)+". Default value forced."); + return 1; + } + } + }catch(...){ + merror("Error in config.cpp: set(ulong &dst, string const& section, string const& entry, uint const& min_limit, uint const& max_limit)"); + return 1; + } +} + +int Tconfig::set(string const& section, string const& entry, bool &dst) const { + try{ + uint return_value; + map<std::string,string>::const_iterator ci = content_.find(section + '/' + entry); + if (ci == content_.end()){ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+" has not been set."); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(dst)+". Default value forced."); + return 1; + } + else{ + return_value=atoi(ci->second.c_str()); + if (return_value==0) + if (toString(return_value)!=ci->second){ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry + +" must have a boolean value: 0 or 1" + +". Can't convert "+ci->second + +" to 0 or 1. "); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(dst)+". Default value forced."); + return 1; + } + if ((return_value==0) or (return_value==1)){ + //everything is ok + merror("Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(return_value)); + dst=return_value; + return 0; + }else{ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry + +" must have a boolean value: 0 or 1"); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(dst)+". Default value forced."); + return 1; + } + } + }catch(...){ + merror("Error in config.cpp: set(bool &dst, string const& section, string const& entry)"); + return 1; + } +} + +void Tconfig::clear(){ + try{ + content_.clear(); + }catch(...){ + merror("Error in config.cpp: clear()"); + } +}
\ No newline at end of file diff --git a/request/config.h b/request/config.h new file mode 100644 index 0000000..9853d4c --- /dev/null +++ b/request/config.h @@ -0,0 +1,54 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __CONFIG_H__ +#define __CONFIG_H__ + +#include <string> +#include <map> +#include <fstream> +#include "str.h" +#include "settings.h" +#include "utils.h" +using namespace std; + +class Tconfig { + private: + map<string,string> content_; + string config_file_name; + public: + Tconfig(string const& file_name): + content_(), + config_file_name("") + {config_file_name=file_name;load_settings_from_config_file();}; + void load_settings_from_config_file(); + int set(string const& section, string const& entry, string &dst) const; + int set(string const& section, string const& entry, ulong &dst, uint const& min_limit, uint const& max_limit) const; + int set(string const& section, string const& entry, bool &dst) const; + void clear(); +}; + +#endif
\ No newline at end of file diff --git a/request/distfile.cpp b/request/distfile.cpp new file mode 100644 index 0000000..21c3ff5 --- /dev/null +++ b/request/distfile.cpp @@ -0,0 +1,742 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "distfile.h" + +//Make the necessary includes and set up the variables: +using namespace std; + +/* +#define R_LM_WAIT_FOR_LOCAL_MIRRORS 100 + +#define R_PF_ADDED_TO_PROXY_QUEUE 101 +#define R_PF_ALREADY_WAS_IN_QUEUE 102 +#define R_PF_DOWNLOADED 103 +#define R_PF_BE_MORE_PATIENT 104 +#define R_PF_ERROR_ADDING_TO_PROXY_QUEUE 105 +#define R_PF_FAILED 106 +#define R_PF_REJECTED 107 + +// 0 for succesfull return of provide_segment() +#define R_R_DOWNLOAD_STARTED 0 +#define R_R_WAITING 108 +#define R_R_DOWNLOADING 109 + +#define R_LM_PF_R_NO_FREE_NETWORK_CONNECTION_FOUND 110 + +#define ALLOW_PROXY_FETCHER_NETWORKS 201 +#define DO_NOT_ALLOW_PROXY_FETCHER_NETWORKS 202 +#define ALLOW_REMOTE_NETWORKS 203 +#define DO_NOT_ALLOW_REMOTE_NETWORKS 204 +#define ALLOW_LOWER_PRIORITY_NETWORKS 205 +*/ + + +bool Tdistfile::allows_new_actions(){ +// if (downloaded) return false; +// else return true; +// int time_left=0; + if (status==DDOWNLOADED){ + debug("No new connection for distfile:"+name+". Distfile has DDOWNLOADED status"); + return false; + } + if (status==DFAILED){ + debug("No new connection for distfile:"+name+". Distfile has DFAILED status"); + return false; + } +// debug("Distfile "+Ppkg_array[pkg_num]->distfile_vector[distfile_num]->name+" allows new connections"); +// debug(" distfile_num:"+toString(distfile_num)); + if (active_connections_num<settings.max_connection_num_per_distfile){ + debug("Allow new connection for ============================================= Distfile "+name); + debug("max_connection limit has not been reached"); + return true; + }else{ + debug("No new connection for distfile: "+name+". It already has " + +toString(active_connections_num) + +" connections => choose another distfile."); + return false; + } +// if (((status==DPROXY_QUEUED) || (status==DPROXY_DOWNLOADING)) && (time_left<100)) return false; +//oterwise allow connections +// DNEW, +// D_NOT_PROXY_REQUESTED, +// DPROXY_REJECTED, +// DPROXY_FAILED, +// DPROXY_DOWNLOADED, +// DWAITING, +// DDOWNLOADING, +} + +void Tdistfile::init(){ + try{ + for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){ + network_distfile_brokers_array[network_num].init(network_num); + } + }catch(...){ + error_log("Error: distfile.cpp: init()"); + } +} +void Tdistfile::load_url_list(json_object* json_array_distfile_urllist){ + try{ + url_count=json_object_array_length(json_array_distfile_urllist); + url_list= new string [url_count]; + for(uint urllist_array_item_num=0;urllist_array_item_num<url_count;urllist_array_item_num++){ + url_list[urllist_array_item_num]=json_object_get_string(json_object_array_get_idx(json_array_distfile_urllist,urllist_array_item_num)); + } + if (url_count){ + num=++stats.distfiles_count; + stats.inc_total_size(size); + } + }catch(...){ + error_log("Error: distfile.cpp: load_url_list()"); + } +} + +bool Tdistfile::check_if_dld(){ + try{ + ifstream filec((settings.distfiles_dir+"/"+name).c_str()); + ulong start = filec.tellg(); + filec.seekg (0, ios::end); + ulong end = filec.tellg(); + ulong d_size; + d_size = end - start; + //debug("seg:"+toString(seg_num)+" Dsize="+toString(downloaded_size)+" seg_size="+toString(segment_size)); + filec.close(); + if (d_size==size){ + status=DDOWNLOADED; +// downloaded=true; + num=++stats.distfiles_count; + stats.inc_total_size(size); + stats.inc_dld_distfiles_count(); + stats.inc_dld_size(size); + debug("Distfile:"+name+" already downloaded"); + return true; + } + else{ + debug("Distfile:"+name+" not downloaded"); + return false; + } + }catch(...){ + error_log("Error: distfile.cpp: check_if_dld()"); + return false; + } +} + +void Tdistfile::load_distfile_from_json(json_object* json_obj_distfile){ + try{ + name=json_object_get_string(json_object_object_get(json_obj_distfile,"name")); + size=atoi(json_object_to_json_string(json_object_object_get(json_obj_distfile,"size"))); + RMD160=json_object_get_string(json_object_object_get(json_obj_distfile,"RMD160")); + SHA1=json_object_get_string(json_object_object_get(json_obj_distfile,"SHA1")); + SHA256=json_object_get_string(json_object_object_get(json_obj_distfile,"SHA256")); + if (not(check_if_dld())){ + json_data=json_object_to_json_string(json_obj_distfile); + split_into_segments(); + load_url_list(json_object_object_get(json_obj_distfile,"url_list")); + } + }catch(...){ + error_log("Error: distfile.cpp: load_distfile_from_json()"); + } +} + +void Tdistfile::split_into_segments(){ + try{ + int range_end; + if (size>settings.max_segment_size){ + segments_count=size/settings.max_segment_size; + // set segments to have equal size, only last segment may be somewhat large (up to segment_count bytes large) + segment_size=size/segments_count; + } + else{ + segment_size=size; + segments_count=1; + }; + dn_segments = new Tsegment [segments_count]; + //none downloaded yet + for (uint segment_num=0; segment_num<segments_count; segment_num++){ + if (segment_num == (segments_count-1)){ + range_end=size-1; + debug("last_segment ends"+toString(range_end)); + } + else{ + range_end=(segment_num+1)*segment_size-1; + debug("segment_range:"+toString(range_end)); + }; + dn_segments[segment_num].set_segment(this, segment_num, name, segment_size, range_end); + if (dn_segments[segment_num].status==SDOWNLOADED) + inc_dld_segments_count(&dn_segments[segment_num]); + } + }catch(...){ + error_log("Error: distfile.cpp: split_into_segments()"); + } +} + +Tdistfile::~Tdistfile(){ + try{ + delete [] url_list; + }catch(...){ + error_log("Error: distfile.cpp: ~Tdistfile()"); + } +} +bool Tdistfile::choose_best_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num){ + try{ + Tmirror *Pcurr_mirror; + Tmirror *Pbest_mirror=0; // the best isn't set let's find it + uint best_mirror_num=-1; // the best isn't set let's find it + + ulong best_mirror_self_rating=-1; + ulong curr_mirror_self_rating; + + for (url_num=0; url_num<url_count; url_num++){ + Pcurr_mirror=find_mirror(strip_mirror_name(url_list[url_num])); + debug("Evaluating url:"+url_list[url_num] + +" active_connections:"+toString(Pcurr_mirror->get_active_num()) + +" connection limit:"+toString(settings.max_connections_num_per_mirror)); + if (Pcurr_mirror->get_active_num()<settings.max_connections_num_per_mirror){ + debug("Url satisfies connection num per mirror limit"); + curr_mirror_self_rating=Pcurr_mirror->mirror_on_the_wall(); + if (curr_mirror_self_rating<best_mirror_self_rating){ + best_mirror_num=url_num; + best_mirror_self_rating=curr_mirror_self_rating; + Pbest_mirror=Pcurr_mirror; + } + if (best_mirror_self_rating==0) + // 0 can not be improved - it's one of the best + break; + } + } + if (Pbest_mirror){ + debug("Downloading from BEST_MIRROR:"+url_list[best_mirror_num]); +// Pbest_mirror->start(); +// active_connections_num++; + connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num); + return R_R_DOWNLOAD_STARTED; + } + else{ + error_log("Can't choose mirror for segment:"+dn_segments[seg_num].file_name); + return 1; + } + }catch(...){ + error_log("Error: distfile.cpp: choose_best_mirror()"); + return 1; + } +} +bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num){ + try{ + long best_mirror_num=-1; // the best isn't set let's find it + bool all_mirrors_failed=true; + ulong best_mirror_self_rating=-1; + ulong curr_mirror_self_rating; + debug("Choosing mirror for network"+toString(network_num)); + for (ulong mirror_num=0; mirror_num<network_array[network_num].benchmarked_mirror_list.size(); mirror_num++){ + debug("Evaluating mirror:"+network_array[network_num].benchmarked_mirror_list[mirror_num].url); + if (network_distfile_brokers_array[network_num].mirror_fails_vector[mirror_num]){ + debug("mirror:"+network_array[network_num].benchmarked_mirror_list[mirror_num].url+" has FAILED status - will be skipped"); + }else{ + all_mirrors_failed=false; + if (network_array[network_num].benchmarked_mirror_list[mirror_num].get_active_num()<settings.max_connections_num_per_mirror){ + curr_mirror_self_rating=network_array[network_num].benchmarked_mirror_list[mirror_num].mirror_on_the_wall(); + if (curr_mirror_self_rating<best_mirror_self_rating){ + best_mirror_num=mirror_num; + best_mirror_self_rating=curr_mirror_self_rating; + } + if (best_mirror_self_rating==0) + // 0 can not be improved - it's one of the best + break; + }else{ + debug("Mirror already has:" + +toString(network_array[network_num].benchmarked_mirror_list[mirror_num].get_active_num()) + +" connections which doesn't meet limit:"+toString(settings.max_connections_num_per_mirror)); + } + } + } + if (best_mirror_num!=-1){ + debug("Downloading from BEST_LOCAL_MIRROR:"+network_array[network_num].benchmarked_mirror_list[best_mirror_num].url); +// active_connections_num++; + connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num); + return R_R_DOWNLOAD_STARTED; + } + else{ + debug("Can't choose LOCAL mirror for segment:"+dn_segments[seg_num].file_name); + error_log("Can't choose LOCAL mirror for segment:"+dn_segments[seg_num].file_name); + if (all_mirrors_failed){ + debug("All local mirrors failed in network#"+toString(network_num)); + error_log("All local mirrors failed in network#"+toString(network_num)); + } + return 1; + } + }catch(...){ + error_log("Error: distfile.cpp: choose_best_local_mirror()"); + return 1; + } +} + +uint Tdistfile::provide_local_network(CURLM* cm, uint connection_num, uint seg_num, uint network_priority){ + try{ + debug("network_priority="+toString(network_priority)); + //choose network +//---------------------------------------------------------------------------------------------------------- +// +// Several criterions can be used here to choose among networks with equal priority: +// min_active_connection, min_ratio_active_to_max_connection, best_speed_accocding_to_stats, etc +// add these options to segget.conf file +// +//---------------------------------------------------------------------------------------------------------- + int best_local_network_num=-1; + bool allow_proxy_fetcher_mirrors=true; + for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){ + if (network_array[network_num].priority==network_priority){ + debug(" network_priority="+toString(network_array[network_num].priority)); + if (network_array[network_num].network_mode==MODE_LOCAL){ + debug(" network#:"+toString(network_num)+" has MODE_LOCAL"); + debug(" phase#"+phaseToString(network_distfile_brokers_array[network_num].phase)); + if (network_distfile_brokers_array[network_num].phase==E_USE_AS_LOCAL_MIRRORS){ + if (network_array[network_num].has_free_connections()){ + debug(" has free connections"); +// debug(" Allowed network#:"+toString(network_num)); + if ((best_local_network_num==-1) + or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){ + best_local_network_num=network_num; + debug(" Replace best LOCAL network to network#:"+toString(network_num)); + } + }else{ + debug(" does NOT have free connections"); + if (network_array[network_num].only_local_when_possible){ + allow_proxy_fetcher_mirrors=false; + debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed"); + } + } + } + } + if (network_array[network_num].network_mode==MODE_PROXY_FETCHER){ + debug(" network#:"+toString(network_num)+" has MODE_PROXY_FETCHER"); + debug(" with phase:"+phaseToString(network_distfile_brokers_array[network_num].phase)); + if ((network_distfile_brokers_array[network_num].phase==E_USE_AS_LOCAL_MIRRORS) + or (network_distfile_brokers_array[network_num].phase==E_PROXY_FETCHER_DOWNLOADED)) + { + debug("phase fits"); + if (network_array[network_num].has_free_connections()){ + debug(" has free connections"); +// debug(" Allowed network#:"+toString(network_num)); + if ((best_local_network_num==-1) + or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){ + best_local_network_num=network_num; + debug(" Replace best LOCAL network to network#:"+toString(network_num)); + } + }else{ + if (network_array[network_num].only_local_when_possible){ + allow_proxy_fetcher_mirrors=false; + debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed"); + } + } + } + } + } + } + if (best_local_network_num!=-1){ + //best network has been found + //work with network + debug(" So best LOCAL network is network#:"+toString(best_local_network_num)); + return choose_best_local_mirror(cm, connection_num, best_local_network_num, seg_num); + }else{ + if (allow_proxy_fetcher_mirrors){ + return ALLOW_REQUESTS_TO_PROXY_FETCHERS; + }else{ + debug("NOT all local mirrors have failed - restricted to local mirrors only."); + return DO_NOT_ALLOW_REQUESTS_TO_PROXY_FETCHERS; + } + } + }catch(...){ + error_log("Error: distfile.cpp: provlocal_networkwork()"); + return 1; + } +} + +uint Tdistfile::request_proxy_fetcher_network(uint network_priority){ +// TO-DO: Add option to go for remote networks or low priority networks even if already DPROXY_QUEUED +// TO-DO: There can be several proxy-fetchers of the same priority level, define a rule to make a choice + try{ + debug("network_priority="+toString(network_priority)); + //choose network + int best_proxy_fetcher_network_num=-1; + bool allow_remote_mirrors=true; + for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){ + if (network_array[network_num].priority==network_priority){ + if (network_array[network_num].network_mode==MODE_PROXY_FETCHER){ + //replace this one by does_not_reject_connections +// if (network_array[network_num].has_free_connections()){ + if ((network_distfile_brokers_array[network_num].phase==E_USE_AS_LOCAL_MIRRORS) + or (network_distfile_brokers_array[network_num].phase==E_ALL_LOCAL_MIRRORS_FAILED)){ + debug("------//////////// Good phase !!!!!!!!!!!!!!!"); + decode_server_response(toString(network_distfile_brokers_array[network_num].proxy_fetcher_response)); + switch (network_distfile_brokers_array[network_num].proxy_fetcher_response){ + case R_PF_DOWNLOADED:break; + case R_PF_ERROR_ADDING_TO_PROXY_QUEUE:break; + case R_PF_FAILED:break; + case R_PF_REJECTED:break; + default:{ + //if less then 30 secs left don't bother proxy-fetcher + if (2000>time_left_from(network_distfile_brokers_array[network_num].last_request_time)){ + debug("====================== Be more patient with proxy-fetcher - network#:"+toString(network_num) + +" because time left:"+toString(time_left_from(network_distfile_brokers_array[network_num].last_request_time))); + return R_PF_BE_MORE_PATIENT; + }else{ +// (network_array[best_proxy_fetcher_network_num].active_connections_num>network_array[network_num].active_connections_num) + best_proxy_fetcher_network_num=network_num; + debug("-------------------------------"); + debug(" Replace best_proxy_fetcher network_num to network#:"+toString(best_proxy_fetcher_network_num)); + }; + break; + } + } + } + if (best_proxy_fetcher_network_num!=-1) break; +/* + }else{ + if (network_array[network_num].only_local_when_possible){ + if (network_distfile_brokers_array[network_num].some_mirrors_have_NOT_failed_yet()){ + allow_remote_mirrors=false; + debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed"); + } + } + } +*/ + } + } + } + if (best_proxy_fetcher_network_num != -1){ + // request from proxy fethcer + network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response=request(best_proxy_fetcher_network_num, json_data); + if ((network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response==R_PF_DOWNLOADED) + and (network_distfile_brokers_array[best_proxy_fetcher_network_num].phase==E_ALL_LOCAL_MIRRORS_FAILED)) + { + network_distfile_brokers_array[best_proxy_fetcher_network_num].phase=E_PROXY_FETCHER_DOWNLOADED; + } + debug("Trying to dowload distfile"+name+" via proxy_fetcher. status="+toString(status)); + if (network_distfile_brokers_array[best_proxy_fetcher_network_num].phase==E_PROXY_FETCHER_DOWNLOADED){ + // start download from the proxy_fetcher + debug(" So best proxy_fetcher_network is network#:"+toString(best_proxy_fetcher_network_num)); + return R_PF_DOWNLOADED; + //choose_best_local_mirror(cm, connection_num, best_proxy_fetcher_network_num, seg_num); + } + //return - don't switch to low priority networks + return network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response; + }else{ + if (allow_remote_mirrors){ + debug(" So best_proxy_fetcher_failed num="+toString(best_proxy_fetcher_network_num)); + return ALLOW_REMOTE_NETWORKS; + }else{ + debug("NOT all proxy-fetcher mirrors have failed - restricted to proxy-fetcher mirrors only."); + return DO_NOT_ALLOW_REMOTE_NETWORKS; + } + } + }catch(...){ + error_log("Error: distfile.cpp: provproxy_fetcher_networkwork()"); + return 1; + } +} + +uint Tdistfile::provide_remote_network(CURLM* cm, uint connection_num, uint seg_num, uint network_priority){ + try{ + debug("network_priority="+toString(network_priority)); + //choose network +//---------------------------------------------------------------------------------------------------------- +// +// Several criterions can be used here to choose among networks with equal priority: +// min_active_connection, min_ratio_active_to_max_connection, best_speed_accocding_to_stats, etc +// add these options to segget.conf file +// +//---------------------------------------------------------------------------------------------------------- + int best_remote_network_num=-1; + for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){ + if (network_array[network_num].priority==network_priority){ + if (network_array[network_num].network_mode==MODE_REMOTE){ + if (network_array[network_num].has_free_connections()){ + if + ((best_remote_network_num==-1) + or + (network_array[best_remote_network_num].active_connections_num>network_array[network_num].active_connections_num)){ + best_remote_network_num=network_num; + debug(" Replace best_remote_network_num to network#:"+toString(network_num)); + } + } + } + } + } + if (best_remote_network_num!=-1){ + //best network has been found + //work with network + debug(" So best network is network#:"+toString(best_remote_network_num)); + return choose_best_mirror(cm, connection_num, best_remote_network_num, seg_num); + } + // haven't found anything suitable + return ALLOW_LOWER_PRIORITY_NETWORKS; + }catch(...){ + error_log("Error: distfile.cpp: provide_remote_network()"); + return 1; + } +} + +int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ + try{ + int result=R_NOT_SET; + for (uint network_priority=10; network_priority>0; network_priority--){ + do { + debug("Switching to local networks with priority:"+toString(network_priority)); + result=provide_local_network(cm, connection_num, seg_num, network_priority); + if (result==ALLOW_REQUESTS_TO_PROXY_FETCHERS){ + debug("Requesting proxy-fetcher networks with priority:"+toString(network_priority)); + result=request_proxy_fetcher_network(network_priority); + switch (result){ + case ALLOW_REMOTE_NETWORKS:{ + debug("Switching to remote networks with priority:"+toString(network_priority)); + result=provide_remote_network(cm, connection_num, seg_num, network_priority); + // if not lower_priority_networks => result found => return it + if (result!=ALLOW_LOWER_PRIORITY_NETWORKS){ + return result; + }; + break; + } + // don't switch to remote and lower priority networks => return results + case DO_NOT_ALLOW_REMOTE_NETWORKS: return result; + case R_PF_DOWNLOADED: {debug("Yep, R_PF_DONWLOADED => LOOP AGAIN"); break;}; + // if already in queue, etc... + // don't switch to remote and lower priority networks => return results + default: return result; + } + }else{ + // don't switch to proxy-fetcher requests, remote, + // and lower priority networks => return results + return result; + } + // try to download from local mirrors again because one of the proxy-fetcher + // claims that distfile has been donwloaded + debug("----- JUST BEFORE THE LOOP ----"); + }while (result==R_PF_DOWNLOADED); + } + // haven't found anything suitable + return R_LM_PF_R_NO_FREE_NETWORK_CONNECTION_FOUND; + }catch(...){ + error_log("Error: distfile.cpp: provide_segment()"); + return 1; + } +} +void Tdistfile::inc_dld_segments_count(Tsegment* current_segment){ + try{ + stats.inc_dld_size(current_segment->segment_size); + if (++dld_segments_count==segments_count) + combine_segments(); + }catch(...){ + error_log("Error: distfile.cpp: inc_dld_segments_count()"); + } +} +long is_symlink_restricted(string distfile_name){ + for(ulong pattern_num=0; pattern_num<settings.provide_mirror_files_restricted_patterns_vector.size(); pattern_num++){ + if (distfile_name.find(settings.provide_mirror_files_restricted_patterns_vector[pattern_num],0)!=distfile_name.npos){ + return pattern_num; + } + } + return -1; +} + +void Tdistfile::symlink_distfile_to_provide_mirror_dir(){ + try{ + long pattern_num=is_symlink_restricted(name); + if (pattern_num>=0){ + log("Symlink to distfile:"+name+" was restricted by pattern <" + +settings.provide_mirror_files_restricted_patterns_vector[pattern_num] + +"> from line "+toString(pattern_num+1)+" of restrict.conf file"); + return; + } + string new_mirror_name; + string old_distfile_name; + try{ + string old_distfile_path; + char current_path[FILENAME_MAX]; + if (!GetCurrentDir(current_path, sizeof(current_path))) + { + return; + } + if (settings.distfiles_dir.find("./",0)==0){ + old_distfile_path=current_path+settings.distfiles_dir.substr(1,settings.distfiles_dir.npos); + }else{ + old_distfile_path=settings.distfiles_dir; + } + new_mirror_name=settings.provide_mirror_dir+"/"+name; + old_distfile_name=old_distfile_path+"/"+name; + try{ + if (!symlink(old_distfile_name.c_str(), new_mirror_name.c_str())){ + log("Distfile:"+old_distfile_path+" was symlinked to the mirror dir:"); + }; + }catch(uint errno){ + switch (errno){ + case EACCES : error_log("Write access to the directory containing "+settings.provide_mirror_dir+" is denied, or one of the directories in the path prefix of "+settings.provide_mirror_dir+" did not allow search permission. (See also path_resolution(7)."); + break; + case EEXIST : error_log("There is already an existing file named "+new_mirror_name+"."); + break; + case EFAULT : error_log(old_distfile_path+" or "+settings.provide_mirror_dir+" points outside your accessible address space."); + break; + case EIO : error_log("A hardware error occurred while reading or writing data on the disk."); + break; + case ELOOP : error_log("Too many symbolic links were encountered in resolving "+settings.provide_mirror_dir+"."); + break; + case ENAMETOOLONG : error_log(old_distfile_path+" or "+settings.provide_mirror_dir+" was too long."); + break; + case ENOENT : error_log("A directory component in "+settings.provide_mirror_dir+" does not exist or is a dangling symbolic link, or "+old_distfile_path+" is the empty string."); + break; + case ENOMEM : error_log("Insufficient kernel memory was available."); + break; + case ENOSPC : error_log("The device containing the file has no room for the new directory entry."); + break; + case ENOTDIR : error_log("A component used as a directory in "+settings.provide_mirror_dir+" is not, in fact, a directory."); + break; + case EPERM : error_log("The file system containing "+settings.provide_mirror_dir+" does not support the creation of symbolic links."); + break; + case EROFS : error_log("The file "+new_mirror_name+" would exist on a read-only file system."); + break; + default: + error_log("Undocumented error while trying to symlink "+old_distfile_name+" to "+new_mirror_name); + } + }catch(...){ + error_log("Undocumented error (error description is not an integer) while trying to symlink "+old_distfile_name+" to "+new_mirror_name); + } + }catch(...){ + error_log("Error in distfile.cpp :: symlink_distfile_to_provide_mirror_dir() while trying to symlink "+old_distfile_name+" to "+new_mirror_name); + } + }catch(...){ + error_log("Error: distfile.cpp: symlink_distfile_to_provide_mirror_dir()"); + } +} +int Tdistfile::combine_segments(){ + try{ + debug("Combining distfile"+name); + ofstream distfile_file; + distfile_file.exceptions (ofstream::failbit | ofstream::badbit); + string distfile_path=settings.distfiles_dir+"/"+name; + try{ + distfile_file.open(distfile_path.c_str(),ofstream::binary|ios::trunc); + }catch(...){ + error_log("Error: distfile.cpp: combine_segments(): opening distfile:"+distfile_path); + status=DFAILED; + return 1; + } + char * buffer; + ulong cur_seg_size; + try{ + for (uint seg_num=0; seg_num < segments_count; seg_num++){ + debug("Joining "+name+" segment "+toString(seg_num)+" "); + ifstream segment_file; + segment_file.exceptions (ofstream::failbit | ofstream::badbit); + try{ + segment_file.open((settings.segments_dir+"/"+dn_segments[seg_num].file_name).c_str(),ifstream::binary); + }catch(...){ + error_log("Error: distfile.cpp: combine_segments(): opening segment:"+settings.segments_dir+"/"+dn_segments[seg_num].file_name); + status=DFAILED; + return 2; + } + try{ + // get size of file + ulong start=segment_file.tellg(); + segment_file.seekg(0,ifstream::end); + ulong end=segment_file.tellg(); + cur_seg_size=end-start; + segment_file.seekg(0); + debug(">>>>>SEG:"+dn_segments[seg_num].file_name+" Start: "+toString(start)+" End: "+toString(end)+" Size: "+toString(cur_seg_size)); + // allocate memory for file content + buffer = new char [cur_seg_size]; + // read content of infile + segment_file.read (buffer,cur_seg_size); + segment_file.close(); + }catch(...){ + error_log("Error: distfile.cpp: combine_segments(): segment is open, but error occured while reading it:"+settings.segments_dir+"/"+dn_segments[seg_num].file_name); + status=DFAILED; + return 3; + } + try{ + // write to outfile + distfile_file.write (buffer,cur_seg_size); + }catch(...){ + error_log("Error: distfile.cpp: combine_segments(): distfile is open, but error occured while writing into it:"+settings.distfiles_dir+"/"+name); + status=DFAILED; + return 4; + } + // release dynamically-allocated memory + delete[] buffer; + if(remove((settings.segments_dir+"/"+dn_segments[seg_num].file_name).c_str()) != 0 ) + error_log("Tdistfile::combine_segments() - Error: can't delete:"+settings.segments_dir+"/"+dn_segments[seg_num].file_name); + else + debug(settings.segments_dir+"/"+dn_segments[seg_num].file_name+" deleted" ); + } + + distfile_file.close(); + stats.inc_dld_distfiles_count(); + log("Distfile "+name+" has been combined"); + }catch(...){ + error_log("Error in distfile.cpp: combine_segments() for distfile:"+settings.distfiles_dir+"/"+name); + status=DFAILED; + return 5; + } + try{ + if (rmd160_ok(settings.distfiles_dir+"/"+name,RMD160)) + log("RMD160 checksum for distfile:"+name+" is [OK]"); + else{ + log("Error: RMD160 checksum for distfile:"+name+" [FAILED]"); + error_log("Error: RMD160 checksum for distfile:"+name+" [FAILED]"); + status=DFAILED; + return 10; + } + + if (sha1_ok(settings.distfiles_dir+"/"+name,SHA1)) + log("SHA1 checksum for distfile:"+name+" is [OK]"); + else{ + log("Error: SHA1 checksum for distfile:"+name+" [FAILED]"); + error_log("Error: SHA1 checksum for distfile:"+name+" [FAILED]"); + status=DFAILED; + return 11; + } + + if (sha256_ok(settings.distfiles_dir+"/"+name,SHA256)) + log("SHA256 checksum for distfile:"+name+" is [OK]"); + else{ + log("Error: SHA256 checksum for distfile:"+name+" [FAILED]"); + error_log("Error: SHA256 checksum for distfile:"+name+" [FAILED]"); + status=DFAILED; + return 12; + } + status=DDOWNLOADED; + if (settings.provide_mirror_dir!="none"){ + symlink_distfile_to_provide_mirror_dir(); + } + }catch(...){ + error_log("Error: distfile.cpp: combine_segments() for segment:"+settings.distfiles_dir+"/"+name+" while checking checksums."); + return 30; + } + }catch(...){ + error_log("Error: distfile.cpp: combine_segments() for segment:"+settings.distfiles_dir+"/"+name+" during procedure."); + return 31; + } + return 0; +}
\ No newline at end of file diff --git a/request/distfile.h b/request/distfile.h new file mode 100644 index 0000000..8d42894 --- /dev/null +++ b/request/distfile.h @@ -0,0 +1,155 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __DISTFILE_H__ +#define __DISTFILE_H__ +#include <errno.h> +#include <stdlib.h> +#include <string.h> +#ifndef WIN32 +# include <unistd.h> +#endif +#include <curl/multi.h> + +#include <stdio.h> +#include <stdlib.h> +#include <fstream> +#include <iostream> +#include <json/json.h> +#ifdef WINDOWS + #include <direct.h> + #define GetCurrentDir _getcwd +#else + #include <unistd.h> + #define GetCurrentDir getcwd +#endif +#include "segment.h" +#include "mirror.h" +#include "checksum.h" +#include "network.h" +#include "networkbroker.h" +#include "segment.h" +#include "response.h" +class Tsegment; + +//#include <sys/types.h> +//#include <sys/socket.h> +//#include <stdio.h> +//#include <netinet/in.h> +//#include <arpa/inet.h> +//#include <sys/ioctl.h> +//#include <unistd.h> +//#include <stdlib.h> +//#include <string> +//#include <string.h> + + +//#include "client.cpp" + +using namespace std; + +typedef unsigned int uint; + +enum Tdistfile_status{ + DNEW, + D_NOT_PROXY_REQUESTED, + DPROXY_REJECTED, + DPROXY_QUEUED, + DPROXY_DOWNLOADING, + DPROXY_DOWNLOADED, + DPROXY_FAILED, + DWAITING, + DDOWNLOADING, + DDOWNLOADED, + DFAILED +}; + +long is_symlink_restricted(string distfile_name); + +class Tdistfile{ + private: + uint dld_segments_count; + bool choose_best_local_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num); + bool choose_best_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num); + public: + Tnetwork_distfile_broker network_distfile_brokers_array[MAX_NETWORKS]; + string json_data; +// bool downloaded; + Tdistfile_status status; + uint active_connections_num; + string *url_list; + uint url_num; + uint segment_num; + uint segments_count; + Tsegment *dn_segments; + string name; + uint num; + ulong size; + string RMD160; + string SHA1; + string SHA256; + uint url_count; + uint segment_size; + Tdistfile(): + dld_segments_count(0), + json_data(""), +// downloaded(0), + status(DNEW), + active_connections_num(0), + url_list(0), + url_num(0), + segment_num(0), + segments_count(0), + dn_segments(0), + name(""), + num(0), + size(0), + RMD160(""), + SHA1(""), + SHA256(""), + url_count(0), + segment_size(settings.max_segment_size) + {init();}; + Tdistfile(const Tdistfile &L); // copy constructor + Tdistfile & operator=(const Tdistfile &L); + ~Tdistfile(); + int request(ulong network_num, string msg); + void init(); + bool allows_new_actions(); + void load_distfile_from_json(json_object* json_obj_distfile); + void load_url_list(json_object* json_array_distfile_urllist); + void split_into_segments(); + uint provide_local_network(CURLM* cm, uint connection_num, uint seg_num, uint network_priority); + uint request_proxy_fetcher_network(uint network_priority); + uint provide_remote_network(CURLM* cm, uint connection_num, uint seg_num, uint network_priority); + int provide_segment(CURLM* cm, uint connection_num, uint seg_num); + void inc_dld_segments_count(Tsegment * current_segment); + void symlink_distfile_to_provide_mirror_dir(); + int combine_segments(); + bool check_if_dld(); +}; + +#endif
\ No newline at end of file diff --git a/request/phase.cpp b/request/phase.cpp new file mode 100644 index 0000000..04aef35 --- /dev/null +++ b/request/phase.cpp @@ -0,0 +1,37 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "phase.h" + +string phaseToString(int phase_num){ + switch(phase_num){ + case E_USE_AS_LOCAL_MIRRORS: return "E_USE_AS_LOCAL_MIRRORS"; + case E_ALL_LOCAL_MIRRORS_FAILED: return "E_ALL_LOCAL_MIRRORS_FAILED"; + case E_PROXY_FETCHER_DOWNLOADED: return "E_PROXY_FETCHER_DOWNLOADED"; + case E_ALL_PROXY_FETCHER_MIRRORS_FAILED: return "E_ALL_PROXY_FETCHER_MIRRORS_FAILED"; + default: return "Unknown phase"; + } +} diff --git a/request/phase.h b/request/phase.h new file mode 100644 index 0000000..221c33d --- /dev/null +++ b/request/phase.h @@ -0,0 +1,43 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __PHASE_H__ +#define __PHASE_H__ + +#include <string> + +using namespace std; + +enum Tnetwork_distfile_broker_phases{ + E_USE_AS_LOCAL_MIRRORS, + E_ALL_LOCAL_MIRRORS_FAILED, + E_PROXY_FETCHER_DOWNLOADED, + E_ALL_PROXY_FETCHER_MIRRORS_FAILED +}; + +string phaseToString(int phase_num); + +#endif
\ No newline at end of file diff --git a/request/pkg.h b/request/pkg.h new file mode 100644 index 0000000..a7a20b6 --- /dev/null +++ b/request/pkg.h @@ -0,0 +1,67 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ +#ifndef __PKG_H__ +#define __PKG_H__ + +#include <stdio.h> +#include <stdlib.h> +#include <fstream> +#include <iostream> +#include <json/json.h> +#include "distfile.h" + +using namespace std; +typedef unsigned int uint; +typedef Tdistfile* Pdistfile; + +#define Q_ADDED_TO_QUEUE 0 +#define Q_ALREADY_EXISTS 1 +#define Q_ERROR 100 + +class Tpkg{ + public: + string name; + string category; + uint distfile_count; + void load_distfile_list(json_object* json_array_distfile_list); + Tpkg(): + Pdistfile_list(), + name(""), + category(""), + distfile_count(0) + {}; + Tpkg(const Tpkg &L); // copy constructor + Tpkg & operator=(const Tpkg &L); + ~Tpkg(); + int push_back_distfile(json_object* json_obj_distfile); + int try_adding_distfile_to_proxy_fetchers_queue(json_object* json_obj_distfile); + void load_pkg_from_json(json_object* json_obj_pkg); +}; + +extern Tpkg **Ppkg_array; +extern Tpkg proxy_fetcher_pkg; + +#endif
\ No newline at end of file diff --git a/request/request.conf b/request/request.conf new file mode 100644 index 0000000..61dac25 --- /dev/null +++ b/request/request.conf @@ -0,0 +1,288 @@ +[folders] +# DISTFILES_DIR +# Define a dir to store distfiles +# Default: +# distfiles_dir=./distfiles +distfiles_dir=./distfiles + +# SEGMENTS_DIR +# Define a dir to store distfiles' segments +# Default: +# segments_dir=./tmp +segments_dir=./tmp + +[pkg_list] +# PKG_LIST_DIR +# Define a dir with pkg.list file +# Default: +# pkg_list_dir=./ +pkg_list_dir=./ + +# SYNOPSIS: del_pkg_list_when_dld_finished=0 | 1 +# - If del_pkg_list_when_dld_finished set to 1: +# Segget deletes pkg.list file, after all distfiles were successfuly fetched. +# Default: +# del_pkg_list_when_dld_finished=1 +del_pkg_list_when_dld_finished=1 + +[portage] +# SYNOPSIS: fetch_only=0 | 1 +# NOT IMPLEMENTED YET: fetch_only=0 + +[distfiles] +# MAX_CONNECTION_NUM_PER_DISTFILE +# Each distfile can have up to max_connection_num_per_distfile simultaneous +# connections. +# Minimum value: 1 +# Maximum value: 20 +# Default: +# max_connection_num_per_distfile=3 +max_connection_num_per_distfile=3 + +[segments] +# MAX_SEGMENT_SIZE +# Define maximum segment size in bytes. +# Minimum value: 10000 +# Maximum value: 10000000 +# Default: +# max_segment_size=500000 +max_segment_size=500000 + +# SYNOPSIS: resume_on=0 | 1 +# - If resume_on set to 1: +# Before starting downloading a segment segget checks whether this segment is +# already downloaded, checks segments size and if size mathces considers this +# segment to be downloaded and skips downloading process. +# - If resume_on set to 0: +# Segget always starts new fetch for a segment regardless of the fact whether it +# is downloaded or not. +# Default: +# resume_on=1 +resume_on=1 + +# MAX_TRIES +# If segment download was unsuccessful, new attempts are made. When attempts +# number reaches max_tries, segment gets FAILED status and error logged to error_log +# Minimum value: 1 +# Maximum value: -1 (for no limit) +# Default: +# max_tries=10 +max_tries=10 + +[connections] +# MAX_CONNECTIONS +# Define maximum number of connections +# Minimum value: 1 +# Maximum value: 20 +# Default: +# max_connections=10 +max_connections=10 + +# CURRENT_SPEED_TIME_INTERVAL_MSECS +# segget transfers may have bursty nature of their traffic. Therefore, while +# measuring current speed, segget actually calculates average speed during +# current_speed_time_interval_msecs time interval, defined in milliseconds. +# Minimum value: 100 +# Maximum value: 60000 +# Default: +# current_speed_time_interval_msecs=1000 +current_speed_time_interval_msecs=1000 + +# NOT IMPLEMENTED YET: max_total_speed=50000 + +[mirrors] +# MAX_CONNECTIONS_NUM_PER_MIRROR +# Define how many simultaneous downloads from one mirror segget is allowed to +# have. While choosing a mirror segget will skip mirrors with +# max_connections_num_per_mirror active downloads. +# Minimum value: 1 +# Maximum value: 10 +# Default: +# max_connections_num_per_mirror=1 +max_connections_num_per_mirror=1 + +# SYNOPSIS: collect_benchmark_stats_on=0 | 1 +# - If set to 1, stats on mirrors performance will be collected. +# default: +# collect_benchmark_stats_on=1 (Note: at the moment can NOT be changed) +collect_benchmark_stats_on=1 + +# SYNOPSIS: use_benchmark_stats=0 | 1 +# If use_benchmark_stats=1 statistics on mirrors is used to rate them +# and therefore improve performance. +# Each time connection from a particular mirror closes mirror->dld_time, +# and mirror->dld_size get increased (in case of unsuccessful connection +# only time gets increased), so avg speed for a mirror can be +# calculated: +# +# mirror->avg_speed=mirror->dld_size/mirror->dld_time. (1) +# +# When new segment is going to be started segget goes through the list of +# the mirrors distfile/segment has, and asks each mirror for self_rating: +# +# "ulong self_rating=mirror->mirror_on_the_wall();". +# +# This way segget chooses a mirror with the best self_rating. To calculate +# self_rating mirrors use the following formula: +# +# self_rating=dld_time/dld_size*honesty. (2) +# +# So mirrors actually say how bad they are. +# Even mirrors can have critical times, so to give mirrors another chance +# honesty was added to the formula (2). honesty can get values in +# interval (0,1]. Each time connection from a mirror opens or closes mirror +# sets its honesty=1. If mirror was asked for self_evaluation with +# mirror->mirror_on_the_wall(), but wasn't chosen its honesty decreases +# somewhat (see [mirror].benchmark_oblivion option), so next time it will +# lie little bit more about how bad it's. +# Default: +# use_benchmark_results=1 +# ***Note: at the moment use_benchmark_results can NOT be set to 0 +use_benchmark_stats=1 + +# BENCHMARK_OBLIVION +# benchmark_oblivion option allows to adjust how fast segget "forgets" +# benchmarking statistics on mirrors performance. +# Each time mirror->mirror_on_the_wall() called, mirror decreases its +# honesty (to have more chances next time) using the following formula: +# +# honesty=honesty*100/(100+settings.benchmark_oblivion) (3) +# +# Therefore, setting benchmark_oblivion=100 will make mirror look twice +# less bad next time mirror->mirror_on_the_wall() called. +# Minimum value: 0 (Beware: setting to 0, will can exclude unlucky mirrors) +# Maximum value: 1000 +# Default: +# benchmark_oblivion=5 +benchmark_oblivion=5 + +[provide_mirror_to_others] +# PROVIDE_MIRROR_DIR +# Define a dir for making symlinks to downloaded distfiles. This dir can be +# used to provide local mirror for other hosts (with help of Apache, vsftp, etc). +# If set to none, segget will not make symlinks. +# Default: +# provide_mirror_dir=none +provide_mirror_dir=./provide_mirror_dir + +# SYNOPSIS: PROVIDE_MIRROR_FILES_RESTRICT_LIST_ON= 0 | 1 +# If PROVIDE_MIRROR_DIR=none this option will be ignored. +# - If set to 1, segget will compare distfile name with the list of forbiden +# patterns from the restricted.conf file. If distfile name contains any of the +# patterns, no symlink will be provided to this distfile. +# Default: +# provide_mirror_files_restrict_list_on=0 +provide_mirror_files_restrict_list_on=0 + +[provide_proxy_fetcher_to_others] +# PROVIDE_PROXY_FETCHER_IP +# Define an ip address segget will use to provide access for tuiclients. +# The parameter should be a string holding your host dotted IP address. +# Default: +# provide_proxy_fetcher_ip=none +provide_proxy_fetcher_ip=127.0.0.1 + +# PROVIDE_PROXY_FETCHER_PORT +# Define a port segget will use to provide access for tuiclients. +# The parameter should be an integer. +# Minimum value: 1 +# Maximum value: 65535 +# Default: +# provide_proxy_fetcher_port=3131 +provide_proxy_fetcher_port=3130 + +[networks] +# NETWORK0_PRIORITY +# Define priority as a value in range from lowest 0 to highest 10. +# Segget tries to use networks with higher priority levels first, and in case of +# failure will switch to networks with lower priority levels. +# Segget will NOT use network if its priority level set to 0. Therefore at least +# one network must have priority level higher than 0. +# Networks with local mirrors usually would have higher priority than that of +# networks with remote mirrors. +# Segget can have up to 10 networks (from network0 to network9). +# Settings for each network should be defined in network#.conf file, where +# instead of # should be network number. For network0 it's network0.conf +# Default: +# network0_priority=10 +# network1_priority=0 +# network2_priority=0 +# network3_priority=0 +# network4_priority=0 +# network5_priority=0 +# network6_priority=0 +# network7_priority=0 +# network8_priority=0 +# network9_priority=0 +network0_priority=0 +network1_priority=0 +network2_priority=0 +network3_priority=0 +network4_priority=7 +network5_priority=0 +network6_priority=0 +network7_priority=0 +network8_priority=0 +network9_priority=0 + +[schedule] + +[user-data] +# USER_AGENT +# Set the User-Agent: header in the http request sent to the remote server. +# This can be used to fool servers or scripts. +# Default: +# user_agent=segget +user_agent=segget + +# Specify the user and password for authentication on a ftp servers. +# NOT IMPLEMENTED YET: ftp-user=anonymous +# NOT IMPLEMENTED YET: ftp-password=me@mail.ru + +[ui_server] +# tuiclient monitors segget's activity by establishing tcp connection +# with segget daemon (ui_server part of it). + +# UI_IP +# Define an ip address segget will use to provide access for tuiclients. +# The parameter should be a string holding your host dotted IP address. +# Default: +# ui_ip=127.0.0.1 +ui_ip=127.0.0.1 + +# UI_PORT +# Define a port segget will use to provide access for tuiclients. +# The parameter should be an integer. +# Minimum value: 1 +# Maximum value: 65535 +# Default: +# ui_port=9999 +ui_port=9999 + +[logs] +# LOGS_DIR +# Define a dir to store log files. +# Default: +# logs_dir=./logs +logs_dir=/home/mona/idfetcha/logs + +# GENERAL_LOG_FILE +# Define a file name to store general log. +# Set to none to disable loggin. +# Default: +# general_log_file=segget.log +general_log_file=segget.log + +# ERROR_LOG_FILE +# Define a file name to store error log. +# Set to none to disable loggin. +# Default: +# error_log_file=segget.log +error_log_file=error.log + +# DEBUG_LOG_FILE +# Define a file name to store debug log. +# Set to none to disable loggin. +# Default: +# debug_log_file=segget.log +debug_log_file=debug.log
\ No newline at end of file diff --git a/request/request.cpp b/request/request.cpp new file mode 100644 index 0000000..9847f75 --- /dev/null +++ b/request/request.cpp @@ -0,0 +1,247 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "request.h" + +int request(string msg){ +// gettimeofday(&network_distfile_brokers_array[network_num].last_request_time, NULL); + int len; + struct sockaddr_in address; + int result; + //Create a socket for the client: + sockfd = socket(AF_INET, SOCK_STREAM, 0); + + //Name the socket, as agreed with the server: + address.sin_family = AF_INET; + address.sin_addr.s_addr = inet_addr(settings.request_ip.c_str()); + address.sin_port = htons(settings.request_port); + len = sizeof(address); + + //Connect your socket to the server’s socket: + result = connect(sockfd, (struct sockaddr *)&address, len); + if(result == -1) { + merror("Can't connect to segget daemon"); + return R_PF_ERROR_ADDING_TO_PROXY_QUEUE; + } + if (msg.length()>90000){return R_PF_ERROR_ADDING_TO_PROXY_QUEUE;}; + char send_buffer[100000]; + // char recv_buffer[256]; + strcpy(send_buffer,msg.c_str()); + //You can now read and write via sockfd: + if (write(sockfd, send_buffer, strlen(send_buffer))!=(int)msg.length()){ + merror("Error in request.cpp: request(): request msg size and sent data size are different."); + }; + + fd_set readfds, testfds; + FD_ZERO(&readfds); + FD_SET(sockfd, &readfds); + testfds = readfds; + + struct timeval response_timeout; + response_timeout.tv_sec=1; + response_timeout.tv_usec=0; + + result = select(FD_SETSIZE, &testfds, (fd_set *)0, + (fd_set *)0, &response_timeout); + + if(FD_ISSET(sockfd,&testfds)) { + int nread; + ioctl(sockfd, FIONREAD, &nread); + + char recv_buffer[1000]; + if(nread == 0) { + close(sockfd); + merror("Error in request.cpp: request(): no response from segget daemon"); + }else{ + if (nread!=read(sockfd, recv_buffer, nread)){ + merror("Error in request.cpp: request(): response msg size and received data size are different."); + }; + return decode_server_response(recv_buffer); + } + }else{ + merror("Error in request.cpp: request(): zero size response from segget daemon"); + } + close(sockfd); + return R_PF_ERROR_ADDING_TO_PROXY_QUEUE; +} + +int load_pkgs(){ + try{ + ifstream json_pkg_list_file; + json_pkg_list_file.exceptions (ofstream::failbit | ofstream::badbit); + try{ + json_pkg_list_file.open((settings.pkg_list_dir+"/pkg.list").c_str()); + }catch(...){ + merror("Error: Can't open "+settings.pkg_list_dir+"/pkg.list"+" file"); + return 1; + } + string buff((std::istreambuf_iterator<char>(json_pkg_list_file)), std::istreambuf_iterator<char>()); + try{ + json_pkg_list_file>>buff; + }catch(std::ifstream::failure e){ + if (!json_pkg_list_file.eof()){ + merror("Error: Can't read pkg.list file: "+(string)e.what()); + return 1; + } + }catch(...){ + perror("Unknown Error: Can't read pkg.list file"); + return 1; + } + try{ + json_pkg_list_file.close(); + }catch(...){ + perror("Error: Can't close pkg.list file"); + } + try{ + json_object *json_array_pkg_list; + json_array_pkg_list = json_tokener_parse(buff.c_str()); + if (is_error(json_array_pkg_list)) + merror("error parsing json:"+toString(json_tokener_errors[-(unsigned long)json_array_pkg_list])); + else { + uint pkg_count=json_object_array_length(json_array_pkg_list); + // create 0 pkg for distfiles to provide proxy-fetcher + for(uint array_item_num=0;array_item_num<pkg_count;array_item_num++){ + json_object* json_obj_pkg=json_object_array_get_idx(json_array_pkg_list,array_item_num); + json_object* json_array_distfile_list=json_object_object_get(json_obj_pkg,"distfile_list"); + ulong distfile_counter=json_object_array_length(json_array_distfile_list); + for(uint distfile_array_item_num=0;distfile_array_item_num<distfile_counter;distfile_array_item_num++){ + json_object* json_obj_distfile=json_object_array_get_idx(json_array_distfile_list,distfile_array_item_num); + request(json_object_to_json_string(json_obj_distfile)); + cout<< json_object_to_json_string(json_object_object_get(json_obj_distfile,"name"))<< endl; + } + } + } + return 0; + }catch(...){ + perror("Error: Can't parse json data from pkg.list file"); + return 1; + } + }catch(...){ + perror("Error in segget.cpp: load_pkgs()"); + return 1; + } +} +/* +void show_pkgs(){ + try{ + for (uint array_item_num=0;array_item_num<stats.pkg_count;array_item_num++){ + cout <<"PKG:"<<array_item_num<<") cat:"<< Ppkg_array[array_item_num]->category <<" name:"<< Ppkg_array[array_item_num]->name <<"\n"; + for(uint distfile_array_item_num=0;distfile_array_item_num<Ppkg_array[array_item_num]->distfile_count;distfile_array_item_num++){ + cout << " "<< distfile_array_item_num<<") distfile_name:"<< Ppkg_array[array_item_num]->Pdistfile_list[distfile_array_item_num]->name<<"\n"; + for(uint urllist_array_item_num=0;urllist_array_item_num<Ppkg_array[array_item_num]->Pdistfile_list[distfile_array_item_num]->url_count;urllist_array_item_num++){ + cout <<" "<<Ppkg_array[array_item_num]->Pdistfile_list[distfile_array_item_num]->url_list[urllist_array_item_num]<<"\n"; + } + } + } + }catch(...){ + error_log("Error in segget.cpp: show_pkgs()"); + } +} +*/ + +void segget_exit(int sig){ + try{ +// for(uint fd = 0; fd <= ui_server.max_fd_num; fd++) { +// close(fd); +// } + close(sockfd); + } + catch(...) + { + //error while clossing server_sockfd + } + cout<<"Exiting. (Cought signal:"+toString(sig)+")\n\n"; + exit(0); +} + +int routine(){ + try{ + signal(SIGABRT,segget_exit);//If program aborts go to assigned function "segget_exit". + signal(SIGTERM,segget_exit);//If program terminates go to assigned function "segget_exit". + signal(SIGINT,segget_exit);//If program terminates go to assigned function "segget_exit". +// prev_time=time((time_t *)NULL); + try{ + //load settings + settings.init(); + } + catch(...) + { + //error while loading settings + } + try{ + load_pkgs(); + }catch(...){ + //error while loading pkgs + } + try{ + //show_pkgs(); +// stats.show_totals(); + }catch(...){ + //error while showing stats + } + try{ +// download_pkgs(); + }catch(...){ + //error while downloading_pkgs + } + return 0; + }catch(...) + { + perror("Error in segget.cpp: routine()"); + } + return 1; +} +int parse_cli_arguments(int argc, char* argv[]){ + try{ + string option,name,value; + cout << "argc = " << argc << endl; + int posEqual; + for(int i = 0; i < argc; i++){ + cout << "argv[" << i << "] = " << argv[i] << endl; + option=argv[i]; + posEqual=option.find('='); + name = trim(option.substr(0,posEqual)); + value = trim(option.substr(posEqual+1)); + if (name=="--conf-dir") {settings.conf_dir=value; continue;}; + } + return 0; + }catch(...){ + perror("Error in segget.cpp: init_curses()"); + } + return 1; +} + +int main(int argc, char* argv[]) +{ + try{ + parse_cli_arguments(argc, argv); + routine(); + exit (0); + }catch(...){ + perror("Error in segget.cpp main()"); + } + exit(1); +}
\ No newline at end of file diff --git a/request/request.h b/request/request.h new file mode 100644 index 0000000..b627394 --- /dev/null +++ b/request/request.h @@ -0,0 +1,77 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __SEGGET_H__ +#define __SEGGET_H__ + +#include <stdio.h> +#include <string.h> +#include <signal.h> +#include <stdlib.h> +#include <stdio.h> +#include <fstream> +#include <iostream> +#include <json/json.h> +#include <pthread.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#include <sys/socket.h> +//#include <stdio.h> +//#include <netinet/in.h> +#include <arpa/inet.h> +#include <sys/ioctl.h> +//#include <unistd.h> +//#include <stdlib.h> +//#include <string> +//#include <string.h> + + +#include "config.h" +#include "settings.h" +//#include "stats.h" +#include "str.h" +#include "utils.h" +#include "response.h" + +using namespace std; +int sockfd; + +int routine(); +void start_daemon_mode(); +int parse_cli_arguments(int argc, char* argv[]); +int init_curses(); +int load_pkgs(); +void show_pkgs(); +int choose_segment(uint connection_num); +int download_pkgs(); +int main(int argc, char* argv[]); +void *print_message_function( void *); + +#endif
\ No newline at end of file diff --git a/request/response.cpp b/request/response.cpp new file mode 100644 index 0000000..10956f4 --- /dev/null +++ b/request/response.cpp @@ -0,0 +1,65 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "response.h" +int decode_server_response(string server_response){ + int int_server_response=atoi(server_response.c_str()); + switch (int_server_response){ + case R_PF_NOT_REQUESTED_YET:{ +// debug("Server response:"+server_response+" - R_PF_NOT_REQUESTED_YET"); + return int_server_response; + }; + case R_PF_REJECTED:{ +// debug("Server response:"+server_response+" - R_PF_REJECTED"); + return int_server_response; + }; + case R_PF_ERROR_ADDING_TO_PROXY_QUEUE:{ +// debug("Server response:"+server_response+" - R_PF_ERROR_ADDING_TO_PROXY_QUEUE"); + return int_server_response; + }; + case R_PF_ADDED_TO_PROXY_QUEUE:{ +// debug("Server response:"+server_response+" - R_PF_ADDED_TO_PROXY_QUEUE"); + return int_server_response; + }; + case R_PF_ALREADY_WAS_IN_QUEUE:{ +// debug("Server response:"+server_response+" - R_PF_ALREADY_WAS_IN_QUEUE"); + return int_server_response; + }; + case R_PF_DOWNLOADED:{ +// debug("Server response:"+server_response+" - R_PF_DOWNLOADED"); + return int_server_response; + }; + case R_PF_FAILED:{ +// debug("Server response:"+server_response+" - R_PF_FAILED"); + return int_server_response; + }; + default :{ +// debug("Server response:"+server_response+" - unknown => R_PF_FAILED"); + return R_PF_FAILED; + }; + } +} + diff --git a/request/response.h b/request/response.h new file mode 100644 index 0000000..d2db9af --- /dev/null +++ b/request/response.h @@ -0,0 +1,61 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __RESPONSE_H__ +#define __RESPONSE_H__ +#include <string> +#include <cstdlib> + +using namespace std; + +#define R_NOT_SET 300 +#define R_LM_WAIT_FOR_LOCAL_MIRRORS 50 + +#define R_PF_NOT_REQUESTED_YET 100 +#define R_PF_ADDED_TO_PROXY_QUEUE 101 +#define R_PF_ALREADY_WAS_IN_QUEUE 102 +#define R_PF_DOWNLOADED 103 +#define R_PF_BE_MORE_PATIENT 104 +#define R_PF_ERROR_ADDING_TO_PROXY_QUEUE 105 +#define R_PF_FAILED 106 +#define R_PF_REJECTED 107 + +// 0 for succesfull return of provide_segment() +#define R_R_DOWNLOAD_STARTED 0 +#define R_R_WAITING 108 +#define R_R_DOWNLOADING 109 + +#define R_LM_PF_R_NO_FREE_NETWORK_CONNECTION_FOUND 110 + +#define ALLOW_REQUESTS_TO_PROXY_FETCHERS 201 +#define DO_NOT_ALLOW_REQUESTS_TO_PROXY_FETCHERS 202 +#define ALLOW_REMOTE_NETWORKS 203 +#define DO_NOT_ALLOW_REMOTE_NETWORKS 204 +#define ALLOW_LOWER_PRIORITY_NETWORKS 205 + +int decode_server_response(string server_response); + +#endif
\ No newline at end of file diff --git a/request/restrict.conf b/request/restrict.conf new file mode 100644 index 0000000..e554a5b --- /dev/null +++ b/request/restrict.conf @@ -0,0 +1,2 @@ +libX +bz2
\ No newline at end of file diff --git a/request/settings.cpp b/request/settings.cpp new file mode 100644 index 0000000..ba9fa7f --- /dev/null +++ b/request/settings.cpp @@ -0,0 +1,44 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "settings.h" + +Tsettings settings; + +void Tsettings::init(){ + try{ + Tconfig conf("segget.conf"); + conf.set("logs","general_log_file",general_log_file); + conf.set("logs","logs_dir",logs_dir); + conf.set("logs","error_log_file",error_log_file); + conf.set("logs","debug_log_file",debug_log_file); + conf.set("request_server","request_ip",request_ip); + conf.set("request_server","request_port",request_port,1,65535); + conf.clear(); + }catch(...){ + perror("Error in settings.cpp: init()"); + } +}
\ No newline at end of file diff --git a/request/settings.h b/request/settings.h new file mode 100644 index 0000000..097969a --- /dev/null +++ b/request/settings.h @@ -0,0 +1,130 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __SETTINGS_H__ +#define __SETTINGS_H__ + +#include <string> +#include "config.h" +//#include "tui.cpp" +#define MAX_CONNECTS 20 /* number of simultaneous transfers */ + +using namespace std; + +class Tsettings{ + public: + string conf_dir; + bool no_daemon_flag; + //folders + string distfiles_dir; + string segments_dir; + //pkg_list + string pkg_list_dir; + bool del_pkg_list_when_dld_finished; + //distfiles + ulong max_connection_num_per_distfile; + //segments + ulong max_segment_size; + bool resume_on; + ulong max_tries; + //connections + ulong max_connections; + ulong connection_timeout; + ulong ftp_response_timeout; + ulong time_out; + ulong low_connection_speed_limit; + ulong low_connection_speed_time; + ulong max_connection_speed; + ulong current_speed_time_interval_msecs; + string bind_interface; + //mirrors + ulong max_connections_num_per_mirror; + ulong benchmark_oblivion; + //provide_mirror_to_others + string provide_mirror_dir; + bool provide_mirror_files_restrict_list_on; + //provide_proxy_fether_to_others + string provide_proxy_fetcher_ip; + ulong provide_proxy_fetcher_port; + //ui_server + string request_ip; + ulong request_port; + //logs + string logs_dir; + string general_log_file; + string error_log_file; + string debug_log_file; + + Tsettings(): + conf_dir("/etc/seggetd"), + no_daemon_flag(false), + //folders + distfiles_dir("./distfiles"), + segments_dir("./tmp"), + //pkg_list + pkg_list_dir("./"), + del_pkg_list_when_dld_finished(1), + //distfiles + max_connection_num_per_distfile(3), + //segments + max_segment_size(500000), + resume_on(1), + max_tries(10), + //connections + max_connections(6), + connection_timeout(15), + ftp_response_timeout(180), + time_out(500), + low_connection_speed_limit(1000), + low_connection_speed_time(10), + max_connection_speed(0), + current_speed_time_interval_msecs(1000), + bind_interface("none"), + //mirrors + max_connections_num_per_mirror(1), + benchmark_oblivion(5), + //provide_mirror_to_others + provide_mirror_dir("none"), + provide_mirror_files_restrict_list_on(0), + //provide_proxy_fether_to_others + provide_proxy_fetcher_ip("127.0.0.1"), + provide_proxy_fetcher_port(9888), + //ui_server + request_ip("127.0.0.1"), + request_port(10000), + //logs + logs_dir("./logs"), + general_log_file("segget.log"), + error_log_file("error.log"), + debug_log_file("debug.log") + {}; + void set_resume(bool resume_setting){resume_on=resume_setting;}; + bool get_resume(){return resume_on;}; + void init(); +}; + +extern Tsettings settings; +#endif
\ No newline at end of file diff --git a/request/shorttypes.h b/request/shorttypes.h new file mode 100644 index 0000000..af0b982 --- /dev/null +++ b/request/shorttypes.h @@ -0,0 +1,32 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef _TYPES_H_ +#define _TYPES_H_ + +typedef unsigned long ulong; + +#endif
\ No newline at end of file diff --git a/request/str.cpp b/request/str.cpp new file mode 100644 index 0000000..d86d12c --- /dev/null +++ b/request/str.cpp @@ -0,0 +1,126 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "str.h" + +string toString(uint t){ + stringstream s; + s << t; + return s.str(); +} +string toString(int t){ + stringstream s; + s << t; + return s.str(); +} +string toString(ulong t){ + stringstream s; + s << t; + return s.str(); +} +string toString(long t){ + stringstream s; + s << t; + return s.str(); +} +string toString(bool t){ + stringstream s; + s << t; + return s.str(); +} +/* +template<typename T> string toString(T t){ + stringstream s; + s << t; + return s.str(); +} +*/ +/* +template<typename T> string field(string prefix,T t, int width){ + try{ + stringstream s1,s2; + s1 << t; + width=width+prefix.length(); + s2.width(width); + s2 << prefix+s1.str(); + return s2.str(); + }catch(...){ + error_log("Error in str.cpp: field()"); + return ""; + } +} +*/ +string field(string prefix,ulong t, int width){ + try{ + stringstream s1,s2; + s1 << t; + width=width+prefix.length(); + s2.width(width); + s2 << prefix+s1.str(); + return s2.str(); + }catch(...){ + perror("Error in str.cpp: field()"); + return ""; + } +} + +int lower_char(int c){ + try{ + return tolower((unsigned char)c); + }catch(...){ + perror("Error in str.cpp: lower_char()"); + return 0; + } +} + +string noupper(string s){ + try{ + transform(s.begin(), s.end(), s.begin(), lower_char); + return s; + }catch(...){ + perror("Error in str.cpp: noupper()"); + return ""; + } +} + +string trim(std::string const& source, char const* delims) { + try{ + string result(source); + string::size_type index = result.find_last_not_of(delims); + if(index != string::npos) + result.erase(++index); + + index = result.find_first_not_of(delims); + if(index != string::npos) + result.erase(0, index); + else + result.erase(); + return result; + }catch(...){ + perror("Error in str.cpp: trim()"); + return ""; + } +}
\ No newline at end of file diff --git a/request/str.h b/request/str.h new file mode 100644 index 0000000..f0c876a --- /dev/null +++ b/request/str.h @@ -0,0 +1,49 @@ + /* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __STR_H__ +#define __STR_H__ +#include <string> +#include <algorithm> +#include <sstream> +#include <stdio.h> +using namespace std; + +//template<typename T> string toString(T t); +//template<typename T> string field(string prefix,T t, int width); + +string trim(std::string const& source, char const* delims = " \t\r\n"); +int lower_char(int c); +string noupper(string s); + +string toString(uint t); +string toString(int t); +string toString(ulong t); +string toString(long t); +string toString(bool t); +string field(string prefix,ulong t, int width); + +#endif
\ No newline at end of file diff --git a/request/utils.cpp b/request/utils.cpp new file mode 100644 index 0000000..bb9b3bf --- /dev/null +++ b/request/utils.cpp @@ -0,0 +1,44 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "utils.h" +void merror(string error_msg){ + perror(error_msg.c_str()); +} + +ulong time_left_from(timeval from_time){ + try{ + timeval now_time; + gettimeofday(&now_time,NULL); + ulong timeleft=(now_time.tv_sec-from_time.tv_sec)*1000+(now_time.tv_usec-from_time.tv_usec)/1000; + if (timeleft<1) + timeleft=1; + return timeleft; + }catch(...){ + merror("Error in utils.cpp: time_left_from()"); + return 1; + } +}
\ No newline at end of file diff --git a/request/utils.h b/request/utils.h new file mode 100644 index 0000000..5e29a20 --- /dev/null +++ b/request/utils.h @@ -0,0 +1,39 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __UTILS_H__ +#define __UTILS_H__ + +#include "sys/time.h" +#include "shorttypes.h" +#include <string> +#include <stdio.h> +#include <cstdlib> +using namespace std; + +void merror(string error_msg); +ulong time_left_from(timeval from_time); +#endif
\ No newline at end of file diff --git a/segget/1 b/segget/1 deleted file mode 100755 index 4d6ff14..0000000 --- a/segget/1 +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -make clean -make -./segget diff --git a/segget/Makefile b/segget/Makefile index eafc4d4..07280ff 100644 --- a/segget/Makefile +++ b/segget/Makefile @@ -18,7 +18,7 @@ all: clean $(BINS) %.o: %.cxx $(CXX) -c -o $@ $(CXXFLAGS) $^ -segget: segget.o connection.o checksum.o config.o distfile.o log.o mirror.o network.o networkbroker.o phase.o pkg.o response.o segment.o settings.o stats.o str.o tui.o utils.o ui_server.o proxyfetcher.o +segget: segget.o connection.o checksum.o config.o distfile.o log.o mirror.o network.o networkbroker.o phase.o pkg.o response.o segment.o settings.o stats.o str.o tui.o utils.o ui_server.o proxyfetcher.o requestserver.o $(CXX) -o $@ -lncurses $(LIBS) $(CIBS) $^ #%: %.o # $(CXX) -o $@ $(LIBS) $(CIBS) $^ diff --git a/segget/pkg.cpp b/segget/pkg.cpp index 3a02f74..b2bf8e9 100644 --- a/segget/pkg.cpp +++ b/segget/pkg.cpp @@ -27,7 +27,7 @@ #include "pkg.h" Tpkg **Ppkg_array; Tpkg proxy_fetcher_pkg; - +Tpkg request_server_pkg; //#define R_DOWNLOAD_STARTED 0 //#define R_ADDED_TO_PROXY_QUEUE 100 @@ -86,6 +86,34 @@ int Tpkg::try_adding_distfile_to_proxy_fetchers_queue(json_object* json_obj_dist return R_PF_ERROR_ADDING_TO_PROXY_QUEUE; } +int Tpkg::try_adding_distfile_to_request_server_queue(json_object* json_obj_distfile){ + string distfile_name; + try{ + for (ulong distfile_num=0; distfile_num<distfile_count; distfile_num++){ + if (Pdistfile_list[distfile_num]->name==distfile_name){ + switch (Pdistfile_list[distfile_num]->status){ + case DDOWNLOADED: { + debug("REQUEST_SERVER: distfile: "+distfile_name+" was downloaded"); + return R_PF_DOWNLOADED; + }; + case DFAILED:{ + debug("REQUEST_SERVER: Failed to download distfile: "+distfile_name); + return R_PF_FAILED; + } + default:{ + debug("REQUEST_SERVER: distfile: "+distfile_name+" was already in queue"); + return R_PF_ALREADY_WAS_IN_QUEUE; + } + } + } + } + return push_back_distfile(json_obj_distfile); + }catch(...){ + error_log("Error in pkg.cpp: pushback_distfile()"); + } + debug("REQUEST_SERVER: Error adding distfile: "+distfile_name+" to seggets's queue"); + return R_PF_ERROR_ADDING_TO_PROXY_QUEUE; +} void Tpkg::load_distfile_list(json_object* json_array_distfile_list){ try{ diff --git a/segget/pkg.h b/segget/pkg.h index c858a72..9446baf 100644 --- a/segget/pkg.h +++ b/segget/pkg.h @@ -58,11 +58,13 @@ class Tpkg{ Tpkg & operator=(const Tpkg &L); ~Tpkg(); int push_back_distfile(json_object* json_obj_distfile); + int try_adding_distfile_to_request_server_queue(json_object* json_obj_distfile); int try_adding_distfile_to_proxy_fetchers_queue(json_object* json_obj_distfile); void load_pkg_from_json(json_object* json_obj_pkg); }; extern Tpkg **Ppkg_array; extern Tpkg proxy_fetcher_pkg; +extern Tpkg request_server_pkg; #endif
\ No newline at end of file diff --git a/segget/proxyfetcher.cpp b/segget/proxyfetcher.cpp index 4abe535..31e9547 100644 --- a/segget/proxyfetcher.cpp +++ b/segget/proxyfetcher.cpp @@ -27,89 +27,95 @@ #include "proxyfetcher.h" void *run_proxy_fetcher_server(void * ){ - int server_sockfd, client_sockfd; - socklen_t server_len, client_len; - struct sockaddr_in server_address; - struct sockaddr_in client_address; - int result; - fd_set readfds, testfds; - - // Create and name a socket for the server: - server_sockfd = socket(AF_INET, SOCK_STREAM, 0); - - int on=1; - setsockopt(server_sockfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); - - server_address.sin_family = AF_INET; - //server_address.sin_addr.s_addr = htonl(INADDR_ANY); - server_address.sin_addr.s_addr = inet_addr(settings.provide_proxy_fetcher_ip.c_str()); - server_address.sin_port = htons(settings.provide_proxy_fetcher_port); - server_len = sizeof(server_address); - - bind(server_sockfd, (struct sockaddr *)&server_address, server_len); - - //Create a connection queue and initialize readfds to handle input from server_sockfd: - listen(server_sockfd, 5); - - FD_ZERO(&readfds); - FD_SET(server_sockfd, &readfds); - - //Now wait for clients and requests. Because you have passed a null pointer as the timeout parameter, no timeout will occur. The program will exit and report an error if select returns a value less than 1: - while(1) { - int fd; - int nread; - testfds = readfds; - - debug("proxyfether is waiting for connections"); - result = select(FD_SETSIZE, &testfds, (fd_set *)0, (fd_set *)0, (struct timeval *) 0); -// debug("proxyfether done waiting"); - - if(result < 1) { - error_log("Error in proxyfetcher.cpp : run"); - exit(1); - } + try{ + int server_sockfd, client_sockfd; + socklen_t server_len, client_len; + struct sockaddr_in server_address; + struct sockaddr_in client_address; + int result; + fd_set readfds, testfds; + + // Create and name a socket for the server: + server_sockfd = socket(AF_INET, SOCK_STREAM, 0); + + int on=1; + setsockopt(server_sockfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); + + server_address.sin_family = AF_INET; + //server_address.sin_addr.s_addr = htonl(INADDR_ANY); + server_address.sin_addr.s_addr = inet_addr(settings.provide_proxy_fetcher_ip.c_str()); + server_address.sin_port = htons(settings.provide_proxy_fetcher_port); + server_len = sizeof(server_address); + + bind(server_sockfd, (struct sockaddr *)&server_address, server_len); + + //Create a connection queue and initialize readfds to handle input from server_sockfd: + listen(server_sockfd, 5); + + FD_ZERO(&readfds); + FD_SET(server_sockfd, &readfds); + + //Now wait for clients and requests. Because you have passed a null pointer as the timeout parameter, no timeout will occur. The program will exit and report an error if select returns a value less than 1: + while(1) { + int fd; + int nread; + testfds = readfds; + + debug("proxyfether is waiting for connections"); + result = select(FD_SETSIZE, &testfds, (fd_set *)0, (fd_set *)0, (struct timeval *) 0); +// debug("proxyfether done waiting"); + + + if(result < 1) { + error_log("Error in proxyfetcher.cpp : run"); + exit(1); + } + + //Once you know you’ve got activity, you can find which descriptor it’s on by checking each in turn using FD_ISSET: + for(fd = 0; fd < FD_SETSIZE; fd++) { + if(FD_ISSET(fd,&testfds)) { + + //If the activity is on server_sockfd, it must be a request for a new connection, and you add the associated client_sockfd to the descriptor set: + + if(fd == server_sockfd) { + debug("new client - read"); + client_len = sizeof(client_address); + client_sockfd = accept(server_sockfd, + (struct sockaddr *)&client_address, &client_len); + FD_SET(client_sockfd, &readfds); + debug("adding client on fd:"+toString(client_sockfd)); - //Once you know you’ve got activity, you can find which descriptor it’s on by checking each in turn using FD_ISSET: - for(fd = 0; fd < FD_SETSIZE; fd++) { - if(FD_ISSET(fd,&testfds)) { - - //If the activity is on server_sockfd, it must be a request for a new connection, and you add the associated client_sockfd to the descriptor set: - - if(fd == server_sockfd) { - debug("new client - read"); - client_len = sizeof(client_address); - client_sockfd = accept(server_sockfd, - (struct sockaddr *)&client_address, &client_len); - FD_SET(client_sockfd, &readfds); - debug("adding client on fd:"+toString(client_sockfd)); - - //If it isn’t the server, it must be client activity. If close is received, the client has gone away, and you remove it from the descriptor set. Otherwise, you “serve” the client as in the previous examples. - }else{ -// debug("old client - read"); - ioctl(fd, FIONREAD, &nread); - if(nread == 0) { - close(fd); - FD_CLR(fd, &readfds); - debug("removing client on fd:"+toString(fd)); + //If it isn’t the server, it must be client activity. If close is received, the client has gone away, and you remove it from the descriptor set. Otherwise, you “serve” the client as in the previous examples. }else{ - char buffer[100000]=""; - if (nread!=read(fd, &buffer, nread)){ - error_log("Error in proxyfetcher.cpp : run_proxy_fetcher_server(): Not all data has been read from proxy-fetcher-client"); +// debug("old client - read"); + ioctl(fd, FIONREAD, &nread); + if(nread == 0) { + close(fd); + FD_CLR(fd, &readfds); + debug("removing client on fd:"+toString(fd)); + }else{ + char buffer[100000]=""; + if (nread!=read(fd, &buffer, nread)){ + error_log("Error in proxyfetcher.cpp : run_proxy_fetcher_server(): Not all data has been read from proxy-fetcher-client"); + } +// debug("serving client - read"); +// debug("serving client on fd"+toString(fd)); + string recv_msg=buffer; + error_log("Received a msg from the client:"+recv_msg); + string send_response; +// char send_buffer[10]=""; + send_response=toString(proxy_fetcher_pkg.try_adding_distfile_to_proxy_fetchers_queue(json_tokener_parse(buffer))); +// if (write(sockfd, send_buffer, strlen(send_buffer))!=(int)msg.length()){ + if (write(fd, send_response.c_str(), send_response.length())!=(int)send_response.length()){ + error_log("Error in proxyfetcher.cpp: run_proxy_fetcher_server(): response msg size and sent data size are different."); + }; } -// debug("serving client - read"); -// debug("serving client on fd"+toString(fd)); - string recv_msg=buffer; - error_log("Received a msg from the client:"+recv_msg); - string send_response; -// char send_buffer[10]=""; - send_response=toString(proxy_fetcher_pkg.try_adding_distfile_to_proxy_fetchers_queue(json_tokener_parse(buffer))); -// if (write(sockfd, send_buffer, strlen(send_buffer))!=(int)msg.length()){ - if (write(fd, send_response.c_str(), send_response.length())!=(int)send_response.length()){ - error_log("Error in proxyfetcher.cpp: run_proxy_fetcher_server(): response msg size and sent data size are different."); - }; } } } } + }catch(...){ + error_log("Error in proxyfetcher.cpp: run_proxy_fetcher_server()"); + return (void*)1; } } diff --git a/segget/requestserver.cpp b/segget/requestserver.cpp new file mode 100644 index 0000000..b53197f --- /dev/null +++ b/segget/requestserver.cpp @@ -0,0 +1,121 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "proxyfetcher.h" + +void *run_request_server(void * ){ + try{ + int server_sockfd, client_sockfd; + socklen_t server_len, client_len; + struct sockaddr_in server_address; + struct sockaddr_in client_address; + int result; + fd_set readfds, testfds; + + // Create and name a socket for the server: + server_sockfd = socket(AF_INET, SOCK_STREAM, 0); + + int on=1; + setsockopt(server_sockfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); + + server_address.sin_family = AF_INET; + //server_address.sin_addr.s_addr = htonl(INADDR_ANY); + server_address.sin_addr.s_addr = inet_addr(settings.request_ip.c_str()); + server_address.sin_port = htons(settings.request_port); + server_len = sizeof(server_address); + + bind(server_sockfd, (struct sockaddr *)&server_address, server_len); + + //Create a connection queue and initialize readfds to handle input from server_sockfd: + listen(server_sockfd, 5); + + FD_ZERO(&readfds); + FD_SET(server_sockfd, &readfds); + + //Now wait for clients and requests. Because you have passed a null pointer as the timeout parameter, no timeout will occur. The program will exit and report an error if select returns a value less than 1: + while(1) { + int fd; + int nread; + testfds = readfds; + + debug("proxyfether is waiting for connections"); + result = select(FD_SETSIZE, &testfds, (fd_set *)0, (fd_set *)0, (struct timeval *) 0); +// debug("proxyfether done waiting"); + + + if(result < 1) { + error_log("Error in proxyfetcher.cpp : run"); + exit(1); + } + + //Once you know you’ve got activity, you can find which descriptor it’s on by checking each in turn using FD_ISSET: + for(fd = 0; fd < FD_SETSIZE; fd++) { + if(FD_ISSET(fd,&testfds)) { + + //If the activity is on server_sockfd, it must be a request for a new connection, and you add the associated client_sockfd to the descriptor set: + + if(fd == server_sockfd) { + debug("new client - read"); + client_len = sizeof(client_address); + client_sockfd = accept(server_sockfd, + (struct sockaddr *)&client_address, &client_len); + FD_SET(client_sockfd, &readfds); + debug("adding client on fd:"+toString(client_sockfd)); + + //If it isn’t the server, it must be client activity. If close is received, the client has gone away, and you remove it from the descriptor set. Otherwise, you “serve” the client as in the previous examples. + }else{ +// debug("old client - read"); + ioctl(fd, FIONREAD, &nread); + if(nread == 0) { + close(fd); + FD_CLR(fd, &readfds); + debug("removing client on fd:"+toString(fd)); + }else{ + char buffer[100000]=""; + if (nread!=read(fd, &buffer, nread)){ + error_log("Error in proxyfetcher.cpp : run_proxy_fetcher_server(): Not all data has been read from proxy-fetcher-client"); + } +// debug("serving client - read"); +// debug("serving client on fd"+toString(fd)); + string recv_msg=buffer; + error_log("Received a msg from the client:"+recv_msg); + string send_response; +// char send_buffer[10]=""; + send_response=toString(request_server_pkg.try_adding_distfile_to_request_server_queue(json_tokener_parse(buffer))); +// if (write(sockfd, send_buffer, strlen(send_buffer))!=(int)msg.length()){ + if (write(fd, send_response.c_str(), send_response.length())!=(int)send_response.length()){ + error_log("Error in proxyfetcher.cpp: run_proxy_fetcher_server(): response msg size and sent data size are different."); + }; + } + } + } + } + } + }catch(...){ + error_log("Error in proxyfetcher.cpp: run_proxy_fetcher_server()"); + return (void*)1; + } +} diff --git a/segget/requestserver.h b/segget/requestserver.h new file mode 100644 index 0000000..0ba2316 --- /dev/null +++ b/segget/requestserver.h @@ -0,0 +1,45 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __REQUESTSERVER_H__ +#define __REQUESTSERVER_H__ + +#include <sys/types.h> +#include <sys/socket.h> +#include <stdio.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <sys/time.h> +#include <sys/ioctl.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <json/json.h> +#include "tui.h" +#include "pkg.h" + +void *run_request_server(void * ); +#endif
\ No newline at end of file diff --git a/segget/segget.cpp b/segget/segget.cpp index 214c452..678a54f 100644 --- a/segget/segget.cpp +++ b/segget/segget.cpp @@ -98,40 +98,45 @@ void show_pkgs(){ } */ int pkg_choose_segment(Tpkg * cur_pkg, uint connection_num){ + try{ uint distfile_num(0); uint segment_num(0); - while(distfile_num<cur_pkg->distfile_count){ -// if (Ppkg_array[pkg_num]->distfile_vector[distfile_num].allows_new_actions()){ - if (cur_pkg->Pdistfile_list[distfile_num]->allows_new_actions()){ - debug("segment_num:"+toString(segment_num)); - debug("segment_count:"+toString(cur_pkg->Pdistfile_list[distfile_num]->segments_count)); - while (segment_num<cur_pkg->Pdistfile_list[distfile_num]->segments_count){ - debug("segment_num:"+toString(segment_num)); -// debug(" segment_num:"+toString(segment_num)); - // segments_in_progress[connection_num]= - // if not(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->get_segment_downloaded_status(segment_num); - debug("Let's get segment status: "+statusToString(cur_pkg->Pdistfile_list[distfile_num]->dn_segments[segment_num].status)); - if (cur_pkg->Pdistfile_list[distfile_num]->dn_segments[segment_num].status==SWAITING){ - if (0==cur_pkg->Pdistfile_list[distfile_num]->provide_segment(cm, connection_num, segment_num)){ - return 0; // download started - }; - // no success with this segment though it's waiting - // probably there will be a problem downloading next segment - // so let's go for the next distfile - // and return to this distfile later - next time pkg_choose_segment is called - break; - }else{ - debug("status is not SWAITING - go for the next segment"); - // haven't managed to provide this segment, go for the next one - segment_num++; - } - } + while(distfile_num<cur_pkg->distfile_count){ +// if (Ppkg_array[pkg_num]->distfile_vector[distfile_num].allows_new_actions()){ + if (cur_pkg->Pdistfile_list[distfile_num]->allows_new_actions()){ + debug("segment_num:"+toString(segment_num)); + debug("segment_count:"+toString(cur_pkg->Pdistfile_list[distfile_num]->segments_count)); + while (segment_num<cur_pkg->Pdistfile_list[distfile_num]->segments_count){ + debug("segment_num:"+toString(segment_num)); +// debug(" segment_num:"+toString(segment_num)); + // segments_in_progress[connection_num]= + // if not(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->get_segment_downloaded_status(segment_num); + debug("Let's get segment status: "+statusToString(cur_pkg->Pdistfile_list[distfile_num]->dn_segments[segment_num].status)); + if (cur_pkg->Pdistfile_list[distfile_num]->dn_segments[segment_num].status==SWAITING){ + if (0==cur_pkg->Pdistfile_list[distfile_num]->provide_segment(cm, connection_num, segment_num)){ + return 0; // download started + }; + // no success with this segment though it's waiting + // probably there will be a problem downloading next segment + // so let's go for the next distfile + // and return to this distfile later - next time pkg_choose_segment is called + break; + }else{ + debug("status is not SWAITING - go for the next segment"); + // haven't managed to provide this segment, go for the next one + segment_num++; + } } - distfile_num++; - segment_num=0; } - distfile_num=0; - return 1; + distfile_num++; + segment_num=0; + } + distfile_num=0; + return 1; + }catch(...){ + error_log("Error: segget.cpp : pkg_choose_segment()"); + return 1; + } } int choose_segment(uint connection_num){ @@ -142,6 +147,9 @@ int choose_segment(uint connection_num){ return 0; } } + if (0==pkg_choose_segment(&request_server_pkg, connection_num)){ + return 0; + } // download distfiles as a proxy-fetcher if (0==pkg_choose_segment(&proxy_fetcher_pkg, connection_num)){ return 0; @@ -282,7 +290,7 @@ int download_pkgs(){ return EXIT_FAILURE; } } -void *print_message_function(void * ){ +void *refresh_tui_screen(void * ){ while (true){ ulong time_diff_msecs=time_left_from(stats.previous_time); if (time_diff_msecs >= settings.current_speed_time_interval_msecs){ @@ -297,7 +305,7 @@ void *print_message_function(void * ){ void launch_tui_thread(){ pthread_t tui_thread; int iret1; - iret1 = pthread_create( &tui_thread, NULL, print_message_function, (void*) NULL); + iret1 = pthread_create( &tui_thread, NULL, refresh_tui_screen, (void*) NULL); } void launch_ui_server_thread(){ @@ -320,6 +328,17 @@ void launch_proxy_fetcher_server_thread(){ } } +void launch_request_server_thread(){ + if (settings.request_ip!="none"){ + pthread_t request_server_thread; + int iret1; + debug_no_msg("Creating request_server_thread."); +// proxy_fetcher_server_thread.init(); + iret1 = pthread_create( &request_server_thread, NULL, run_request_server, (void*) NULL); + debug_no_msg("request_server_thread launched"); + } +} + void segget_exit(int sig){ try{ endwin(); @@ -367,6 +386,11 @@ int routine(){ error_log_no_msg("Error in segget.cpp launch_ui_server() failed"); } try{ + launch_request_server_thread(); + }catch(...){ + error_log_no_msg("Error in segget.cpp launch_proxy_fetcher_server_thread failed"); + } + try{ launch_proxy_fetcher_server_thread(); }catch(...){ error_log_no_msg("Error in segget.cpp launch_proxy_fetcher_server_thread failed"); diff --git a/segget/segget.h b/segget/segget.h index 45b4abd..4edf30e 100644 --- a/segget/segget.h +++ b/segget/segget.h @@ -55,6 +55,7 @@ #include "utils.h" #include "ui_server.h" #include "proxyfetcher.h" +#include "requestserver.h" using namespace std; @@ -70,6 +71,10 @@ int pkg_choose_segment(Tpkg * cur_pkg, uint connection_num); int choose_segment(uint connection_num); int download_pkgs(); int main(int argc, char* argv[]); -void *print_message_function( void *); +void *refresh_tui_screen( void *); +void launch_tui_thread(); +void launch_ui_server_thread(); +void launch_proxy_fetcher_server_thread(); +void launch_request_server_thread(); #endif
\ No newline at end of file diff --git a/segget/settings.cpp b/segget/settings.cpp index 5a44ad4..d16507c 100644 --- a/segget/settings.cpp +++ b/segget/settings.cpp @@ -120,6 +120,9 @@ void Tsettings::init(){ conf.set("ui_server","ui_ip",ui_ip); conf.set("ui_server","ui_port",ui_port,1,65535); + conf.set("request_server","request_ip",request_ip); + conf.set("request_server","request_port",request_port,1,65535); + conf.clear(); }catch(...){ error_log_no_msg("Error in settings.cpp: init()"); diff --git a/segget/settings.h b/segget/settings.h index 28d910e..93f381a 100644 --- a/segget/settings.h +++ b/segget/settings.h @@ -76,6 +76,9 @@ class Tsettings{ //ui_server string ui_ip; ulong ui_port; + //request_server + string request_ip; + ulong request_port; //logs string logs_dir; string general_log_file; @@ -120,6 +123,9 @@ class Tsettings{ //ui_server ui_ip("127.0.0.1"), ui_port(9999), + //request_server + request_ip("127.0.0.1"), + request_port(10000), //logs logs_dir("./logs"), general_log_file("segget.log"), |