diff options
author | 2010-07-20 01:49:05 +0300 | |
---|---|---|
committer | 2010-07-20 01:49:05 +0300 | |
commit | 759b1cd98abe95b240328cea5e5d966f7d982f97 (patch) | |
tree | f52c23b8660bec16a817795ed0ac75f1d04874a6 | |
parent | Improve network selection algorithm. (diff) | |
download | idfetch-759b1cd98abe95b240328cea5e5d966f7d982f97.tar.gz idfetch-759b1cd98abe95b240328cea5e5d966f7d982f97.tar.bz2 idfetch-759b1cd98abe95b240328cea5e5d966f7d982f97.zip |
Fix: segget freezes while trying to download files via proxy-fetcher
-rw-r--r-- | segget/Makefile | 3 | ||||
-rw-r--r-- | segget/connection.cpp | 32 | ||||
-rw-r--r-- | segget/connection.h | 2 | ||||
-rw-r--r-- | segget/distfile.cpp | 109 | ||||
-rw-r--r-- | segget/distfile.h | 3 | ||||
-rw-r--r-- | segget/mirror.cpp | 2 | ||||
-rw-r--r-- | segget/network0_mirrors.conf | 5 | ||||
-rw-r--r-- | segget/network1_mirrors.conf | 2 | ||||
-rw-r--r-- | segget/networkbroker.cpp | 5 | ||||
-rw-r--r-- | segget/networkbroker.h | 10 | ||||
-rw-r--r-- | segget/phase.cpp | 37 | ||||
-rw-r--r-- | segget/phase.h | 43 | ||||
-rw-r--r-- | segget/pkg.cpp | 3 | ||||
-rw-r--r-- | segget/response.cpp | 65 | ||||
-rw-r--r-- | segget/response.h | 61 | ||||
-rw-r--r-- | segget/segget.conf | 6 | ||||
-rw-r--r-- | segget/segget.cpp | 29 | ||||
-rw-r--r-- | segget/segment.cpp | 39 | ||||
-rw-r--r-- | segget/segment.h | 3 |
19 files changed, 342 insertions, 117 deletions
diff --git a/segget/Makefile b/segget/Makefile index fe15e2c..ed965bf 100644 --- a/segget/Makefile +++ b/segget/Makefile @@ -18,8 +18,7 @@ all: clean $(BINS) %.o: %.cxx $(CXX) -c -o $@ $(CXXFLAGS) $^ -segget: segget.o connection.o checksum.o config.o distfile.o mirror.o network.o networkbroker.o pkg.o segment.o \ - settings.o stats.o str.o tui.o utils.o ui_server.o proxyfetcher.o +segget: segget.o connection.o checksum.o config.o distfile.o mirror.o network.o networkbroker.o phase.o pkg.o response.o segment.o settings.o stats.o str.o tui.o utils.o ui_server.o proxyfetcher.o $(CXX) -o $@ -lncurses $(LIBS) $(CIBS) $^ #%: %.o # $(CXX) -o $@ $(LIBS) $(CIBS) $^ diff --git a/segget/connection.cpp b/segget/connection.cpp index d2b0e2c..108c991 100644 --- a/segget/connection.cpp +++ b/segget/connection.cpp @@ -28,6 +28,7 @@ uint Tconnection::total_connections=0; Tconnection connection_array[MAX_CONNECTS]; time_t prev_time; + void init_connections(){ for (ulong connection_num=0; connection_num<MAX_CONNECTS; connection_num++){ connection_array[connection_num].connection_num=connection_num; @@ -47,25 +48,44 @@ void Tconnection::start(CURLM *cm, uint network_number, uint distfile_num, Tsegm active=true; debug("Connecting network"+toString(network_num)); - if (network_array[network_number].network_mode==MODE_PROXY_FETCHER){ - connection_start_time_network_phase_for_pf_networks=segment->parent_distfile->network_distfile_brokers_array[network_number].phase; + if (network_array[network_num].network_mode==MODE_PROXY_FETCHER){ + connection_start_time_network_phase_for_pf_networks=segment->parent_distfile->network_distfile_brokers_array[network_num].phase; + } + string url; + if (network_array[network_num].network_mode!=MODE_REMOTE){ + url=network_array[network_num].benchmarked_mirror_list[mirror_num].url+segment->parent_distfile->name; + debug(" URL:"+url); + }else{ + url=segment->parent_distfile->url_list[mirror_num]; } + + network_array[network_num].benchmarked_mirror_list[best_mirror_num].start(); network_array[network_num].connect(); - segment->prepare_for_connection(cm, connection_num, network_num, distfile_num, mirror_num); + segment->prepare_for_connection(cm, connection_num, network_num, distfile_num, url); debug("Started connection for distfile: "+segment->parent_distfile->name); }catch(...){ error_log("Error in connection.cpp: start()"); } } -void Tconnection::stop(uint connection_result){ +void Tconnection::stop(int connection_result){ try{ - debug("Finished connection for distfile"+segment->parent_distfile->name+" Status"+toString(connection_result)); + debug("Finished connection for distfile: "+segment->parent_distfile->name+" Segment#:"+toString(segment->segment_num)+" Network#"+toString(network_num)+" Status: "+toString(connection_result)); + error_log("Finished connection for distfile: "+segment->parent_distfile->name+" Segment#:"+toString(segment->segment_num)+" Network#"+toString(network_num)+" Status: "+toString(connection_result)); + msg_clean_connection(connection_num); active=false; network_array[network_num].disconnect(); +// network_array[network_num].benchmarked_mirror_list[mirror_num].stop(); segment->segment_file.close(); + if (connection_result==0){ + if (! segment->segment_verification_is_ok()){ + debug("curl_lies - there is a problem downloading segment"); + error_log("curl_lies - there is a problem downloading segment"); + connection_result=100; + } + } Tdistfile* prnt_distfile=segment->parent_distfile; prnt_distfile->active_connections_num--; @@ -124,7 +144,7 @@ void Tconnection::stop(uint connection_result){ // no error => count this one and start new log("Succesfully downloaded "+segment->file_name+" on connection#"+toString(connection_num)); debug(" Successful download "+segment->url); - Pcurr_mirror=find_mirror(strip_mirror_name(segment->url)); +// already done earlier in this function Pcurr_mirror=find_mirror(strip_mirror_name(segment->url)); Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),segment->segment_size); segment->status=SDOWNLOADED; prnt_distfile->inc_dld_segments_count(segment); diff --git a/segget/connection.h b/segget/connection.h index d20ad01..03d502f 100644 --- a/segget/connection.h +++ b/segget/connection.h @@ -60,7 +60,7 @@ class Tconnection{ start_time(), segment(0){}; void start(CURLM *cm, uint network_number, uint distfile_num, Tsegment *started_segment, uint best_mirror_num); - void stop(uint connection_result); + void stop(int connection_result); void inc_bytes_per_last_interval(ulong new_bytes_count); void show_connection_progress(ulong time_diff); }; diff --git a/segget/distfile.cpp b/segget/distfile.cpp index 16d98f0..b27a52f 100644 --- a/segget/distfile.cpp +++ b/segget/distfile.cpp @@ -29,8 +29,6 @@ //Make the necessary includes and set up the variables: using namespace std; -int Tdistfile::decode_server_response(string server_response){ - /* #define R_LM_WAIT_FOR_LOCAL_MIRRORS 100 @@ -56,39 +54,6 @@ int Tdistfile::decode_server_response(string server_response){ #define ALLOW_LOWER_PRIORITY_NETWORKS 205 */ -int int_server_response=atoi(server_response.c_str()); - switch (int_server_response){ - case R_PF_REJECTED:{ - debug("Server response:"+server_response+" - R_PF_REJECTED"); - return int_server_response; - }; - case R_PF_ERROR_ADDING_TO_PROXY_QUEUE:{ - debug("Server response:"+server_response+" - R_PF_ERROR_ADDING_TO_PROXY_QUEUE"); - return int_server_response; - }; - case R_PF_ADDED_TO_PROXY_QUEUE:{ - debug("Server response:"+server_response+" - R_PF_ADDED_TO_PROXY_QUEUE"); - return int_server_response; - }; - case R_PF_ALREADY_WAS_IN_QUEUE:{ - debug("Server response:"+server_response+" - R_PF_ALREADY_WAS_IN_QUEUE"); - return int_server_response; - }; - case R_PF_DOWNLOADED:{ - debug("Server response:"+server_response+" - R_PF_DOWNLOADED"); - return int_server_response; - }; - case R_PF_FAILED:{ - debug("Server response:"+server_response+" - R_PF_FAILED"); - return int_server_response; - }; - default :{ - debug("Server response:"+server_response+" - unknown => R_PF_FAILED"); - return R_PF_FAILED; - }; - } -} - int Tdistfile::request(ulong network_num, string msg){ gettimeofday(&network_distfile_brokers_array[network_num].last_request_time, NULL); int sockfd; @@ -326,7 +291,9 @@ bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint ne long best_mirror_num=-1; // the best isn't set let's find it ulong best_mirror_self_rating=-1; ulong curr_mirror_self_rating; + debug("Choosing mirror for network"+toString(network_num)); for (ulong mirror_num=0; mirror_num<network_array[network_num].benchmarked_mirror_list.size(); mirror_num++){ + debug("Evaluating mirror:"+network_array[network_num].benchmarked_mirror_list[mirror_num].url); if (network_array[network_num].benchmarked_mirror_list[mirror_num].get_active_num()<settings.max_connections_num_per_mirror){ curr_mirror_self_rating=network_array[network_num].benchmarked_mirror_list[mirror_num].mirror_on_the_wall(); if (curr_mirror_self_rating<best_mirror_self_rating){ @@ -336,16 +303,23 @@ bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint ne if (best_mirror_self_rating==0) // 0 can not be improved - it's one of the best break; + }else{ + debug("Mirror already has:" + +toString(network_array[network_num].benchmarked_mirror_list[mirror_num].get_active_num()) + +" connections which doesn't meet limit:"+toString(settings.max_connections_num_per_mirror)); } } if (best_mirror_num!=-1){ debug("Downloading from BEST_LOCAL_MIRROR:"+network_array[network_num].benchmarked_mirror_list[best_mirror_num].url); - network_array[network_num].benchmarked_mirror_list[best_mirror_num].start(); + + + active_connections_num++; connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num); return R_R_DOWNLOAD_STARTED; } else{ + debug("Can't choose LOCAL mirror for segment:"+dn_segments[seg_num].file_name); error_log("Can't choose LOCAL mirror for segment:"+dn_segments[seg_num].file_name); return 1; } @@ -372,8 +346,11 @@ uint Tdistfile::provide_local_network(CURLM* cm, uint connection_num, uint seg_n if (network_array[network_num].priority==network_priority){ debug(" network_priority="+toString(network_array[network_num].priority)); if (network_array[network_num].network_mode==MODE_LOCAL){ + debug(" network#:"+toString(network_num)+" has MODE_LOCAL"); + debug(" phase#"+phaseToString(network_distfile_brokers_array[network_num].phase)); if (network_distfile_brokers_array[network_num].phase==E_USE_AS_LOCAL_MIRRORS){ if (network_array[network_num].has_free_connections()){ + debug(" has free connections"); // debug(" Allowed network#:"+toString(network_num)); if ((best_local_network_num==-1) or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){ @@ -381,6 +358,7 @@ uint Tdistfile::provide_local_network(CURLM* cm, uint connection_num, uint seg_n debug(" Replace best LOCAL network to network#:"+toString(network_num)); } }else{ + debug(" does NOT have free connections"); if (network_array[network_num].only_local_when_possible){ allow_proxy_fetcher_mirrors=false; debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed"); @@ -388,12 +366,15 @@ uint Tdistfile::provide_local_network(CURLM* cm, uint connection_num, uint seg_n } } } - if (network_array[network_num].network_mode==MODE_PROXY_FETCHER) - { + if (network_array[network_num].network_mode==MODE_PROXY_FETCHER){ + debug(" network#:"+toString(network_num)+" has MODE_PROXY_FETCHER"); + debug(" with phase:"+phaseToString(network_distfile_brokers_array[network_num].phase)); if ((network_distfile_brokers_array[network_num].phase==E_USE_AS_LOCAL_MIRRORS) or (network_distfile_brokers_array[network_num].phase==E_PROXY_FETCHER_DOWNLOADED)) { + debug("phase fits"); if (network_array[network_num].has_free_connections()){ + debug(" has free connections"); // debug(" Allowed network#:"+toString(network_num)); if ((best_local_network_num==-1) or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){ @@ -442,23 +423,32 @@ uint Tdistfile::request_proxy_fetcher_network(uint network_priority){ if (network_array[network_num].network_mode==MODE_PROXY_FETCHER){ //replace this one by does_not_reject_connections // if (network_array[network_num].has_free_connections()){ - if ((best_proxy_fetcher_network_num==-1) - -// or + if ((network_distfile_brokers_array[network_num].phase==E_USE_AS_LOCAL_MIRRORS) + or (network_distfile_brokers_array[network_num].phase==E_ALL_LOCAL_MIRRORS_FAILED)){ + debug("------//////////// Good phase !!!!!!!!!!!!!!!"); + decode_server_response(toString(network_distfile_brokers_array[network_num].proxy_fetcher_response)); + switch (network_distfile_brokers_array[network_num].proxy_fetcher_response){ + case R_PF_DOWNLOADED:break; + case R_PF_ERROR_ADDING_TO_PROXY_QUEUE:break; + case R_PF_FAILED:break; + case R_PF_REJECTED:break; + default:{ + //if less then 30 secs left don't bother proxy-fetcher + if (2000>time_left_from(network_distfile_brokers_array[network_num].last_request_time)){ + debug("====================== Be more patient with proxy-fetcher - network#:"+toString(network_num) + +" because time left:"+toString(time_left_from(network_distfile_brokers_array[network_num].last_request_time))); + return R_PF_BE_MORE_PATIENT; + }else{ // (network_array[best_proxy_fetcher_network_num].active_connections_num>network_array[network_num].active_connections_num) - - ){ - switch (network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response){ - case R_PF_DOWNLOADED:break; - case R_PF_ERROR_ADDING_TO_PROXY_QUEUE:break; - case R_PF_FAILED:break; - case R_PF_REJECTED:break; - default:{ best_proxy_fetcher_network_num=network_num; + debug("-------------------------------"); debug(" Replace best_proxy_fetcher network_num to network#:"+toString(best_proxy_fetcher_network_num)); - break; - } + }; + break; } + } + } + if (best_proxy_fetcher_network_num!=-1) break; /* }else{ if (network_array[network_num].only_local_when_possible){ @@ -469,15 +459,10 @@ uint Tdistfile::request_proxy_fetcher_network(uint network_priority){ } } */ - } } } } if (best_proxy_fetcher_network_num != -1){ - //if less then 30 secs left don't bother proxy-fetcher - if (30000>time_left_from(network_distfile_brokers_array[best_proxy_fetcher_network_num].last_request_time)){ - return R_PF_BE_MORE_PATIENT; - }else{ // request from proxy fethcer network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response=request(best_proxy_fetcher_network_num, json_data); if ((network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response==R_PF_DOWNLOADED) @@ -494,7 +479,6 @@ uint Tdistfile::request_proxy_fetcher_network(uint network_priority){ } //return - don't switch to low priority networks return network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response; - } }else{ if (allow_remote_mirrors){ debug(" So best_proxy_fetcher_failed num="+toString(best_proxy_fetcher_network_num)); @@ -555,10 +539,11 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ try{ int result=R_NOT_SET; for (uint network_priority=10; network_priority>0; network_priority--){ - { + do { + debug("Switching to local networks with priority:"+toString(network_priority)); result=provide_local_network(cm, connection_num, seg_num, network_priority); if (result==ALLOW_REQUESTS_TO_PROXY_FETCHERS){ - debug("Switching to proxy-fetcher networks with priority:"+toString(network_priority)); + debug("Requesting proxy-fetcher networks with priority:"+toString(network_priority)); result=request_proxy_fetcher_network(network_priority); switch (result){ case ALLOW_REMOTE_NETWORKS:{ @@ -572,7 +557,10 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ } // don't switch to remote and lower priority networks => return results case DO_NOT_ALLOW_REMOTE_NETWORKS: return result; - case R_PF_DOWNLOADED: break; + case R_PF_DOWNLOADED: {debug("Yep, R_PF_DONWLOADED => LOOP AGAIN"); break;}; + // if already in queue, etc... + // don't switch to remote and lower priority networks => return results + default: return result; } }else{ // don't switch to proxy-fetcher requests, remote, @@ -581,6 +569,7 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ } // try to download from local mirrors again because one of the proxy-fetcher // claims that distfile has been donwloaded + debug("----- JUST BEFORE THE LOOP ----"); }while (result==R_PF_DOWNLOADED); } // haven't found anything suitable diff --git a/segget/distfile.h b/segget/distfile.h index ea92fe3..8d42894 100644 --- a/segget/distfile.h +++ b/segget/distfile.h @@ -52,7 +52,7 @@ #include "network.h" #include "networkbroker.h" #include "segment.h" -#include "responses.h" +#include "response.h" class Tsegment; //#include <sys/types.h> @@ -136,7 +136,6 @@ class Tdistfile{ Tdistfile(const Tdistfile &L); // copy constructor Tdistfile & operator=(const Tdistfile &L); ~Tdistfile(); - int decode_server_response(string server_response); int request(ulong network_num, string msg); void init(); bool allows_new_actions(); diff --git a/segget/mirror.cpp b/segget/mirror.cpp index defec75..b23bd5f 100644 --- a/segget/mirror.cpp +++ b/segget/mirror.cpp @@ -54,7 +54,7 @@ void Tmirror::stop(ulong time, uint size){ dld_time+=time/1000; dld_size+=size; honesty=1; - debug(toString(time)+"---"+toString(size)); + debug("Connnection to Mirror stopped: Time spent:"+toString(time)+"--- Size downloaded:"+toString(size)); active_num--; }catch(...){ error_log("Error in mirror.cpp: stop()"); diff --git a/segget/network0_mirrors.conf b/segget/network0_mirrors.conf index 4164689..529a268 100644 --- a/segget/network0_mirrors.conf +++ b/segget/network0_mirrors.conf @@ -1,4 +1 @@ -ftp://192.168.56.11/ -ftp://192.168.56.12/ -ftp://192.168.56.13/ -ftp://192.168.56.14/
\ No newline at end of file +ftp://192.168.56.101/
\ No newline at end of file diff --git a/segget/network1_mirrors.conf b/segget/network1_mirrors.conf index 7628506..9406219 100644 --- a/segget/network1_mirrors.conf +++ b/segget/network1_mirrors.conf @@ -1 +1 @@ -http://192.168.0.1/
\ No newline at end of file +http://127.0.0.1/
\ No newline at end of file diff --git a/segget/networkbroker.cpp b/segget/networkbroker.cpp index 8062ab6..760bcfe 100644 --- a/segget/networkbroker.cpp +++ b/segget/networkbroker.cpp @@ -25,6 +25,11 @@ */ #include "networkbroker.h" + +string phaseToString(int phase_num); + + + void Tnetwork_distfile_broker::init(ulong network_number){ try{ network_num=network_number; diff --git a/segget/networkbroker.h b/segget/networkbroker.h index bb6ba49..602cdb0 100644 --- a/segget/networkbroker.h +++ b/segget/networkbroker.h @@ -29,16 +29,10 @@ #include <vector> #include "shorttypes.h" #include "network.h" -#include "responses.h" +#include "response.h" +#include "phase.h" using namespace std; -enum Tnetwork_distfile_broker_phases{ - E_USE_AS_LOCAL_MIRRORS, - E_ALL_LOCAL_MIRRORS_FAILED, - E_PROXY_FETCHER_DOWNLOADED, - E_ALL_PROXY_FETCHER_MIRRORS_FAILED -}; - class Tnetwork_distfile_broker{ public: // map<string,Tmirror *> benchmarked_mirror_list; diff --git a/segget/phase.cpp b/segget/phase.cpp new file mode 100644 index 0000000..04aef35 --- /dev/null +++ b/segget/phase.cpp @@ -0,0 +1,37 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "phase.h" + +string phaseToString(int phase_num){ + switch(phase_num){ + case E_USE_AS_LOCAL_MIRRORS: return "E_USE_AS_LOCAL_MIRRORS"; + case E_ALL_LOCAL_MIRRORS_FAILED: return "E_ALL_LOCAL_MIRRORS_FAILED"; + case E_PROXY_FETCHER_DOWNLOADED: return "E_PROXY_FETCHER_DOWNLOADED"; + case E_ALL_PROXY_FETCHER_MIRRORS_FAILED: return "E_ALL_PROXY_FETCHER_MIRRORS_FAILED"; + default: return "Unknown phase"; + } +} diff --git a/segget/phase.h b/segget/phase.h new file mode 100644 index 0000000..221c33d --- /dev/null +++ b/segget/phase.h @@ -0,0 +1,43 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __PHASE_H__ +#define __PHASE_H__ + +#include <string> + +using namespace std; + +enum Tnetwork_distfile_broker_phases{ + E_USE_AS_LOCAL_MIRRORS, + E_ALL_LOCAL_MIRRORS_FAILED, + E_PROXY_FETCHER_DOWNLOADED, + E_ALL_PROXY_FETCHER_MIRRORS_FAILED +}; + +string phaseToString(int phase_num); + +#endif
\ No newline at end of file diff --git a/segget/pkg.cpp b/segget/pkg.cpp index 72cf539..3a02f74 100644 --- a/segget/pkg.cpp +++ b/segget/pkg.cpp @@ -56,7 +56,8 @@ int Tpkg::try_adding_distfile_to_proxy_fetchers_queue(json_object* json_obj_dist string distfile_name; try{ distfile_name=json_object_get_string(json_object_object_get(json_obj_distfile,"name")); - if (is_symlink_restricted(distfile_name)){ + if (is_symlink_restricted(distfile_name)!=-1){ + debug("PROXY_FETCHER: distfile: "+distfile_name+" restricted (name matches restriting pattern"); return R_PF_REJECTED; } for (ulong distfile_num=0; distfile_num<distfile_count; distfile_num++){ diff --git a/segget/response.cpp b/segget/response.cpp new file mode 100644 index 0000000..5db952e --- /dev/null +++ b/segget/response.cpp @@ -0,0 +1,65 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "response.h" +int decode_server_response(string server_response){ + int int_server_response=atoi(server_response.c_str()); + switch (int_server_response){ + case R_PF_NOT_REQUESTED_YET:{ + debug("Server response:"+server_response+" - R_PF_NOT_REQUESTED_YET"); + return int_server_response; + }; + case R_PF_REJECTED:{ + debug("Server response:"+server_response+" - R_PF_REJECTED"); + return int_server_response; + }; + case R_PF_ERROR_ADDING_TO_PROXY_QUEUE:{ + debug("Server response:"+server_response+" - R_PF_ERROR_ADDING_TO_PROXY_QUEUE"); + return int_server_response; + }; + case R_PF_ADDED_TO_PROXY_QUEUE:{ + debug("Server response:"+server_response+" - R_PF_ADDED_TO_PROXY_QUEUE"); + return int_server_response; + }; + case R_PF_ALREADY_WAS_IN_QUEUE:{ + debug("Server response:"+server_response+" - R_PF_ALREADY_WAS_IN_QUEUE"); + return int_server_response; + }; + case R_PF_DOWNLOADED:{ + debug("Server response:"+server_response+" - R_PF_DOWNLOADED"); + return int_server_response; + }; + case R_PF_FAILED:{ + debug("Server response:"+server_response+" - R_PF_FAILED"); + return int_server_response; + }; + default :{ + debug("Server response:"+server_response+" - unknown => R_PF_FAILED"); + return R_PF_FAILED; + }; + } +} + diff --git a/segget/response.h b/segget/response.h new file mode 100644 index 0000000..77ec9fe --- /dev/null +++ b/segget/response.h @@ -0,0 +1,61 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __RESPONSE_H__ +#define __RESPONSE_H__ + +#include "tui.h" + +using namespace std; + +#define R_NOT_SET 300 +#define R_LM_WAIT_FOR_LOCAL_MIRRORS 50 + +#define R_PF_NOT_REQUESTED_YET 100 +#define R_PF_ADDED_TO_PROXY_QUEUE 101 +#define R_PF_ALREADY_WAS_IN_QUEUE 102 +#define R_PF_DOWNLOADED 103 +#define R_PF_BE_MORE_PATIENT 104 +#define R_PF_ERROR_ADDING_TO_PROXY_QUEUE 105 +#define R_PF_FAILED 106 +#define R_PF_REJECTED 107 + +// 0 for succesfull return of provide_segment() +#define R_R_DOWNLOAD_STARTED 0 +#define R_R_WAITING 108 +#define R_R_DOWNLOADING 109 + +#define R_LM_PF_R_NO_FREE_NETWORK_CONNECTION_FOUND 110 + +#define ALLOW_REQUESTS_TO_PROXY_FETCHERS 201 +#define DO_NOT_ALLOW_REQUESTS_TO_PROXY_FETCHERS 202 +#define ALLOW_REMOTE_NETWORKS 203 +#define DO_NOT_ALLOW_REMOTE_NETWORKS 204 +#define ALLOW_LOWER_PRIORITY_NETWORKS 205 + +int decode_server_response(string server_response); + +#endif
\ No newline at end of file diff --git a/segget/segget.conf b/segget/segget.conf index 26b39dd..21e28cd 100644 --- a/segget/segget.conf +++ b/segget/segget.conf @@ -76,7 +76,7 @@ max_tries=10 # Maximum value: 20 # Default: # max_connections=10 -max_connections=20 +max_connections=3 # CURRENT_SPEED_TIME_INTERVAL_MSECS # segget transfers may have bursty nature of their traffic. Therefore, while @@ -172,7 +172,7 @@ provide_mirror_dir=./provide_mirror_dir # patterns, no symlink will be provided to this distfile. # Default: # provide_mirror_files_restrict_list_on=0 -provide_mirror_files_restrict_list_on=1 +provide_mirror_files_restrict_list_on=0 [provide_proxy_fetcher_to_others] # PROVIDE_PROXY_FETCHER_IP @@ -216,7 +216,7 @@ provide_proxy_fetcher_port=3130 # network9_priority=0 network0_priority=10 network1_priority=9 -network2_priority=8 +network2_priority=0 network3_priority=0 network4_priority=0 network5_priority=0 diff --git a/segget/segget.cpp b/segget/segget.cpp index 0754806..4cf5f9d 100644 --- a/segget/segget.cpp +++ b/segget/segget.cpp @@ -103,7 +103,8 @@ int pkg_choose_segment(Tpkg * cur_pkg, uint connection_num){ while(distfile_num<cur_pkg->distfile_count){ // if (Ppkg_array[pkg_num]->distfile_vector[distfile_num].allows_new_actions()){ if (cur_pkg->Pdistfile_list[distfile_num]->allows_new_actions()){ - debug("Distfile "+cur_pkg->Pdistfile_list[distfile_num]->name+" allows new connections"); + debug("============================================= Distfile " + +cur_pkg->Pdistfile_list[distfile_num]->name+" allows new connections"); // debug("Distfile "+Ppkg_array[pkg_num]->distfile_vector[distfile_num]->name+" allows new connections"); // debug(" distfile_num:"+toString(distfile_num)); if (cur_pkg->Pdistfile_list[distfile_num]->active_connections_num<settings.max_connection_num_per_distfile){ @@ -115,16 +116,21 @@ int pkg_choose_segment(Tpkg * cur_pkg, uint connection_num){ // debug(" segment_num:"+toString(segment_num)); // segments_in_progress[connection_num]= // if not(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->get_segment_downloaded_status(segment_num); - debug("Let's get segment status"+statusToString(cur_pkg->Pdistfile_list[distfile_num]->dn_segments[segment_num].status)); + debug("Let's get segment status: "+statusToString(cur_pkg->Pdistfile_list[distfile_num]->dn_segments[segment_num].status)); if (cur_pkg->Pdistfile_list[distfile_num]->dn_segments[segment_num].status==SWAITING){ - if ( ! cur_pkg->Pdistfile_list[distfile_num]->provide_segment(cm, connection_num, segment_num)){ + if (0==cur_pkg->Pdistfile_list[distfile_num]->provide_segment(cm, connection_num, segment_num)){ return 0; // download started }; + // no success with this segment though it's waiting + // probably there will be a problem downloading next segment + // so let's go for the next distfile + // and return to this distfile later - next time pkg_choose_segment is called + break; }else{ debug("status is not SWAITING - go for the next segment"); + // haven't managed to provide this segment, go for the next one + segment_num++; } - // haven't managed to provide this segment, go for the next one - segment_num++; } }else{ debug(" distfile "+cur_pkg->Pdistfile_list[distfile_num]->name+" has " @@ -143,12 +149,12 @@ int choose_segment(uint connection_num){ try{ for (uint pkg_num=0; pkg_num<stats.pkg_count; pkg_num++){ // debug("pkg_num:"+toString(pkg_num)); - if (! pkg_choose_segment(Ppkg_array[pkg_num], connection_num)){ + if (0==pkg_choose_segment(Ppkg_array[pkg_num], connection_num)){ return 0; } } // download distfiles as a proxy-fetcher - if (! pkg_choose_segment(&proxy_fetcher_pkg, connection_num)){ + if (0==pkg_choose_segment(&proxy_fetcher_pkg, connection_num)){ return 0; } // for (uint array_item_num=0;array_item_num<pkg_count;array_item_num++){ @@ -193,19 +199,20 @@ int download_pkgs(){ U=1; while (U) { // Use free connections to download segments connections + debug("Entering connection activation cycle"); for (uint connection_num = 0; connection_num < settings.max_connections; ++connection_num) { debug("connection_num:"+toString(connection_num)); if ( ! connection_array[connection_num].active){ -// if ( debug("connection is not active - choosing segment"); choose_segment(connection_num); -// ) -// break; +// // activate only one connection +// break; } else{ debug("connection is active"); } }; + debug("Exit connection activation sycle"); while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(cm, &U)){}; if (U) { FD_ZERO(&R); @@ -244,7 +251,7 @@ int download_pkgs(){ Tsegment *current_segment; CURL *e = msg->easy_handle; curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, ¤t_segment); - uint connection_result=msg->data.result; + int connection_result=msg->data.result; string result_msg_text="RESULT:"+toString(connection_result)+" "+curl_easy_strerror(msg->data.result)+" while downloading segment"; msg_status1(current_segment->connection_num,current_segment->segment_num,result_msg_text); curl_multi_remove_handle(cm, e); diff --git a/segget/segment.cpp b/segget/segment.cpp index 45116cf..cab6343 100644 --- a/segget/segment.cpp +++ b/segget/segment.cpp @@ -37,6 +37,25 @@ string statusToString(Tsegment_status the_status){ default :return "UNKNOWN STATUS"; } } +bool Tsegment::segment_verification_is_ok(){ + try{ + ulong downloaded_size; + ifstream file((settings.segments_dir+"/"+file_name).c_str(), ios::in|ios::binary); + ulong start = file.tellg(); + file.seekg (0, ios::end); + ulong end = file.tellg(); + downloaded_size = end - start; + debug("seg:"+toString(segment_num)+" Dsize="+toString(downloaded_size)+" seg_size="+toString(segment_size)); + file.close(); + if (downloaded_size==segment_size){ + return true; + } + }catch(...){ + error_log("Error in segment.cpp: segment_verification_is_ok()"); + } + return false; +} + void Tsegment::set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfile_name, ulong default_seg_size, ulong range_end){ try{ parent_distfile=prnt_distfile; @@ -44,17 +63,9 @@ void Tsegment::set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfi segment_size=range_end-seg_num*default_seg_size+1; range=toString(seg_num*default_seg_size)+"-"+toString(range_end); file_name="."+distfile_name+".seg"+toString(seg_num); - ulong downloaded_size; if (settings.get_resume()){ //check if downloaded - ifstream file((settings.segments_dir+"/"+file_name).c_str(), ios::in|ios::binary); - ulong start = file.tellg(); - file.seekg (0, ios::end); - ulong end = file.tellg(); - downloaded_size = end - start; - debug("seg:"+toString(seg_num)+" Dsize="+toString(downloaded_size)+" seg_size="+toString(segment_size)); - file.close(); - if (downloaded_size==segment_size){ + if (segment_verification_is_ok()){ status=SDOWNLOADED; debug("seg:"+toString(seg_num)+" Downloaded"); } @@ -67,15 +78,11 @@ void Tsegment::set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfi error_log("Error in segment.cpp: prepare_for_connection()"); } } -void Tsegment::prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, uint mirror_num){ +void Tsegment::prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, string url_address){ try{ // debug("NETWORK:"+toString(network_num)+(network_array[network_num].use_own_mirror_list_only_on ? " - LOCAL": " - REMOTE")); - if (network_array[network_num].network_mode==MODE_LOCAL){ - url=network_array[network_num].benchmarked_mirror_list[mirror_num].url+parent_distfile->name; - debug(" URL:"+url); - }else{ - url=parent_distfile->url_list[mirror_num]; - } + // if MODE_LOCAL or MODE_PROXY_FETCHER + url=url_address; msg_connecting(con_num,distfile_num, segment_num,"Downloading from "+url); segments_in_progress[con_num]=this; status=SDOWNLOADING; diff --git a/segget/segment.h b/segget/segment.h index af00f34..bdb6dcc 100644 --- a/segget/segment.h +++ b/segget/segment.h @@ -84,8 +84,9 @@ class Tsegment{ Tsegment(const Tsegment &L); // copy constructor Tsegment & operator=(const Tsegment &L); ~Tsegment(); + bool segment_verification_is_ok(); void set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfile_name, ulong default_seg_size, ulong range_end); - void prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, uint mirror_num); + void prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, string url); string get_file_name(){return file_name;}; int add_easy_handle_to_multi(CURLM *cm, uint network_num); }; |