"""Starez, an update-platform for linux Copyright (C) 2007 Hylke Donker This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. """ import urllib2 import threading import time ## Class that supports threaded downloading with resuming # # The class HttpDownload is a multithreaded class(which means, that it's derived from # threading.Thread), that allows you to download files through HTTP in a seperate thread. # It also supports resuming, after a perticular byte(and automaticly appends the new bytes # to the file). class HttpDownload(threading.Thread): ## Constructor, downloads url and writes it to output_file(or to self.m_data if # output_file is None). Resume download after byte resume_byte. Chunk size is the # maximum size of the content-length of each request. The constructor initializes # some variables such as locks, the parameters of this function and sets some other # variables too. def __init__(self, url, output_file = None, resume_byte = 0, chunk_size = 128000, length = 0): threading.Thread.__init__(self) print "INIT!!!!" ## Whether this class should stop download data-chunks self.m_stop = False ## Whether the current thread is paused or not self.m_paused = False ## If the received data should not be written to a file, it's written to self.m_data self.m_data = "" ## The number of bytes this class has written(so it's not the overall number of bytes written) self.m_TotallyWritten = 0 ## Lock to guard the variable self.m_data self.m_DataLock = threading.Lock() ## Lock to guard the variable self.m_TotallyWritten self.m_TotalWrittenLock = threading.Lock() if resume_byte != 0 and (int(resume_byte) % int(chunk_size) != 0 or float(resume_byte) / float(chunk_size) == 0.0): print "ERROR1: Resume byte incorrect!" # constructor parameters ## The url to the file to download self.m_url = url ## The file to which the received data will be written self.m_output_file = output_file ## Start downloading at byte self.m_start_byte self.m_start_byte = int(resume_byte) ## The number of bytes to download per request self.m_chunk_size = int(chunk_size) ## Whether this class has encountered an error(for example when urllib2 couldn't connect to a server) self.m_error = False ## The total length of the file(not the size to download this session) self.m_total_length = int(length) ## Contains a pair: (bytes, callback) if the callback is set. It calls callback every bytes that is downloaded self.m_callback_info = None ## Whether this class is in debug mode(prints more information) self.m_debug = False print "self.m_start_byte(=" + str(self.m_start_byte) + ") % self.m_chunk_size(=" + str(self.m_chunk_size) + "): " + str(self.m_start_byte % self.m_chunk_size) if self.m_start_byte % self.m_chunk_size != 0: print "ERROR2: Resume byte incorrect!" ## This function will be called by the start function of threading.Thread. You're # supposed to overwrite this function. In this case it downloads the file from the # webserver, and writes it to the file(using the parameters from the constructor). def run(self): file_object = None if self.m_output_file: # If we should not resume downloading then overwrite it all if self.m_start_byte == 0: file_object = open(self.m_output_file, "wb") # Otherwise append the file else: #file_object = open(self.m_output_file, "r+") #file_object.seek(self.m_start_byte) file_object = open(self.m_output_file, "ab+") #file_object.seek(0, 2) #self.m_start_byte = file_object.tell() #file_object.seek(0, 0) # Byte counter, so we know when to call the callback function callback_byte_counter = 0 while not self.m_stop: #if self.m_paused and not self.m_stop: # time.sleep(0.1) # return download_from = int(self.m_start_byte) + int(self.m_TotallyWritten) download_to = int(self.m_start_byte) + int(self.m_TotallyWritten) + self.m_chunk_size # If we know the total length and we try to download more bytes than the server has, we reset the download_to to the max if self.m_total_length > 0 and download_to > self.m_total_length: download_to = self.m_total_length # Create a download request with the range header set, to say what bytes we want to download request = urllib2.Request(self.m_url, headers = { "Range" : "bytes=" + str(download_from) + "-" + str(download_to-1) }) data_object = None try: data_object = urllib2.urlopen(request) except Exception, msg: # If we were unable to download, then return to the end of this function print "Error downloading data: " + str(msg) self.m_error = True return # Extract the length of the file and read the data from the file-object data_chunk = data_object.read() self.m_total_length = int(data_object.info()["content-range"].split("/")[1]) # If the byte range is zero, obiously something went wrong, so we retry if self.m_total_length == 0: print "Warning: Something went wrong while downloading data chunk(did not receive length). Retrying chunk..." continue # If we do not try to download the last chunk and the total bytes we downloaded are not dividable by the # chunk-size(a chunk of data we downloaded was not of length self.m_chunk_size), then something went wrong. if download_to != self.m_total_length and (len(data_chunk) + self.m_TotallyWritten + self.m_start_byte) % self.m_chunk_size != 0: print "Warning: Something went wrong while downloading data chunk. Retrying chunk..." # print debugging output if self.m_debug: if self.m_start_byte % self.m_chunk_size != 0: print "CRAP!" print "Dump:\n--" print "Range: bytes=" + str(download_from) + "-" + str(download_to-1) print "self.m_total_length: " + str(self.m_total_length) print "len(data_chunk): " + str(len(data_chunk)) print "self.m_chunk_size: " + str(self.m_chunk_size) print "totally downloaded: " + str(len(data_chunk) + self.m_TotallyWritten + self.m_start_byte) print "self.m_TotallyWritten: " + str(self.m_TotallyWritten) print "self.m_start_byte: " + str(self.m_start_byte) print "(len(data_chunk) + self.m_TotallyWritten + self.m_start_byte) % self.m_chunk_size: " + str((len(data_chunk) + self.m_TotallyWritten + self.m_start_byte) % self.m_chunk_size) print "--" continue # skip the rest of the loop, retry # If we've chosen to write to a file, then do so, otherwise write to our carefully locked data buffer if self.m_output_file: file_object.write(data_chunk) else: self.m_DataLock.acquire() self.m_data += data_chunk self.m_DataLock.release() # Update our variable that holds the number of downloaded bytes from this session self.m_TotalWrittenLock.acquire() self.m_TotallyWritten += len(data_chunk) self.m_TotalWrittenLock.release() # Also update out callback byte counter callback_byte_counter += len(data_chunk) #if self.m_callback_info != None and (callback_byte_counter >= self.m_callback_info[0] or self.m_TotallyWritten >= self.m_total_length-self.m_callback_info[0] ): # If we have a callback set, and we have downloaded enough bytes or this is the last chunk, then call the callback function if self.m_callback_info != None and (callback_byte_counter >= self.m_callback_info[0] or download_to-download_from < self.m_chunk_size): if self.m_TotallyWritten + self.m_start_byte == self.m_total_length: print "100%" # 'reset' our callback byte counter callback_byte_counter -= self.m_callback_info[0] percentage = float(self.m_TotallyWritten+self.m_start_byte) / float(self.m_total_length) * 100.0 # Call the callback(with extra argument when appropiately) if self.m_callback_info[2] > -1: self.m_callback_info[1](percentage, self.m_TotallyWritten, self.m_TotallyWritten + self.m_start_byte, self.m_total_length, self.m_callback_info[2]) else: self.m_callback_info[1](percentage, self.m_TotallyWritten, self.m_total_length - self.m_start_byte, self.m_total_length) # If we've downloaded the whole file, then stop the loop if self.m_TotallyWritten + self.m_start_byte == self.m_total_length: print "done" self.m_stop = True data_object.close() # Find out whether we've actually written as much as the variable told us file_object.seek(0, 2) if file_object.tell() != self.m_TotallyWritten + self.m_start_byte: print "ERROR: File ies not compatible" if self.m_output_file: file_object.close() ## Set a callback function. The callbackfunction callback gets called every num_bytes # bytes. The callback receives the parameters: percenatege, TotalWrittenBytes(from # this session), the remaining-bytes to download and the total length of the of the download. def SetNotifier(self, callback, num_bytes = 128000, arg = -1): self.m_callback_info = (num_bytes, callback, arg)