###########################################################
#
# Apple Quicktime Trailers library using iTunes XML files
# !!! Do not lauch this script! It won't do anything anyway.
# This file is used by the main script file: QuickTimeBrowser.py
#
# @author Mouton <mouton_x@abuserz.com>
# @date 2005-10-10
# @version 1.37
#
###########################################################

########################################
## Change those if needed ##############

# Use threads to download trailers information ? (True/False)
THREADED_PARSING = True

# Number of threads to use. If XBMC hangs while downloading data, try to decrease this.
# Don't use more than 7.
MAX_THREADS = 7

## Config ends here ####################
########################################

maintainer = 'Mouton <mouton_x@abuserz.com>'
import sys
from pickle import HIGHEST_PROTOCOL, Pickler, Unpickler
from os import getcwd, mkdir, path
from re import search as reSearch, IGNORECASE as reIGNORECASE
from string import capitalize, find, lower, replace, rfind, strip
from time import localtime, time
from xbmc import output
from xbmcgui import DialogProgress as xbmcguiDialogProgress, Dialog as xbmcguiDialog
try: from xbmcgui import Emulating
except: Emulating = False
## Following 2 Lines are imporant
sys.path.append(replace(getcwd(), ';', ''))
sys.path.append(replace(getcwd(), ';', '') + "\\_xmlplus.zip")
# global progress dialog
dialogProgress = xbmcguiDialogProgress()

# defines
POSTER_WIDTH = 134
POSTER_HEIGHT = 193

UNIT_DEBUG = 0
UNIT_START = 1
UNIT_END = 2

MOVIE_TYPE_NONE = 0
MOVIE_TYPE_EXCLUSIVE = 1
MOVIE_TYPE_OPENING = 2
MOVIE_TYPE_NEW = 3

USER_AGENT = 'iTunes/4.7'

BASE_URL = 'http://www.apple.com'
MAIN_INDEX_URL = BASE_URL + '/moviesxml/h/index.xml'
EXCLUDED_MOVIE_URLS = ('info.xml')
STANDARD_URLS = ('small.xml','medium.xml','large.xml','fullscreen.xml')

# Initial values
allMovies = {}
allCategories = {}
allStudios = {}
exclusiveMovies = []
openingMovies = []
newestMovies = []
newMovies = []

today = "%8d" % int("%04d%02d%02d" % localtime(time())[:3])

# Those will be overwritten by the values assigned in the main script file
DEBUG = 0
HTTP_TIMEOUT = 15.0
DOWNLOAD_ATTEMPTS = 5

# Use 0 to get human-readable data files
PICKLE_FORMAT = 0
#PICKLE_FORMAT = HIGHEST_PROTOCOL

DATA_DIR = path.join(replace(getcwd(), ';', ''), 'QuickTimeBrowser')
print('DATA_DIR = '+DATA_DIR)
if not path.exists(DATA_DIR):
    try:
        mkdir(DATA_DIR)
    except Exception:
        pass

# Remove invalid characters from the movie name to be able to use it as a file name
def CleanFilename(text):
    result = replace(text.encode('ascii','replace'), '\\','_')
    result = replace(result, '/','_')
    result = replace(result, ':','_')
    result = replace(result, '*','_')
    result = replace(result, '?','_')
    result = replace(result, '"','_')
    result = replace(result, '<','_')
    result = replace(result, '>','_')
    result = replace(result, '|','_')
    result = replace(result, '&','_')
    result = replace(result, ';','_')
    return result[:32].encode('ascii')

def getDocFromUrl(url):
    feeddata = download_file(url)
    printDebug('Parsing XML file using Minidom',UNIT_START)
    from xml.dom.minidom import parseString
    doc = parseString(feeddata)
    printDebug('Parsing XML file using Minidom',UNIT_END)
    return doc

def printError(text):
    text = text.encode('ascii','replace')
    output(text)

def printDebug(text,mode=UNIT_DEBUG):
    if not DEBUG: return
    if (mode == UNIT_START):
        text = '[START] ' + text
    elif (mode == UNIT_END):
        text = ' [END]  ' + text
    text = text.encode('ascii','replace')
    output(text)

def url_size(url):
    m = reSearch('http://([^/]*)(.*)', url, reIGNORECASE)
    if m != None:
        server = m.group(1)
        path = m.group(2)

    from httplib import HTTP
    http = HTTP(server)
    http.putrequest("HEAD", path)
    http.putheader("Host", server)
    http.putheader("User-Agent", USER_AGENT)
    http.endheaders()
    errcode, errmsg, headers = http.getreply()
    if errcode == 200:
        m = reSearch('Content-Length: ([0-9]*)', str(headers), reIGNORECASE)
        if m != None:
            size = m.group(1)
        printDebug('Size of '+url+' = '+size)
        return int(size)
    else:
        printDebug('URL doesn\'t exists: ' + url)
        return 0

def url_exists(url):
    m = reSearch('http://([^/]*)(.*)', url, reIGNORECASE)
    if m != None:
        server = m.group(1)
        path = m.group(2)

    from httplib import HTTP
    http = HTTP(server)
    http.putrequest("HEAD", path)
    http.putheader("Host", server)
    http.putheader("User-Agent", USER_AGENT)
    http.endheaders()
    errcode, errmsg, headers = http.getreply()
    if errcode == 200:
        printDebug('URL exists: ' + url)
        return 1
    else:
        printDebug('URL doesn\'t exists: ' + url)
        return 0

def download_file(url,stream=False):
    from socket import setdefaulttimeout
    from urllib2 import build_opener, Request, URLError
    printDebug('Reading file at ' + url, UNIT_START)

    def try_download(url):
        result = None
        try:
            setdefaulttimeout(HTTP_TIMEOUT)
            request = Request(url)
            request.add_header('User-Agent', USER_AGENT)
            opener = build_opener()
            if stream:
                result = opener.open(request)
            else:
                result = opener.open(request).read()
        except URLError, e:
            printError('Caught URLError in download_file for url: ' + url)
            printError('Exception Reason: ' + str(e))
            if e.reason[1] == 'getaddrinfo failed':
                printError('Cause: Undefined or not working DNS server.')
                xbmcguiDialog().ok('Error','Undefined or not working DNS server.\nPlease verify your network settings.')
            raise URLError, e
        except Exception, e:
            printError('Caught exception in download_file for url: ' + url)
            printError('Exception Reason: ' + str(e))
            raise Exception, e
        return result

    for i in range(1, DOWNLOAD_ATTEMPTS+1):
        printDebug('Download attempt #' + str(i))
        result = try_download(url)
        if result != None: break

    if result == None:
        printError('Error reading file at ' + url)
        xbmcguiDialog().ok('ERROR!','Failed to download file at\n' + url)

    printDebug('Reading file at ' + url, UNIT_END)

    return result

class Config:
    def __init__(self):
        self.download = False
        self.save = False
        self.favorites = []

    def load(self):
        if FileExists(path.join(DATA_DIR,'config.data')):
            input_file = open(path.join(DATA_DIR,'config.data'), 'rb')
            config = Unpickler(input_file).load()
            input_file.close()
            self.download = config.download
            self.save = config.save
            self.favorites = config.favorites

    def dump(self):
        output_file = open(path.join(DATA_DIR,'config.data'), 'wb')
        Pickler(output_file, 0).dump(self)
        output_file.close()

class Category:
    def __init__(self, name, url):
        self.index = url
        self.name = name
        self.url = url
        self.parsed = 0
        return

class Studio:
    def __init__(self, name, url):
        self.index = url
        self.name = name
        self.url = url
        return

class Thumb:
    def __init__(self,url,width,height):
        self.index = url
        self.url = url
        self.width = width
        self.height = height
        self.loaded = 0
        self.localFile = ''
        return

class Movie:
    def __init__(self, name, url, movieType=MOVIE_TYPE_NONE, studio=None, categories=[], actors='', thumb=None, rating=''):
        self.index = url
        self.title = self.clean_text(name)
        self.xmlUrl = url
        self.movieType = movieType
        self.studio = studio
        if studio != None:
            addStudio(studio)
        self.categories = categories
        for c in categories:
            addCategory(c)
        self.actors = actors
        self.thumb = thumb
        self.rating = rating
        self.hasCustomTrailer = 0
        self.trailers = []
        self.loaded = today	# Used to find new trailers
        return

    def clean_text(self,text):
        text = replace(text, '&amp;', '&')
        text = replace(text, '&hearts;', '<3')
        text = replace(text, '&#146;', "'")
        return text

    def loadThumb(self):
        return

    def loadMovieTrailersUrls(self):
        if len(self.trailers) > 0: return	# Already loaded

        printDebug('Finding trailers for movie: ' + self.title,UNIT_START)

        movieIndexPageDoc = getDocFromUrl(self.xmlUrl)

        elements = getElements('/Document/ScrollView/MatrixView/HBoxView/VBoxView/HBoxView/VBoxView/MatrixView/HBoxView/HBoxView/VBoxView/HBoxView/VBoxView/HBoxView/VBoxView/GotoURL',movieIndexPageDoc.documentElement)
        if len(elements) > 0:
            urlAttr = elements[0].getAttribute('url')
            if find(urlAttr,'http') != -1:
                url = urlAttr
            elif find(urlAttr,'/') != -1:
                url = BASE_URL + urlAttr
            else:
                url = replace(self.xmlUrl,'index.xml','') + urlAttr
            self.xmlUrl = url
            printDebug('Looking for trailers in ' + url)
            movieIndexPageDoc = getDocFromUrl(url)

        elements = getElements('/Document/ScrollView/MatrixView/View/VBoxView/HBoxView/MatrixView/HBoxView/VBoxView/HBoxView/GotoURL',movieIndexPageDoc.documentElement)
        if len(elements) == 0:
            elements = getElements('/Document/ScrollView/MatrixView/VBoxView/HBoxView/View/GotoURL',movieIndexPageDoc.documentElement)
        if len(elements) == 0:
            printError('Found 0 URL in ' + self.xmlUrl + " for movie '" + self.title + "'")
        for element in elements:
            if element.getAttribute('url') not in EXCLUDED_MOVIE_URLS:
                urlAttr = element.getAttribute('url')
                if find(urlAttr,'http') != -1:
                    url = urlAttr
                elif find(urlAttr,'/') != -1:
                    url = BASE_URL + urlAttr
                else:
                    url = replace(self.xmlUrl,'index.xml','') + urlAttr
                if url not in self.trailers:
                    if lower(urlAttr) not in STANDARD_URLS:
                        self.hasCustomTrailer = 1
                    self.trailers.append(url)
        if len(self.trailers) > 4:
            self.hasCustomTrailer = 1
        movieIndexPageDoc.unlink()
        del movieIndexPageDoc

        self.trailers.sort(sort_trailers_ascending)

        printDebug('Found trailers: ')
        for trailer in self.trailers:
            printDebug(trailer)

        printDebug('Finding trailers for movie: ' + self.title,UNIT_END)
        return

    def getMov(self,index):
        url = self.trailers[index]
        printDebug('Getting .mov URL from URL: ' + url)
        m = reSearch('(.+_.+\.mov)\|(.*)', url, reIGNORECASE)
        if m != None:
            printDebug('The .mov URL is already cached: ' + m.group(1))
            return m.group(1).encode('ascii')
        dialogProgress.create("Please wait...","Finding trailer URL.")
        movieDoc = getDocFromUrl(url)
        movieLink = getValue('//MovieView/@url',movieDoc.documentElement)
        movieDoc.unlink()
        del movieDoc
        m = reSearch('(.*)_i([0-9]+\.mov)', movieLink, reIGNORECASE)
        if m != None:
            movieLink = m.group(1) + '_m' + m.group(2)
        movieLink = movieLink.encode('ascii')
        self.trailers[index] = movieLink + '|' + getTrailerName(url)
        dialogProgress.close()
        return movieLink

def getTrailerName(url):
    m = reSearch('(.*)\|(.*)', url, reIGNORECASE)
    if m != None:
        return m.group(2).encode('ascii','replace')
    name = url[rfind(url,'/') + 1:]
    name = capitalize(replace(name,'.xml',''))
    return name.encode('ascii','replace')

def getText(node):
    rc = ''
    for node in node.childNodes:
        if node.nodeType == node.TEXT_NODE:
            rc = rc + node.data
    return rc

def getElement(xpathText,element):
    try:
        from xml.xpath import Evaluate
    except Exception, e:
        xbmcguiDialog().ok('Error','Error loading needed module from _xmlplus.\nCheck the README.txt on how to install this script correctly.')
        printError('_xmlplus isn\'t installed correctly.')
        printError('Error: ' + str(e))

    tempNodes = Evaluate(xpathText, element)
    if tempNodes == None:
        return None
    else:
        return tempNodes[0]

def getValue(xpathText,element):
    from xml.xpath import Evaluate
    tempNodes = Evaluate(xpathText, element)
    if len(tempNodes) == 0:
        return ''
    else:
        return strip(getText(tempNodes[0]))

def getElements(xpathText,element):
    from xml.xpath import Evaluate
    val = Evaluate(xpathText, element)
    return val

def addMovie(movie):
    global allMovies
    if allMovies == None:
        allMovies = {}
    if not allMovies.has_key(movie.index):
        allMovies[movie.index] = movie
    else:
        # Merge old movie with new one in case new infos was found
        m = allMovies[movie.index]
        if m.movieType == MOVIE_TYPE_NONE:
            m.movieType = movie.movieType
        if m.studio == None:
            m.studio = movie.studio
        if len(movie.categories) > 0:
            for category in movie.categories:
                found = False
                for category2 in m.categories:
                    if category2.index == category.index:
                        found = True
                        break
                if not found:
                    m.categories.append(category)
        if m.actors == '':
            m.actors = movie.actors
        if m.rating == '':
            m.rating = movie.rating
        if m.thumb == None and not movie.thumb == None:
            m.thumb = movie.thumb
            m.loadThumb()
        if len(movie.trailers) > 0:
            m.trailers = movie.trailers
        if m.loaded == 1:
            m.loaded = movie.loaded

def addCategory(category):
    if not allCategories.has_key(category.index):
        allCategories[category.index] = category

def addStudio(studio):
    if not allStudios.has_key(studio.index):
        allStudios[studio.index] = studio

def loadMoviesInfoFromIndexUrl(url,movie_type=MOVIE_TYPE_NONE):
    doc = getDocFromUrl(url)
    loadMoviesInfoFromDocument(doc,movie_type)

def loadMoviesInfoFromDocument(movieIndexPageDoc,movie_type=MOVIE_TYPE_NONE):
    elements = getElements('/Document/ScrollView/MatrixView/MatrixView/VBoxView/MatrixView/HBoxView/VBoxView/MatrixView',movieIndexPageDoc.documentElement)
    for element in elements:
        fullTitle = getValue('VBoxView/HBoxView/TextView/GotoURL/B',element);
        if rfind(fullTitle,' - ') != -1:
            title = fullTitle[:rfind(fullTitle,' - ')]
        else:
            title = fullTitle
        url = BASE_URL + getValue('GotoURL/@url',element);
        thumbUrl = getValue('GotoURL/PictureView/@url',element);
        studio_name = getValue('VBoxView/TextView[1]/GotoURL',element);
        studio_url = getValue('VBoxView/TextView[1]/GotoURL/@url',element);
        studio = Studio(studio_name,studio_url)
        actors = getValue('VBoxView/TextView[2]',element);
        categories = []
        elements = getElements('VBoxView/TextView[3]/GotoURL',element);
        for element in elements:
            name = getValue('.',element)
            cat_url = BASE_URL + element.getAttribute('url')
            category = Category(name,cat_url)
            categories.append(category)
        rating = getValue('VBoxView/TextView[4]',element);

        t = Thumb(thumbUrl, POSTER_WIDTH, POSTER_HEIGHT)
        m = Movie(title, url, movie_type, studio, categories, actors, t, rating)
        printDebug('New Trailer: Title = ' + m.title)
        addMovie(m)

def saveDataFiles(load_before_save=False,quiet=False,window=None,reset_movie_types=False):
    from shutil import move
    global allMovies

    printDebug('Saving QuickTimeBrowser data files', UNIT_START)
    if window:
        window.labelStatus.setLabel('Saving data files...')
        window.addControl(window.labelStatus)
    if not quiet:
        dialogProgress.create("Saving", 'Saving QuickTimeBrowser data files')

    backup_allMovies = allMovies

    # Merge back all new informations into allMovies
    for list_movies in exclusiveMovies,openingMovies,newestMovies,newMovies:
        for movie in list_movies:
            if reset_movie_types:
                movie.movieType = MOVIE_TYPE_NONE
            addMovie(movie)

    # Get the movies on disk and merge with the ones in memory
    if load_before_save:
        current_movies = allMovies
        input_file = open(path.join(DATA_DIR,'allMovies.data'), 'rb')
        allMovies = Unpickler(input_file).load()
        if reset_movie_types:
            for key,movie in allMovies.iteritems():
                movie.movieType = MOVIE_TYPE_NONE
        input_file.close()
        if current_movies != None and len(current_movies) > 0:
            for key,movie in current_movies.iteritems():
                addMovie(movie)

    filename = path.join(DATA_DIR,'allMovies.data')
    output_file = open(filename+'.tmp', 'wb')
    Pickler(output_file, PICKLE_FORMAT).dump(allMovies)
    output_file.close()
    move(filename+'.tmp',filename)
    printDebug('Saved ' + str(len(allMovies)) + ' movie trailers.')

    delAllMovies()
    allMovies = backup_allMovies

    filename = path.join(DATA_DIR,'allCategories.data')
    output_file = open(filename+'.tmp', 'wb')
    Pickler(output_file, PICKLE_FORMAT).dump(allCategories)
    output_file.close()
    move(filename+'.tmp',filename)

    filename = path.join(DATA_DIR,'allStudios.data')
    output_file = open(filename+'.tmp', 'wb')
    Pickler(output_file, PICKLE_FORMAT).dump(allStudios)
    output_file.close()
    move(filename+'.tmp',filename)

    if window:
        window.removeControl(window.labelStatus)
    if not quiet:
        dialogProgress.close()
    printDebug('Saving QuickTimeBrowser data files', UNIT_END)
    return True

def FileExists(filename):
    from glob import glob
    return len(glob(filename)) > 0

def loadDataFiles(ResetTypes=True):
    global allMovies, allCategories, allStudios
    if FileExists(path.join(DATA_DIR,'allMovies.data')):
        printDebug('Loading QuickTimeBrowser data files', UNIT_START)
        dialogProgress.create("Loading", 'Loading QuickTimeBrowser data files')
        input_file = open(path.join(DATA_DIR,'allMovies.data'), 'rb')
        allMovies = Unpickler(input_file).load()
        input_file.close()
        printDebug('Loaded ' + str(len(allMovies)) + ' movie trailers.')

        input_file = open(path.join(DATA_DIR,'allCategories.data'), 'rb')
        allCategories = Unpickler(input_file).load()
        input_file.close()
        printDebug('Loaded ' + str(len(allCategories)) + ' categories.')

        input_file = open(path.join(DATA_DIR,'allStudios.data'), 'rb')
        allStudios = Unpickler(input_file).load()
        input_file.close()
        printDebug('Loaded ' + str(len(allStudios)) + ' studios.')

        if ResetTypes:
            printDebug('Reseting movies types.')
            # Resets all movieType; Exclusive, Opening and New trailers might not be in those categories anymore
            # Will always reload those sections
            for key,movie in allMovies.iteritems():
                movie.movieType = MOVIE_TYPE_NONE

        dialogProgress.close()
        printDebug('Loading QuickTimeBrowser data files', UNIT_END)
        return True
    else:
        parse(loaded=False,update_all=True)
    return False

def parse(loaded=False,update_all=False):
    global MAX_THREADS
    if DEBUG and MAX_THREADS > 3:
            MAX_THREADS = 3

    global allMovies
    categories_urls = []
    if update_all:
        allMovies = {}
        if loaded:
            for list_movies in exclusiveMovies,openingMovies,newestMovies:
                for movie in list_movies:
                    addMovie(movie)
            for category in getAllCategories():
                categories_urls.append(category.url)

    if not loaded or not update_all:
        try:
            dialogProgress.create("Parsing", 'Exclusive Movie Trailers')
            doc = getDocFromUrl(MAIN_INDEX_URL)
            baseNode = getElement( '/Document/View/ScrollView/MatrixView/View/VBoxView/MatrixView/VBoxView[1]/VBoxView/VBoxView', doc.documentElement )

            # Excusive Movie Trailers
            exclusiveNodes = getElements('VBoxView[0]/View/View/VBoxView[1]/View/VBoxView[1]/VBoxView/VBoxView/Include', baseNode)
            for element in exclusiveNodes:
                url = BASE_URL + element.getAttribute('url')
                title = getValue('TextView[1]', element)
                m = Movie(title, url, MOVIE_TYPE_EXCLUSIVE)
                printDebug('New Exclusive Trailer: Title = ' + m.title)
                addMovie(m)
            dialogProgress.close()

            # Opening This Week Movie Trailers
            dialogProgress.create("Parsing", 'Opening This Week Movie Trailers')
            thisWeekNodes = getElements('VBoxView[0]/View/View/VBoxView[1]/View/VBoxView[1]/VBoxView/Include', baseNode)
            for element in thisWeekNodes:
                url = BASE_URL + element.getAttribute('url')
                title = getValue('TextView[1]/B', element)
                m = Movie(title, url, MOVIE_TYPE_OPENING)
                printDebug('New Opening Trailer: Title = ' + m.title)
                addMovie(m)
            dialogProgress.close()

            # Will take those from the trailers data instead...
            # But we still need the URL to find all trailers
            if update_all:
                dialogProgress.create("Parsing", 'Listing Categories')
                categoriesNodes = getElements('VBoxView[1]/VBoxView[4]/MatrixView/View/VBoxView/HBoxView/TextView/GotoURL',baseNode)
                for element in categoriesNodes:
                    url = BASE_URL + element.getAttribute('url')
                    categories_urls.append(url)
                dialogProgress.close()

            dialogProgress.create("Parsing", 'New Movie Trailers')
            newMoviesUrl = replace(MAIN_INDEX_URL,'index.xml','') + getValue('VBoxView[2]/View/MatrixView/TextView[2]/GotoURL/@url',baseNode)
            printDebug('New Movies Index URL = ' + newMoviesUrl)
            loadMoviesInfoFromIndexUrl(newMoviesUrl,MOVIE_TYPE_NEW)
            dialogProgress.close()

            doc.unlink()
            del doc

            # All trailers will only be loaded the 1st time, or when the user forces full update ('Update All' button)
            if loaded and not update_all:
                echoDebugInfos()
                saveDataFiles(load_before_save=True,reset_movie_types=True)
                loadMovieVars()
                return
        except Exception,e:
            dialogProgress.close()
            xbmcguiDialog().ok('Error','Error while parsing new trailers.\nError: ' + str(e))
            printError('Error while parsing new trailers.')
            printError('Error: ' + str(e))
            return -1

    printDebug('Parsing data for all categories.', UNIT_START)

    def downloadCategory(url):
        m = reSearch('(http://.*_)[0-9]+\.xml', url, reIGNORECASE)
        if m != None:
            category_url = m.group(1)
        else:
            printError('Error parsing category URL: ' + url)
            return

        printDebug('Loading all movies from category URL ' + url, UNIT_START)
        for i in range(1,100):
            url = category_url + str(i) + '.xml'
            if not url_exists(url): break
            data = download_file(url)
            try:
                from xml.dom.minidom import parseString
                doc = parseString(data)
            except Exception, e:
                printError('Error catched while parsing XML to Document.')
                printError('Error: ' + str(e))
                break
                continue
            loadMoviesInfoFromDocument(doc)
            doc.unlink()
            del doc
            global cat_step
            cat_step = cat_step + 1
            current_pc = int(cat_step*100/92)
            if current_pc > 100:
                dialogProgress.update(100-(current_pc-100))
            else:
                dialogProgress.update(current_pc)
        printDebug('Loading all movies from category URL ' + url, UNIT_END)
        return

    def check_threads(subthreads):
        from time import sleep
        sleep(1.000)
        while len(subthreads) > 0:
            for subThread in subthreads:
                if not subThread.isAlive():
                    subthreads.remove(subThread)
            sleep(0.100)
        return

    try:
        global cat_step
        cat_step = 0
        if THREADED_PARSING:
            from threading import Thread
            dialogProgress.create("Parsing", 'All Categories Movie Trailers')
            subthreads = []

            checkSubThread = Thread(target=check_threads, args=(subthreads,))
            checkSubThread.start()

            # Start MAX_THREADS threads at the same time
            from time import sleep
            for category_url in categories_urls:
                subThread = Thread(target=downloadCategory, args=(category_url,))
                subThread.start()
                subthreads.append(subThread)
                sleep(0.500)
                while len(subthreads) >= MAX_THREADS:
                    sleep(0.100)

            # Wait for all threads to complete
            checkSubThread.join()
            dialogProgress.close()
        else:
            for category_url in categories_urls:
                dialogProgress.create("Parsing", 'Movie Trailers from \n' + category_url)
                downloadCategory(category_url)
                dialogProgress.close()

        printDebug('Parsing data for all categories.', UNIT_END)
        echoDebugInfos()
        saveDataFiles()
        loadMovieVars()
    except Exception,e:
        dialogProgress.close()
        xbmcguiDialog().ok('Error','Error while parsing all categories', 'Error: ' + str(e))
        printError('Error while parsing all categories.')
        printError('Error: '+str(e))
        return -1
    return 0

def loadMovieVars():
    global exclusiveMovies, openingMovies, newestMovies, newMovies
    exclusiveMovies = getAllMovies(MOVIE_TYPE_EXCLUSIVE,False)
    openingMovies = getAllMovies(MOVIE_TYPE_OPENING,False)
    newestMovies = getAllMovies(MOVIE_TYPE_NEW,False)
    newMovies = getAllNewMovies(False)
    delAllMovies()	# To free some RAM...

def sort_trailers_ascending(first, second):
    i = 0
    first_value = -1
    second_value = -1
    for url in STANDARD_URLS:
        if find(lower(first), url) != -1:
            first_value = i
        if find(lower(second), url) != -1:
            second_value = i
        i = i + 1
    return cmp(first_value, second_value)

def sort_movies_ascending(first, second):
    return cmp(first.title, second.title)

def sort_name_ascending(first, second):
    return cmp(first.name, second.name)

def loadAllMovies():
    dialogProgress.create("Loading", 'Loading QuickTimeBrowser data files')
    global allMovies
    input_file = open(path.join(DATA_DIR,'allMovies.data'), 'rb')
    allMovies = Unpickler(input_file).load()
    input_file.close()
    dialogProgress.close()

def delAllMovies():
    global allMovies
    del allMovies
    allMovies = None
    from gc import collect
    collect()
    printDebug('All Movies removed to free more RAM...')

def getAllMovies(movieType=None,sorted=False):
    del_when_done = False
    if allMovies == None:
        loadAllMovies()
        del_when_done = True
    movies = []
    for key,movie in allMovies.iteritems():
        if movieType == None:
            movies.append(movie)
        elif movieType != None and movie.movieType == movieType:
            movies.append(movie)
    if sorted:
        movies.sort(sort_movies_ascending)
    if del_when_done:
        delAllMovies()
    return movies

def getAllNewMovies(sorted=False):
    movies = []
    for key,movie in allMovies.iteritems():
        if movie.loaded == today:
            movies.append(movie)
    if sorted:
        movies.sort(sort_movies_ascending)
    return movies

def getAllMoviesFromStudio(studio):
    del_when_done = False
    if allMovies == None:
        loadAllMovies()
        del_when_done = True
    movies = []
    for key,movie in allMovies.iteritems():
        if movie.studio != None and movie.studio.index == studio.index:
            movies.append(movie)
    movies.sort(sort_movies_ascending)
    if del_when_done:
        delAllMovies()
    return movies

def getAllMoviesFromCategory(category):
    del_when_done = False
    if allMovies == None:
        loadAllMovies()
        del_when_done = True
    movies = []
    for key,movie in allMovies.iteritems():
        for cat in movie.categories:
            if cat.index == category.index:
                movies.append(movie)
                break
    movies.sort(sort_movies_ascending)
    if del_when_done:
        delAllMovies()
    return movies

def getAllCategories():
    categories = []
    for key,category in allCategories.iteritems():
        categories.append(category)
    categories.sort(sort_name_ascending)
    return categories

def getAllStudios():
    studios = []
    for key,studio in allStudios.iteritems():
        studios.append(studio)
    studios.sort(sort_name_ascending)
    return studios

def echoDebugInfos():
    printDebug('Movie Trailers Found: ' + str(len(allMovies)))
    '''movies = []
    for key,movie in allMovies.iteritems():
        movies.append(movie)
    movies.sort(sort_movies_ascending)
    for movie in movies:
        printDebug('Trailer = ' + movie.title)
    '''
    printDebug('Categories Found: ' + str(len(allCategories)))
    '''categories = []
    for key,category in allCategories.iteritems():
        categories.append(category)
    categories.sort(sort_name_ascending)
    for category in categories:
        printDebug('Category = ' + category.name)
    '''
    printDebug('Studios Found: ' + str(len(allStudios)))
    '''studios = []
    for key,studio in allStudios.iteritems():
        studios.append(studio)
    studios.sort(sort_name_ascending)
    for studio in studios:
        printDebug('Studio = ' + studio.name)
    '''
    return