using System; using System.Collections.Generic; using System.Web; using System.IO; using System.Text.RegularExpressions; using System.Diagnostics; using System.Net; using KLiPSCH; namespace ConsoleApplication1 { class Program { static void Main(string[] args) { Stopwatch time = new Stopwatch(); time.Start(); Console.WriteLine("GETTING RAGE PAGE"); TVRage rage = new TVRage(); rage.runParser(); Console.WriteLine("GOT RAGE PAGE"); //runString("imdbSearch.html",searchSrc); Console.WriteLine("FINISHED IN --> " + time.ElapsedMilliseconds); time.Stop(); } static void runString(string filename,string data) { TextWriter tw = new StreamWriter(filename); tw.Write(data); tw.Close(); System.Diagnostics.Process.Start(@filename); } } } namespace KLiPSCH { class http { private HttpWebRequest _request; private HttpWebResponse _response; private CookieContainer _cookieJar; private string _src; private string _url; public http(string url) { _url = url; } public http(string url,CookieContainer cookieJar) { _url = url; _cookieJar = cookieJar; } public void open() { try { _request = (HttpWebRequest)WebRequest.Create(_url); } catch (Exception e) { Console.WriteLine("ERROR: could not create WebRequest: \n" + e); return; } if (_cookieJar == null) { _cookieJar = new CookieContainer(); } _request.CookieContainer = _cookieJar; //try reading the data from to the socket try { //get the response _response = (HttpWebResponse)_request.GetResponse(); StreamReader streamReader = new StreamReader(_response.GetResponseStream()); _src = streamReader.ReadToEnd(); streamReader.Dispose(); _response.Close(); } catch (Exception e) { Console.WriteLine("ERROR: could not read from the socket: \n" + e); return; } } public void postForm(string formString) { if (formString == null) { return; } try { _request = (HttpWebRequest)WebRequest.Create(_url); } catch (Exception e) { Console.WriteLine("ERROR: could not create WebRequest: \n" + e); return; } if (_cookieJar == null) { _cookieJar = new CookieContainer(); } _request.CookieContainer = _cookieJar; _request.Method = "POST"; _request.ContentType = "application/x-www-form-urlencoded"; _request.AllowAutoRedirect = false; byte[] bytes = System.Text.Encoding.ASCII.GetBytes(formString); _request.ContentLength = bytes.Length; //try writing the form data to the socket try { System.IO.Stream os = _request.GetRequestStream(); os.Write(bytes,0,bytes.Length); os.Dispose(); } catch (Exception e) { Console.WriteLine("ERROR: could not write to socket: \n" + e); return; } //try reading the data from to the socket try { //get the response _response = (HttpWebResponse)_request.GetResponse(); StreamReader streamReader = new StreamReader(_response.GetResponseStream()); _src = streamReader.ReadToEnd(); streamReader.Dispose(); _response.Close(); } catch (Exception e) { Console.WriteLine("ERROR: could not read from the socket: \n" + e); return; } //printCookies(); } public HttpWebRequest getRequest() { return _request; } public HttpWebResponse getResponse() { return _response; } public CookieContainer getCookieJar() { return _cookieJar; } public String getSource() { return _src; } public String getUrl() { return _url; } } public enum Genre : int { Action = 1, Adventure = 2, Children = 3, Comedy = 4, //Cooking/Food = 5, Crime = 6, Drama = 7, Family = 8, Fantasy = 9, //Financial/Business = 10, Horror = 11, Music = 12, Mystery = 13, Romance = 14, //Sci-Fi = 15, Sketch = 16, Soaps = 17, Sports = 18, //Tech/Gaming = 19, Thriller = 20, Western = 21 } public enum Classification : int { Animation = 1, //Award Show = 2, Documentary = 3, //Game Show = 4, //Mini-Series/Movie = 5, News = 6, Reality = 7, Scripted = 8, Sports = 9, //Talk Shows = 10, Variety } class Genres : List { public Genres() { } } class Show { private String _title; private Uri _URL; private Genres _genres; private Classification _classification; public Show() { } public String Title { get { return _title; } set { _title = (String)value; } } public Uri URL { get { return _URL; } set { _URL = (Uri)value; } } public Genres Genres { get { return _genres; } set { _genres = (Genres)value; } } public Classification Classification { get { return _classification; } set { _classification = (Classification)value; } } } class TVRage { private String _listURL = "http://tvrage.com/all.php"; public TVRage() { } public bool runParser() { http rageAll = new http(_listURL); rageAll.open(); String theSrc = rageAll.getSource(); if (theSrc == null) { return false; } String startParseRegex = "([\\d]*?) shows were found with the selected filter options"; String endParseRegex = "Selected Filter Options: "; int numMatches = 0; Regex startReg = new Regex(startParseRegex,RegexOptions.Multiline | RegexOptions.IgnoreCase); Match lastMatch = startReg.Match(theSrc); if (lastMatch != null) { int startPos = lastMatch.Index; //Console.Write("START MATCHED AT: " + startPos + "of " + theSrc.Length + "\n"); theSrc = theSrc.Substring(startPos); //Console.Write("NEW LENGTH IS: " + theSrc.Length + "\n"); Regex endReg = new Regex(endParseRegex,RegexOptions.Multiline | RegexOptions.IgnoreCase); Match endMatch = endReg.Match(theSrc); if (endMatch != null) { int endPos = endMatch.Index; //Console.Write("END MATCHED AT: " + endPos + "of " + theSrc.Length + "\n"); theSrc = theSrc.Substring(0,endPos); //Console.Write("NEW LENGTH IS: " + theSrc.Length + "\n"); Regex linksReg = new Regex("(.*?)",RegexOptions.Multiline | RegexOptions.IgnoreCase); Match linkMatch = linksReg.Match(theSrc); while (linkMatch.Success) { numMatches++; try { Show thisShow = parseLink(linkMatch.Groups[1].Value); } catch (UriFormatException e) { if (linkMatch.Groups[1].Value == null) { Console.WriteLine("url found is null for link " + numMatches + "\n" + e); } else { Console.WriteLine("invlaid url format found in " + numMatches + "-> " + linkMatch.Groups[1].Value + "\n" + e); } } linkMatch = linkMatch.NextMatch(); } } else { Console.WriteLine("something is wrong with tvrage src file \n"); } } Console.Write("FOUND " + numMatches + " MATCHES \n"); return true; } private Show parseLink(String url) { try { Uri showURL = new Uri(url); Console.WriteLine("READING " + showURL + " ...."); } catch (UriFormatException e) { //throw UriFormatException e; } http checkPage = new http(url); checkPage.open(); String theSrc = checkPage.getSource(); Show thisShow = new Show(); Regex r1 = new Regex(".+ :: (.*?) :: .+",RegexOptions.Multiline | RegexOptions.IgnoreCase); Match titleMatch = r1.Match(theSrc); if (titleMatch != null) { Console.WriteLine("FOUND TITLE --> " + titleMatch.Groups[1].Value); thisShow.Title = titleMatch.Groups[1].Value; } Regex r2 = new Regex("Show Classification\\: (.*?)",RegexOptions.Multiline | RegexOptions.IgnoreCase); Match classMatch = r2.Match(theSrc); if (classMatch != null) { Console.WriteLine("FOUND CLASS --> " + classMatch.Groups[1].Value); Classification classifcation = (Classification)classMatch.Groups[1].Value; //thisShow.Classification = classMatch.Groups[1].Value; } Regex r3 = new Regex("Genre\\: (.*?)",RegexOptions.Multiline | RegexOptions.IgnoreCase); Match genreMatch = r3.Match(theSrc); if (genreMatch != null) { Genres genres = new Genres(); Console.WriteLine("FOUND GENRES --> " + genreMatch.Groups[1].Value); //thisShow.Genres = } return thisShow; } } }