#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <netdb.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <sys/socket.h>

#define PORT 80
#define MAXDATASIZE 2048

char get[1024] = "GET \0";

int main(int argc, char *argv[])
{
        int sockfd, numbytes;
        char buf[MAXDATASIZE];
        struct hostent *he;
        struct sockaddr_in their_addr;

        if (argc < 2 || argc > 3)
        {
                fprintf(stderr, "usage: robots <hostname> <path to robots>(if not in root)\n");
                exit(1);
        }

        if (argc == 3)
        {
                strcat(get, argv[2]);
                strcat(get, "/robots.txt HTTP/1.1\r\n\r\n");
        } else {
                strcat(get, "/robots.txt HTTP/1.1\r\nHost: ");
                strcat(get, argv[1]);
                strcat(get, "\r\nUser-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.8) Gecko/20061107 Fedora/1.5.0.8-1.fc6 Firefox/1.5.0.8\r\n"
                            "Accept: text/plain;q=0.8\r\n"
                            "Accept-Language: en-us,en;q=0.5\r\n"
                            "Accept-Encoding: deflate\r\n"
                            "Accept-Charset: utf-8;q=0.7\r\n"
                            "Keep-Alive: 300\r\n"
                            "Connection: keep-alive\r\n\r\n");
        }

        if ((he = gethostbyname(argv[1])) == NULL)
        {
                herror("gethostbyname");
                exit(1);
        }

        if ((sockfd = socket(PF_INET, SOCK_STREAM, 0)) == -1)
        {
                perror("socket");
                exit(1);
        }

        their_addr.sin_family = AF_INET;
        their_addr.sin_port = htons(PORT);
        their_addr.sin_addr = *((struct in_addr *)he->h_addr);
        memset(&(their_addr.sin_zero), '\0', 8);

        if (connect(sockfd, (struct sockaddr *) &their_addr, sizeof(struct sockaddr)) == -1)
        {
                perror("connect");
                exit(1);
        }
        if (send(sockfd, get, strlen(get), 0) == -1)
                perror("send");

        while (numbytes = recv(sockfd, buf, MAXDATASIZE-1, 0))
        {
                if (numbytes == 0)
                        break;

                if (numbytes == -1)
                {
                        perror("recv");
                        exit(1);
                }

                printf("%s", buf);
        }

        printf("\n");

        close(sockfd);

        return 0;
}