File: includes/htmlSaveComplete.php

<?php
/**
 * The htmlSaveComplete class can be used to save specified URLs completely in single file
 * by converting images to data URIs and extracting all CSS.
 *
 * Author: Sarfraz Ahmed
 * http://sarfraznawaz.wordpress.com
 *
 *
 * NOTE: If you are using this class, please keep above author information intact. Thanks
 */

/* Issues
 ---------------------------------------------------------
Does not convert images from @import URLs to data URIs, removed this (eg not using getImportStyles function)
due to performance issue.
*/

set_time_limit(300); // no more than 5 minutes !!!

require_once 'includes/content_extractor.php'; # class to extract "main" content part of given page
require_once 'includes/url_to_absolute.php'; # or implement your own function to convert relative URLs to absolute

class htmlSaveComplete
{

    # set debug mode on or off with 1, 0 or true, false respectively
    const DEBUG = 0;

    # holds debug data
    private $debugOutput = array();
    # url to save complete page from
    private $url = '';
    # set user agent mode on or off with 1, 0 or true, false respectively
    private $useUserAgent = FALSE;
    # holds parsed html
    private $html = '';
    # holds DOM object
    private $dom = '';

    /**
     *  The constructor function. Allows setting url to save data from and whether to use user agent.
     *
     * @param $url - url to save complete page from.
     * @param bool $useUserAgent - whether to use user agent to get page data
     *
     * @throws exception - throws an exception if provided url isn't in proper format
     */
    public function __construct($url, $useUserAgent = FALSE)
    {
        # see if Data URIs are supported by the browser
        $isDatauriSupported = preg_match('#(Opera|Gecko|MSIE 8)#', $_SERVER['HTTP_USER_AGENT']);

        if (! $isDatauriSupported)
        {
            throw new Exception('Your Browser does not support Data URIs');
        }

        # validate the URL
        if (! filter_var($url, FILTER_VALIDATE_URL))
        {
            throw new Exception('Invalid URL. Make sure to specify http(s) part.');
        }

        $this->url = $url;
        $this->useUserAgent = $useUserAgent;

        # suppress DOM parsing errors
        libxml_use_internal_errors(TRUE);

        $this->dom = new DOMDocument();
        $this->dom->preserveWhiteSpace = FALSE;
        # avoid strict error checking
        $this->dom->strictErrorChecking = FALSE;
    }

    /**
     * Gets complete page data and returns generated string
     *
     * @param bool $keepjs - whether to keep javascript
     * @param bool $contentOnly - whether to extract main content part of the page only
     * @param bool $compress - whether to remove extra whitespac
     *
     * @return string|void
     */
    public function getCompletePage($keepjs = FALSE, $contentOnly = FALSE, $compress = FALSE)
    {
        $scriptBuffer = '';
        $cssBuffer = '<style>';

        if ($this->useUserAgent)
        {
            $this->html = $this->getUrlContents($this->url);
        }
        else
        {
            $this->html = file_get_contents($this->url);
        }

        # get document stylesheets
        $stylesheets = $this->getStyleSheets();

        foreach ($stylesheets as $stylesheet)
        {
            $cssBuffer .= $this->getContents($this->getFullUrl($stylesheet)) . "\r\n\r\n";
        }

        # get @import URLS and merge it in CSS
//      $importURLs = $this->getImportStyles();
//
//      foreach ( $importURLs as $importURL )
//      {
//          $cssBuffer .= $this->getContents( $this->getFullUrl($importURL) ) . "\r\n\r\n";
//      }

        $cssBuffer .= '</style>' . "\r\n\r\n";

        # get document scripts
        if ($keepjs)
        {
            $scriptBuffer .= '<script>';
            $scripts = $this->getScripts();

            foreach ($scripts as $script)
            {
                $scriptBuffer .= $this->getContents($this->getFullUrl($script)) . "\r\n\r\n";
            }

            $scriptBuffer .= '</script>' . "\r\n\r\n";
        }

        # convert URLs from CSS styles to data URIs
        $cssBuffer = $this->toDataUri($cssBuffer);

        # remove useless stuff such as <link>, <meta> and <script> tags
        $this->removeUseless($keepjs);

        # convert URLs from @import styles to data URIs
        $this->html = $this->toDataUri($this->html);

        # see if we need to extract main content part
        if ($contentOnly)
        {
            $extractor = new ContentExtractor();
            $this->html = $extractor->extract($this->html);
        }

        # convert <img> tags to data URIs
        $this->convertImageToDataUri();

        # convert all relative links for <a> tags to absolute
        $this->toAbsoluteURLs();

        # finally join the css and html and inssert information header
        if (strlen($this->html) > 300)
        { # we did get some html back
            $this->html = $cssBuffer . $scriptBuffer . $this->html;
            $this->insertHeader();

            if (self::DEBUG)
            {
                $this->showDebugInfo();
                exit();
            }

            if ($compress)
            {
                return $this->compress($this->html);
            }
            else
            {
                return $this->html;
            }
        }
        else
        {
            return '';
        }
    }

    /**
     * Converts images to data URIs
     */
    private function convertImageToDataUri()
    {
        $tags = $this->getTags('//img');
        $tagsLength = $tags->length;

        # loop over all <img> tags and convert them to data uri
        for ($i = 0; $i < $tagsLength; $i ++)
        {
            $tag = $tags->item($i);
            $src = $this->getFullUrl($tag->getAttribute('src'));

            if ($this->remote_file_exists($src))
            {
                $dataUri = $this->imageToDataUri($src);
                $tag->setAttribute('src', $dataUri);
            }
        }

        # now save html with converted images
        $this->html = $this->dom->saveHTML();
    }

    /**
     * Returns tags list for specified selector
     *
     * @param $selector - xpath selector expression
     *
     * @return DOMNodeList
     */
    private function getTags($selector)
    {
        $this->dom->loadHTML($this->html);
        $xpath = new DOMXpath($this->dom);
        $tags = $xpath->query($selector);

        # free memory
        libxml_use_internal_errors(FALSE);
        libxml_use_internal_errors(TRUE);
        libxml_clear_errors();
        unset($xpath);
        $xpath = NULL;

        return $tags;
    }

    /**
     * Converts URLs with format url(....) to data URIs
     *
     * @param $html
     *
     * @return mixed
     */
    private function toDataUri($html)
    {
        # convert css URLs to data URIs
        $html = preg_replace_callback('#(url\([\'\"]?)([^\"\'\)]+)([\"\']?\))#', array($this, 'createDataUri'), $html);

        return $html;
    }

    /**
     * Inserts htmlSaveComplete information header on very start of page
     */
    private function insertHeader()
    {
        $header = '<!-- This page was saved with htmlSaveComplete (http://sarfraznawaz.wordpress.com) -->' . "\r\n\r\n";
        $this->html = $header . $this->html;
    }

    /**
     * Checks whether or not remote file exists
     *
     * @param $url
     *
     * @return bool
     */
    private function remote_file_exists($url)
    {
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        # don't download content
        curl_setopt($ch, CURLOPT_NOBODY, 1);
        curl_setopt($ch, CURLOPT_FAILONERROR, 1);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);

        if (curl_exec($ch) !== FALSE)
        {
            return TRUE;
        }

        return FALSE;
    }

    /**
     * Converts images from <img> tags to data URIs
     *
     * @param $path - image path eg src value
     *
     * @return string - generated data uri
     */
    private function imageToDataUri($path)
    {
        $fileType = trim(strtolower(pathinfo($path, PATHINFO_EXTENSION)));
        $mimType = $fileType;

        # since jpg/jpeg images have image/jpeg mime-type
        if (! $fileType || $fileType === 'jpg')
        {
            $mimType = 'jpeg';
        }
        else
        {
            if ($fileType === 'ico')
            {
                $mimType = 'x-icon';
            }
        }

        # make sure that it is an image and convert to data uri
        if (preg_match('#^(gif|png|jp[e]?g|bmp)$#i', $fileType) || $this->isImage($path))
        {
            # in case of images from gravatar, etc
            if ($mimType === 'php' || stripos($mimType, 'php') !== FALSE)
            {
                $mimType = 'jpeg';
            }

            $data = $this->getContents($path);
            $base64 = 'data:image/' . $mimType . ';base64,' . base64_encode($data);

            return $base64;
        }
    }

    /**
     * Removes <link>, <meta> and <script> tags from generated page
     */
    private function removeUseless($keepjs)
    {
        # remove empty lines
        //$this->html = preg_replace('#(\r\n[ \t]*){2,}#', "\r\n", $this->html);

        # remove @import declarations
        //preg_replace('#(@import url\([\'\"]?)([^\"\'\)]+)([\"\']?\))#', '', $this->html);

        # fix showing up of garbage characters
        $this->html = mb_convert_encoding($this->html, 'HTML-ENTITIES', 'UTF-8');

        $tags = $this->getTags('//meta | //link | //script');

        $tagsLength = $tags->length;

        # get all <link>, <meta> and <script> tags and remove them
        for ($i = 0; $i < $tagsLength; $i ++)
        {
            $tag = $tags->item($i);

            # delete only external scripts
            if (strtolower($tag->nodeName) === 'script')
            {
                if ($keepjs)
                {
                    if ($tag->getAttribute('src') !== '')
                    {
                        $tag->parentNode->removeChild($tag);
                    }
                }
                else
                {
                    $tag->parentNode->removeChild($tag);
                }
            }
            elseif (strtolower($tag->nodeName) === 'meta')
            {
                # keep the charset meta
                if (stripos($tag->getAttribute('content'), 'charset') === FALSE)
                {
                    $tag->parentNode->removeChild($tag);
                }
            }
            else
            {
                $tag->parentNode->removeChild($tag);
            }
        }

        $this->html = $this->dom->saveHTML();
    }

    /**
     * Gets all external stylesheets of the page
     *
     * @return array
     */
    private function getStyleSheets()
    {
        $styleSheets = array();
        $links = $this->getTags('//link[contains(@rel, "stylesheet")]');

        foreach ($links as $link)
        {
            if (self::DEBUG)
            {
                $this->debugStore(array('stylesheet' => $link->getAttribute('href')));
            }

            array_push($styleSheets, $link->getAttribute('href'));
        }

        return $styleSheets;
    }

    /**
     * Gets all external scripts of the page.
     *
     * @return array
     */
    private function getScripts()
    {
        $scripts = array();
        $links = $this->getTags('//script[contains(@src, "")]');

        foreach ($links as $link)
        {
            //$src = preg_replace('#\?.*#', '', $link->getAttribute('src'));
            $src = $link->getAttribute('src');

            if (strpos($src, '.js') !== FALSE)
            {
                if (self::DEBUG)
                {
                    $this->debugStore(array('script' => $src));
                }

                array_push($scripts, $src);
            }
        }

        return $scripts;
    }

    /**
     * Gets all
     *
     * @import URLs
     *
     * @return array
     */
    private function getImportStyles()
    {
        $importURLs = array();
        $styles = $this->getTags('//style');

        foreach ($styles as $style)
        {
            $content = $style->textContent;
            preg_match_all('#(@import url\([\'\"]?)([^\"\'\)]+)([\"\']?\))#', $content, $matches);

            if (isset($matches[2]) && count($matches[2]))
            {
                foreach ($matches[2] as $importUrl)
                {
                    $importURLs[] = $importUrl;
                }
            }
        }

        return $importURLs;
    }

    /**
     * Converts relative <a> tag paths to absolute paths
     */
    private function toAbsoluteURLs()
    {
        $links = $this->getTags('//a');

        foreach ($links as $link)
        {
            $link->setAttribute('href', $this->getFullUrl($link->getAttribute('href')));
        }

        $this->html = $this->dom->saveHTML();
    }

    /**
     * Compresses generated page by removing extra whitespace
     */
    private function compress($string)
    {
        # remove whitespace
        return str_replace(array("\r\n", "\r", "\n", "\t", '  ', '    ', '    '), '', $string);
    }

    /**
     * Gets content for given url
     *
     * @param $url
     *
     * @return string
     */
    private function getContents($url)
    {
        $data = @file_get_contents($url);

        if ($data)
        {
            return $data;
        }

        return @file_get_contents(trim($url));
    }

    /**
     * Converts matched URLs to data URIs
     *
     * @param $matches
     *
     * @return string
     */
    private function createDataUri($matches)
    {
        $fileType = explode('.', $matches[2]);
        $fileType = trim(strtolower($fileType[count($fileType) - 1]));

        # replace ?whatever=value from extensions
        $fileType = preg_replace('#\?.*#', '', $fileType);
        $mimeType = $fileType;

        # since jpg/jpeg images have image/jpeg mime-type
        if ($fileType === 'jpg')
        {
            $mimeType = 'jpeg';
        }
        else
        {
            if ($fileType === 'ico')
            {
                $mimeType = 'x-icon';
            }
            else
            {
                if ($fileType === 'css')
                {
                    $mimeType = 'css';
                }
            }
        }

        $datauri = $this->getFullUrl($matches[2]);

        #if the file is an image from CSS URLs convert it to data uri
        if (preg_match('#^(gif|png|jp[e]?g|bmp|css)$#i', $fileType))
        {
            if (self::DEBUG)
            {
                $this->debugStore(array('datauri' => $datauri));
            }

            $data = $this->getContents($datauri);

            if (! $data)
            {
                # return whatever there was originally
                return $matches[0];
            }

            $data = base64_encode($data);

            $mime = $fileType === 'css' ? 'text' : 'image';

            return $matches[1] . 'data:' . $mime . '/' . $mimeType . ';base64,' . $data . $matches[3];
        }
        else
        {
            # return whatever there was originally
            return $matches[0];
        }

    }

    /**
     * Gets content for given url using curl and optionally using user agent
     *
     * @param $url
     * @param int $timeout
     * @param string $userAgent
     *
     * @return int|mixed
     */
    private function getUrlContents(
        $url,
        $timeout = 0,
        $userAgent = 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.215 Safari/534.10'
    ) {
        $rawHtml = curl_init();
        curl_setopt($rawHtml, CURLOPT_URL, $url);
        curl_setopt($rawHtml, CURLOPT_RETURNTRANSFER, 1); # return result as string rather than direct output
        curl_setopt($rawHtml, CURLOPT_CONNECTTIMEOUT, $timeout); # set the timeout
        curl_setopt($rawHtml, CURLOPT_USERAGENT, $userAgent); # set our 'user agent'

        curl_setopt($rawHtml, CURLOPT_SSL_VERIFYPEER, false);

        $output = curl_exec($rawHtml);
        curl_close($rawHtml);

        if (! $output)
        {
            return - 1;
        }

        return $output;
    }

    /**
     * Converts relative URLs to absolute URLs
     *
     * @param $url
     *
     * @return bool|string
     */
    private function getFullUrl($url)
    {
        if (strpos($url, '//') === FALSE)
        {
            return url_to_absolute($this->url, $url);
        }

        return $url;
    }

    /**
     * Checks if provided path is an image
     *
     * @param $path
     *
     * @return bool
     */
    private function isImage($path)
    {
        list($width) = @getimagesize($path);

        if (isset($width) && $width)
        {
            return TRUE;
        }

        return FALSE;
    }

    /**
     * Stores debug information
     *
     * @param $data
     */
    private function debugStore($data)
    {
        if (self::DEBUG)
        {
            if (is_array($data))
            {
                foreach ($data as $key => $value)
                {
                    if (isset($this->debugOutput[$key]))
                    {
                        $this->debugOutput[$key . count($this->debugOutput)] = $value;
                    }
                    else
                    {
                        $this->debugOutput[$key] = $value;
                    }
                }
            }
            else
            {
                $this->debugOutput[] = $data;
            }
        }
    }

    /**
     * Shows debug information
     */
    private function showDebugInfo()
    {
        echo '<pre>';
        ksort($this->debugOutput);
        print_r($this->debugOutput);
        echo '</pre>';
    }

}