curlmaster

Wrapper for the cURL extension to handle GET, POST, HEAD requests. Features include (optionally compressed) response caching, caching override, ETAG support, cookie storage, SSL certificate verification.

Ordinary request GET:

$curlm=new curlmaster;

$curlm->CacheDir='/srv/cache';
$curlm->ca_file='/srv/cert-ca/cacert.pem';
// Method GET is default.
$response = $curlm->Request('https://www.google.com/');

var_export($response);

...produces this response, which is not being cached.

/*
array (
  'library' => 'peterkahl\\curlmaster\\curlmaster',
  'library-version' => '7.1.2.2',
  'origin' => 'new',
  'timestamp' => 1601043487,
  'exectime' => '0.092391',
  'status' => '200',
  'forced' => false,
  'cachingtime' => 0,
  'cachecompress' => false,
  'filename' => '/srv/cache/curlmaster_response_f648fed2369cff9219f1c65e54701c.json',
  'cookiefile' => '/srv/cache/curlmaster_cookie_www.google.com.cookie',
  'request' =>
  array (
    'method' => 'GET',
    'url' => 'https://www.google.com/',
    'user-agent' => 'Mozilla/5.0 (curlmaster/7.1.2.2; +https://github.com/peterkahl/curlmaster)',
    'headers' =>
    array (
    ),
    'etag-enable' => false,
    'ca-file' => '/srv/cert-ca/cacert.pem',
    'cipher' => '',
    'post-data' => '',
  ),
  'verbose' =>
  array (
    0 => '*   Trying 2a00:1450:4001:81f::2004:443...',
    1 => '* TCP_NODELAY set',
    2 => '* Connected to www.google.com (2a00:1450:4001:81f::2004) port 443 (#0)',
    3 => '* ALPN, offering h2',
    4 => '* ALPN, offering http/1.1',
    5 => '* successfully set certificate verify locations:',
    6 => '*   CAfile: /srv/cert-ca/cacert.pem',
    7 => '  CApath: /etc/ssl/certs',
    8 => '* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384',
    9 => '* ALPN, server accepted to use h2',
    10 => '* Server certificate:',
    11 => '*  subject: C=US; ST=California; L=Mountain View; O=Google LLC; CN=www.google.com',
    12 => '*  start date: Sep  3 06:42:46 2020 GMT',
    13 => '*  expire date: Nov 26 06:42:46 2020 GMT',
    14 => '*  subjectAltName: host "www.google.com" matched cert\'s "www.google.com"',
    15 => '*  issuer: C=US; O=Google Trust Services; CN=GTS CA 1O1',
    16 => '*  SSL certificate verify ok.',
    17 => '* Using HTTP2, server supports multi-use',
    18 => '* Connection state changed (HTTP/2 confirmed)',
    19 => '* Copying HTTP/2 data in stream buffer to connection buffer after upgrade: len=0',
    20 => '* Using Stream ID: 1 (easy handle 0x55c5b37ebf80)',
    21 => '> GET / HTTP/2',
    22 => 'Host: www.google.com',
    23 => 'user-agent: Mozilla/5.0 (curlmaster/7.1.2.2; +https://github.com/peterkahl/curlmaster)',
    24 => 'accept: /',
    25 => 'accept-encoding: deflate, gzip, br',
    26 => 'cookie: NID=204=mKHrdeEZ7MsIM_tAU0UbcXlfdzx6lsXvLES8lH54G27cTF9skpRfAOEWk9XqM6Ks-MQKZY-9aBQRMTCDSToCkP3K578C7cbdxIiTbq9ZRcqDp5nP5Uke2sZ5d8Lo9W_aw-soUOXQKM8qnG1F2by3MhACB29kOPGsDVmjJAt4oRw',
    27 => '',
    28 => '* old SSL session ID is stale, removing',
    29 => '* Connection state changed (MAX_CONCURRENT_STREAMS == 100)!',
    30 => '< HTTP/2 200 ',
    31 => '< date: Fri, 25 Sep 2020 14:18:07 GMT',
    32 => '< expires: -1',
    33 => '< cache-control: private, max-age=0',
    34 => '< content-type: text/html; charset=UTF-8',
    35 => '< content-encoding: gzip',
    36 => '< server: gws',
    37 => '< content-length: 15675',
    38 => '< x-xss-protection: 0',
    39 => '< x-frame-options: SAMEORIGIN',
    40 => '< alt-svc: h3-Q050=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-27=":443"; ma=2592000,h3-T051=":443"; ma=2592000,h3-T050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"',
    41 => '< ',
    42 => '* Closing connection 0',
    43 => '',
  ),
  'response' =>
  array (
    'timestamp' => 1601043487,
    'exectime' => '0.092378',
    'status' => '200',
    'cachingtime' => 0,
    'headers' =>
    array (
      'status' => 'HTTP/2 200',
      'date' => 'Fri, 25 Sep 2020 14:18:07 GMT',
      'expires' => '-1',
      'cache-control' => 'private, max-age=0',
      'content-type' => 'text/html; charset=UTF-8',
      'content-encoding' => 'gzip',
      'server' => 'gws',
      'content-length' => '15675',
      'x-xss-protection' => '0',
      'x-frame-options' => 'SAMEORIGIN',
      'alt-svc' => 'h3-Q050=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-27=":443"; ma=2592000,h3-T051=":443"; ma=2592000,h3-T050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"',
    ),
    'error-num' => 0,
    'error-verb' => 'CURLE_OK',
    'etag' => '',
    'last-modified' => 0,
    'body' => '<!doctype html>...[truncated]...</body></html>',
  ),
)
*/

Force caching of response:

$curlm=new curlmaster;

$curlm->CacheDir='/srv/cache';
$curlm->ca_file='/srv/cert-ca/cacert.pem';
$curlm->ForcedCacheMaxAge=3600;
$response = $curlm->Request('https://www.google.com/');

...produces this response, which is being cached as JSON.

{
    "library": "peterkahl\\curlmaster\\curlmaster",
    "library-version": "7.1.2.2",
    "origin": "new",
    "timestamp": 1601044020,
    "exectime": "0.120096",
    "status": "200",
    "forced": false,
    "cachingtime": 3600,
    "cachecompress": false,
    "filename": "\/srv\/cache\/curlmaster_response_f648fed2369cff9219f1c65e54701c.json",
    "cookiefile": "\/srv\/cache\/curlmaster_cookie_www.google.com.cookie",
    "request": {
        "method": "GET",
        "url": "https:\/\/www.google.com\/",
        "user-agent": "Mozilla\/5.0 (curlmaster\/7.1.2.2; +https:\/\/github.com\/peterkahl\/curlmaster)",
        "headers": [],
        "etag-enable": false,
        "ca-file": "\/srv\/cert-ca\/cacert.pem",
        "cipher": "",
        "post-data": ""
    },
    "verbose": [
        "*   Trying 2a00:1450:4001:81f::2004:443...",
        "* TCP_NODELAY set",
        "* Connected to www.google.com (2a00:1450:4001:81f::2004) port 443 (#0)",
        "* ALPN, offering h2",
        "* ALPN, offering http\/1.1",
        "* successfully set certificate verify locations:",
        "*   CAfile: \/srv\/cert-ca\/cacert.pem",
        "  CApath: \/etc\/ssl\/certs",
        "* SSL connection using TLSv1.3 \/ TLS_AES_256_GCM_SHA384",
        "* ALPN, server accepted to use h2",
        "* Server certificate:",
        "*  subject: C=US; ST=California; L=Mountain View; O=Google LLC; CN=www.google.com",
        "*  start date: Sep  3 06:42:46 2020 GMT",
        "*  expire date: Nov 26 06:42:46 2020 GMT",
        "*  subjectAltName: host \"www.google.com\" matched cert's \"www.google.com\"",
        "*  issuer: C=US; O=Google Trust Services; CN=GTS CA 1O1",
        "*  SSL certificate verify ok.",
        "* Using HTTP2, server supports multi-use",
        "* Connection state changed (HTTP\/2 confirmed)",
        "* Copying HTTP\/2 data in stream buffer to connection buffer after upgrade: len=0",
        "* Using Stream ID: 1 (easy handle 0x5635d76eaf80)",
        "> GET \/ HTTP\/2",
        "Host: www.google.com",
        "user-agent: Mozilla\/5.0 (curlmaster\/7.1.2.2; +https:\/\/github.com\/peterkahl\/curlmaster)",
        "accept: \/",
        "accept-encoding: deflate, gzip, br",
        "cookie: NID=204=mKHrdeEZ7MsIM_tAU0UbcXlfdzx6lsXvLES8lH54G27cTF9skpRfAOEWk9XqM6Ks-MQKZY-9aBQRMTCDSToCkP3K578C7cbdxIiTbq9ZRcqDp5nP5Uke2sZ5d8Lo9W_aw-soUOXQKM8qnG1F2by3MhACB29kOPGsDVmjJAt4oRw",
        "",
        "* old SSL session ID is stale, removing",
        "* Connection state changed (MAX_CONCURRENT_STREAMS == 100)!",
        "< HTTP\/2 200 ",
        "< date: Fri, 25 Sep 2020 14:27:00 GMT",
        "< expires: -1",
        "< cache-control: private, max-age=0",
        "< content-type: text\/html; charset=UTF-8",
        "< content-encoding: gzip",
        "< server: gws",
        "< content-length: 15675",
        "< x-xss-protection: 0",
        "< x-frame-options: SAMEORIGIN",
        "< alt-svc: h3-Q050=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-27=\":443\"; ma=2592000,h3-T051=\":443\"; ma=2592000,h3-T050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",
        "< ",
        "* Closing connection 0",
        ""
    ],
    "response": {
        "timestamp": 1601044020,
        "exectime": "0.120084",
        "status": "200",
        "cachingtime": 0,
        "headers": {
            "status": "HTTP\/2 200",
            "date": "Fri, 25 Sep 2020 14:27:00 GMT",
            "expires": "-1",
            "cache-control": "private, max-age=0",
            "content-type": "text\/html; charset=UTF-8",
            "content-encoding": "gzip",
            "server": "gws",
            "content-length": "15675",
            "x-xss-protection": "0",
            "x-frame-options": "SAMEORIGIN",
            "alt-svc": "h3-Q050=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-27=\":443\"; ma=2592000,h3-T051=\":443\"; ma=2592000,h3-T050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\""
        },
        "error-num": 0,
        "error-verb": "CURLE_OK",
        "etag": "",
        "last-modified": 0,
        "body": "<!doctype html>...[truncated]...<\/body><\/html>"
    }
}

Usage example, method POST:

use peterkahl\curlmaster\curlmaster;

$curlm = new curlmaster;
$curlm->CacheDir = '/srv/cache';
$curlm->ca_file = '/srv/cert-ca/cacert.pem';
$post_data = array(
  'user' => 'admin',
  'pwd'  => 'oracle',
);

$response = $curlm->Request('https://whatever.anything/login', 'POST', $post_data);

Usage example, method HEAD:

use peterkahl\curlmaster\curlmaster;

$curlm = new curlmaster;
$curlm->CacheDir = '/srv/cache';
$curlm->ca_file = '/srv/cert-ca/cacert.pem';
$curlm->headers = array('accept: text/html');
$response = $curlm->Request('https://github.com/', 'HEAD');

/*
The resulting response is an array. HEAD responses are not cached.

array (
  'library' => 'peterkahl\\curlmaster\\curlmaster',
  'library-version' => '7.1.1',
  'origin' => 'new',
  'timestamp' => 1600930605,
  'exectime' => '0.330269',
  'status' => '200',
  'forced' => false,
  'cachingtime' => -1,
  'filename' => '/srv/cache/curlmaster_response_ba5453820c9f8f87fec8b54e7540bf.json',
  'cookiefile' => '/srv/cache/curlmaster_cookie_github.com.cookie',
  'request' =>
  array (
    'method' => 'HEAD',
    'url' => 'https://github.com/',
    'user-agent' => 'Mozilla/5.0 (curlmaster/7.1.1; +https://github.com/peterkahl/curlmaster)',
    'headers' =>
    array (
      0 => 'accept: text/html',
    ),
    'etag-enable' => false,
    'ca-file' => '/srv/cert-ca/cacert.pem',
    'cipher' => '',
    'post-data' => '',
  ),
  'verbose' =>
  array (
    0 => '*   Trying 140.82.121.4:443...',
    1 => '* TCP_NODELAY set',
    2 => '* Connected to github.com (140.82.121.4) port 443 (#0)',
    3 => '* ALPN, offering h2',
    4 => '* ALPN, offering http/1.1',
    5 => '* successfully set certificate verify locations:',
    6 => '*   CAfile: /srv/cert-ca/cacert.pem',
    7 => '  CApath: /etc/ssl/certs',
    8 => '* SSL connection using TLSv1.3 / TLS_AES_128_GCM_SHA256',
    9 => '* ALPN, server accepted to use http/1.1',
    10 => '* Server certificate:',
    11 => '*  subject: C=US; ST=California; L=San Francisco; O=GitHub, Inc.; CN=github.com',
    12 => '*  start date: May  5 00:00:00 2020 GMT',
    13 => '*  expire date: May 10 12:00:00 2022 GMT',
    14 => '*  subjectAltName: host "github.com" matched cert\'s "github.com"',
    15 => '*  issuer: C=US; O=DigiCert Inc; OU=www.digicert.com; CN=DigiCert SHA2 High Assurance Server CA',
    16 => '*  SSL certificate verify ok.',
    17 => '> HEAD / HTTP/1.1',
    18 => 'Host: github.com',
    19 => 'User-Agent: Mozilla/5.0 (curlmaster/7.1.1; +https://github.com/peterkahl/curlmaster)',
    20 => 'Cookie: logged_in=no; _octo=GH1.1.586556027.1600930239',
    21 => 'accept: text/html',
    22 => '',
    23 => '* old SSL session ID is stale, removing',
    24 => '* Mark bundle as not supporting multiuse',
    25 => '< HTTP/1.1 200 OK',
    26 => '< date: Thu, 24 Sep 2020 06:56:45 GMT',
    27 => '< content-type: text/html; charset=utf-8',
    28 => '< server: GitHub.com',
    29 => '< status: 200 OK',
    30 => '< vary: X-PJAX, Accept-Encoding, Accept, X-Requested-With, Accept-Encoding',
    31 => '< etag: W/"4da62c2bc712cef0599ed6a7f550f4d2"',
    32 => '< cache-control: max-age=0, private, must-revalidate',
    33 => '< strict-transport-security: max-age=31536000; includeSubdomains; preload',
    34 => '< x-frame-options: deny',
    35 => '< x-content-type-options: nosniff',
    36 => '< x-xss-protection: 1; mode=block',
    37 => '< referrer-policy: origin-when-cross-origin, strict-origin-when-cross-origin',
    38 => '< expect-ct: max-age=2592000, report-uri="https://api.github.com/_private/browser/errors"',
    39 => '< content-security-policy: default-src \'none\'; base-uri \'self\'; block-all-mixed-content; connect-src \'self\' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com www.google-analytics.com github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com; font-src github.githubassets.com; form-action \'self\' github.com gist.github.com; frame-ancestors \'none\'; frame-src render.githubusercontent.com; img-src \'self\' data: github.githubassets.com identicons.github.com collector.githubapp.com github-cloud.s3.amazonaws.com *.githubusercontent.com customer-stories-feed.github.com spotlights-feed.github.com; manifest-src \'self\'; media-src github.githubassets.com; script-src github.githubassets.com; style-src \'unsafe-inline\' github.githubassets.com; worker-src github.com/socket-worker.js gist.github.com/socket-worker.js',
    40 => '* Added cookie _gh_sess="YT1whftzqTDiFdxWK6B8KsA6%2FF%2Fo4DhCEPXJ8%2FA%2FB97tPOmGTQ9aOEwwrlntWc8THlcYj91gzSznAgR%2BGMHR%2BIRfOW7ErgXFbJZpNrzwn5KjLpdPQHoz3bkLvZiKQ%2B7xiXBt4IcsO7wdJN4zrHcehaZgTTELIEVrIkLOoV3ZktjVMBhJf3SkBF3of%2B3zPbM2UjAfEDrVK%2FaKM3YXgwDbsKoAdWsrqgOO4Gv%2FprqpT2VqSgPSwCNnF3kzz4ivaTf%2B8IHIS7lAHlr5N82STNzLUg%3D%3D--lyzCNBt0J30oi58f--CFgqDV2Di71wa7gw%2BGmk3A%3D%3D" for domain github.com, path /, expire 0',
    41 => '< Set-Cookie: _gh_sess=YT1whftzqTDiFdxWK6B8KsA6%2FF%2Fo4DhCEPXJ8%2FA%2FB97tPOmGTQ9aOEwwrlntWc8THlcYj91gzSznAgR%2BGMHR%2BIRfOW7ErgXFbJZpNrzwn5KjLpdPQHoz3bkLvZiKQ%2B7xiXBt4IcsO7wdJN4zrHcehaZgTTELIEVrIkLOoV3ZktjVMBhJf3SkBF3of%2B3zPbM2UjAfEDrVK%2FaKM3YXgwDbsKoAdWsrqgOO4Gv%2FprqpT2VqSgPSwCNnF3kzz4ivaTf%2B8IHIS7lAHlr5N82STNzLUg%3D%3D--lyzCNBt0J30oi58f--CFgqDV2Di71wa7gw%2BGmk3A%3D%3D; Path=/; HttpOnly; Secure; SameSite=Lax',
    42 => '< Accept-Ranges: bytes',
    43 => '< X-GitHub-Request-Id: AF42:E4AE:14B52F0:1CD1320:5F6C432C',
    44 => '< ',
    45 => '* Closing connection 0',
    46 => '',
  ),
  'response' =>
  array (
    'timestamp' => 1600930605,
    'exectime' => '0.330255',
    'status' => '200',
    'cachingtime' => 0,
    'headers' =>
    array (
      'status' => 'HTTP/1.1 200 OK',
      'date' => 'Thu, 24 Sep 2020 06:56:45 GMT',
      'content-type' => 'text/html; charset=utf-8',
      'server' => 'GitHub.com',
      'status-1' => '200 OK',
      'vary' => 'X-PJAX, Accept-Encoding, Accept, X-Requested-With, Accept-Encoding',
      'etag' => 'W/"4da62c2bc712cef0599ed6a7f550f4d2"',
      'cache-control' => 'max-age=0, private, must-revalidate',
      'strict-transport-security' => 'max-age=31536000; includeSubdomains; preload',
      'x-frame-options' => 'deny',
      'x-content-type-options' => 'nosniff',
      'x-xss-protection' => '1; mode=block',
      'referrer-policy' => 'origin-when-cross-origin, strict-origin-when-cross-origin',
      'expect-ct' => 'max-age=2592000, report-uri="https://api.github.com/_private/browser/errors"',
      'content-security-policy' => 'default-src \'none\'; base-uri \'self\'; block-all-mixed-content; connect-src \'self\' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com www.google-analytics.com github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com; font-src github.githubassets.com; form-action \'self\' github.com gist.github.com; frame-ancestors \'none\'; frame-src render.githubusercontent.com; img-src \'self\' data: github.githubassets.com identicons.github.com collector.githubapp.com github-cloud.s3.amazonaws.com *.githubusercontent.com customer-stories-feed.github.com spotlights-feed.github.com; manifest-src \'self\'; media-src github.githubassets.com; script-src github.githubassets.com; style-src \'unsafe-inline\' github.githubassets.com; worker-src github.com/socket-worker.js gist.github.com/socket-worker.js',
      'set-cookie' => '_gh_sess=YT1whftzqTDiFdxWK6B8KsA6%2FF%2Fo4DhCEPXJ8%2FA%2FB97tPOmGTQ9aOEwwrlntWc8THlcYj91gzSznAgR%2BGMHR%2BIRfOW7ErgXFbJZpNrzwn5KjLpdPQHoz3bkLvZiKQ%2B7xiXBt4IcsO7wdJN4zrHcehaZgTTELIEVrIkLOoV3ZktjVMBhJf3SkBF3of%2B3zPbM2UjAfEDrVK%2FaKM3YXgwDbsKoAdWsrqgOO4Gv%2FprqpT2VqSgPSwCNnF3kzz4ivaTf%2B8IHIS7lAHlr5N82STNzLUg%3D%3D--lyzCNBt0J30oi58f--CFgqDV2Di71wa7gw%2BGmk3A%3D%3D; Path=/; HttpOnly; Secure; SameSite=Lax',
      'accept-ranges' => 'bytes',
      'x-github-request-id' => 'AF42:E4AE:14B52F0:1CD1320:5F6C432C',
    ),
    'error-num' => 0,
    'error-verb' => 'CURLE_OK',
    'etag' => 'W/"4da62c2bc712cef0599ed6a7f550f4d2"',
    'last-modified' => 0,
    'body' => '

',
  ),
)
*/

Cache Purging:

Best to set this up on a crontab job.

use peterkahl\curlmaster\curlmaster;

$curlm = new curlmaster;
$curlm->CacheDir = '/srv/cache';
$curlm->PurgeCache();