cache-extractor.py

# cache-extractor.py
#

import os
import time
import hashlib
import binascii
import zlib
import json
from urllib.parse import urlparse
from datetime import datetime
from bs4 import BeautifulSoup as bs

md5 = hashlib.md5()

# set cache_dir from which to extract files
# cache_dir = "./data/squid3"
cache_dir = "/var/spool/squid"

# set whitelist for http headers
whitelist = ['Location','Content-Type','Content-Encoding','Content-Language','Expires','Date','Last-Modified','Vary','Mime-Version','Server','Content-Length','Keep-Alive','Cache-Control','Access-Control-Allow-Origin','ETag','P3P','Set-Cookie','Via','Aka-DNS-Name','Content-Security-Policy','Content-Security-Policy-Report-Only','X-Cache','X-Amz-Cf-Id','X-Served-By','X-Powered-By','X-CDN','X-Frame-Options','X-Content-Type-Options','X-Varnish','Strict-Transport-Security']

# hex byte parse of squid_meta, headers, payload from a cache_file
def parse_cache_file(cache_file):

    # open the cache_file on disk and read
    with open(cache_file, 'rb') as cache_file_raw:
        
        # create dictionary to store all parsed fields
        cache_file_parsed = {}
        cache_file_parsed['references'] = []

        # store file_path and swap_filen
        cache_file_parsed['file_path'] = cache_file
        cache_file_parsed['swap_filen'] = cache_file.split("/").pop()

        # parse url, headers, payload from cache_file
        got_squid_meta = False
        payload = b""
        byte = cache_file_raw.read(1)
        str_buff = byte

        # read all file bytes
        while byte != "":
            # check for \r\n (next header)
            # hex bytes 0d0a = \r\n
            if byte.encode('hex') == "0d":
                byte2 = cache_file_raw.read(1)
                if byte2.encode('hex') == "0a":

                    # if got_squid_meta this is a header
                    if got_squid_meta:
                        try:
                            key = str_buff.split(":",1)[0].strip()
                            value = str_buff.split(":",1)[1].strip()
                            # only add header if in whitelist
                            if key in whitelist:
                                cache_file_parsed[key] = value
                        except IndexError:
                            print("Error: IndexError in parse_cache_file at cache_file: %s"
                                % cache_file)

                    # parse_squid_meta from buffer
                    else:
                        cache_file_parsed.update(
                            parse_squid_meta(str_buff))
                        got_squid_meta = True
                    
                    str_buff = ""
                    
                    # check for double \r\n (payload)
                    word2 = cache_file_raw.read(2)
                    if word2.encode('hex') == "0d0a":
                        # payload, read to EOF
                        payload = cache_file_raw.read()

                    # word2 starts next header
                    else:
                        byte = ""
                        str_buff = word2

                # not a new header, add bytes to buffer
                else:
                    str_buff = str_buff + byte + byte2

            # not a new header, add byte to buffer
            else:
                str_buff = str_buff + byte

            # read another byte
            byte = cache_file_raw.read(1)

        # md5 response payload
        md5sum = md5.new()
        md5sum.update(payload)
        cache_file_parsed['payload_md5'] = md5sum.hexdigest()

        # add a ref for HTTP redirects (301,302)
        if "Location" in cache_file_parsed.keys():
            cache_file_parsed['references'].append(
                {'ref': cache_file_parsed['Location'],
                'type': 'redirect'})

        # check for gzip file signatures
        # hex bytes 1f8b08 = gzip signature
        if payload.encode('hex')[:6] == "1f8b08":
            payload = decompress_gzip(payload)
        
        # check for zip file signature
        # hex bytes 504b0304 = zip signature (PKZIP archive_1)
        if payload.encode('hex')[:6] == "504b0304":
            payload = decompress_pk(payload)

        # if content is text/html parse the payload for references
        if "Content-Type" in cache_file_parsed.keys():
            if cache_file_parsed['Content-Type'] == "text/html":
                cache_file_parsed['references'] = (
                cache_file_parsed['references']
                + parse_references(payload,cache_file_parsed))

        # convert some header date formats to standard unix timestamp
        cache_file_parsed = convert_time_strings(cache_file_parsed)

        # md5 response payload after any decompression
        md5sum2 = md5.new()
        md5sum2.update(payload)
        cache_file_parsed['payload_md5_decompressed'] = md5sum2.hexdigest()

        # return cache_file_parsed data structure
        return cache_file_parsed

# parse squid_meta in cache_file header
def parse_squid_meta(squid_meta):
    
    squid_meta_parsed = {}

    # parse squid_meta from cache_file
    cache_key = squid_meta[11:27] # md5 cache_key (hex string)
    timestamp = squid_meta[32:39] # timestamp (host endianness bytes)
    lastref = squid_meta[40:47] # lastref (host endianness bytes)
    expires = squid_meta[48:55] # expires (host endianness bytes)
    lastmod = squid_meta[56:63] # lastmod (host endianness bytes)
    refcount = squid_meta[72:73] # refcount (host endianness bytes)
    flags = squid_meta[74:76] # flags (host endianness bytes)

    url_ver_code = binascii.b2a_hex(squid_meta[81:]) # remainder of string
    url = ""
    http_ver = 0
    http_code = 0
    try:
        url = binascii.a2b_hex( # url (string)
            url_ver_code[:url_ver_code.find("000a")]) # 000a terminated
        http_ver = binascii.a2b_hex( # http_ver (string)
            url_ver_code[url_ver_code.find("48545450"):] # 48545450 = HTTP
            ).split()[0]
        http_code = int(binascii.a2b_hex( # http_code (string)
            url_ver_code[url_ver_code.find("48545450"):] # 48545450 = HTTP
            ).split()[1])
        url_parse = urlparse(url)
        
        # try to get root domain from hostname
        root_domain = url_parse.hostname.split(".")
        root_domain = ".".join(len(root_domain[-2]) < 4 and root_domain[-3:] or root_domain[-2:])

        squid_meta_parsed['meta_cache_key'] = binascii.b2a_hex(cache_key)
        squid_meta_parsed['meta_timestamp'] =  int(binascii.b2a_hex(timestamp[::-1]), 16)
        squid_meta_parsed['meta_lastref'] =  int(binascii.b2a_hex(lastref[::-1]), 16)
        squid_meta_parsed['meta_expires'] = int(binascii.b2a_hex(expires[::-1]), 16)
        squid_meta_parsed['meta_lastmod'] = int(binascii.b2a_hex(lastmod[::-1]), 16)
        squid_meta_parsed['meta_refcount'] = int(binascii.b2a_hex(refcount[::-1]), 16)
        squid_meta_parsed['meta_flags'] = int(binascii.b2a_hex(flags[::-1]), 16)
        squid_meta_parsed['url'] = url
        squid_meta_parsed['url_host'] = url_parse.netloc
        squid_meta_parsed['url_tld'] = url_parse.netloc.rpartition(".")[2]
        squid_meta_parsed['url_domain'] = root_domain
        squid_meta_parsed['url_scheme'] = url_parse.scheme
        squid_meta_parsed['url_path'] = url_parse.path.rpartition("/")[0]
        squid_meta_parsed['url_file'] = url_parse.path.rpartition("/")[2]
        squid_meta_parsed['http_ver'] = http_ver
        squid_meta_parsed['http_code'] = http_code

    except IndexError as e:
        print("Error: IndexError in parse_squid_meta at cache_key: %s"
             % binascii.b2a_hex(cache_key))
    except ValueError as e:
        print("Error: ValueError in parse_squid_meta at cache_key: %s"
            % binascii.b2a_hex(cache_key))
    except TypeError as e:
        print("Error: TypeError in parse_squid_meta at cache_key: %s"
            % binascii.b2a_hex(cache_key))

    return squid_meta_parsed

# decompress a gzipped response payload
def decompress_gzip(response_payload):
    decompressed = ""
    try:
        decompressed = zlib.decompress(response_payload, zlib.MAX_WBITS | 16)
    except zlib.error:
        print("Error: zlib decompression error in decompress_gzip")
    return decompressed

# decompress deflate/zip/PK response payload
def decompress_pk(response_payload):
    decompressed = ""
    try:
        decompressed = zlib.decompress(response_payload, -zlib.MAX_WBITS)
    except zlib.error:
        print("Error: zlib decompression error in decompress_pk")
    return decompressed

# parse out references from response_payload (currently HTML only)
def parse_references(response_payload,cache_file_parsed):
    
    # create a list of references
    references = []
    
    # create a new BeautifulSoup object from the HTML
    soup = bs(response_payload, 'html.parser')  

    # get all hrefs
    for ref in soup.find_all('a'):
        insert = {}
        insert['ref'] = convert_abs_url(ref.get('href'),cache_file_parsed)
        insert['type'] = "href"
        if not insert['ref'] == None:
            references.append(insert)
    for ref in soup.find_all('link'):
        insert = {}
        insert['ref'] = convert_abs_url(ref.get('href'),cache_file_parsed)
        insert['type'] = "href"
        if not insert['ref'] == None:
            references.append(insert)

    # get all srcs
    for ref in soup.find_all('img'):
        insert = {}
        insert['ref'] = convert_abs_url(ref.get('src'),cache_file_parsed)
        insert['type'] = "src"
        if not insert['ref'] == None:
            references.append(insert)

    # return a list containing dicts with refs and their types
    return references

# standardize urls in refs to absolutes. cfp is cache_file_parsed dict
def convert_abs_url(ref,cfp):

    # don't try to convert a ref that doesn't exist
    if ref == None:
        return None
    # already absolute, probably
    if ref[:4] == "http":
        return (ref).encode('ascii','ignore')
    # relative (root)
    if ref[:1] == "/":
        return (cfp['url_scheme']+"://"+cfp['url_host']+ref).encode('ascii','ignore')
    # relative (traveral..)
    if ref[:3] == "../":
        trimmed_path = cfp['url_path'].rpartition("/")[0]
        return (cfp['url_scheme']+"://"+cfp['url_host']+trimmed_path+ref[2:]).encode('ascii','ignore')
    # relative (more traveral..)
        if ref[:2] == "./":
                return (cfp['url_scheme']+"://"+cfp['url_host']+cfp['url_path']+ref[1:]).encode('ascii','ignore')
    # relative
    return (cfp['url_scheme']+"://"+cfp['url_host']+cfp['url_path']+"/"+ref).encode('ascii','ignore')

# standardize time formats to unix time
def convert_time_strings(cache_file_parsed):
    
    # list of header values we want to convert
    time_headers = ["Expires", "Date", "Last-Modified"]

    for key in time_headers:
        if key in cache_file_parsed:
            try:
                timestring = cache_file_parsed[key]
                # try to parse datetime string and write as unix timm
                cache_file_parsed[key] = int(time.mktime(
                    datetime.strptime(timestring, "%a, %d %b %Y %H:%M:%S GMT").timetuple()))
            except ValueError:
                print("Error: ValueError in convert_time_strings at cache_key: %s"
                    % cache_file_parsed['meta_cache_key'])

    return cache_file_parsed


# print out the configured cache_dir
print("cache_dir: '%s'" % cache_dir)

# get list of all cache_files
cache_files = []
for path, dirs, files in os.walk(cache_dir):
    if path != cache_dir:
        for cache_file in files:
            cache_files.append(("%s/%s" % (path, cache_file)))

# iterate through all cache_files to extract squid_meta, headers, and payload
output = open('cache-extractor.json', 'w')
for cache_file in cache_files:
    try:
        output.write(json.dumps(parse_cache_file(cache_file))+"\n") # just write this to a file for now, manual cleanup
    except UnicodeDecodeError:
        print("Error: UnicodeDecodeError because unicode has to make everything difficult")
output.close()