First pass. Works on SP2013.

spider.py

+#!/usr/bin/env python
+import sys, os, time, re, getpass
+import urllib
+import argparse
+from subprocess import Popen, PIPE
+
+
+def string_strip(st, strip):
+    if st.startswith(strip):
+        return st[len(strip):]
+    return st
+
+def string_rstrip(st, strip):
+    if st.endswith(strip):
+        return st[:len(st)-len(strip)]
+    return st
+
+def get_location_subfolders(site,location):
+    url = site+'/'+location
+    subfolder_re = location+'/[^/]*(?:/Forms/AllItems.aspx)'
+
+    cmd = ['curl','--url',url,'--location-trusted','--ntlm','-K','-']
+    p = Popen(cmd,stdin=PIPE,stdout=PIPE,stderr=PIPE)
+    o,_ = p.communicate(curl_config)
+
+    folders = [string_rstrip(x,'Forms/AllItems.aspx') for x in re.findall(subfolder_re,o)]
+    folders = [string_rstrip(x,'/') for x in folders]
+    folders = list(filter(bool,list(set([string_strip(x,location+'/') for x in folders]))))
+    return folders
+
+def get_folder_contents(site,location,subfolder):
+    subfolder_url = site+'/'+location+'/'+'/'.join([urllib.quote(x) for x in subfolder.split('/')])
+    subfolder_file_re = '(?<=\")/'+location+'/'+subfolder+'/[^\"]*'
+    subfolder_folder_re = '(?<=&quot;)/'+location+'/'+subfolder+'/[^&]*(?:&quot;)*'
+
+    cmd = ['curl','--url',subfolder_url,'--location-trusted','--ntlm','-K','-']
+    p = Popen(cmd,stdin=PIPE,stdout=PIPE,stderr=PIPE)
+    o,_ = p.communicate(curl_config)
+
+    files = re.findall(subfolder_file_re,o)
+    files = [string_strip(x,'/'+location+'/'+subfolder+'/') for x in files]
+    files = list(filter(lambda x: x != 'Forms/AllItems.aspx', files))
+    folders = [string_rstrip(x,'&quot;') for x in re.findall(subfolder_folder_re,o)]
+    folders = list(set([string_strip(x,'/'+location+'/'+subfolder+'/') for x in folders]))
+    return files, folders
+
+def get_file(site, location, subfolder, path):
+    subfolder_url = site+'/'+location+'/'+'/'.join([urllib.quote(x) for x in subfolder.split('/')])
+    file_url = subfolder_url +'/'+ urllib.quote(path)
+    cmd = ['curl','--url',file_url,'--location-trusted','--ntlm','-K','-']
+    with open(subfolder+'/'+path,'w') as f:
+        p = Popen(cmd,stdin=PIPE,stdout=PIPE,stderr=PIPE)
+        o,_ = p.communicate(curl_config)
+        f.write(o)
+def get_folder(site, location, subfolder):
+    fi, fo = get_folder_contents(site,location,subfolder)
+    if len(fi+fo) > 0:   
+        try:
+            os.makedirs(subfolder)
+        except:
+            pass
+    for f in fi:
+        print('get_file', site, location, subfolder, f)
+        get_file(site, location, subfolder, f)
+    for f in fo:
+        new_subfolder = subfolder + '/'+f
+        print('get_folder', site, location, new_subfolder)
+        get_folder(site, location, new_subfolder)
+
+def get_location(site, location):
+    print(site, location)
+    subfo = get_location_subfolders(site,location)
+    for f in subfo:
+        print('get_folder', site, location, f)
+        get_folder(site,location,f)
+
+curl_config=''
+def main():
+    global curl_config
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-u", "--user", help='User to spider as')
+    parser.add_argument("-s", "--site", help='Name of the site hosting this sharepoint site e.g. https://sharepoint.org.example')
+    parser.add_argument("-l", "--location", help='Path of the sharepoint location e.g. sites/projects/MySite')
+    args = parser.parse_args()
+    passwd = getpass.getpass('Password for {}: '.format(args.user))
+    curl_config = ' '.join(['--user',user+':'+passwd])
+    get_location(site,location)
+
+if __name__='__main__':
+    main()