Release 0.5.4

servefile still works with python3.6, we just no longer test this.

.github/workflows/run-tox.yml

@@ -0,0 +1,37 @@
+name: Run Tox
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: [2.7, 3.7, 3.8, 3.9, "3.10", 3.11]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install Tox
+        run: pip install tox
+      - name: Run Tox
+        run: tox -e py
+  pep8:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Install Tox
+        run: pip install tox
+      - name: Run Tox pep8
+        run: "tox -e pep8"

ChangeLog

@@ -1,6 +1,42 @@
 servefile changelog
 ===================
 
+2023-01-23 v0.5.4
+-----------------
+
+	0.5.4 released
+
+	* code reformatting for better maintainability
+	* upload to uploaddir instead of /tmp for large files
+	* add python3.10 / python3.11 support
+	* drop python3.6 support
+
+
+2021-11-18 v0.5.3
+-----------------
+
+	0.5.3 released
+
+	* improved test performance
+
+
+2021-09-08 v0.5.2
+-----------------
+
+	0.5.2 released
+
+	* fixed bug where exception was shown on transmission abort with python3
+	* fixed wrong/outdated pyopenssl package names
+	* tests are now using a free non-default port to avoid clashes; if
+	  wished the ports can be set from outside by specifying the
+	  environment variables SERVEFILE_DEFAULT_PORT and
+	  SERVEFILE_SECONDARY_PORT
+	* fixed broken redirect when filename contained umlauts or other characters
+	  that should have been quoted
+	* fixed broken special char handling in directory listing for python2
+	* drop python3.5 support
+	* fixed PUT uploads with python3 and documented PUT-uploads with curl
+
 
 2020-10-30 v0.5.1
 -----------------

servefile.1

@@ -1,4 +1,4 @@
-.TH SERVEFILE 1 "September 2020" "servefile 0.5.1" "User Commands"
+.TH SERVEFILE 1 "January 2023" "servefile 0.5.4" "User Commands"
 
 .SH NAME
 servefile \- small HTTP-Server for temporary file transfer
@@ -28,8 +28,8 @@ In upload mode with \fB\-u\fR servefile creates a directory and saves all
 uploaded files into that directory. When uploading with curl or wget the
 filename is extracted from the path part of the url used for the upload.
 
-For SSL support python-openssl (pyssl) needs to be installed. If no key and
-cert is given, servefile will generate a key pair for you and display its
+For SSL support pyopenssl (python3-openssl) needs to be installed. If no key
+and cert is given, servefile will generate a key pair for you and display its
 fingerprint.
 
 In \fB--tar\fR mode the given file or directory will be packed on (each)

servefile/servefile.py

@@ -7,11 +7,10 @@
 
 from __future__ import print_function
 
-__version__ = '0.5.1'
+__version__ = '0.5.4'
 
 import argparse
 import base64
-import cgi
 import datetime
 import io
 import mimetypes
@@ -21,7 +20,9 @@ import select
 import socket
 from subprocess import Popen, PIPE
 import sys
+import tempfile
 import time
+import warnings
 
 # fix imports for python2/python3
 try:
@@ -42,11 +43,18 @@ try:
 except ImportError:
     pass
 
+with warnings.catch_warnings():
+    warnings.filterwarnings("ignore", category=DeprecationWarning)
+    # scheduled for removal in python3.13, used for FieldStorage
+    import cgi
+
+
 def getDateStrNow():
     """ Get the current time formatted for HTTP header """
     now = datetime.datetime.fromtimestamp(time.mktime(time.gmtime()))
     return now.strftime("%a, %d %b %Y %H:%M:%S GMT")
 
+
 class FileBaseHandler(BaseHTTPServer.BaseHTTPRequestHandler):
     fileName = None
     blockSize = 1024 * 1024
@@ -60,7 +68,7 @@ class FileBaseHandler(BaseHTTPServer.BaseHTTPRequestHandler):
             fileName = self.fileName
         if unquote(self.path) != "/" + fileName:
             self.send_response(302)
-			self.send_header('Location', '/' + fileName)
+            self.send_header('Location', '/' + quote(fileName))
             self.end_headers()
             return True
         return False
@@ -106,7 +114,7 @@ class FileBaseHandler(BaseHTTPServer.BaseHTTPRequestHandler):
                     except ValueError:
                         return (False, None)
 
-					if fromto[0] >= fileLength or fromto[0] < 0 or fromto[1] >= fileLength or fromto[1]-fromto[0] < 0:
+                    if fromto[0] >= fileLength or fromto[0] < 0 or fromto[1] >= fileLength or fromto[1] - fromto[0] < 0:
                         # oops, already done! (requested range out of range)
                         self.send_response(416)
                         self.send_header('Content-Range', 'bytes */%d' % fileLength)
@@ -141,7 +149,7 @@ class FileBaseHandler(BaseHTTPServer.BaseHTTPRequestHandler):
             # now we can wind the file *brrrrrr*
             myfile.seek(fromto[0])
 
-		if fromto != None:
+        if fromto is not None:
             self.send_response(216)
             self.send_header('Content-Range', 'bytes %d-%d/%d' % (fromto[0], fromto[1], fileLength))
             fileLength = fromto[1] - fromto[0] + 1
@@ -162,8 +170,8 @@ class FileBaseHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         return True
 
     def getChunk(self, myfile, fromto):
-		if fromto and myfile.tell()+self.blockSize >= fromto[1]:
-			readsize = fromto[1]-myfile.tell()+1
+        if fromto and myfile.tell() + self.blockSize >= fromto[1]:
+            readsize = fromto[1] - myfile.tell() + 1
         else:
             readsize = self.blockSize
         return myfile.read(readsize)
@@ -246,7 +254,7 @@ class TarFileHandler(FileBaseHandler):
         # give the process a short time to find out if it can
         # pack/compress the file
         time.sleep(0.05)
-		if tarCmd.poll() != None and tarCmd.poll() != 0:
+        if tarCmd.poll() is not None and tarCmd.poll() != 0:
             # something went wrong
             print("Error while compressing '%s'. Aborting request." % self.target)
             self.send_response(500)
@@ -416,7 +424,7 @@ class DirListingHandler(FileBaseHandler):
             </tr>
         </thead>
         <tbody>
-		""" % {'path': os.path.normpath(unquote(self.path))}
+        """ % {'path': os.path.normpath(unquote(self.path))}  # noqa: E501
         footer = """</tbody></table></div>
 <div class="footer"><a href="http://seba-geek.de/stuff/servefile/">servefile %(version)s</a></div>
 <script>
@@ -502,7 +510,7 @@ class DirListingHandler(FileBaseHandler):
         dir_items = list()
         file_items = list()
 
-		for item in [".."] + sorted(os.listdir(path), key=lambda x:x.lower()):
+        for item in [".."] + sorted(os.listdir(path), key=lambda x: x.lower()):
             # create path to item
             itemPath = os.path.join(path, item)
 
@@ -524,10 +532,7 @@ class DirListingHandler(FileBaseHandler):
             target_items.append((item, itemPath, stat))
 
         # Directories first, then files
-		for (tuple_list, is_dir) in (
-				(dir_items, True),
-				(file_items, False),
-				):
+        for (tuple_list, is_dir) in ((dir_items, True), (file_items, False)):
             for (item, itemPath, stat) in tuple_list:
                 self._appendToListing(content, item, itemPath, stat, is_dir=is_dir)
 
@@ -542,7 +547,9 @@ class DirListingHandler(FileBaseHandler):
         self.send_header("Content-Length", str(len(listing)))
         self.send_header('Connection', 'close')
         self.end_headers()
-		self.wfile.write(listing.encode())
+        if sys.version_info.major >= 3:
+            listing = listing.encode()
+        self.wfile.write(listing)
 
     def convertSize(self, size):
         for ext in "KMGT":
@@ -607,8 +614,25 @@ class FilePutter(BaseHTTPServer.BaseHTTPRequestHandler):
         # create FieldStorage object for multipart parsing
         env = os.environ
         env['REQUEST_METHOD'] = "POST"
-		fstorage = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ=env)
-		if not "file" in fstorage:
+        targetDir = self.targetDir
+
+        class CustomFieldStorage(cgi.FieldStorage):
+
+            def make_file(self, *args, **kwargs):
+                """Overwritten to use a named file and the upload directory
+
+                Python 2.7 has an unused "binary" argument while Python 3 does
+                not have any arguments. Python 2.7 does not have a
+                self._binary_file attribute.
+                """
+                if sys.version_info.major == 2 or self._binary_file:
+                    return tempfile.NamedTemporaryFile("wb+", dir=targetDir)
+                else:
+                    return tempfile.NamedTemporaryFile(
+                        "w+", encoding=self.encoding, newline='\n', dir=targetDir)
+
+        fstorage = CustomFieldStorage(fp=self.rfile, headers=self.headers, environ=env)
+        if "file" not in fstorage:
             self.sendResponse(400, "No file found in request.")
             return
 
@@ -617,7 +641,14 @@ class FilePutter(BaseHTTPServer.BaseHTTPRequestHandler):
             self.sendResponse(400, "Filename was empty or invalid")
             return
 
-		# write file down to disk, send a 200 afterwards
+        # put the file at the right place, send 200 afterwards
+        if getattr(fstorage["file"].file, "name", None):
+            # the sent file was large, so we can just hard link the temporary
+            # file and are done
+            os.link(fstorage["file"].file.name, destFileName)
+        else:
+            # write file to disk. it was small enough so no temporary file was
+            # created
             target = open(destFileName, "wb")
             bytesLeft = length
             while bytesLeft > 0:
@@ -635,7 +666,7 @@ class FilePutter(BaseHTTPServer.BaseHTTPRequestHandler):
         http://host:8080/testfile will cause the file to be named testfile. If
         no filename is given, a random name will be generated.
 
-		Files can be uploaded with e.g. curl -X POST -d @file <url> .
+        Files can be uploaded with e.g. curl -T file <url> .
         """
         length = self.getContentLength()
         if length < 0:
@@ -652,11 +683,11 @@ class FilePutter(BaseHTTPServer.BaseHTTPRequestHandler):
             return
 
         # Sometimes clients want to be told to continue with their transfer
-		if self.headers.getheader("Expect") == "100-continue":
+        if self.headers.get("Expect") == "100-continue":
             self.send_response(100)
             self.end_headers()
 
-		target = open(cleanFileName, "w")
+        target = open(cleanFileName, "wb")
         bytesLeft = int(self.headers['Content-Length'])
         while bytesLeft > 0:
             bytesToRead = min(self.blockSize, bytesLeft)
@@ -675,7 +706,8 @@ class FilePutter(BaseHTTPServer.BaseHTTPRequestHandler):
             self.sendResponse(411, "Content-Length was invalid or not set.")
             return -1
         if self.maxUploadSize > 0 and length > self.maxUploadSize:
-			self.sendResponse(413, "Your file was too big! Maximum allowed size is %d byte. <a href=\"/\">back</a>" % self.maxUploadSize)
+            self.sendResponse(413, "Your file was too big! Maximum allowed size is %d byte. <a href=\"/\">back</a>" %
+                                   self.maxUploadSize)
             return -1
         return length
 
@@ -712,9 +744,11 @@ class FilePutter(BaseHTTPServer.BaseHTTPRequestHandler):
             return extraDestFileName
         # never reached
 
+
 class ThreadedHTTPServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer):
     def handle_error(self, request, client_address):
-		print("%s ABORTED transmission (Reason: %s)" % (client_address[0], sys.exc_value))
+        _, exc_value, _ = sys.exc_info()
+        print("%s ABORTED transmission (Reason: %s)" % (client_address[0], exc_value))
 
 
 def catchSSLErrors(BaseSSLClass):
@@ -786,6 +820,7 @@ class SecureHandler():
             self.rfile = socket._fileobject(self.request, "rb", self.rbufsize)
             self.wfile = socket._fileobject(self.request, "wb", self.wbufsize)
 
+
 class ServeFileException(Exception):
     pass
 
@@ -810,7 +845,8 @@ class ServeFile():
 
         if self.serveMode not in range(self._NUM_MODES):
             self.serveMode = None
-			raise ValueError("Unknown serve mode, needs to be MODE_SINGLE, MODE_SINGLETAR, MODE_UPLOAD or MODE_DIRLIST.")
+            raise ValueError("Unknown serve mode, needs to be MODE_SINGLE, "
+                             "MODE_SINGLETAR, MODE_UPLOAD or MODE_DIRLIST.")
 
     def setIPv4(self, ipv4):
         """ En- or disable ipv4 """
@@ -824,23 +860,23 @@ class ServeFile():
         """ Get IPs from all interfaces via ip or ifconfig. """
         # ip and ifconfig sometimes are located in /sbin/
         os.environ['PATH'] += ':/sbin:/usr/sbin'
-		proc = Popen(r"ip addr|" + \
-					  "sed -n -e 's/.*inet6\{0,1\} \([0-9.a-fA-F:]\+\).*/\\1/ p'|" + \
-					  "grep -v '^fe80\|^127.0.0.1\|^::1'", \
+        proc = Popen(r"ip addr|"
+                     r"sed -n -e 's/.*inet6\{0,1\} \([0-9.a-fA-F:]\+\).*/\1/ p'|"
+                     r"grep -v '^fe80\|^127.0.0.1\|^::1'",
                      shell=True, stdout=PIPE, stderr=PIPE)
         if proc.wait() != 0:
             # ip failed somehow, falling back to ifconfig
             oldLang = os.environ.get("LC_ALL", None)
             os.environ['LC_ALL'] = "C"
-			proc = Popen(r"ifconfig|" + \
-						  "sed -n 's/.*inet6\{0,1\}\( addr:\)\{0,1\} \{0,1\}\([0-9a-fA-F.:]*\).*/" + \
-						  "\\2/p'|" + \
-						  "grep -v '^fe80\|^127.0.0.1\|^::1'", \
+            proc = Popen(r"ifconfig|"
+                         r"sed -n 's/.*inet6\{0,1\}\( addr:\)\{0,1\} \{0,1\}\([0-9a-fA-F.:]*\).*/"
+                         r"\2/p'|"
+                         r"grep -v '^fe80\|^127.0.0.1\|^::1'",
                          shell=True, stdout=PIPE, stderr=PIPE)
             if oldLang:
                 os.environ['LC_ALL'] = oldLang
             else:
-				del(os.environ['LC_ALL'])
+                del os.environ['LC_ALL']
             if proc.wait() != 0:
                 # we couldn't find any ip address
                 proc = None
@@ -857,7 +893,7 @@ class ServeFile():
         return None
 
     def setSSLKeys(self, cert, key):
-		""" Set SSL cert/key. Can be either path to file or pyssl X509/PKey object. """
+        """ Set SSL cert/key. Can be either path to file or pyopenssl X509/PKey object. """
         self.cert = cert
         self.key = key
 
@@ -883,7 +919,7 @@ class ServeFile():
         req = crypto.X509Req()
         subj = req.get_subject()
         subj.CN = "127.0.0.1"
-		subj.O = "servefile laboratories"
+        subj.O = "servefile laboratories"  # noqa: E741
         subj.OU = "servefile"
 
         # generate altnames
@@ -904,7 +940,7 @@ class ServeFile():
         # with the same serial ==> we just use the seconds as serial.
         cert.set_serial_number(int(time.time()))
         cert.gmtime_adj_notBefore(0)
-		cert.gmtime_adj_notAfter(365*24*60*60)
+        cert.gmtime_adj_notAfter(365 * 24 * 60 * 60)
         cert.set_issuer(req.get_subject())
         cert.set_subject(req.get_subject())
         cert.add_extensions([ext])
@@ -948,7 +984,8 @@ class ServeFile():
                 server = SecureThreadedHTTPServer(self._getCert(), self._getKey(),
                                                   (listenIp, self.port), handler, bind_and_activate=False)
             except SSL.Error as e:
-				raise ServeFileException("SSL error: Could not read SSL public/private key from file(s) (error was: \"%s\")" % (e[0][0][2],))
+                raise ServeFileException("SSL error: Could not read SSL public/private key "
+                                         "from file(s) (error was: \"%s\")" % (e[0][0][2],))
         else:
             server = ThreadedHTTPServer((listenIp, self.port), handler,
                                         bind_and_activate=False)
@@ -981,7 +1018,7 @@ class ServeFile():
             print("Serving \"%s\" for uploads at port %d." % (self.target, self.port))
 
         # print urls with local network adresses
-		print("\nSome addresses %s will be available at:" % \
+        print("\nSome addresses %s will be available at:" %
               ("this file" if (self.serveMode != self.MODE_UPLOAD) else "the uploadform", ))
         ips = self.getIPs()
         if not ips or len(ips) == 0 or ips[0] == '':
@@ -1038,7 +1075,8 @@ class ServeFile():
                 try:
                     os.mkdir(self.target)
                 except (IOError, OSError) as e:
-					raise ServeFileException("Error: Could not create directory '%s' for uploads, %r" % (self.target, str(e)))
+                    raise ServeFileException("Error: Could not create directory '%s' for uploads, %r" %
+                                             (self.target, str(e)))
             else:
                 raise ServeFileException("Error: Upload directory already exists and is a file.")
             FilePutter.targetDir = os.path.abspath(self.target)
@@ -1057,6 +1095,7 @@ class ServeFile():
             AuthenticationHandler.authString = self.auth
             if self.authrealm:
                 AuthenticationHandler.realm = self.authrealm
+
             class AuthenticatedHandler(AuthenticationHandler, handler):
                 pass
             handler = AuthenticatedHandler
@@ -1102,7 +1141,8 @@ class AuthenticationHandler():
             self.send_response(401)
             self.send_header("WWW-Authenticate", "Basic realm=\"%s\"" % self.realm)
             self.send_header("Connection", "close")
-			errorMsg = "<html><head><title>401 - Unauthorized</title></head><body><h1>401 - Unauthorized</h1></body></html>"
+            errorMsg = ("<html><head><title>401 - Unauthorized</title></head>"
+                        "<body><h1>401 - Unauthorized</h1></body></html>")
             self.send_header("Content-Length", str(len(errorMsg)))
             self.end_headers()
             self.wfile.write(errorMsg.encode())
@@ -1112,32 +1152,35 @@ def main():
     parser = argparse.ArgumentParser(prog='servefile', description='Serve a single file via HTTP.')
     parser.add_argument('--version', action='version', version='%(prog)s ' + __version__)
     parser.add_argument('target', metavar='file/directory', type=str)
-	parser.add_argument('-p', '--port', type=int, default=8080, \
+    parser.add_argument('-p', '--port', type=int, default=8080,
                         help='Port to listen on')
-	parser.add_argument('-u', '--upload', action="store_true", default=False, \
+    parser.add_argument('-u', '--upload', action="store_true", default=False,
                         help="Enable uploads to a given directory")
-	parser.add_argument('-s', '--max-upload-size', type=str, \
+    parser.add_argument('-s', '--max-upload-size', type=str,
                         help="Limit upload size in kB. Size modifiers are allowed, e.g. 2G, 12MB, 1B")
-	parser.add_argument('-l', '--list-dir', action="store_true", default=False, \
+    parser.add_argument('-l', '--list-dir', action="store_true", default=False,
                         help="Show directory indexes and allow access to all subdirectories")
-	parser.add_argument('--ssl', action="store_true", default=False, \
+    parser.add_argument('--ssl', action="store_true", default=False,
                         help="Enable SSL. If no key/cert is specified one will be generated")
-	parser.add_argument('--key', type=str, \
-	                    help="Keyfile to use for SSL. If no cert is given with --cert the keyfile will also be searched for a cert")
-	parser.add_argument('--cert', type=str, \
+    parser.add_argument('--key', type=str,
+                        help="Keyfile to use for SSL. If no cert is given with --cert the keyfile "
+                             "will also be searched for a cert")
+    parser.add_argument('--cert', type=str,
                         help="Certfile to use for SSL")
-	parser.add_argument('-a', '--auth', type=str, metavar='user:password', \
+    parser.add_argument('-a', '--auth', type=str, metavar='user:password',
                         help="Set user and password for HTTP basic authentication")
-	parser.add_argument('--realm', type=str, default=None,\
+    parser.add_argument('--realm', type=str, default=None,
                         help="Set a realm for HTTP basic authentication")
-	parser.add_argument('-t', '--tar', action="store_true", default=False, \
-	                    help="Enable on the fly tar creation for given file or directory. Note: Download continuation will not be available")
-	parser.add_argument('-c', '--compression', type=str, metavar='method', \
-	                    default="none", \
-	                    help="Set compression method, only in combination with --tar. Can be one of %s" % ", ".join(TarFileHandler.compressionMethods))
-	parser.add_argument('-4', '--ipv4-only', action="store_true", default=False, \
+    parser.add_argument('-t', '--tar', action="store_true", default=False,
+                        help="Enable on the fly tar creation for given file or directory. "
+                             "Note: Download continuation will not be available")
+    parser.add_argument('-c', '--compression', type=str, metavar='method',
+                        default="none",
+                        help="Set compression method, only in combination with --tar. "
+                             "Can be one of %s" % ", ".join(TarFileHandler.compressionMethods))
+    parser.add_argument('-4', '--ipv4-only', action="store_true", default=False,
                         help="Listen on IPv4 only")
-	parser.add_argument('-6', '--ipv6-only', action="store_true", default=False, \
+    parser.add_argument('-6', '--ipv6-only', action="store_true", default=False,
                         help="Listen on IPv6 only")
 
     args = parser.parse_args()
@@ -1153,7 +1196,7 @@ def main():
         sys.exit(1)
 
     if args.max_upload_size:
-		sizeRe = re.match("^(\d+(?:[,.]\d+)?)(?:([bkmgtpe])(?:(?<!b)b?)?)?$", args.max_upload_size.lower())
+        sizeRe = re.match(r"^(\d+(?:[,.]\d+)?)(?:([bkmgtpe])(?:(?<!b)b?)?)?$", args.max_upload_size.lower())
         if not sizeRe:
             print("Error: Your max upload size param is broken. Try something like 3M or 2.5Gb.")
             sys.exit(1)
@@ -1166,7 +1209,7 @@ def main():
             sys.exit(1)
 
     if args.ssl and not HAVE_SSL:
-		print("Error: SSL is not available, please install pyssl (python-openssl).")
+        print("Error: SSL is not available, please install pyopenssl (python3-openssl).")
         sys.exit(1)
 
     if args.cert and not args.key:
@@ -1179,8 +1222,9 @@ def main():
 
     if args.auth:
         dpos = args.auth.find(":")
-		if dpos <= 0 or dpos == (len(args.auth)-1):
-			print("Error: User and password for HTTP basic authentication need to be both at least one character and have to be separated by a \":\".")
+        if dpos <= 0 or dpos == (len(args.auth) - 1):
+            print("Error: User and password for HTTP basic authentication need to be both "
+                  "at least one character and have to be separated by a \":\".")
             sys.exit(1)
 
     if args.realm and not args.auth:
@@ -1251,4 +1295,3 @@ def main():
 
 if __name__ == '__main__':
     main()
-

setup.py

@@ -11,7 +11,7 @@ setup(
     long_description=long_description,
     long_description_content_type='text/markdown',
     platforms='posix',
-    version='0.5.1',
+    version='0.5.4',
     license='GPLv3 or later',
     url='https://github.com/sebageek/servefile/',
     author='Sebastian Lohff',
@@ -38,10 +38,11 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         'Topic :: Communications',
         'Topic :: Communications :: File Sharing',
         'Topic :: Internet',

tests/test_servefile.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 import io
 import os
 import pytest
@@ -8,18 +9,36 @@ import sys
 import tarfile
 import time
 import urllib3
+from requests.exceptions import ConnectionError
 
 # crudly written to learn more about pytest and to have a base for refactoring
 
 
 if sys.version_info.major >= 3:
     from pathlib import Path
+    from urllib.parse import quote
     connrefused_exc = ConnectionRefusedError
 else:
     from pathlib2 import Path
+    from urllib import quote
     connrefused_exc = socket.error
 
 
+def _get_port_from_env(var_name, default):
+    port = int(os.environ.get(var_name, default))
+    if port == 0:
+        # do a one-time port selection for a free port, use it for all tests
+        s = socket.socket()
+        s.bind(('', 0))
+        port = s.getsockname()[1]
+        s.close()
+    return port
+
+
+SERVEFILE_DEFAULT_PORT = _get_port_from_env('SERVEFILE_DEFAULT_PORT', 0)
+SERVEFILE_SECONDARY_PORT = _get_port_from_env('SERVEFILE_SECONDARY_PORT', 0)
+
+
 @pytest.fixture
 def run_servefile():
     instances = []
@@ -34,9 +53,12 @@ def run_servefile():
             # call servefile as python module
             servefile_path = ['-m', 'servefile']
 
+        # use non-default default port, if one is given via env (and none via args)
+        if '-p' not in args and '--port' not in args:
+            args.extend(['-p', str(SERVEFILE_DEFAULT_PORT)])
+
         print("running {} with args {}".format(", ".join(servefile_path), args))
         p = subprocess.Popen([sys.executable] + servefile_path + args, **kwargs)
-        time.sleep(kwargs.get('timeout', 0.3))
         instances.append(p)
 
         return p
@@ -62,22 +84,26 @@ def datadir(tmp_path):
                 _datadir(v, new_path)
             else:
                 if hasattr(v, 'decode'):
-                    v = v.decode()  # python2 compability
+                    v = v.decode('utf-8')  # python2 compability
                 (path / k).write_text(v)
 
         return path
     return _datadir
 
 
-def make_request(path='/', host='localhost', port=8080, method='get', protocol='http', **kwargs):
+def make_request(path='/', host='localhost', port=SERVEFILE_DEFAULT_PORT, method='get', protocol='http',
+                 encoding='utf-8', **kwargs):
     url = '{}://{}:{}{}'.format(protocol, host, port, path)
     print('Calling {} on {} with {}'.format(method, url, kwargs))
     r = getattr(requests, method)(url, **kwargs)
 
+    if r.encoding is None and encoding:
+        r.encoding = encoding
+
     return r
 
 
-def check_download(expected_data=None, path='/', fname=None, status_code=200, **kwargs):
+def check_download(expected_data=None, path='/', fname=None, **kwargs):
     if fname is None:
         fname = os.path.basename(path)
     r = make_request(path, **kwargs)
@@ -91,6 +117,22 @@ def check_download(expected_data=None, path='/', fname=None, status_code=200, **
     return r  # for additional tests
 
 
+def _retry_while(exception, function, timeout=2):
+    now = time.time  # float seconds since epoch
+
+    def wrapped(*args, **kwargs):
+        timeout_after = now() + timeout
+        while True:
+            try:
+                return function(*args, **kwargs)
+            except exception:
+                if now() >= timeout_after:
+                    raise
+                time.sleep(0.1)
+
+    return wrapped
+
+
 def _test_version(run_servefile, standalone):
     # we expect the version on stdout (python3.4+) or stderr(python2.6-3.3)
     s = run_servefile('--version', standalone=standalone, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@@ -104,7 +146,7 @@ def _test_version(run_servefile, standalone):
         version = s.stdout.readline().decode().strip()
 
     # hardcode version as string until servefile is a module
-    assert version == 'servefile 0.5.1'
+    assert version == 'servefile 0.5.4'
 
 
 def test_version(run_servefile):
@@ -121,7 +163,7 @@ def test_correct_headers(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(str(p))
 
-    r = make_request()
+    r = _retry_while(ConnectionError, make_request)()
     assert r.status_code == 200
     assert r.headers.get('Content-Type') == 'application/octet-stream'
     assert r.headers.get('Content-Disposition') == 'attachment; filename="testfile"'
@@ -134,7 +176,7 @@ def test_redirect_and_download(run_servefile, datadir):
     run_servefile(str(p))
 
     # redirect
-    r = make_request(allow_redirects=False)
+    r = _retry_while(ConnectionError, make_request)(allow_redirects=False)
     assert r.status_code == 302
     assert r.headers.get('Location') == '/testfile'
 
@@ -142,12 +184,29 @@ def test_redirect_and_download(run_servefile, datadir):
     check_download(data, fname='testfile')
 
 
+def test_redirect_and_download_with_umlaut(run_servefile, datadir):
+    data = "NÖÖT NÖÖT"
+    filename = "tästføile"
+    p = datadir({filename: data}) / filename
+    run_servefile(str(p))
+
+    # redirect
+    r = _retry_while(ConnectionError, make_request)(allow_redirects=False)
+    assert r.status_code == 302
+    assert r.headers.get('Location') == '/{}'.format(quote(filename))
+
+    # normal download
+    if sys.version_info.major < 3:
+        data = unicode(data, 'utf-8')
+    check_download(data, fname=filename)
+
+
 def test_specify_port(run_servefile, datadir):
     data = "NOOT NOOT"
     p = datadir({'testfile': data}) / 'testfile'
-    run_servefile([str(p), '-p', '8081'])
+    run_servefile([str(p), '-p', str(SERVEFILE_SECONDARY_PORT)])
 
-    check_download(data, fname='testfile', port=8081)
+    _retry_while(ConnectionError, check_download)(data, fname='testfile', port=SERVEFILE_SECONDARY_PORT)
 
 
 def test_ipv4_only(run_servefile, datadir):
@@ -155,11 +214,11 @@ def test_ipv4_only(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile([str(p), '-4'])
 
-    check_download(data, fname='testfile', host='127.0.0.1')
+    _retry_while(ConnectionError, check_download)(data, fname='testfile', host='127.0.0.1')
 
     sock = socket.socket(socket.AF_INET6)
     with pytest.raises(connrefused_exc):
-        sock.connect(("::1", 8080))
+        sock.connect(("::1", SERVEFILE_DEFAULT_PORT))
 
 
 def test_big_download(run_servefile, datadir):
@@ -168,7 +227,7 @@ def test_big_download(run_servefile, datadir):
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(str(p))
 
-    check_download(data, fname='testfile')
+    _retry_while(ConnectionError, check_download)(data, fname='testfile')
 
 
 def test_authentication(run_servefile, datadir):
@@ -177,11 +236,11 @@ def test_authentication(run_servefile, datadir):
 
     run_servefile([str(p), '-a', 'user:password'])
     for auth in [('foo', 'bar'), ('user', 'wrong'), ('unknown', 'password')]:
-        r = make_request(auth=auth)
+        r = _retry_while(ConnectionError, make_request)(auth=auth)
         assert '401 - Unauthorized' in r.text
         assert r.status_code == 401
 
-    check_download(data, fname='testfile', auth=('user', 'password'))
+    _retry_while(ConnectionError, check_download)(data, fname='testfile', auth=('user', 'password'))
 
 
 def test_serve_directory(run_servefile, datadir):
@@ -190,6 +249,7 @@ def test_serve_directory(run_servefile, datadir):
         'bar': {'thisisaverylongfilenamefortestingthatthisstillworksproperly': 'jup!'},
         'noot': 'still data in here',
         'bigfile': 'x' * (10 * 1024 ** 2),
+        'möwe': 'KRAKRAKRAKA',
     }
     p = datadir(d)
     run_servefile([str(p), '-l'])
@@ -197,12 +257,12 @@ def test_serve_directory(run_servefile, datadir):
     # check if all files are in directory listing
     # (could be made more sophisticated with beautifulsoup)
     for path in '/', '/../':
-        r = make_request(path)
+        r = _retry_while(ConnectionError, make_request)(path)
         for k in d:
-            assert k in r.text
+            assert quote(k) in r.text
 
     for fname, content in d['foo'].items():
-        check_download(content, '/foo/' + fname)
+        _retry_while(ConnectionError, check_download)(content, '/foo/' + fname)
 
     r = make_request('/unknown')
     assert r.status_code == 404
@@ -224,7 +284,7 @@ def test_serve_relative_directory(run_servefile, datadir):
     # check if all files are in directory listing
     # (could be made more sophisticated with beautifulsoup)
     for path in '/', '/../':
-        r = make_request(path)
+        r = _retry_while(ConnectionError, make_request)(path)
         for k in d:
             assert k in r.text
 
@@ -248,14 +308,14 @@ def test_upload(run_servefile, tmp_path):
 
     run_servefile(['-u', str(uploaddir)])
 
-    # check that servefile created the directory
-    assert uploaddir.is_dir()
-
     # check upload form present
-    r = make_request()
+    r = _retry_while(ConnectionError, make_request)()
     assert r.status_code == 200
     assert 'multipart/form-data' in r.text
 
+    # check that servefile created the directory
+    assert uploaddir.is_dir()
+
     # upload file
     files = {'file': ('haiku.txt', data)}
     r = make_request(method='post', files=files)
@@ -271,6 +331,13 @@ def test_upload(run_servefile, tmp_path):
     with open(str(uploaddir / 'haiku.txt(1)')) as f:
         assert f.read() == data
 
+    # upload file using PUT
+    r = make_request("/haiku.txt", method='put', data=data)
+    assert r.status_code == 201
+    assert 'OK!' in r.text
+    with open(str(uploaddir / 'haiku.txt(2)')) as f:
+        assert f.read() == data
+
 
 def test_upload_size_limit(run_servefile, tmp_path):
     uploaddir = tmp_path / 'upload'
@@ -278,7 +345,7 @@ def test_upload_size_limit(run_servefile, tmp_path):
 
     # upload file that is too big
     files = {'file': ('toobig', "x" * 2049)}
-    r = make_request(method='post', files=files)
+    r = _retry_while(ConnectionError, make_request)(method='post', files=files)
     assert 'Your file was too big' in r.text
     assert r.status_code == 413
     assert not (uploaddir / 'toobig').exists()
@@ -290,6 +357,20 @@ def test_upload_size_limit(run_servefile, tmp_path):
     assert r.status_code == 200
 
 
+def test_upload_large_file(run_servefile, tmp_path):
+    # small files end up in BytesIO while large files get temporary files. this
+    # test makes sure we hit the large file codepath at least once
+    uploaddir = tmp_path / 'upload'
+    run_servefile(['-u', str(uploaddir)])
+
+    data = "asdf" * 1024
+    files = {'file': ('more_data.txt', data)}
+    r = _retry_while(ConnectionError, make_request)(method='post', files=files)
+    assert r.status_code == 200
+    with open(str(uploaddir / 'more_data.txt')) as f:
+        assert f.read() == data
+
+
 def test_tar_mode(run_servefile, datadir):
     d = {
         'foo': {
@@ -303,7 +384,7 @@ def test_tar_mode(run_servefile, datadir):
     # test redirect?
 
     # test contents of tar file
-    r = make_request()
+    r = _retry_while(ConnectionError, make_request)()
     assert r.status_code == 200
     tar = tarfile.open(fileobj=io.BytesIO(r.content))
     assert len(tar.getmembers()) == 3
@@ -319,7 +400,7 @@ def test_tar_compression(run_servefile, datadir):
     p = datadir(d)
     run_servefile(['-c', 'gzip', '-t', str(p / 'foo')])
 
-    r = make_request()
+    r = _retry_while(ConnectionError, make_request)()
     assert r.status_code == 200
     tar = tarfile.open(fileobj=io.BytesIO(r.content), mode='r:gz')
     assert len(tar.getmembers()) == 1
@@ -329,7 +410,6 @@ def test_https(run_servefile, datadir):
     data = "NOOT NOOT"
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(['--ssl', str(p)])
-    time.sleep(0.2)  # time for generating ssl certificates
 
     # fingerprint = None
     # while not fingerprint:
@@ -344,7 +424,7 @@ def test_https(run_servefile, datadir):
 
     # assert fingerprint
     urllib3.disable_warnings()
-    check_download(data, protocol='https', verify=False)
+    _retry_while(ConnectionError, check_download)(data, protocol='https', verify=False)
 
 
 def test_https_big_download(run_servefile, datadir):
@@ -352,7 +432,27 @@ def test_https_big_download(run_servefile, datadir):
     data = "x" * (10 * 1024 ** 2)
     p = datadir({'testfile': data}) / 'testfile'
     run_servefile(['--ssl', str(p)])
-    time.sleep(0.2)  # time for generating ssl certificates
 
     urllib3.disable_warnings()
-    check_download(data, protocol='https', verify=False)
+    _retry_while(ConnectionError, check_download)(data, protocol='https', verify=False)
+
+
+def test_abort_download(run_servefile, datadir):
+    data = "x" * (10 * 1024 ** 2)
+    p = datadir({'testfile': data}) / 'testfile'
+    env = os.environ.copy()
+    env['PYTHONUNBUFFERED'] = '1'
+    proc = run_servefile(str(p), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)
+
+    # provoke a connection abort
+    # hopefully the buffers will not fill up with all of the 10mb
+    sock = socket.socket(socket.AF_INET)
+    _retry_while(connrefused_exc, sock.connect)(("localhost", SERVEFILE_DEFAULT_PORT))
+    sock.send(b"GET /testfile HTTP/1.0\n\n")
+    resp = sock.recv(100)
+    assert resp != b''
+    sock.close()
+    time.sleep(0.1)
+    proc.kill()
+    out = proc.stdout.read().decode()
+    assert "127.0.0.1 ABORTED transmission" in out

tox.ini

@@ -1,9 +1,19 @@
 [tox]
-envlist = py27,py36,py37,py38
+envlist = py27,py37,py38,py39,py310,py311,pep8
 
 [testenv]
 deps =
         pathlib2; python_version<"3"
         pytest
         requests
-commands = pytest --tb=short {posargs}
+        flake8
+commands = pytest -v --tb=short {posargs}
+
+[testenv:pep8]
+commands = flake8 servefile/ {posargs}
+
+[flake8]
+show-source = True
+max-line-length = 120
+ignore = E123,E125,E241,E402,E741,W503,W504,H301
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build