OwlCyberSecurity - MANAGER

Edit File: test_newclient.py

# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.

"""
Tests for L{twisted.web._newclient}.
"""

from __future__ import division, absolute_import
__metaclass__ = type

from zope.interface import implementer
from zope.interface.verify import verifyObject

from twisted.python.failure import Failure
from twisted.internet.interfaces import IConsumer, IPushProducer
from twisted.internet.error import ConnectionDone, ConnectionLost
from twisted.internet.defer import Deferred, succeed, fail, CancelledError
from twisted.internet.protocol import Protocol
from twisted.protocols.basic import LineReceiver
from twisted.trial.unittest import TestCase
from twisted.test.proto_helpers import (
    AccumulatingProtocol,
    EventLoggingObserver,
    StringTransport,
    StringTransportWithDisconnection,
    )
from twisted.web._newclient import UNKNOWN_LENGTH, STATUS, HEADER, BODY, DONE
from twisted.web._newclient import HTTPParser, HTTPClientParser
from twisted.web._newclient import BadResponseVersion, ParseError
from twisted.web._newclient import ChunkedEncoder
from twisted.web._newclient import WrongBodyLength, RequestNotSent
from twisted.web._newclient import ConnectionAborted
from twisted.web._newclient import BadHeaders, ExcessWrite
from twisted.web._newclient import TransportProxyProducer, LengthEnforcingConsumer, makeStatefulDispatcher
from twisted.web.client import (
    HTTP11ClientProtocol,
    PotentialDataLoss,
    Request,
    RequestGenerationFailed,
    RequestTransmissionFailed,
    Response,
    ResponseDone,
    ResponseFailed,
    ResponseNeverReceived,
    )
from twisted.web.http_headers import Headers
from twisted.web.http import _DataLoss
from twisted.web.iweb import IBodyProducer, IResponse
from twisted.web.test.requesthelper import (
    bytesLinearWhitespaceComponents,
    sanitizedBytes,
)
from twisted.logger import globalLogPublisher

class ArbitraryException(Exception):
    """
    A unique, arbitrary exception type which L{twisted.web._newclient} knows
    nothing about.
    """

class AnotherArbitraryException(Exception):
    """
    Similar to L{ArbitraryException} but with a different identity.
    """

# A re-usable Headers instance for tests which don't really care what headers
# they're sending.
_boringHeaders = Headers({b'host': [b'example.com']})

def assertWrapperExceptionTypes(self, deferred, mainType, reasonTypes):
    """
    Assert that the given L{Deferred} fails with the exception given by
    C{mainType} and that the exceptions wrapped by the instance of C{mainType}
    it fails with match the list of exception types given by C{reasonTypes}.

This is a helper for testing failures of exceptions which subclass
    L{_newclient._WrapperException}.

@param self: A L{TestCase} instance which will be used to make the
        assertions.

@param deferred: The L{Deferred} which is expected to fail with
        C{mainType}.

@param mainType: A L{_newclient._WrapperException} subclass which will be
        trapped on C{deferred}.

@param reasonTypes: A sequence of exception types which will be trapped on
        the resulting C{mainType} exception instance's C{reasons} sequence.

@return: A L{Deferred} which fires with the C{mainType} instance
        C{deferred} fails with, or which fails somehow.
    """
    def cbFailed(err):
        for reason, type in zip(err.reasons, reasonTypes):
            reason.trap(type)
        self.assertEqual(len(err.reasons), len(reasonTypes),
                         "len(%s) != len(%s)" % (err.reasons, reasonTypes))
        return err
    d = self.assertFailure(deferred, mainType)
    d.addCallback(cbFailed)
    return d

def assertResponseFailed(self, deferred, reasonTypes):
    """
    A simple helper to invoke L{assertWrapperExceptionTypes} with a C{mainType}
    of L{ResponseFailed}.
    """
    return assertWrapperExceptionTypes(self, deferred, ResponseFailed, reasonTypes)

def assertRequestGenerationFailed(self, deferred, reasonTypes):
    """
    A simple helper to invoke L{assertWrapperExceptionTypes} with a C{mainType}
    of L{RequestGenerationFailed}.
    """
    return assertWrapperExceptionTypes(self, deferred, RequestGenerationFailed, reasonTypes)

def assertRequestTransmissionFailed(self, deferred, reasonTypes):
    """
    A simple helper to invoke L{assertWrapperExceptionTypes} with a C{mainType}
    of L{RequestTransmissionFailed}.
    """
    return assertWrapperExceptionTypes(self, deferred, RequestTransmissionFailed, reasonTypes)

def justTransportResponse(transport):
    """
    Helper function for creating a Response which uses the given transport.
    All of the other parameters to L{Response.__init__} are filled with
    arbitrary values.  Only use this method if you don't care about any of
    them.
    """
    return Response((b'HTTP', 1, 1), 200, b'OK', _boringHeaders, transport)

class MakeStatefulDispatcherTests(TestCase):
    """
    Tests for L{makeStatefulDispatcher}.
    """
    def test_functionCalledByState(self):
        """
        A method defined with L{makeStatefulDispatcher} invokes a second
        method based on the current state of the object.
        """
        class Foo:
            _state = 'A'

def bar(self):
                pass
            bar = makeStatefulDispatcher('quux', bar)

def _quux_A(self):
                return 'a'

def _quux_B(self):
                return 'b'

stateful = Foo()
        self.assertEqual(stateful.bar(), 'a')
        stateful._state = 'B'
        self.assertEqual(stateful.bar(), 'b')
        stateful._state = 'C'
        self.assertRaises(RuntimeError, stateful.bar)

class _HTTPParserTests(object):
    """
    Base test class for L{HTTPParser} which is responsible for the bulk of
    the task of parsing HTTP bytes.
    """
    sep = None

def test_statusCallback(self):
        """
        L{HTTPParser} calls its C{statusReceived} method when it receives a
        status line.
        """
        status = []
        protocol = HTTPParser()
        protocol.statusReceived = status.append
        protocol.makeConnection(StringTransport())
        self.assertEqual(protocol.state, STATUS)
        protocol.dataReceived(b'HTTP/1.1 200 OK' + self.sep)
        self.assertEqual(status, [b'HTTP/1.1 200 OK'])
        self.assertEqual(protocol.state, HEADER)

def _headerTestSetup(self):
        header = {}
        protocol = HTTPParser()
        protocol.headerReceived = header.__setitem__
        protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'HTTP/1.1 200 OK' + self.sep)
        return header, protocol

def test_headerCallback(self):
        """
        L{HTTPParser} calls its C{headerReceived} method when it receives a
        header.
        """
        header, protocol = self._headerTestSetup()
        protocol.dataReceived(b'X-Foo:bar' + self.sep)
        # Cannot tell it's not a continue header until the next line arrives
        # and is not a continuation
        protocol.dataReceived(self.sep)
        self.assertEqual(header, {b'X-Foo': b'bar'})
        self.assertEqual(protocol.state, BODY)

def test_continuedHeaderCallback(self):
        """
        If a header is split over multiple lines, L{HTTPParser} calls
        C{headerReceived} with the entire value once it is received.
        """
        header, protocol = self._headerTestSetup()
        protocol.dataReceived(b'X-Foo: bar' + self.sep)
        protocol.dataReceived(b' baz' + self.sep)
        protocol.dataReceived(b'\tquux' + self.sep)
        protocol.dataReceived(self.sep)
        self.assertEqual(header, {b'X-Foo': b'bar baz\tquux'})
        self.assertEqual(protocol.state, BODY)

def test_fieldContentWhitespace(self):
        """
        Leading and trailing linear whitespace is stripped from the header
        value passed to the C{headerReceived} callback.
        """
        header, protocol = self._headerTestSetup()
        value = self.sep.join([b' \t ', b' bar \t', b' \t', b''])
        protocol.dataReceived(b'X-Bar:' + value)
        protocol.dataReceived(b'X-Foo:' + value)
        protocol.dataReceived(self.sep)
        self.assertEqual(header, {b'X-Foo': b'bar',
                                  b'X-Bar': b'bar'})

def test_allHeadersCallback(self):
        """
        After the last header is received, L{HTTPParser} calls
        C{allHeadersReceived}.
        """
        called = []
        header, protocol = self._headerTestSetup()
        def allHeadersReceived():
            called.append(protocol.state)
            protocol.state = STATUS
        protocol.allHeadersReceived = allHeadersReceived
        protocol.dataReceived(self.sep)
        self.assertEqual(called, [HEADER])
        self.assertEqual(protocol.state, STATUS)

def test_noHeaderCallback(self):
        """
        If there are no headers in the message, L{HTTPParser} does not call
        C{headerReceived}.
        """
        header, protocol = self._headerTestSetup()
        protocol.dataReceived(self.sep)
        self.assertEqual(header, {})
        self.assertEqual(protocol.state, BODY)

def test_headersSavedOnResponse(self):
        """
        All headers received by L{HTTPParser} are added to
        L{HTTPParser.headers}.
        """
        protocol = HTTPParser()
        protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'HTTP/1.1 200 OK' + self.sep)
        protocol.dataReceived(b'X-Foo: bar' + self.sep)
        protocol.dataReceived(b'X-Foo: baz' + self.sep)
        protocol.dataReceived(self.sep)
        expected = [(b'X-Foo', [b'bar', b'baz'])]
        self.assertEqual(expected, list(protocol.headers.getAllRawHeaders()))

def test_connectionControlHeaders(self):
        """
        L{HTTPParser.isConnectionControlHeader} returns C{True} for headers
        which are always connection control headers (similar to "hop-by-hop"
        headers from RFC 2616 section 13.5.1) and C{False} for other headers.
        """
        protocol = HTTPParser()
        connHeaderNames = [
            b'content-length', b'connection', b'keep-alive', b'te', b'trailers',
            b'transfer-encoding', b'upgrade', b'proxy-connection']

for header in connHeaderNames:
            self.assertTrue(
                protocol.isConnectionControlHeader(header),
                "Expecting %r to be a connection control header, but "
                "wasn't" % (header,))
        self.assertFalse(
            protocol.isConnectionControlHeader(b"date"),
            "Expecting the arbitrarily selected 'date' header to not be "
            "a connection control header, but was.")

def test_switchToBodyMode(self):
        """
        L{HTTPParser.switchToBodyMode} raises L{RuntimeError} if called more
        than once.
        """
        protocol = HTTPParser()
        protocol.makeConnection(StringTransport())
        protocol.switchToBodyMode(object())
        self.assertRaises(RuntimeError, protocol.switchToBodyMode, object())

class HTTPParserRFCComplaintDelimeterTests(_HTTPParserTests, TestCase):
    """
    L{_HTTPParserTests} using standard CR LF newlines.
    """
    sep = b'\r\n'

class HTTPParserNonRFCComplaintDelimeterTests(_HTTPParserTests, TestCase):
    """
    L{_HTTPParserTests} using bare LF newlines.
    """
    sep = b'\n'

class HTTPClientParserTests(TestCase):
    """
    Tests for L{HTTPClientParser} which is responsible for parsing HTTP
    response messages.
    """
    def test_parseVersion(self):
        """
        L{HTTPClientParser.parseVersion} parses a status line into its three
        components.
        """
        protocol = HTTPClientParser(None, None)
        self.assertEqual(
            protocol.parseVersion(b'CANDY/7.2'),
            (b'CANDY', 7, 2))

def test_parseBadVersion(self):
        """
        L{HTTPClientParser.parseVersion} raises L{ValueError} when passed an
        unparsable version.
        """
        protocol = HTTPClientParser(None, None)
        e = BadResponseVersion
        f = protocol.parseVersion

def checkParsing(s):
            exc = self.assertRaises(e, f, s)
            self.assertEqual(exc.data, s)

checkParsing(b'foo')
        checkParsing(b'foo/bar/baz')

checkParsing(b'foo/')
        checkParsing(b'foo/..')

checkParsing(b'foo/a.b')
        checkParsing(b'foo/-1.-1')

def test_responseStatusParsing(self):
        """
        L{HTTPClientParser.statusReceived} parses the version, code, and phrase
        from the status line and stores them on the response object.
        """
        request = Request(b'GET', b'/', _boringHeaders, None)
        protocol = HTTPClientParser(request, None)
        protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'HTTP/1.1 200 OK\r\n')
        self.assertEqual(protocol.response.version, (b'HTTP', 1, 1))
        self.assertEqual(protocol.response.code, 200)
        self.assertEqual(protocol.response.phrase, b'OK')

def test_responseStatusWithoutPhrase(self):
        """
        L{HTTPClientParser.statusReceived} can parse a status line without a
        phrase (though such lines are a violation of RFC 7230, section 3.1.2;
        nevertheless some broken servers omit the phrase).
        """
        request = Request(b'GET', b'/', _boringHeaders, None)
        protocol = HTTPClientParser(request, None)
        protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'HTTP/1.1 200\r\n')
        self.assertEqual(protocol.response.version, (b'HTTP', 1, 1))
        self.assertEqual(protocol.response.code, 200)
        self.assertEqual(protocol.response.phrase, b'')

def test_badResponseStatus(self):
        """
        L{HTTPClientParser.statusReceived} raises L{ParseError} if it is called
        with a status line which cannot be parsed.
        """
        protocol = HTTPClientParser(None, None)

def checkParsing(s):
            exc = self.assertRaises(ParseError, protocol.statusReceived, s)
            self.assertEqual(exc.data, s)

# If there are fewer than two whitespace-delimited parts to the status
        # line, it is not valid and cannot be parsed.
        checkParsing(b'foo')

# If the response code is not an integer, the status line is not valid
        # and cannot be parsed.
        checkParsing(b'HTTP/1.1 bar OK')

def _noBodyTest(self, request, status, response):
        """
        Assert that L{HTTPClientParser} parses the given C{response} to
        C{request}, resulting in a response with no body and no extra bytes and
        leaving the transport in the producing state.

@param request: A L{Request} instance which might have caused a server
            to return the given response.
        @param status: A string giving the status line of the response to be
            parsed.
        @param response: A string giving the response to be parsed.

@return: A C{dict} of headers from the response.
        """
        header = {}
        finished = []
        body = []
        bodyDataFinished = []
        protocol = HTTPClientParser(request, finished.append)
        protocol.headerReceived = header.__setitem__
        transport = StringTransport()
        protocol.makeConnection(transport)
        # Deliver just the status to initialize the response object so we can
        # monkey-patch it to observe progress of the response parser.
        protocol.dataReceived(status)
        protocol.response._bodyDataReceived = body.append
        protocol.response._bodyDataFinished = (
            lambda: bodyDataFinished.append(True))
        protocol.dataReceived(response)
        self.assertEqual(transport.producerState, u'producing')
        self.assertEqual(protocol.state, DONE)
        self.assertEqual(body, [])
        self.assertEqual(finished, [b''])
        self.assertEqual(bodyDataFinished, [True])
        self.assertEqual(protocol.response.length, 0)
        return header

def test_headResponse(self):
        """
        If the response is to a HEAD request, no body is expected, the body
        callback is not invoked, and the I{Content-Length} header is passed to
        the header callback.
        """
        request = Request(b'HEAD', b'/', _boringHeaders, None)
        status = b'HTTP/1.1 200 OK\r\n'
        response = (
            b'Content-Length: 10\r\n'
            b'\r\n')
        header = self._noBodyTest(request, status, response)
        self.assertEqual(header, {b'Content-Length': b'10'})

def test_noContentResponse(self):
        """
        If the response code is I{NO CONTENT} (204), no body is expected and
        the body callback is not invoked.
        """
        request = Request(b'GET', b'/', _boringHeaders, None)
        status = b'HTTP/1.1 204 NO CONTENT\r\n'
        response = b'\r\n'
        self._noBodyTest(request, status, response)

def test_notModifiedResponse(self):
        """
        If the response code is I{NOT MODIFIED} (304), no body is expected and
        the body callback is not invoked.
        """
        request = Request(b'GET', b'/', _boringHeaders, None)
        status = b'HTTP/1.1 304 NOT MODIFIED\r\n'
        response = b'\r\n'
        self._noBodyTest(request, status, response)

def test_responseHeaders(self):
        """
        The response headers are added to the response object's C{headers}
        L{Headers} instance.
        """
        protocol = HTTPClientParser(
            Request(b'GET', b'/', _boringHeaders, None),
            lambda rest: None)
        protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'HTTP/1.1 200 OK\r\n')
        protocol.dataReceived(b'X-Foo: bar\r\n')
        protocol.dataReceived(b'\r\n')
        self.assertEqual(
            protocol.connHeaders,
            Headers({}))
        self.assertEqual(
            protocol.response.headers,
            Headers({b'x-foo': [b'bar']}))
        self.assertIdentical(protocol.response.length, UNKNOWN_LENGTH)

def test_responseHeadersMultiline(self):
        """
        The multi-line response headers are folded and added to the response
        object's C{headers} L{Headers} instance.
        """
        protocol = HTTPClientParser(
            Request(b'GET', b'/', _boringHeaders, None),
            lambda rest: None)
        protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'HTTP/1.1 200 OK\r\n')
        protocol.dataReceived(b'X-Multiline: a\r\n')
        protocol.dataReceived(b'    b\r\n')
        protocol.dataReceived(b'\r\n')
        self.assertEqual(
            protocol.connHeaders,
            Headers({}))
        self.assertEqual(
            protocol.response.headers,
            Headers({b'x-multiline': [b'a    b']}))
        self.assertIdentical(protocol.response.length, UNKNOWN_LENGTH)

def test_connectionHeaders(self):
        """
        The connection control headers are added to the parser's C{connHeaders}
        L{Headers} instance.
        """
        protocol = HTTPClientParser(
            Request(b'GET', b'/', _boringHeaders, None),
            lambda rest: None)
        protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'HTTP/1.1 200 OK\r\n')
        protocol.dataReceived(b'Content-Length: 123\r\n')
        protocol.dataReceived(b'Connection: close\r\n')
        protocol.dataReceived(b'\r\n')
        self.assertEqual(
            protocol.response.headers,
            Headers({}))
        self.assertEqual(
            protocol.connHeaders,
            Headers({b'content-length': [b'123'],
                     b'connection': [b'close']}))
        self.assertEqual(protocol.response.length, 123)

def test_headResponseContentLengthEntityHeader(self):
        """
        If a HEAD request is made, the I{Content-Length} header in the response
        is added to the response headers, not the connection control headers.
        """
        protocol = HTTPClientParser(
            Request(b'HEAD', b'/', _boringHeaders, None),
            lambda rest: None)
        protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'HTTP/1.1 200 OK\r\n')
        protocol.dataReceived(b'Content-Length: 123\r\n')
        protocol.dataReceived(b'\r\n')
        self.assertEqual(
            protocol.response.headers,
            Headers({b'content-length': [b'123']}))
        self.assertEqual(
            protocol.connHeaders,
            Headers({}))
        self.assertEqual(protocol.response.length, 0)

def test_contentLength(self):
        """
        If a response includes a body with a length given by the
        I{Content-Length} header, the bytes which make up the body are passed
        to the C{_bodyDataReceived} callback on the L{HTTPParser}.
        """
        finished = []
        protocol = HTTPClientParser(
            Request(b'GET', b'/', _boringHeaders, None),
            finished.append)
        transport = StringTransport()
        protocol.makeConnection(transport)
        protocol.dataReceived(b'HTTP/1.1 200 OK\r\n')
        body = []
        protocol.response._bodyDataReceived = body.append
        protocol.dataReceived(b'Content-Length: 10\r\n')
        protocol.dataReceived(b'\r\n')

# Incidentally, the transport should be paused now.  It is the response
        # object's responsibility to resume this when it is ready for bytes.
        self.assertEqual(transport.producerState, u'paused')

self.assertEqual(protocol.state, BODY)
        protocol.dataReceived(b'x' * 6)
        self.assertEqual(body, [b'x' * 6])
        self.assertEqual(protocol.state, BODY)
        protocol.dataReceived(b'y' * 4)
        self.assertEqual(body, [b'x' * 6, b'y' * 4])
        self.assertEqual(protocol.state, DONE)
        self.assertEqual(finished, [b''])

def test_zeroContentLength(self):
        """
        If a response includes a I{Content-Length} header indicating zero bytes
        in the response, L{Response.length} is set accordingly and no data is
        delivered to L{Response._bodyDataReceived}.
        """
        finished = []
        protocol = HTTPClientParser(
            Request(b'GET', b'/', _boringHeaders, None),
            finished.append)

protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'HTTP/1.1 200 OK\r\n')

body = []
        protocol.response._bodyDataReceived = body.append

protocol.dataReceived(b'Content-Length: 0\r\n')
        protocol.dataReceived(b'\r\n')

self.assertEqual(protocol.state, DONE)
        self.assertEqual(body, [])
        self.assertEqual(finished, [b''])
        self.assertEqual(protocol.response.length, 0)

def test_multipleContentLengthHeaders(self):
        """
        If a response includes multiple I{Content-Length} headers,
        L{HTTPClientParser.dataReceived} raises L{ValueError} to indicate that
        the response is invalid and the transport is now unusable.
        """
        protocol = HTTPClientParser(
            Request(b'GET', b'/', _boringHeaders, None),
            None)

protocol.makeConnection(StringTransport())
        self.assertRaises(
            ValueError,
            protocol.dataReceived,
            b'HTTP/1.1 200 OK\r\n'
            b'Content-Length: 1\r\n'
            b'Content-Length: 2\r\n'
            b'\r\n')

def test_extraBytesPassedBack(self):
        """
        If extra bytes are received past the end of a response, they are passed
        to the finish callback.
        """
        finished = []
        protocol = HTTPClientParser(
            Request(b'GET', b'/', _boringHeaders, None),
            finished.append)

protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'HTTP/1.1 200 OK\r\n')
        protocol.dataReceived(b'Content-Length: 0\r\n')
        protocol.dataReceived(b'\r\nHere is another thing!')
        self.assertEqual(protocol.state, DONE)
        self.assertEqual(finished, [b'Here is another thing!'])

def test_extraBytesPassedBackHEAD(self):
        """
        If extra bytes are received past the end of the headers of a response
        to a HEAD request, they are passed to the finish callback.
        """
        finished = []
        protocol = HTTPClientParser(
            Request(b'HEAD', b'/', _boringHeaders, None),
            finished.append)

protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'HTTP/1.1 200 OK\r\n')
        protocol.dataReceived(b'Content-Length: 12\r\n')
        protocol.dataReceived(b'\r\nHere is another thing!')
        self.assertEqual(protocol.state, DONE)
        self.assertEqual(finished, [b'Here is another thing!'])

def test_chunkedResponseBody(self):
        """
        If the response headers indicate the response body is encoded with the
        I{chunked} transfer encoding, the body is decoded according to that
        transfer encoding before being passed to L{Response._bodyDataReceived}.
        """
        finished = []
        protocol = HTTPClientParser(
            Request(b'GET', b'/', _boringHeaders, None),
            finished.append)
        protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'HTTP/1.1 200 OK\r\n')

body = []
        protocol.response._bodyDataReceived = body.append

protocol.dataReceived(b'Transfer-Encoding: chunked\r\n')
        protocol.dataReceived(b'\r\n')

# No data delivered yet
        self.assertEqual(body, [])

# Cannot predict the length of a chunked encoded response body.
        self.assertIdentical(protocol.response.length, UNKNOWN_LENGTH)

# Deliver some chunks and make sure the data arrives
        protocol.dataReceived(b'3\r\na')
        self.assertEqual(body, [b'a'])
        protocol.dataReceived(b'bc\r\n')
        self.assertEqual(body, [b'a', b'bc'])

# The response's _bodyDataFinished method should be called when the last
        # chunk is received.  Extra data should be passed to the finished
        # callback.
        protocol.dataReceived(b'0\r\n\r\nextra')
        self.assertEqual(finished, [b'extra'])

def test_unknownContentLength(self):
        """
        If a response does not include a I{Transfer-Encoding} or a
        I{Content-Length}, the end of response body is indicated by the
        connection being closed.
        """
        finished = []
        protocol = HTTPClientParser(
            Request(b'GET', b'/', _boringHeaders, None), finished.append)
        transport = StringTransport()
        protocol.makeConnection(transport)
        protocol.dataReceived(b'HTTP/1.1 200 OK\r\n')

body = []
        protocol.response._bodyDataReceived = body.append

protocol.dataReceived(b'\r\n')
        protocol.dataReceived(b'foo')
        protocol.dataReceived(b'bar')
        self.assertEqual(body, [b'foo', b'bar'])
        protocol.connectionLost(ConnectionDone(u"simulated end of connection"))
        self.assertEqual(finished, [b''])

def test_contentLengthAndTransferEncoding(self):
        """
        According to RFC 2616, section 4.4, point 3, if I{Content-Length} and
        I{Transfer-Encoding: chunked} are present, I{Content-Length} MUST be
        ignored
        """
        finished = []
        protocol = HTTPClientParser(
            Request(b'GET', b'/', _boringHeaders, None), finished.append)
        transport = StringTransport()
        protocol.makeConnection(transport)
        protocol.dataReceived(b'HTTP/1.1 200 OK\r\n')

body = []
        protocol.response._bodyDataReceived = body.append

protocol.dataReceived(
            b'Content-Length: 102\r\n'
            b'Transfer-Encoding: chunked\r\n'
            b'\r\n'
            b'3\r\n'
            b'abc\r\n'
            b'0\r\n'
            b'\r\n')

self.assertEqual(body, [b'abc'])
        self.assertEqual(finished, [b''])

def test_connectionLostBeforeBody(self):
        """
        If L{HTTPClientParser.connectionLost} is called before the headers are
        finished, the C{_responseDeferred} is fired with the L{Failure} passed
        to C{connectionLost}.
        """
        transport = StringTransport()
        protocol = HTTPClientParser(Request(b'GET', b'/', _boringHeaders,
            None), None)
        protocol.makeConnection(transport)
        # Grab this here because connectionLost gets rid of the attribute
        responseDeferred = protocol._responseDeferred
        protocol.connectionLost(Failure(ArbitraryException()))

return assertResponseFailed(
            self, responseDeferred, [ArbitraryException])

def test_connectionLostWithError(self):
        """
        If one of the L{Response} methods called by
        L{HTTPClientParser.connectionLost} raises an exception, the exception
        is logged and not re-raised.
        """
        logObserver = EventLoggingObserver.createWithCleanup(
            self,
            globalLogPublisher
        )
        transport = StringTransport()
        protocol = HTTPClientParser(Request(b'GET', b'/', _boringHeaders, None),
                                    None)
        protocol.makeConnection(transport)

response = []
        protocol._responseDeferred.addCallback(response.append)
        protocol.dataReceived(
            b'HTTP/1.1 200 OK\r\n'
            b'Content-Length: 1\r\n'
            b'\r\n')
        response = response[0]

# Arrange for an exception
        def fakeBodyDataFinished(err=None):
            raise ArbitraryException()
        response._bodyDataFinished = fakeBodyDataFinished

protocol.connectionLost(None)
        self.assertEquals(1, len(logObserver))
        event = logObserver[0]
        f = event["log_failure"]
        self.assertIsInstance(f.value, ArbitraryException)
        self.flushLoggedErrors(ArbitraryException)

def test_noResponseAtAll(self):
        """
        If no response at all was received and the connection is lost, the
        resulting error is L{ResponseNeverReceived}.
        """
        protocol = HTTPClientParser(
            Request(b'HEAD', b'/', _boringHeaders, None),
            lambda ign: None)
        d = protocol._responseDeferred

protocol.makeConnection(StringTransport())
        protocol.connectionLost(ConnectionLost())
        return self.assertFailure(d, ResponseNeverReceived)

def test_someResponseButNotAll(self):
        """
        If a partial response was received and the connection is lost, the
        resulting error is L{ResponseFailed}, but not
        L{ResponseNeverReceived}.
        """
        protocol = HTTPClientParser(
            Request(b'HEAD', b'/', _boringHeaders, None),
            lambda ign: None)
        d = protocol._responseDeferred

protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'2')
        protocol.connectionLost(ConnectionLost())
        return self.assertFailure(d, ResponseFailed).addCallback(
            self.assertIsInstance, ResponseFailed)

def test_1XXResponseIsSwallowed(self):
        """
        If a response in the 1XX range is received it just gets swallowed and
        the parser resets itself.
        """
        sample103Response = (
            b'HTTP/1.1 103 Early Hints\r\n'
            b'Server: socketserver/1.0.0\r\n'
            b'Link: </other/styles.css>; rel=preload; as=style\r\n'
            b'Link: </other/action.js>; rel=preload; as=script\r\n'
            b'\r\n'
        )

# The response should have been erased
        self.assertTrue(getattr(protocol, 'response', None) is None)
        self.assertEqual(protocol.state, STATUS)
        self.assertEqual(len(list(protocol.headers.getAllRawHeaders())), 0)
        self.assertEqual(len(list(protocol.connHeaders.getAllRawHeaders())), 0)
        self.assertTrue(protocol._everReceivedData)

def test_1XXFollowedByFinalResponseOnlyEmitsFinal(self):
        """
        When a 1XX response is swallowed, the final response that follows it is
        the only one that gets sent to the application.
        """
        sample103Response = (
            b'HTTP/1.1 103 Early Hints\r\n'
            b'Server: socketserver/1.0.0\r\n'
            b'Link: </other/styles.css>; rel=preload; as=style\r\n'
            b'Link: </other/action.js>; rel=preload; as=script\r\n'
            b'\r\n'
        )
        following200Response = (
            b'HTTP/1.1 200 OK\r\n'
            b'Content-Length: 123\r\n'
            b'\r\n'
        )

protocol = HTTPClientParser(
            Request(b'GET', b'/', _boringHeaders, None),
            lambda ign: None
        )
        protocol.makeConnection(StringTransport())
        protocol.dataReceived(sample103Response + following200Response)

self.assertEqual(protocol.response.code, 200)
        self.assertEqual(
            protocol.response.headers,
            Headers({}))
        self.assertEqual(
            protocol.connHeaders,
            Headers({b'content-length': [b'123']}))
        self.assertEqual(protocol.response.length, 123)

def test_multiple1XXResponsesAreIgnored(self):
        """
        It is acceptable for multiple 1XX responses to come through, all of
        which get ignored.
        """
        sample103Response = (
            b'HTTP/1.1 103 Early Hints\r\n'
            b'Server: socketserver/1.0.0\r\n'
            b'Link: </other/styles.css>; rel=preload; as=style\r\n'
            b'Link: </other/action.js>; rel=preload; as=script\r\n'
            b'\r\n'
        )
        following200Response = (
            b'HTTP/1.1 200 OK\r\n'
            b'Content-Length: 123\r\n'
            b'\r\n'
        )

def test_ignored1XXResponseCausesLog(self):
        """
        When a 1XX response is ignored, Twisted emits a log.
        """
        logObserver = EventLoggingObserver.createWithCleanup(
            self,
            globalLogPublisher
        )
        sample103Response = (
            b'HTTP/1.1 103 Early Hints\r\n'
            b'Server: socketserver/1.0.0\r\n'
            b'Link: </other/styles.css>; rel=preload; as=style\r\n'
            b'Link: </other/action.js>; rel=preload; as=script\r\n'
            b'\r\n'
        )

self.assertEquals(1, len(logObserver))
        event = logObserver[0]
        self.assertEquals(
            event['log_format'],
            "Ignoring unexpected {code} response"
        )
        self.assertEquals(event['code'], 103)

class SlowRequest:
    """
    L{SlowRequest} is a fake implementation of L{Request} which is easily
    controlled externally (for example, by code in a test method).

@ivar stopped: A flag indicating whether C{stopWriting} has been called.

@ivar finished: After C{writeTo} is called, a L{Deferred} which was
        returned by that method.  L{SlowRequest} will never fire this
        L{Deferred}.
    """
    method = b'GET'
    stopped = False
    persistent = False

def writeTo(self, transport):
        self.finished = Deferred()
        return self.finished

def stopWriting(self):
        self.stopped = True

class SimpleRequest:
    """
    L{SimpleRequest} is a fake implementation of L{Request} which writes a
    short, fixed string to the transport passed to its C{writeTo} method and
    returns a succeeded L{Deferred}.  This vaguely emulates the behavior of a
    L{Request} with no body producer.
    """
    persistent = False

def writeTo(self, transport):
        transport.write(b'SOME BYTES')
        return succeed(None)

class HTTP11ClientProtocolTests(TestCase):
    """
    Tests for the HTTP 1.1 client protocol implementation,
    L{HTTP11ClientProtocol}.
    """
    def setUp(self):
        """
        Create an L{HTTP11ClientProtocol} connected to a fake transport.
        """
        self.transport = StringTransport()
        self.protocol = HTTP11ClientProtocol()
        self.protocol.makeConnection(self.transport)

def test_request(self):
        """
        L{HTTP11ClientProtocol.request} accepts a L{Request} and calls its
        C{writeTo} method with its own transport.
        """
        self.protocol.request(SimpleRequest())
        self.assertEqual(self.transport.value(), b'SOME BYTES')

def test_secondRequest(self):
        """
        The second time L{HTTP11ClientProtocol.request} is called, it returns a
        L{Deferred} which immediately fires with a L{Failure} wrapping a
        L{RequestNotSent} exception.
        """
        self.protocol.request(SlowRequest())
        def cbNotSent(ignored):
            self.assertEqual(self.transport.value(), b'')
        d = self.assertFailure(
            self.protocol.request(SimpleRequest()), RequestNotSent)
        d.addCallback(cbNotSent)
        return d

def test_requestAfterConnectionLost(self):
        """
        L{HTTP11ClientProtocol.request} returns a L{Deferred} which immediately
        fires with a L{Failure} wrapping a L{RequestNotSent} if called after
        the protocol has been disconnected.
        """
        self.protocol.connectionLost(
            Failure(ConnectionDone(u"sad transport")))
        def cbNotSent(ignored):
            self.assertEqual(self.transport.value(), b'')
        d = self.assertFailure(
            self.protocol.request(SimpleRequest()), RequestNotSent)
        d.addCallback(cbNotSent)
        return d

def test_failedWriteTo(self):
        """
        If the L{Deferred} returned by L{Request.writeTo} fires with a
        L{Failure}, L{HTTP11ClientProtocol.request} disconnects its transport
        and returns a L{Deferred} which fires with a L{Failure} of
        L{RequestGenerationFailed} wrapping the underlying failure.
        """
        class BrokenRequest:
            persistent = False
            def writeTo(self, transport):
                return fail(ArbitraryException())

d = self.protocol.request(BrokenRequest())
        def cbFailed(ignored):
            self.assertTrue(self.transport.disconnecting)
            # Simulate what would happen if the protocol had a real transport
            # and make sure no exception is raised.
            self.protocol.connectionLost(
                Failure(ConnectionDone(u"you asked for it")))
        d = assertRequestGenerationFailed(self, d, [ArbitraryException])
        d.addCallback(cbFailed)
        return d

def test_synchronousWriteToError(self):
        """
        If L{Request.writeTo} raises an exception,
        L{HTTP11ClientProtocol.request} returns a L{Deferred} which fires with
        a L{Failure} of L{RequestGenerationFailed} wrapping that exception.
        """
        class BrokenRequest:
            persistent = False
            def writeTo(self, transport):
                raise ArbitraryException()

d = self.protocol.request(BrokenRequest())
        return assertRequestGenerationFailed(self, d, [ArbitraryException])

def test_connectionLostDuringRequestGeneration(self, mode=None):
        """
        If L{HTTP11ClientProtocol}'s transport is disconnected before the
        L{Deferred} returned by L{Request.writeTo} fires, the L{Deferred}
        returned by L{HTTP11ClientProtocol.request} fires with a L{Failure} of
        L{RequestTransmissionFailed} wrapping the underlying failure.
        """
        request = SlowRequest()
        d = self.protocol.request(request)
        d = assertRequestTransmissionFailed(self, d, [ArbitraryException])

# The connection hasn't been lost yet.  The request should still be
        # allowed to do its thing.
        self.assertFalse(request.stopped)

self.protocol.connectionLost(Failure(ArbitraryException()))

# Now the connection has been lost.  The request should have been told
        # to stop writing itself.
        self.assertTrue(request.stopped)

if mode == 'callback':
            request.finished.callback(None)
        elif mode == 'errback':
            request.finished.errback(Failure(AnotherArbitraryException()))
            errors = self.flushLoggedErrors(AnotherArbitraryException)
            self.assertEqual(len(errors), 1)
        else:
            # Don't fire the writeTo Deferred at all.
            pass
        return d

def test_connectionLostBeforeGenerationFinished(self):
        """
        If the request passed to L{HTTP11ClientProtocol} finishes generation
        successfully after the L{HTTP11ClientProtocol}'s connection has been
        lost, nothing happens.
        """
        return self.test_connectionLostDuringRequestGeneration('callback')

def test_connectionLostBeforeGenerationFailed(self):
        """
        If the request passed to L{HTTP11ClientProtocol} finished generation
        with an error after the L{HTTP11ClientProtocol}'s connection has been
        lost, nothing happens.
        """
        return self.test_connectionLostDuringRequestGeneration('errback')

def test_errorMessageOnConnectionLostBeforeGenerationFailedDoesNotConfuse(self):
        """
        If the request passed to L{HTTP11ClientProtocol} finished generation
        with an error after the L{HTTP11ClientProtocol}'s connection has been
        lost, an error is logged that gives a non-confusing hint to user on what
        went wrong.
        """
        logObserver = EventLoggingObserver.createWithCleanup(
            self,
            globalLogPublisher
        )

def check(ignore):
            self.assertEquals(1, len(logObserver))
            event = logObserver[0]
            self.assertIn("log_failure", event)
            self.assertEqual(event["log_format"],
                             u'Error writing request, but not in valid state '
                             u'to finalize request: {state}')
            self.assertEqual(event["state"], 'CONNECTION_LOST')

return self.test_connectionLostDuringRequestGeneration(
            'errback').addCallback(check)

def test_receiveSimplestResponse(self):
        """
        When a response is delivered to L{HTTP11ClientProtocol}, the
        L{Deferred} previously returned by the C{request} method is called back
        with a L{Response} instance and the connection is closed.
        """
        d = self.protocol.request(Request(b'GET', b'/', _boringHeaders, None))
        def cbRequest(response):
            self.assertEqual(response.code, 200)
            self.assertEqual(response.headers, Headers())
            self.assertTrue(self.transport.disconnecting)
            self.assertEqual(self.protocol.state, u'QUIESCENT')
        d.addCallback(cbRequest)
        self.protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"Content-Length: 0\r\n"
            b"Connection: close\r\n"
            b"\r\n")
        return d

def test_receiveResponseHeaders(self):
        """
        The headers included in a response delivered to L{HTTP11ClientProtocol}
        are included on the L{Response} instance passed to the callback
        returned by the C{request} method.
        """
        d = self.protocol.request(Request(b'GET', b'/', _boringHeaders, None))
        def cbRequest(response):
            expected = Headers({b'x-foo': [b'bar', b'baz']})
            self.assertEqual(response.headers, expected)
        d.addCallback(cbRequest)
        self.protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"X-Foo: bar\r\n"
            b"X-Foo: baz\r\n"
            b"\r\n")
        return d

def test_receiveResponseBeforeRequestGenerationDone(self):
        """
        If response bytes are delivered to L{HTTP11ClientProtocol} before the
        L{Deferred} returned by L{Request.writeTo} fires, those response bytes
        are parsed as part of the response.

The connection is also closed, because we're in a confusing state, and
        therefore the C{quiescentCallback} isn't called.
        """
        quiescentResult = []
        transport = StringTransport()
        protocol = HTTP11ClientProtocol(quiescentResult.append)
        protocol.makeConnection(transport)

request = SlowRequest()
        d = protocol.request(request)
        protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"X-Foo: bar\r\n"
            b"Content-Length: 6\r\n"
            b"\r\n"
            b"foobar")
        def cbResponse(response):
            p = AccumulatingProtocol()
            whenFinished = p.closedDeferred = Deferred()
            response.deliverBody(p)
            self.assertEqual(
                protocol.state, u'TRANSMITTING_AFTER_RECEIVING_RESPONSE')
            self.assertTrue(transport.disconnecting)
            self.assertEqual(quiescentResult, [])
            return whenFinished.addCallback(
                lambda ign: (response, p.data))
        d.addCallback(cbResponse)
        def cbAllResponse(result):
            response, body = result
            self.assertEqual(response.version, (b'HTTP', 1, 1))
            self.assertEqual(response.code, 200)
            self.assertEqual(response.phrase, b'OK')
            self.assertEqual(response.headers, Headers({b'x-foo': [b'bar']}))
            self.assertEqual(body, b"foobar")

# Also nothing bad should happen if the request does finally
            # finish, even though it is completely irrelevant.
            request.finished.callback(None)

d.addCallback(cbAllResponse)
        return d

def test_receiveResponseHeadersTooLong(self):
        """
        The connection is closed when the server respond with a header which
        is above the maximum line.
        """
        transport = StringTransportWithDisconnection()
        protocol = HTTP11ClientProtocol()
        transport.protocol = protocol
        protocol.makeConnection(transport)

longLine = b'a' * LineReceiver.MAX_LENGTH
        d = protocol.request(Request(b'GET', b'/', _boringHeaders, None))

protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"X-Foo: " + longLine + b"\r\n"
            b"X-Ignored: ignored\r\n"
            b"\r\n"
            )

# For now, there is no signal that something went wrong, just a
        # connection which is closed in what looks like a clean way.
        # L{LineReceiver.lineLengthExceeded} just calls loseConnection
        # without giving any reason.
        return assertResponseFailed(self, d, [ConnectionDone])

def test_connectionLostAfterReceivingResponseBeforeRequestGenerationDone(self):
        """
        If response bytes are delivered to L{HTTP11ClientProtocol} before the
        request completes, calling C{connectionLost} on the protocol will
        result in protocol being moved to C{'CONNECTION_LOST'} state.
        """
        request = SlowRequest()
        d = self.protocol.request(request)
        self.protocol.dataReceived(
            b"HTTP/1.1 400 BAD REQUEST\r\n"
            b"Content-Length: 9\r\n"
            b"\r\n"
            b"tisk tisk")
        def cbResponse(response):
            p = AccumulatingProtocol()
            whenFinished = p.closedDeferred = Deferred()
            response.deliverBody(p)
            return whenFinished.addCallback(
                lambda ign: (response, p.data))
        d.addCallback(cbResponse)
        def cbAllResponse(ignore):
            request.finished.callback(None)
            # Nothing dire will happen when the connection is lost
            self.protocol.connectionLost(Failure(ArbitraryException()))
            self.assertEqual(self.protocol._state, u'CONNECTION_LOST')
        d.addCallback(cbAllResponse)
        return d

def test_receiveResponseBody(self):
        """
        The C{deliverBody} method of the response object with which the
        L{Deferred} returned by L{HTTP11ClientProtocol.request} fires can be
        used to get the body of the response.
        """
        protocol = AccumulatingProtocol()
        whenFinished = protocol.closedDeferred = Deferred()
        requestDeferred = self.protocol.request(Request(b'GET', b'/', _boringHeaders, None))

self.protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"Content-Length: 6\r\n"
            b"\r")

# Here's what's going on: all the response headers have been delivered
        # by this point, so the request Deferred can fire with a Response
        # object.  The body is yet to come, but that's okay, because the
        # Response object is how you *get* the body.
        result = []
        requestDeferred.addCallback(result.append)

self.assertEqual(result, [])
        # Deliver the very last byte of the response.  It is exactly at this
        # point which the Deferred returned by request should fire.
        self.protocol.dataReceived(b"\n")
        response = result[0]

response.deliverBody(protocol)

self.protocol.dataReceived(b"foo")
        self.protocol.dataReceived(b"bar")

def cbAllResponse(ignored):
            self.assertEqual(protocol.data, b"foobar")
            protocol.closedReason.trap(ResponseDone)
        whenFinished.addCallback(cbAllResponse)
        return whenFinished

def test_responseBodyFinishedWhenConnectionLostWhenContentLengthIsUnknown(
        self):
        """
        If the length of the response body is unknown, the protocol passed to
        the response's C{deliverBody} method has its C{connectionLost}
        method called with a L{Failure} wrapping a L{PotentialDataLoss}
        exception.
        """
        requestDeferred = self.protocol.request(Request(b'GET', b'/',
            _boringHeaders, None))
        self.protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"\r\n")

result = []
        requestDeferred.addCallback(result.append)
        response = result[0]

protocol = AccumulatingProtocol()
        response.deliverBody(protocol)

self.protocol.dataReceived(b"foo")
        self.protocol.dataReceived(b"bar")

self.assertEqual(protocol.data, b"foobar")
        self.protocol.connectionLost(
            Failure(ConnectionDone(u"low-level transport disconnected")))

protocol.closedReason.trap(PotentialDataLoss)

def test_chunkedResponseBodyUnfinishedWhenConnectionLost(self):
        """
        If the final chunk has not been received when the connection is lost
        (for any reason), the protocol passed to C{deliverBody} has its
        C{connectionLost} method called with a L{Failure} wrapping the
        exception for that reason.
        """
        requestDeferred = self.protocol.request(Request(b'GET', b'/',
            _boringHeaders, None))
        self.protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"Transfer-Encoding: chunked\r\n"
            b"\r\n")

result = []
        requestDeferred.addCallback(result.append)
        response = result[0]

protocol = AccumulatingProtocol()
        response.deliverBody(protocol)

self.protocol.dataReceived(b"3\r\nfoo\r\n")
        self.protocol.dataReceived(b"3\r\nbar\r\n")

self.assertEqual(protocol.data, b"foobar")

self.protocol.connectionLost(Failure(ArbitraryException()))

return assertResponseFailed(
            self, fail(protocol.closedReason), [ArbitraryException, _DataLoss])

def test_parserDataReceivedException(self):
        """
        If the parser L{HTTP11ClientProtocol} delivers bytes to in
        C{dataReceived} raises an exception, the exception is wrapped in a
        L{Failure} and passed to the parser's C{connectionLost} and then the
        L{HTTP11ClientProtocol}'s transport is disconnected.
        """
        requestDeferred = self.protocol.request(Request(b'GET', b'/',
            _boringHeaders, None))
        self.protocol.dataReceived(b'unparseable garbage goes here\r\n')
        d = assertResponseFailed(self, requestDeferred, [ParseError])
        def cbFailed(exc):
            self.assertTrue(self.transport.disconnecting)
            self.assertEqual(
                exc.reasons[0].value.data, b'unparseable garbage goes here')

# Now do what StringTransport doesn't do but a real transport would
            # have, call connectionLost on the HTTP11ClientProtocol.  Nothing
            # is asserted about this, but it's important for it to not raise an
            # exception.
            self.protocol.connectionLost(Failure(ConnectionDone(u"it is done")))

d.addCallback(cbFailed)
        return d

def test_proxyStopped(self):
        """
        When the HTTP response parser is disconnected, the
        L{TransportProxyProducer} which was connected to it as a transport is
        stopped.
        """
        requestDeferred = self.protocol.request(Request(b'GET', b'/',
            _boringHeaders, None))
        transport = self.protocol._parser.transport
        self.assertIdentical(transport._producer, self.transport)
        self.protocol._disconnectParser(
            Failure(ConnectionDone(u"connection done")))
        self.assertIdentical(transport._producer, None)
        return assertResponseFailed(self, requestDeferred, [ConnectionDone])

def test_abortClosesConnection(self):
        """
        L{HTTP11ClientProtocol.abort} will tell the transport to close its
        connection when it is invoked, and returns a C{Deferred} that fires
        when the connection is lost.
        """
        transport = StringTransport()
        protocol = HTTP11ClientProtocol()
        protocol.makeConnection(transport)
        r1 = []
        r2 = []
        protocol.abort().addCallback(r1.append)
        protocol.abort().addCallback(r2.append)
        self.assertEqual((r1, r2), ([], []))
        self.assertTrue(transport.disconnecting)

# Disconnect protocol, the Deferreds will fire:
        protocol.connectionLost(Failure(ConnectionDone()))
        self.assertEqual(r1, [None])
        self.assertEqual(r2, [None])

def test_abortAfterConnectionLost(self):
        """
        L{HTTP11ClientProtocol.abort} called after the connection is lost
        returns a C{Deferred} that fires immediately.
        """
        transport = StringTransport()
        protocol = HTTP11ClientProtocol()
        protocol.makeConnection(transport)
        protocol.connectionLost(Failure(ConnectionDone()))

result = []
        protocol.abort().addCallback(result.append)
        self.assertEqual(result, [None])
        self.assertEqual(protocol._state, u"CONNECTION_LOST")

def test_abortBeforeResponseBody(self):
        """
        The Deferred returned by L{HTTP11ClientProtocol.request} will fire
        with a L{ResponseFailed} failure containing a L{ConnectionAborted}
        exception, if the connection was aborted before all response headers
        have been received.
        """
        transport = StringTransport()
        protocol = HTTP11ClientProtocol()
        protocol.makeConnection(transport)
        result = protocol.request(Request(b'GET', b'/', _boringHeaders, None))
        protocol.abort()
        self.assertTrue(transport.disconnecting)
        protocol.connectionLost(Failure(ConnectionDone()))
        return assertResponseFailed(self, result, [ConnectionAborted])

def test_abortAfterResponseHeaders(self):
        """
        When the connection is aborted after the response headers have
        been received and the L{Response} has been made available to
        application code, the response body protocol's C{connectionLost}
        method will be invoked with a L{ResponseFailed} failure containing a
        L{ConnectionAborted} exception.
        """
        # We need to set StringTransport to lenient mode because we'll call
        # resumeProducing on it after the connection is aborted. That's ok:
        # for real transports nothing will happen.
        transport = StringTransport(lenient=True)
        protocol = HTTP11ClientProtocol()
        protocol.makeConnection(transport)
        result = protocol.request(Request(b'GET', b'/', _boringHeaders, None))

protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"Content-Length: 1\r\n"
            b"\r\n"
            )

testResult = Deferred()

class BodyDestination(Protocol):
            """
            A body response protocol which immediately aborts the HTTP
            connection.
            """
            def connectionMade(self):
                """
                Abort the HTTP connection.
                """
                protocol.abort()

def connectionLost(self, reason):
                """
                Make the reason for the losing of the connection available to
                the unit test via C{testResult}.
                """
                testResult.errback(reason)

def deliverBody(response):
            """
            Connect the L{BodyDestination} response body protocol to the
            response, and then simulate connection loss after ensuring that
            the HTTP connection has been aborted.
            """
            response.deliverBody(BodyDestination())
            self.assertTrue(transport.disconnecting)
            protocol.connectionLost(Failure(ConnectionDone()))

def checkError(error):
            self.assertIsInstance(error.response, Response)

result.addCallback(deliverBody)
        deferred = assertResponseFailed(self, testResult,
                                        [ConnectionAborted, _DataLoss])
        return deferred.addCallback(checkError)

def test_quiescentCallbackCalled(self):
        """
        If after a response is done the {HTTP11ClientProtocol} stays open and
        returns to QUIESCENT state, all per-request state is reset and the
        C{quiescentCallback} is called with the protocol instance.

This is useful for implementing a persistent connection pool.

The C{quiescentCallback} is called *before* the response-receiving
        protocol's C{connectionLost}, so that new requests triggered by end of
        first request can re-use a persistent connection.
        """
        quiescentResult = []
        def callback(p):
            self.assertEqual(p, protocol)
            self.assertEqual(p.state, u"QUIESCENT")
            quiescentResult.append(p)

transport = StringTransport()
        protocol = HTTP11ClientProtocol(callback)
        protocol.makeConnection(transport)

requestDeferred = protocol.request(
            Request(b'GET', b'/', _boringHeaders, None, persistent=True))
        protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"Content-length: 3\r\n"
            b"\r\n")

# Headers done, but still no quiescent callback:
        self.assertEqual(quiescentResult, [])

result = []
        requestDeferred.addCallback(result.append)
        response = result[0]

# When response body is done (i.e. connectionLost is called), note the
        # fact in quiescentResult:
        bodyProtocol = AccumulatingProtocol()
        bodyProtocol.closedDeferred = Deferred()
        bodyProtocol.closedDeferred.addCallback(
            lambda ign: quiescentResult.append(u"response done"))

response.deliverBody(bodyProtocol)
        protocol.dataReceived(b"abc")
        bodyProtocol.closedReason.trap(ResponseDone)
        # Quiescent callback called *before* protocol handling the response
        # body gets its connectionLost called:
        self.assertEqual(quiescentResult, [protocol, u"response done"])

# Make sure everything was cleaned up:
        self.assertEqual(protocol._parser, None)
        self.assertEqual(protocol._finishedRequest, None)
        self.assertEqual(protocol._currentRequest, None)
        self.assertEqual(protocol._transportProxy, None)
        self.assertEqual(protocol._responseDeferred, None)

def test_transportProducingWhenQuiescentAfterFullBody(self):
        """
        The C{quiescentCallback} passed to L{HTTP11ClientProtocol} will only be
        invoked once that protocol is in a state similar to its initial state.
        One of the aspects of this initial state is the producer-state of its
        transport; an L{HTTP11ClientProtocol} begins with a transport that is
        producing, i.e. not C{pauseProducing}'d.

Therefore, when C{quiescentCallback} is invoked the protocol will still
        be producing.
        """
        quiescentResult = []
        def callback(p):
            self.assertEqual(p, protocol)
            self.assertEqual(p.state, u"QUIESCENT")
            quiescentResult.append(p)

transport = StringTransport()
        protocol = HTTP11ClientProtocol(callback)
        protocol.makeConnection(transport)
        requestDeferred = protocol.request(
            Request(b'GET', b'/', _boringHeaders, None, persistent=True))
        protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"Content-length: 3\r\n"
            b"\r\n"
            b"BBB" # _full_ content of the response.
        )

response = self.successResultOf(requestDeferred)
        # Sanity check: response should have full response body, just waiting
        # for deliverBody
        self.assertEqual(response._state, u'DEFERRED_CLOSE')

# The transport is quiescent, because the response has been received.
        # If we were connection pooling here, it would have been returned to
        # the pool.
        self.assertEqual(len(quiescentResult), 1)

# And that transport is totally still reading, right? Because it would
        # leak forever if it were sitting there disconnected from the
        # reactor...
        self.assertEqual(transport.producerState, u'producing')

def test_quiescentCallbackCalledEmptyResponse(self):
        """
        The quiescentCallback is called before the request C{Deferred} fires,
        in cases where the response has no body.
        """
        quiescentResult = []
        def callback(p):
            self.assertEqual(p, protocol)
            self.assertEqual(p.state, u"QUIESCENT")
            quiescentResult.append(p)

transport = StringTransport()
        protocol = HTTP11ClientProtocol(callback)
        protocol.makeConnection(transport)

requestDeferred = protocol.request(
            Request(b'GET', b'/', _boringHeaders, None, persistent=True))
        requestDeferred.addCallback(quiescentResult.append)
        protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"Content-length: 0\r\n"
            b"\r\n")

self.assertEqual(len(quiescentResult), 2)
        self.assertIdentical(quiescentResult[0], protocol)
        self.assertIsInstance(quiescentResult[1], Response)

def test_quiescentCallbackNotCalled(self):
        """
        If after a response is done the {HTTP11ClientProtocol} returns a
        C{Connection: close} header in the response, the C{quiescentCallback}
        is not called and the connection is lost.
        """
        quiescentResult = []
        transport = StringTransport()
        protocol = HTTP11ClientProtocol(quiescentResult.append)
        protocol.makeConnection(transport)

requestDeferred = protocol.request(
            Request(b'GET', b'/', _boringHeaders, None, persistent=True))
        protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"Content-length: 0\r\n"
            b"Connection: close\r\n"
            b"\r\n")

result = []
        requestDeferred.addCallback(result.append)
        response = result[0]

bodyProtocol = AccumulatingProtocol()
        response.deliverBody(bodyProtocol)
        bodyProtocol.closedReason.trap(ResponseDone)
        self.assertEqual(quiescentResult, [])
        self.assertTrue(transport.disconnecting)

def test_quiescentCallbackNotCalledNonPersistentQuery(self):
        """
        If the request was non-persistent (i.e. sent C{Connection: close}),
        the C{quiescentCallback} is not called and the connection is lost.
        """
        quiescentResult = []
        transport = StringTransport()
        protocol = HTTP11ClientProtocol(quiescentResult.append)
        protocol.makeConnection(transport)

requestDeferred = protocol.request(
            Request(b'GET', b'/', _boringHeaders, None, persistent=False))
        protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"Content-length: 0\r\n"
            b"\r\n")

result = []
        requestDeferred.addCallback(result.append)
        response = result[0]

def test_quiescentCallbackThrows(self):
        """
        If C{quiescentCallback} throws an exception, the error is logged and
        protocol is disconnected.
        """
        def callback(p):
            raise ZeroDivisionError()

logObserver = EventLoggingObserver.createWithCleanup(
            self,
            globalLogPublisher
        )

transport = StringTransport()
        protocol = HTTP11ClientProtocol(callback)
        protocol.makeConnection(transport)

result = []
        requestDeferred.addCallback(result.append)
        response = result[0]
        bodyProtocol = AccumulatingProtocol()
        response.deliverBody(bodyProtocol)
        bodyProtocol.closedReason.trap(ResponseDone)

self.assertEquals(1, len(logObserver))
        event = logObserver[0]
        f = event["log_failure"]
        self.assertIsInstance(f.value, ZeroDivisionError)

self.flushLoggedErrors(ZeroDivisionError)
        self.assertTrue(transport.disconnecting)

def test_cancelBeforeResponse(self):
        """
        The L{Deferred} returned by L{HTTP11ClientProtocol.request} will fire
        with a L{ResponseNeverReceived} failure containing a L{CancelledError}
        exception if the request was cancelled before any response headers were
        received.
        """
        transport = StringTransport()
        protocol = HTTP11ClientProtocol()
        protocol.makeConnection(transport)
        result = protocol.request(Request(b'GET', b'/', _boringHeaders, None))
        result.cancel()
        self.assertTrue(transport.disconnected)
        return assertWrapperExceptionTypes(
            self, result, ResponseNeverReceived, [CancelledError])

def test_cancelDuringResponse(self):
        """
        The L{Deferred} returned by L{HTTP11ClientProtocol.request} will fire
        with a L{ResponseFailed} failure containing a L{CancelledError}
        exception if the request was cancelled before all response headers were
        received.
        """
        transport = StringTransport()
        protocol = HTTP11ClientProtocol()
        protocol.makeConnection(transport)
        result = protocol.request(Request(b'GET', b'/', _boringHeaders, None))
        protocol.dataReceived(b"HTTP/1.1 200 OK\r\n")
        result.cancel()
        self.assertTrue(transport.disconnected)
        return assertResponseFailed(self, result, [CancelledError])

def assertCancelDuringBodyProduction(self, producerLength):
        """
        The L{Deferred} returned by L{HTTP11ClientProtocol.request} will fire
        with a L{RequestGenerationFailed} failure containing a
        L{CancelledError} exception if the request was cancelled before a
        C{bodyProducer} has finished producing.
        """
        transport = StringTransport()
        protocol = HTTP11ClientProtocol()
        protocol.makeConnection(transport)
        producer = StringProducer(producerLength)

nonLocal = {'cancelled': False}
        def cancel(ign):
            nonLocal['cancelled'] = True
        def startProducing(consumer):
            producer.consumer = consumer
            producer.finished = Deferred(cancel)
            return producer.finished
        producer.startProducing = startProducing

result = protocol.request(Request(b'POST', b'/bar', _boringHeaders,
                                          producer))
        producer.consumer.write(b'x' * 5)
        result.cancel()
        self.assertTrue(transport.disconnected)
        self.assertTrue(nonLocal['cancelled'])
        return assertRequestGenerationFailed(self, result, [CancelledError])

def test_cancelDuringBodyProduction(self):
        """
        The L{Deferred} returned by L{HTTP11ClientProtocol.request} will fire
        with a L{RequestGenerationFailed} failure containing a
        L{CancelledError} exception if the request was cancelled before a
        C{bodyProducer} with an explicit length has finished producing.
        """
        return self.assertCancelDuringBodyProduction(10)

def test_cancelDuringChunkedBodyProduction(self):
        """
        The L{Deferred} returned by L{HTTP11ClientProtocol.request} will fire
        with a L{RequestGenerationFailed} failure containing a
        L{CancelledError} exception if the request was cancelled before a
        C{bodyProducer} with C{UNKNOWN_LENGTH} has finished producing.
        """
        return self.assertCancelDuringBodyProduction(UNKNOWN_LENGTH)

@implementer(IBodyProducer)
class StringProducer:
    """
    L{StringProducer} is a dummy body producer.

@ivar stopped: A flag which indicates whether or not C{stopProducing} has
        been called.
    @ivar consumer: After C{startProducing} is called, the value of the
        C{consumer} argument to that method.
    @ivar finished: After C{startProducing} is called, a L{Deferred} which was
        returned by that method.  L{StringProducer} will never fire this
        L{Deferred}.
    """

stopped = False

def __init__(self, length):
        self.length = length

def startProducing(self, consumer):
        self.consumer = consumer
        self.finished = Deferred()
        return self.finished

def stopProducing(self):
        self.stopped = True

class RequestTests(TestCase):
    """
    Tests for L{Request}.
    """
    def setUp(self):
        self.transport = StringTransport()

def test_sendSimplestRequest(self):
        """
        L{Request.writeTo} formats the request data and writes it to the given
        transport.
        """
        Request(b'GET', b'/', _boringHeaders, None).writeTo(self.transport)
        self.assertEqual(
            self.transport.value(),
            b"GET / HTTP/1.1\r\n"
            b"Connection: close\r\n"
            b"Host: example.com\r\n"
            b"\r\n")

def test_sendSimplestPersistentRequest(self):
        """
        A pesistent request does not send 'Connection: close' header.
        """
        req = Request(b'GET', b'/', _boringHeaders, None, persistent=True)
        req.writeTo(self.transport)
        self.assertEqual(
            self.transport.value(),
            b"GET / HTTP/1.1\r\n"
            b"Host: example.com\r\n"
            b"\r\n")

def test_sendRequestHeaders(self):
        """
        L{Request.writeTo} formats header data and writes it to the given
        transport.
        """
        headers = Headers({b'x-foo': [b'bar', b'baz'],
                           b'host': [b'example.com']})
        Request(b'GET', b'/foo', headers, None).writeTo(self.transport)
        lines = self.transport.value().split(b'\r\n')
        self.assertEqual(lines[0], b"GET /foo HTTP/1.1")
        self.assertEqual(lines[-2:], [b"", b""])
        del lines[0], lines[-2:]
        lines.sort()
        self.assertEqual(
            lines,
            [b"Connection: close",
             b"Host: example.com",
             b"X-Foo: bar",
             b"X-Foo: baz"])

def test_sanitizeLinearWhitespaceInRequestHeaders(self):
        """
        Linear whitespace in request headers is replaced with a single
        space.
        """
        for component in bytesLinearWhitespaceComponents:
            headers = Headers({component: [component],
                               b"host": [b"example.invalid"]})
            transport = StringTransport()
            Request(b'GET', b'/foo', headers, None).writeTo(transport)
            lines = transport.value().split(b'\r\n')
            self.assertEqual(lines[0], b"GET /foo HTTP/1.1")
            self.assertEqual(lines[-2:], [b"", b""])
            del lines[0], lines[-2:]
            lines.remove(b"Connection: close")
            lines.remove(b"Host: example.invalid")
            sanitizedHeaderLine = b": ".join([sanitizedBytes, sanitizedBytes])
            self.assertEqual(lines, [sanitizedHeaderLine])

def test_sendChunkedRequestBody(self):
        """
        L{Request.writeTo} uses chunked encoding to write data from the request
        body producer to the given transport.  It registers the request body
        producer with the transport.
        """
        producer = StringProducer(UNKNOWN_LENGTH)
        request = Request(b'POST', b'/bar', _boringHeaders, producer)
        request.writeTo(self.transport)

self.assertNotIdentical(producer.consumer, None)
        self.assertIdentical(self.transport.producer, producer)
        self.assertTrue(self.transport.streaming)

self.assertEqual(
            self.transport.value(),
            b"POST /bar HTTP/1.1\r\n"
            b"Connection: close\r\n"
            b"Transfer-Encoding: chunked\r\n"
            b"Host: example.com\r\n"
            b"\r\n")
        self.transport.clear()

producer.consumer.write(b'x' * 3)
        producer.consumer.write(b'y' * 15)
        producer.finished.callback(None)
        self.assertIdentical(self.transport.producer, None)
        self.assertEqual(
            self.transport.value(),
            b"3\r\n"
            b"xxx\r\n"
            b"f\r\n"
            b"yyyyyyyyyyyyyyy\r\n"
            b"0\r\n"
            b"\r\n")

def test_sendChunkedRequestBodyWithError(self):
        """
        If L{Request} is created with a C{bodyProducer} without a known length
        and the L{Deferred} returned from its C{startProducing} method fires
        with a L{Failure}, the L{Deferred} returned by L{Request.writeTo} fires
        with that L{Failure} and the body producer is unregistered from the
        transport.  The final zero-length chunk is not written to the
        transport.
        """
        producer = StringProducer(UNKNOWN_LENGTH)
        request = Request(b'POST', b'/bar', _boringHeaders, producer)
        writeDeferred = request.writeTo(self.transport)
        self.transport.clear()
        producer.finished.errback(ArbitraryException())
        def cbFailed(ignored):
            self.assertEqual(self.transport.value(), b"")
            self.assertIdentical(self.transport.producer, None)
        d = self.assertFailure(writeDeferred, ArbitraryException)
        d.addCallback(cbFailed)
        return d

def test_sendRequestBodyWithLength(self):
        """
        If L{Request} is created with a C{bodyProducer} with a known length,
        that length is sent as the value for the I{Content-Length} header and
        chunked encoding is not used.
        """
        producer = StringProducer(3)
        request = Request(b'POST', b'/bar', _boringHeaders, producer)
        request.writeTo(self.transport)

self.assertNotIdentical(producer.consumer, None)
        self.assertIdentical(self.transport.producer, producer)
        self.assertTrue(self.transport.streaming)

self.assertEqual(
            self.transport.value(),
            b"POST /bar HTTP/1.1\r\n"
            b"Connection: close\r\n"
            b"Content-Length: 3\r\n"
            b"Host: example.com\r\n"
            b"\r\n")
        self.transport.clear()

producer.consumer.write(b'abc')
        producer.finished.callback(None)
        self.assertIdentical(self.transport.producer, None)
        self.assertEqual(self.transport.value(), b"abc")

def _sendRequestEmptyBodyWithLength(self, method):
        """
        Verify that the message generated by a L{Request} initialized with
        the given method and C{None} as the C{bodyProducer} includes
        I{Content-Length: 0} in the header.

@param method: The HTTP method issue in the request.
        @type method: L{bytes}
        """
        request = Request(method, b"/foo", _boringHeaders, None)
        request.writeTo(self.transport)

self.assertEqual(
            self.transport.value(),
            method + b" /foo HTTP/1.1\r\n"
            b"Connection: close\r\n"
            b"Content-Length: 0\r\n"
            b"Host: example.com\r\n"
            b"\r\n")

def test_sendPUTRequestEmptyBody(self):
        """
        If I{PUT} L{Request} is created without a C{bodyProducer},
        I{Content-Length: 0} is included in the header and chunked
        encoding is not used.
        """
        self._sendRequestEmptyBodyWithLength(b"PUT")

def test_sendPOSTRequestEmptyBody(self):
        """
        If I{POST} L{Request} is created without a C{bodyProducer},
        I{Content-Length: 0} is included in the header and chunked
        encoding is not used.
        """
        self._sendRequestEmptyBodyWithLength(b"POST")

def test_sendRequestBodyWithTooFewBytes(self):
        """
        If L{Request} is created with a C{bodyProducer} with a known length and
        the producer does not produce that many bytes, the L{Deferred} returned
        by L{Request.writeTo} fires with a L{Failure} wrapping a
        L{WrongBodyLength} exception.
        """
        producer = StringProducer(3)
        request = Request(b'POST', b'/bar', _boringHeaders, producer)
        writeDeferred = request.writeTo(self.transport)
        producer.consumer.write(b'ab')
        producer.finished.callback(None)
        self.assertIdentical(self.transport.producer, None)
        return self.assertFailure(writeDeferred, WrongBodyLength)

def _sendRequestBodyWithTooManyBytesTest(self, finisher):
        """
        Verify that when too many bytes have been written by a body producer
        and then the body producer's C{startProducing} L{Deferred} fires that
        the producer is unregistered from the transport and that the
        L{Deferred} returned from L{Request.writeTo} is fired with a L{Failure}
        wrapping a L{WrongBodyLength}.

@param finisher: A callable which will be invoked with the body
            producer after too many bytes have been written to the transport.
            It should fire the startProducing Deferred somehow.
        """
        producer = StringProducer(3)
        request = Request(b'POST', b'/bar', _boringHeaders, producer)
        writeDeferred = request.writeTo(self.transport)

producer.consumer.write(b'ab')

# The producer hasn't misbehaved yet, so it shouldn't have been
        # stopped.
        self.assertFalse(producer.stopped)

producer.consumer.write(b'cd')

# Now the producer *has* misbehaved, so we should have tried to
        # make it stop.
        self.assertTrue(producer.stopped)

# The transport should have had the producer unregistered from it as
        # well.
        self.assertIdentical(self.transport.producer, None)

def cbFailed(exc):
            # The "cd" should not have been written to the transport because
            # the request can now locally be recognized to be invalid.  If we
            # had written the extra bytes, the server could have decided to
            # start processing the request, which would be bad since we're
            # going to indicate failure locally.
            self.assertEqual(
                self.transport.value(),
                b"POST /bar HTTP/1.1\r\n"
                b"Connection: close\r\n"
                b"Content-Length: 3\r\n"
                b"Host: example.com\r\n"
                b"\r\n"
                b"ab")
            self.transport.clear()

# Subsequent writes should be ignored, as should firing the
            # Deferred returned from startProducing.
            self.assertRaises(ExcessWrite, producer.consumer.write, b'ef')

# Likewise, if the Deferred returned from startProducing fires,
            # this should more or less be ignored (aside from possibly logging
            # an error).
            finisher(producer)

# There should have been nothing further written to the transport.
            self.assertEqual(self.transport.value(), b"")

d = self.assertFailure(writeDeferred, WrongBodyLength)
        d.addCallback(cbFailed)
        return d

def test_sendRequestBodyWithTooManyBytes(self):
        """
        If L{Request} is created with a C{bodyProducer} with a known length and
        the producer tries to produce more than than many bytes, the
        L{Deferred} returned by L{Request.writeTo} fires with a L{Failure}
        wrapping a L{WrongBodyLength} exception.
        """
        def finisher(producer):
            producer.finished.callback(None)
        return self._sendRequestBodyWithTooManyBytesTest(finisher)

def test_sendRequestBodyErrorWithTooManyBytes(self):
        """
        If L{Request} is created with a C{bodyProducer} with a known length and
        the producer tries to produce more than than many bytes, the
        L{Deferred} returned by L{Request.writeTo} fires with a L{Failure}
        wrapping a L{WrongBodyLength} exception.
        """
        logObserver = EventLoggingObserver.createWithCleanup(
            self,
            globalLogPublisher
        )

def finisher(producer):
            producer.finished.errback(ArbitraryException())
            event = logObserver[0]
            self.assertIn("log_failure", event)
            f = event["log_failure"]
            self.assertIsInstance(f.value, ArbitraryException)
            errors = self.flushLoggedErrors(ArbitraryException)
            self.assertEqual(len(errors), 1)
        return self._sendRequestBodyWithTooManyBytesTest(finisher)

def test_sendRequestBodyErrorWithConsumerError(self):
        """
        Though there should be no way for the internal C{finishedConsuming}
        L{Deferred} in L{Request._writeToBodyProducerContentLength} to fire a
        L{Failure} after the C{finishedProducing} L{Deferred} has fired, in
        case this does happen, the error should be logged with a message about
        how there's probably a bug in L{Request}.

This is a whitebox test.
        """
        logObserver = EventLoggingObserver.createWithCleanup(
            self,
            globalLogPublisher
        )
        producer = StringProducer(3)
        request = Request(b'POST', b'/bar', _boringHeaders, producer)
        request.writeTo(self.transport)

finishedConsuming = producer.consumer._finished

producer.consumer.write(b'abc')
        producer.finished.callback(None)

finishedConsuming.errback(ArbitraryException())
        event = logObserver[0]
        self.assertIn("log_failure", event)
        f = event["log_failure"]
        self.assertIsInstance(f.value, ArbitraryException)
        self.assertEqual(len(self.flushLoggedErrors(ArbitraryException)), 1)

def _sendRequestBodyFinishedEarlyThenTooManyBytes(self, finisher):
        """
        Verify that if the body producer fires its Deferred and then keeps
        writing to the consumer that the extra writes are ignored and the
        L{Deferred} returned by L{Request.writeTo} fires with a L{Failure}
        wrapping the most appropriate exception type.
        """
        producer = StringProducer(3)
        request = Request(b'POST', b'/bar', _boringHeaders, producer)
        writeDeferred = request.writeTo(self.transport)

producer.consumer.write(b'ab')
        finisher(producer)
        self.assertIdentical(self.transport.producer, None)
        self.transport.clear()
        self.assertRaises(ExcessWrite, producer.consumer.write, b'cd')
        self.assertEqual(self.transport.value(), b"")
        return writeDeferred

def test_sendRequestBodyFinishedEarlyThenTooManyBytes(self):
        """
        If the request body producer indicates it is done by firing the
        L{Deferred} returned from its C{startProducing} method but then goes on
        to write too many bytes, the L{Deferred} returned by {Request.writeTo}
        fires with a L{Failure} wrapping L{WrongBodyLength}.
        """
        def finisher(producer):
            producer.finished.callback(None)
        return self.assertFailure(
            self._sendRequestBodyFinishedEarlyThenTooManyBytes(finisher),
            WrongBodyLength)

def test_sendRequestBodyErroredEarlyThenTooManyBytes(self):
        """
        If the request body producer indicates an error by firing the
        L{Deferred} returned from its C{startProducing} method but then goes on
        to write too many bytes, the L{Deferred} returned by {Request.writeTo}
        fires with that L{Failure} and L{WrongBodyLength} is logged.
        """
        def finisher(producer):
            producer.finished.errback(ArbitraryException())
        return self.assertFailure(
            self._sendRequestBodyFinishedEarlyThenTooManyBytes(finisher),
            ArbitraryException)

def test_sendChunkedRequestBodyFinishedThenWriteMore(self, _with=None):
        """
        If the request body producer with an unknown length tries to write
        after firing the L{Deferred} returned by its C{startProducing} method,
        the C{write} call raises an exception and does not write anything to
        the underlying transport.
        """
        producer = StringProducer(UNKNOWN_LENGTH)
        request = Request(b'POST', b'/bar', _boringHeaders, producer)
        writeDeferred = request.writeTo(self.transport)
        producer.finished.callback(_with)
        self.transport.clear()

self.assertRaises(ExcessWrite, producer.consumer.write, b'foo')
        self.assertEqual(self.transport.value(), b"")
        return writeDeferred

def test_sendChunkedRequestBodyFinishedWithErrorThenWriteMore(self):
        """
        If the request body producer with an unknown length tries to write
        after firing the L{Deferred} returned by its C{startProducing} method
        with a L{Failure}, the C{write} call raises an exception and does not
        write anything to the underlying transport.
        """
        d = self.test_sendChunkedRequestBodyFinishedThenWriteMore(
            Failure(ArbitraryException()))
        return self.assertFailure(d, ArbitraryException)

def test_sendRequestBodyWithError(self):
        """
        If the L{Deferred} returned from the C{startProducing} method of the
        L{IBodyProducer} passed to L{Request} fires with a L{Failure}, the
        L{Deferred} returned from L{Request.writeTo} fails with that
        L{Failure}.
        """
        producer = StringProducer(5)
        request = Request(b'POST', b'/bar', _boringHeaders, producer)
        writeDeferred = request.writeTo(self.transport)

# Sanity check - the producer should be registered with the underlying
        # transport.
        self.assertIdentical(self.transport.producer, producer)
        self.assertTrue(self.transport.streaming)

producer.consumer.write(b'ab')
        self.assertEqual(
            self.transport.value(),
            b"POST /bar HTTP/1.1\r\n"
            b"Connection: close\r\n"
            b"Content-Length: 5\r\n"
            b"Host: example.com\r\n"
            b"\r\n"
            b"ab")

self.assertFalse(self.transport.disconnecting)
        producer.finished.errback(Failure(ArbitraryException()))

# Disconnection is handled by a higher level.  Request should leave the
        # transport alone in this case.
        self.assertFalse(self.transport.disconnecting)

# Oh.  Except it should unregister the producer that it registered.
        self.assertIdentical(self.transport.producer, None)

return self.assertFailure(writeDeferred, ArbitraryException)

def test_hostHeaderRequired(self):
        """
        L{Request.writeTo} raises L{BadHeaders} if there is not exactly one
        I{Host} header and writes nothing to the given transport.
        """
        request = Request(b'GET', b'/', Headers({}), None)
        self.assertRaises(BadHeaders, request.writeTo, self.transport)
        self.assertEqual(self.transport.value(), b'')

request = Request(b'GET', b'/',
            Headers({b'Host': [b'example.com', b'example.org']}), None)
        self.assertRaises(BadHeaders, request.writeTo, self.transport)
        self.assertEqual(self.transport.value(), b'')

def test_stopWriting(self):
        """
        L{Request.stopWriting} calls its body producer's C{stopProducing}
        method.
        """
        producer = StringProducer(3)
        request = Request(b'GET', b'/', _boringHeaders, producer)
        request.writeTo(self.transport)
        self.assertFalse(producer.stopped)
        request.stopWriting()
        self.assertTrue(producer.stopped)

def test_brokenStopProducing(self):
        """
        If the body producer's C{stopProducing} method raises an exception,
        L{Request.stopWriting} logs it and does not re-raise it.
        """
        logObserver = EventLoggingObserver.createWithCleanup(
            self,
            globalLogPublisher
        )
        producer = StringProducer(3)
        def brokenStopProducing():
            raise ArbitraryException(u"stopProducing is busted")
        producer.stopProducing = brokenStopProducing

request = Request(b'GET', b'/', _boringHeaders, producer)
        request.writeTo(self.transport)
        request.stopWriting()
        self.assertEqual(
            len(self.flushLoggedErrors(ArbitraryException)), 1)
        self.assertEquals(1, len(logObserver))
        event = logObserver[0]
        self.assertIn("log_failure", event)
        f = event["log_failure"]
        self.assertIsInstance(f.value, ArbitraryException)

class LengthEnforcingConsumerTests(TestCase):
    """
    Tests for L{LengthEnforcingConsumer}.
    """
    def setUp(self):
        self.result = Deferred()
        self.producer = StringProducer(10)
        self.transport = StringTransport()
        self.enforcer = LengthEnforcingConsumer(
            self.producer, self.transport, self.result)

def test_write(self):
        """
        L{LengthEnforcingConsumer.write} calls the wrapped consumer's C{write}
        method with the bytes it is passed as long as there are fewer of them
        than the C{length} attribute indicates remain to be received.
        """
        self.enforcer.write(b'abc')
        self.assertEqual(self.transport.value(), b'abc')
        self.transport.clear()
        self.enforcer.write(b'def')
        self.assertEqual(self.transport.value(), b'def')

def test_finishedEarly(self):
        """
        L{LengthEnforcingConsumer._noMoreWritesExpected} raises
        L{WrongBodyLength} if it is called before the indicated number of bytes
        have been written.
        """
        self.enforcer.write(b'x' * 9)
        self.assertRaises(WrongBodyLength, self.enforcer._noMoreWritesExpected)

def test_writeTooMany(self, _unregisterAfter=False):
        """
        If it is called with a total number of bytes exceeding the indicated
        limit passed to L{LengthEnforcingConsumer.__init__},
        L{LengthEnforcingConsumer.write} fires the L{Deferred} with a
        L{Failure} wrapping a L{WrongBodyLength} and also calls the
        C{stopProducing} method of the producer.
        """
        self.enforcer.write(b'x' * 10)
        self.assertFalse(self.producer.stopped)
        self.enforcer.write(b'x')
        self.assertTrue(self.producer.stopped)
        if _unregisterAfter:
            self.enforcer._noMoreWritesExpected()
        return self.assertFailure(self.result, WrongBodyLength)

def test_writeAfterNoMoreExpected(self):
        """
        If L{LengthEnforcingConsumer.write} is called after
        L{LengthEnforcingConsumer._noMoreWritesExpected}, it calls the
        producer's C{stopProducing} method and raises L{ExcessWrite}.
        """
        self.enforcer.write(b'x' * 10)
        self.enforcer._noMoreWritesExpected()
        self.assertFalse(self.producer.stopped)
        self.assertRaises(ExcessWrite, self.enforcer.write, b'x')
        self.assertTrue(self.producer.stopped)

def test_finishedLate(self):
        """
        L{LengthEnforcingConsumer._noMoreWritesExpected} does nothing (in
        particular, it does not raise any exception) if called after too many
        bytes have been passed to C{write}.
        """
        return self.test_writeTooMany(True)

def test_finished(self):
        """
        If L{LengthEnforcingConsumer._noMoreWritesExpected} is called after
        the correct number of bytes have been written it returns L{None}.
        """
        self.enforcer.write(b'x' * 10)
        self.assertIdentical(self.enforcer._noMoreWritesExpected(), None)

def test_stopProducingRaises(self):
        """
        If L{LengthEnforcingConsumer.write} calls the producer's
        C{stopProducing} because too many bytes were written and the
        C{stopProducing} method raises an exception, the exception is logged
        and the L{LengthEnforcingConsumer} still errbacks the finished
        L{Deferred}.
        """
        def brokenStopProducing():
            StringProducer.stopProducing(self.producer)
            raise ArbitraryException(u"stopProducing is busted")
        self.producer.stopProducing = brokenStopProducing

def cbFinished(ignored):
            self.assertEqual(
                len(self.flushLoggedErrors(ArbitraryException)), 1)
        d = self.test_writeTooMany()
        d.addCallback(cbFinished)
        return d

class RequestBodyConsumerTests(TestCase):
    """
    Tests for L{ChunkedEncoder} which sits between an L{ITransport} and a
    request/response body producer and chunked encodes everything written to
    it.
    """
    def test_interface(self):
        """
        L{ChunkedEncoder} instances provide L{IConsumer}.
        """
        self.assertTrue(
            verifyObject(IConsumer, ChunkedEncoder(StringTransport())))

def test_write(self):
        """
        L{ChunkedEncoder.write} writes to the transport the chunked encoded
        form of the bytes passed to it.
        """
        transport = StringTransport()
        encoder = ChunkedEncoder(transport)
        encoder.write(b'foo')
        self.assertEqual(transport.value(), b'3\r\nfoo\r\n')
        transport.clear()
        encoder.write(b'x' * 16)
        self.assertEqual(transport.value(), b'10\r\n' + b'x' * 16 + b'\r\n')

def test_producerRegistration(self):
        """
        L{ChunkedEncoder.registerProducer} registers the given streaming
        producer with its transport and L{ChunkedEncoder.unregisterProducer}
        writes a zero-length chunk to its transport and unregisters the
        transport's producer.
        """
        transport = StringTransport()
        producer = object()
        encoder = ChunkedEncoder(transport)
        encoder.registerProducer(producer, True)
        self.assertIdentical(transport.producer, producer)
        self.assertTrue(transport.streaming)
        encoder.unregisterProducer()
        self.assertIdentical(transport.producer, None)
        self.assertEqual(transport.value(), b'0\r\n\r\n')

class TransportProxyProducerTests(TestCase):
    """
    Tests for L{TransportProxyProducer} which proxies the L{IPushProducer}
    interface of a transport.
    """
    def test_interface(self):
        """
        L{TransportProxyProducer} instances provide L{IPushProducer}.
        """
        self.assertTrue(
            verifyObject(IPushProducer, TransportProxyProducer(None)))

def test_stopProxyingUnreferencesProducer(self):
        """
        L{TransportProxyProducer.stopProxying} drops the reference to the
        wrapped L{IPushProducer} provider.
        """
        transport = StringTransport()
        proxy = TransportProxyProducer(transport)
        self.assertIdentical(proxy._producer, transport)
        proxy.stopProxying()
        self.assertIdentical(proxy._producer, None)

def test_resumeProducing(self):
        """
        L{TransportProxyProducer.resumeProducing} calls the wrapped
        transport's C{resumeProducing} method unless told to stop proxying.
        """
        transport = StringTransport()
        transport.pauseProducing()

proxy = TransportProxyProducer(transport)
        # The transport should still be paused.
        self.assertEqual(transport.producerState, u'paused')
        proxy.resumeProducing()
        # The transport should now be resumed.
        self.assertEqual(transport.producerState, u'producing')

transport.pauseProducing()
        proxy.stopProxying()

# The proxy should no longer do anything to the transport.
        proxy.resumeProducing()
        self.assertEqual(transport.producerState, u'paused')

def test_pauseProducing(self):
        """
        L{TransportProxyProducer.pauseProducing} calls the wrapped transport's
        C{pauseProducing} method unless told to stop proxying.
        """
        transport = StringTransport()

proxy = TransportProxyProducer(transport)
        # The transport should still be producing.
        self.assertEqual(transport.producerState, u'producing')
        proxy.pauseProducing()
        # The transport should now be paused.
        self.assertEqual(transport.producerState, u'paused')

transport.resumeProducing()
        proxy.stopProxying()

# The proxy should no longer do anything to the transport.
        proxy.pauseProducing()
        self.assertEqual(transport.producerState, u'producing')

def test_stopProducing(self):
        """
        L{TransportProxyProducer.stopProducing} calls the wrapped transport's
        C{stopProducing} method unless told to stop proxying.
        """
        transport = StringTransport()
        proxy = TransportProxyProducer(transport)
        # The transport should still be producing.
        self.assertEqual(transport.producerState, u'producing')
        proxy.stopProducing()
        # The transport should now be stopped.
        self.assertEqual(transport.producerState, u'stopped')

transport = StringTransport()
        proxy = TransportProxyProducer(transport)
        proxy.stopProxying()
        proxy.stopProducing()
        # The transport should not have been stopped.
        self.assertEqual(transport.producerState, u'producing')

def test_loseConnectionWhileProxying(self):
        """
        L{TransportProxyProducer.loseConnection} calls the wrapped transport's
        C{loseConnection}.
        """
        transport = StringTransportWithDisconnection()
        protocol = AccumulatingProtocol()
        protocol.makeConnection(transport)
        transport.protocol = protocol
        proxy = TransportProxyProducer(transport)
        # Transport is connected and production.
        self.assertTrue(transport.connected)
        self.assertEqual(transport.producerState, u'producing')

proxy.loseConnection()

# The transport is not explicitly stopped, but requested to
        # disconnect.
        self.assertEqual(transport.producerState, u'producing')
        self.assertFalse(transport.connected)

def test_loseConnectionNotProxying(self):
        """
        L{TransportProxyProducer.loseConnection} does nothing when the
        proxy is not active.
        """
        transport = StringTransportWithDisconnection()
        protocol = AccumulatingProtocol()
        protocol.makeConnection(transport)
        transport.protocol = protocol
        proxy = TransportProxyProducer(transport)
        proxy.stopProxying()
        self.assertTrue(transport.connected)

proxy.loseConnection()

# The transport is not touched, when not proxying.
        self.assertTrue(transport.connected)

class ResponseTests(TestCase):
    """
    Tests for L{Response}.
    """

def test_verifyInterface(self):
        """
        L{Response} instances provide L{IResponse}.
        """
        response = justTransportResponse(StringTransport())
        self.assertTrue(verifyObject(IResponse, response))

def test_makeConnection(self):
        """
        The L{IProtocol} provider passed to L{Response.deliverBody} has its
        C{makeConnection} method called with an L{IPushProducer} provider
        hooked up to the response as an argument.
        """
        producers = []
        transport = StringTransport()
        class SomeProtocol(Protocol):
            def makeConnection(self, producer):
                producers.append(producer)

consumer = SomeProtocol()
        response = justTransportResponse(transport)
        response.deliverBody(consumer)
        [theProducer] = producers
        theProducer.pauseProducing()
        self.assertEqual(transport.producerState, u'paused')
        theProducer.resumeProducing()
        self.assertEqual(transport.producerState, u'producing')

def test_dataReceived(self):
        """
        The L{IProtocol} provider passed to L{Response.deliverBody} has its
        C{dataReceived} method called with bytes received as part of the
        response body.
        """
        bytes = []
        class ListConsumer(Protocol):
            def dataReceived(self, data):
                bytes.append(data)

consumer = ListConsumer()
        response = justTransportResponse(StringTransport())
        response.deliverBody(consumer)

response._bodyDataReceived(b'foo')
        self.assertEqual(bytes, [b'foo'])

def test_connectionLost(self):
        """
        The L{IProtocol} provider passed to L{Response.deliverBody} has its
        C{connectionLost} method called with a L{Failure} wrapping
        L{ResponseDone} when the response's C{_bodyDataFinished} method is
        called.
        """
        lost = []
        class ListConsumer(Protocol):
            def connectionLost(self, reason):
                lost.append(reason)

consumer = ListConsumer()
        response = justTransportResponse(StringTransport())
        response.deliverBody(consumer)

response._bodyDataFinished()
        lost[0].trap(ResponseDone)
        self.assertEqual(len(lost), 1)

# The protocol reference should be dropped, too, to facilitate GC or
        # whatever.
        self.assertIdentical(response._bodyProtocol, None)

def test_bufferEarlyData(self):
        """
        If data is delivered to the L{Response} before a protocol is registered
        with C{deliverBody}, that data is buffered until the protocol is
        registered and then is delivered.
        """
        bytes = []
        class ListConsumer(Protocol):
            def dataReceived(self, data):
                bytes.append(data)

protocol = ListConsumer()
        response = justTransportResponse(StringTransport())
        response._bodyDataReceived(b'foo')
        response._bodyDataReceived(b'bar')
        response.deliverBody(protocol)
        response._bodyDataReceived(b'baz')
        self.assertEqual(bytes, [b'foo', b'bar', b'baz'])
        # Make sure the implementation-detail-byte-buffer is cleared because
        # not clearing it wastes memory.
        self.assertIdentical(response._bodyBuffer, None)

def test_multipleStartProducingFails(self):
        """
        L{Response.deliverBody} raises L{RuntimeError} if called more than
        once.
        """
        response = justTransportResponse(StringTransport())
        response.deliverBody(Protocol())
        self.assertRaises(RuntimeError, response.deliverBody, Protocol())

def test_startProducingAfterFinishedFails(self):
        """
        L{Response.deliverBody} raises L{RuntimeError} if called after
        L{Response._bodyDataFinished}.
        """
        response = justTransportResponse(StringTransport())
        response.deliverBody(Protocol())
        response._bodyDataFinished()
        self.assertRaises(RuntimeError, response.deliverBody, Protocol())

def test_bodyDataReceivedAfterFinishedFails(self):
        """
        L{Response._bodyDataReceived} raises L{RuntimeError} if called after
        L{Response._bodyDataFinished} but before L{Response.deliverBody}.
        """
        response = justTransportResponse(StringTransport())
        response._bodyDataFinished()
        self.assertRaises(RuntimeError, response._bodyDataReceived, b'foo')

def test_bodyDataReceivedAfterDeliveryFails(self):
        """
        L{Response._bodyDataReceived} raises L{RuntimeError} if called after
        L{Response._bodyDataFinished} and after L{Response.deliverBody}.
        """
        response = justTransportResponse(StringTransport())
        response._bodyDataFinished()
        response.deliverBody(Protocol())
        self.assertRaises(RuntimeError, response._bodyDataReceived, b'foo')

def test_bodyDataFinishedAfterFinishedFails(self):
        """
        L{Response._bodyDataFinished} raises L{RuntimeError} if called more
        than once.
        """
        response = justTransportResponse(StringTransport())
        response._bodyDataFinished()
        self.assertRaises(RuntimeError, response._bodyDataFinished)

def test_bodyDataFinishedAfterDeliveryFails(self):
        """
        L{Response._bodyDataFinished} raises L{RuntimeError} if called after
        the body has been delivered.
        """
        response = justTransportResponse(StringTransport())
        response._bodyDataFinished()
        response.deliverBody(Protocol())
        self.assertRaises(RuntimeError, response._bodyDataFinished)

def test_transportResumed(self):
        """
        L{Response.deliverBody} resumes the HTTP connection's transport
        after passing it to the consumer's C{makeConnection} method.
        """
        transportState = []
        class ListConsumer(Protocol):
            def makeConnection(self, transport):
                transportState.append(transport.producerState)

transport = StringTransport()
        transport.pauseProducing()
        protocol = ListConsumer()
        response = justTransportResponse(transport)
        self.assertEqual(transport.producerState, u'paused')
        response.deliverBody(protocol)
        self.assertEqual(transportState, [u'paused'])
        self.assertEqual(transport.producerState, u'producing')

def test_bodyDataFinishedBeforeStartProducing(self):
        """
        If the entire body is delivered to the L{Response} before the
        response's C{deliverBody} method is called, the protocol passed to
        C{deliverBody} is immediately given the body data and then
        disconnected.
        """
        transport = StringTransport()
        response = justTransportResponse(transport)
        response._bodyDataReceived(b'foo')
        response._bodyDataReceived(b'bar')
        response._bodyDataFinished()

protocol = AccumulatingProtocol()
        response.deliverBody(protocol)
        self.assertEqual(protocol.data, b'foobar')
        protocol.closedReason.trap(ResponseDone)

def test_finishedWithErrorWhenConnected(self):
        """
        The L{Failure} passed to L{Response._bodyDataFinished} when the response
        is in the I{connected} state is passed to the C{connectionLost} method
        of the L{IProtocol} provider passed to the L{Response}'s
        C{deliverBody} method.
        """
        transport = StringTransport()
        response = justTransportResponse(transport)

protocol = AccumulatingProtocol()
        response.deliverBody(protocol)

# Sanity check - this test is for the connected state
        self.assertEqual(response._state, u'CONNECTED')
        response._bodyDataFinished(Failure(ArbitraryException()))

protocol.closedReason.trap(ArbitraryException)

def test_finishedWithErrorWhenInitial(self):
        """
        The L{Failure} passed to L{Response._bodyDataFinished} when the response
        is in the I{initial} state is passed to the C{connectionLost} method of
        the L{IProtocol} provider passed to the L{Response}'s C{deliverBody}
        method.
        """
        transport = StringTransport()
        response = justTransportResponse(transport)

# Sanity check - this test is for the initial state
        self.assertEqual(response._state, u'INITIAL')
        response._bodyDataFinished(Failure(ArbitraryException()))

protocol = AccumulatingProtocol()
        response.deliverBody(protocol)

protocol.closedReason.trap(ArbitraryException)