#!/usr/bin/python # -*- coding: utf-8 -*- # :Id: $Id: smartquotes.py 7716 2013-08-21 21:54:57Z milde $ # :Copyright: © 2010 Günter Milde, # original `SmartyPants`_: © 2003 John Gruber # smartypants.py: © 2004, 2007 Chad Miller # :Maintainer: docutils-develop@lists.sourceforge.net # :License: Released under the terms of the `2-Clause BSD license`_, in short: # # Copying and distribution of this file, with or without modification, # are permitted in any medium without royalty provided the copyright # notices and this notice are preserved. # This file is offered as-is, without any warranty. # # .. _2-Clause BSD license: http://www.spdx.org/licenses/BSD-2-Clause r""" ======================== SmartyPants for Docutils ======================== Synopsis ======== Smart-quotes for Docutils. The original "SmartyPants" is a free web publishing plug-in for Movable Type, Blosxom, and BBEdit that easily translates plain ASCII punctuation characters into "smart" typographic punctuation characters. `smartypants.py`, endeavours to be a functional port of SmartyPants to Python, for use with Pyblosxom_. `smartquotes.py` is an adaption of Smartypants to Docutils_. By using Unicode characters instead of HTML entities for typographic quotes, it works for any output format that supports Unicode. Authors ======= `John Gruber`_ did all of the hard work of writing this software in Perl for `Movable Type`_ and almost all of this useful documentation. `Chad Miller`_ ported it to Python to use with Pyblosxom_. Adapted to Docutils_ by Günter Milde Additional Credits ================== Portions of the SmartyPants original work are based on Brad Choate's nifty MTRegex plug-in. `Brad Choate`_ also contributed a few bits of source code to this plug-in. Brad Choate is a fine hacker indeed. `Jeremy Hedley`_ and `Charles Wiltgen`_ deserve mention for exemplary beta testing of the original SmartyPants. `Rael Dornfest`_ ported SmartyPants to Blosxom. .. _Brad Choate: http://bradchoate.com/ .. _Jeremy Hedley: http://antipixel.com/ .. _Charles Wiltgen: http://playbacktime.com/ .. _Rael Dornfest: http://raelity.org/ Copyright and License ===================== SmartyPants_ license (3-Clause BSD license): Copyright (c) 2003 John Gruber (http://daringfireball.net/) All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name "SmartyPants" nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. This software is provided by the copyright holders and contributors "as is" and any express or implied warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose are disclaimed. In no event shall the copyright owner or contributors be liable for any direct, indirect, incidental, special, exemplary, or consequential damages (including, but not limited to, procurement of substitute goods or services; loss of use, data, or profits; or business interruption) however caused and on any theory of liability, whether in contract, strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this software, even if advised of the possibility of such damage. smartypants.py license (2-Clause BSD license): smartypants.py is a derivative work of SmartyPants. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. This software is provided by the copyright holders and contributors "as is" and any express or implied warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose are disclaimed. In no event shall the copyright owner or contributors be liable for any direct, indirect, incidental, special, exemplary, or consequential damages (including, but not limited to, procurement of substitute goods or services; loss of use, data, or profits; or business interruption) however caused and on any theory of liability, whether in contract, strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this software, even if advised of the possibility of such damage. .. _John Gruber: http://daringfireball.net/ .. _Chad Miller: http://web.chad.org/ .. _Pyblosxom: http://pyblosxom.bluesock.org/ .. _SmartyPants: http://daringfireball.net/projects/smartypants/ .. _Movable Type: http://www.movabletype.org/ .. _2-Clause BSD license: http://www.spdx.org/licenses/BSD-2-Clause .. _Docutils: http://docutils.sf.net/ Description =========== SmartyPants can perform the following transformations: - Straight quotes ( " and ' ) into "curly" quote characters - Backticks-style quotes (\`\`like this'') into "curly" quote characters - Dashes (``--`` and ``---``) into en- and em-dash entities - Three consecutive dots (``...`` or ``. . .``) into an ellipsis entity This means you can write, edit, and save your posts using plain old ASCII straight quotes, plain dashes, and plain dots, but your published posts (and final HTML output) will appear with smart quotes, em-dashes, and proper ellipses. SmartyPants does not modify characters within ``
``, ````, ````,
```` or ``
He said, "'Quoted' words in a larger quote."

    text = re.sub(r""""'(?=\w)""", smart.opquote+smart.osquote, text)
    text = re.sub(r"""'"(?=\w)""", smart.osquote+smart.opquote, text)

    # Special case for decade abbreviations (the '80s):
    text = re.sub(r"""\b'(?=\d{2}s)""", smart.csquote, text)

    close_class = r"""[^\ \t\r\n\[\{\(\-]"""
    dec_dashes = r"""–|—"""

    # Get most opening single quotes:
    opening_single_quotes_regex = re.compile(r"""
                    (
                            \s          |   # a whitespace char, or
                                   |   # a non-breaking space entity, or
                            --          |   # dashes, or
                            &[mn]dash;  |   # named dash entities
                            %s          |   # or decimal entities
                            &\#x201[34];    # or hex
                    )
                    '                 # the quote
                    (?=\w)            # followed by a word character
                    """ % (dec_dashes,), re.VERBOSE)
    text = opening_single_quotes_regex.sub(r'\1'+smart.osquote, text)

    closing_single_quotes_regex = re.compile(r"""
                    (%s)
                    '
                    (?!\s | s\b | \d)
                    """ % (close_class,), re.VERBOSE)
    text = closing_single_quotes_regex.sub(r'\1'+smart.csquote, text)

    closing_single_quotes_regex = re.compile(r"""
                    (%s)
                    '
                    (\s | s\b)
                    """ % (close_class,), re.VERBOSE)
    text = closing_single_quotes_regex.sub(r'\1%s\2' % smart.csquote, text)

    # Any remaining single quotes should be opening ones:
    text = re.sub(r"""'""", smart.osquote, text)

    # Get most opening double quotes:
    opening_double_quotes_regex = re.compile(r"""
                    (
                            \s          |   # a whitespace char, or
                                   |   # a non-breaking space entity, or
                            --          |   # dashes, or
                            &[mn]dash;  |   # named dash entities
                            %s          |   # or decimal entities
                            &\#x201[34];    # or hex
                    )
                    "                 # the quote
                    (?=\w)            # followed by a word character
                    """ % (dec_dashes,), re.VERBOSE)
    text = opening_double_quotes_regex.sub(r'\1'+smart.opquote, text)

    # Double closing quotes:
    closing_double_quotes_regex = re.compile(r"""
                    #(%s)?   # character that indicates the quote should be closing
                    "
                    (?=\s)
                    """ % (close_class,), re.VERBOSE)
    text = closing_double_quotes_regex.sub(smart.cpquote, text)

    closing_double_quotes_regex = re.compile(r"""
                    (%s)   # character that indicates the quote should be closing
                    "
                    """ % (close_class,), re.VERBOSE)
    text = closing_double_quotes_regex.sub(r'\1'+smart.cpquote, text)

    # Any remaining quotes should be opening ones.
    text = re.sub(r'"', smart.opquote, text)

    return text


def educateBackticks(text, language='en'):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with ``backticks'' -style double quotes
                translated into HTML curly quote entities.
    Example input:  ``Isn't this fun?''
    Example output: “Isn't this fun?“;
    """
    smart = smartchars(language)

    text = re.sub(r"""``""", smart.opquote, text)
    text = re.sub(r"""''""", smart.cpquote, text)
    return text


def educateSingleBackticks(text, language='en'):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with `backticks' -style single quotes
                translated into HTML curly quote entities.

    Example input:  `Isn't this fun?'
    Example output: ‘Isn’t this fun?’
    """
    smart = smartchars(language)

    text = re.sub(r"""`""", smart.osquote, text)
    text = re.sub(r"""'""", smart.csquote, text)
    return text


def educateDashes(text):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each instance of "--" translated to
                an em-dash character.
    """

    text = re.sub(r"""---""", smartchars.endash, text) # en  (yes, backwards)
    text = re.sub(r"""--""", smartchars.emdash, text) # em (yes, backwards)
    return text


def educateDashesOldSchool(text):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each instance of "--" translated to
                an en-dash character, and each "---" translated to
                an em-dash character.
    """

    text = re.sub(r"""---""", smartchars.emdash, text)
    text = re.sub(r"""--""", smartchars.endash, text)
    return text


def educateDashesOldSchoolInverted(text):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each instance of "--" translated to
                an em-dash character, and each "---" translated to
                an en-dash character. Two reasons why: First, unlike the
                en- and em-dash syntax supported by
                EducateDashesOldSchool(), it's compatible with existing
                entries written before SmartyPants 1.1, back when "--" was
                only used for em-dashes.  Second, em-dashes are more
                common than en-dashes, and so it sort of makes sense that
                the shortcut should be shorter to type. (Thanks to Aaron
                Swartz for the idea.)
    """
    text = re.sub(r"""---""", smartchars.endash, text)    # em
    text = re.sub(r"""--""", smartchars.emdash, text)    # en
    return text



def educateEllipses(text):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each instance of "..." translated to
                an ellipsis character.

    Example input:  Huh...?
    Example output: Huh…?
    """

    text = re.sub(r"""\.\.\.""", smartchars.ellipsis, text)
    text = re.sub(r"""\. \. \.""", smartchars.ellipsis, text)
    return text


def stupefyEntities(text, language='en'):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each SmartyPants character translated to
                its ASCII counterpart.

    Example input:  “Hello — world.”
    Example output: "Hello -- world."
    """
    smart = smartchars(language)

    text = re.sub(smart.endash, "-", text)  # en-dash
    text = re.sub(smart.emdash, "--", text) # em-dash

    text = re.sub(smart.osquote, "'", text)  # open single quote
    text = re.sub(smart.csquote, "'", text)  # close single quote

    text = re.sub(smart.opquote, '"', text)  # open double quote
    text = re.sub(smart.cpquote, '"', text)  # close double quote

    text = re.sub(smart.ellipsis, '...', text)# ellipsis

    return text


def processEscapes(text, restore=False):
    r"""
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with after processing the following backslash
                escape sequences. This is useful if you want to force a "dumb"
                quote or other character to appear.

                Escape  Value
                ------  -----
                \\      \
                \"      "
                \'      '
                \.      .
                \-      -
                \`      `
    """
    replacements = ((r'\\', r'\'),
                    (r'\"', r'"'),
                    (r"\'", r'''),
                    (r'\.', r'.'),
                    (r'\-', r'-'),
                    (r'\`', r'`'))
    if restore:
        for (ch, rep) in replacements:
            text = text.replace(rep, ch[1])
    else:
        for (ch, rep) in replacements:
            text = text.replace(ch, rep)

    return text


def tokenize(text):
    """
    Parameter:  String containing HTML markup.
    Returns:    An iterator that yields the tokens comprising the input
                string. Each token is either a tag (possibly with nested,
                tags contained therein, such as , or a
                run of text between tags. Each yielded element is a
                two-element tuple; the first is either 'tag' or 'text';
                the second is the actual value.

    Based on the _tokenize() subroutine from Brad Choate's MTRegex plugin.
        
    """

    pos = 0
    length = len(text)
    # tokens = []

    depth = 6
    nested_tags = "|".join(['(?:<(?:[^<>]',] * depth) + (')*>)' * depth)
    #match = r"""(?:  ) |  # comments
    #               (?: <\? .*? \?> ) |  # directives
    #               %s  # nested tags       """ % (nested_tags,)
    tag_soup = re.compile(r"""([^<]*)(<[^>]*>)""")

    token_match = tag_soup.search(text)

    previous_end = 0
    while token_match is not None:
        if token_match.group(1):
            yield ('text', token_match.group(1))

        yield ('tag', token_match.group(2))

        previous_end = token_match.end()
        token_match = tag_soup.search(text, token_match.end())

    if previous_end < len(text):
        yield ('text', text[previous_end:])



if __name__ == "__main__":

    import locale

    try:
        locale.setlocale(locale.LC_ALL, '')
    except:
        pass

    from docutils.core import publish_string
    docstring_html = publish_string(__doc__, writer_name='html')

    print docstring_html


    # Unit test output goes out stderr.
    import unittest
    sp = smartyPants

    class TestSmartypantsAllAttributes(unittest.TestCase):
        # the default attribute is "1", which means "all".

        def test_dates(self):
            self.assertEqual(sp("1440-80's"), u"1440-80’s")
            self.assertEqual(sp("1440-'80s"), u"1440-‘80s")
            self.assertEqual(sp("1440---'80s"), u"1440–‘80s")
            self.assertEqual(sp("1960s"), "1960s")  # no effect.
            self.assertEqual(sp("1960's"), u"1960’s")
            self.assertEqual(sp("one two '60s"), u"one two ‘60s")
            self.assertEqual(sp("'60s"), u"‘60s")

        def test_ordinal_numbers(self):
            self.assertEqual(sp("21st century"), "21st century")  # no effect.
            self.assertEqual(sp("3rd"), "3rd")  # no effect.

        def test_educated_quotes(self):
            self.assertEqual(sp('''"Isn't this fun?"'''), u'“Isn’t this fun?”')

        def test_html_tags(self):
            text = 'more'
            self.assertEqual(sp(text), text)

    unittest.main()




__author__ = "Chad Miller "
__version__ = "1.5_1.6: Fri, 27 Jul 2007 07:06:40 -0400"
__url__ = "http://wiki.chad.org/SmartyPantsPy"
__description__ = "Smart-quotes, smart-ellipses, and smart-dashes for weblog entries in pyblosxom"