1. Advisor
  2. ftfy
  3. functions
  4. ftfy.fix_text_segment
View all ftfy analysis

How to use the ftfy.fix_text_segment function in ftfy

To help you get started, we’ve selected a few ftfy examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github LuminosoInsight / python-ftfy / tests / test_entities.py View on Github external
def test_entities():
    example = '&\n\n&'
    assert fix_text(example) == '&\n\n&'
    assert fix_text_segment(example) == '&\n\n&'

    assert fix_text(example, fix_entities=True) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=True) == '&\n\n&'

    assert fix_text(example, fix_entities=False) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=False) == '&\n\n&'

    assert fix_text_segment('<>', fix_entities=False) == '<>'
    assert fix_text_segment('<>', fix_entities=True) == '<>'
    assert fix_text_segment('<>') == '<>'
    assert fix_text_segment('jednocześnie') == 'jednocześnie'
    assert fix_text_segment('JEDNOCZEŚNIE') == 'JEDNOCZEŚNIE'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('broken') == 'broken\x81'
    assert unescape_html('euro €') == 'euro €'
    assert unescape_html('not an entity x6;') == 'not an entity x6;'
github LuminosoInsight / python-ftfy / tests / test_entities.py View on Github external
assert fix_text(example) == '&\n\n&'
    assert fix_text_segment(example) == '&\n\n&'

    assert fix_text(example, fix_entities=True) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=True) == '&\n\n&'

    assert fix_text(example, fix_entities=False) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=False) == '&\n\n&'

    assert fix_text_segment('<>', fix_entities=False) == '<>'
    assert fix_text_segment('<>', fix_entities=True) == '<>'
    assert fix_text_segment('<>') == '<>'
    assert fix_text_segment('jednocześnie') == 'jednocześnie'
    assert fix_text_segment('JEDNOCZEŚNIE') == 'JEDNOCZEŚNIE'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('broken') == 'broken\x81'
    assert unescape_html('euro €') == 'euro €'
    assert unescape_html('not an entity x6;') == 'not an entity x6;'
github LuminosoInsight / python-ftfy / tests / test_entities.py View on Github external
assert fix_text_segment(example) == '&\n\n&'

    assert fix_text(example, fix_entities=True) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=True) == '&\n\n&'

    assert fix_text(example, fix_entities=False) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=False) == '&\n\n&'

    assert fix_text_segment('<>', fix_entities=False) == '<>'
    assert fix_text_segment('<>', fix_entities=True) == '<>'
    assert fix_text_segment('<>') == '<>'
    assert fix_text_segment('jednocześnie') == 'jednocześnie'
    assert fix_text_segment('JEDNOCZEŚNIE') == 'JEDNOCZEŚNIE'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('broken') == 'broken\x81'
    assert unescape_html('euro €') == 'euro €'
    assert unescape_html('not an entity x6;') == 'not an entity x6;'
github LuminosoInsight / python-ftfy / tests / test_entities.py View on Github external
def test_entities():
    example = '&\n\n&'
    assert fix_text(example) == '&\n\n&'
    assert fix_text_segment(example) == '&\n\n&'

    assert fix_text(example, fix_entities=True) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=True) == '&\n\n&'

    assert fix_text(example, fix_entities=False) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=False) == '&\n\n&'

    assert fix_text_segment('<>', fix_entities=False) == '<>'
    assert fix_text_segment('<>', fix_entities=True) == '<>'
    assert fix_text_segment('<>') == '<>'
    assert fix_text_segment('jednocześnie') == 'jednocześnie'
    assert fix_text_segment('JEDNOCZEŚNIE') == 'JEDNOCZEŚNIE'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('broken') == 'broken\x81'
    assert unescape_html('euro €') == 'euro €'
    assert unescape_html('not an entity x6;') == 'not an entity x6;'
github LuminosoInsight / python-ftfy / tests / test_entities.py View on Github external
def test_entities():
    example = '&\n\n&'
    assert fix_text(example) == '&\n\n&'
    assert fix_text_segment(example) == '&\n\n&'

    assert fix_text(example, fix_entities=True) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=True) == '&\n\n&'

    assert fix_text(example, fix_entities=False) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=False) == '&\n\n&'

    assert fix_text_segment('<>', fix_entities=False) == '<>'
    assert fix_text_segment('<>', fix_entities=True) == '<>'
    assert fix_text_segment('<>') == '<>'
    assert fix_text_segment('jednocześnie') == 'jednocześnie'
    assert fix_text_segment('JEDNOCZEŚNIE') == 'JEDNOCZEŚNIE'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('broken') == 'broken\x81'
    assert unescape_html('euro €') == 'euro €'
github LuminosoInsight / python-ftfy / tests / test_entities.py View on Github external
def test_entities():
    example = '&\n\n&'
    assert fix_text(example) == '&\n\n&'
    assert fix_text_segment(example) == '&\n\n&'

    assert fix_text(example, fix_entities=True) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=True) == '&\n\n&'

    assert fix_text(example, fix_entities=False) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=False) == '&\n\n&'

    assert fix_text_segment('<>', fix_entities=False) == '<>'
    assert fix_text_segment('<>', fix_entities=True) == '<>'
    assert fix_text_segment('<>') == '<>'
    assert fix_text_segment('jednocześnie') == 'jednocześnie'
    assert fix_text_segment('JEDNOCZEŚNIE') == 'JEDNOCZEŚNIE'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('broken') == 'broken\x81'
    assert unescape_html('euro €') == 'euro €'
    assert unescape_html('not an entity x6;') == 'not an entity x6;'
github LuminosoInsight / python-ftfy / tests / test_entities.py View on Github external
def test_entities():
    example = '&\n\n&'
    assert fix_text(example) == '&\n\n&'
    assert fix_text_segment(example) == '&\n\n&'

    assert fix_text(example, fix_entities=True) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=True) == '&\n\n&'

    assert fix_text(example, fix_entities=False) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=False) == '&\n\n&'

    assert fix_text_segment('<>', fix_entities=False) == '<>'
    assert fix_text_segment('<>', fix_entities=True) == '<>'
    assert fix_text_segment('<>') == '<>'
    assert fix_text_segment('jednocześnie') == 'jednocześnie'
    assert fix_text_segment('JEDNOCZEŚNIE') == 'JEDNOCZEŚNIE'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('broken') == 'broken\x81'
    assert unescape_html('euro €') == 'euro €'
    assert unescape_html('not an entity x6;') == 'not an entity x6;'
github LuminosoInsight / python-ftfy / tests / test_entities.py View on Github external
example = '&\n\n&'
    assert fix_text(example) == '&\n\n&'
    assert fix_text_segment(example) == '&\n\n&'

    assert fix_text(example, fix_entities=True) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=True) == '&\n\n&'

    assert fix_text(example, fix_entities=False) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=False) == '&\n\n&'

    assert fix_text_segment('<>', fix_entities=False) == '<>'
    assert fix_text_segment('<>', fix_entities=True) == '<>'
    assert fix_text_segment('<>') == '<>'
    assert fix_text_segment('jednocześnie') == 'jednocześnie'
    assert fix_text_segment('JEDNOCZEŚNIE') == 'JEDNOCZEŚNIE'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('broken') == 'broken\x81'
    assert unescape_html('euro €') == 'euro €'
    assert unescape_html('not an entity x6;') == 'not an entity x6;'
github LuminosoInsight / python-ftfy / tests / test_entities.py View on Github external
def test_entities():
    example = '&\n\n&'
    assert fix_text(example) == '&\n\n&'
    assert fix_text_segment(example) == '&\n\n&'

    assert fix_text(example, fix_entities=True) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=True) == '&\n\n&'

    assert fix_text(example, fix_entities=False) == '&\n\n&'
    assert fix_text_segment(example, fix_entities=False) == '&\n\n&'

    assert fix_text_segment('<>', fix_entities=False) == '<>'
    assert fix_text_segment('<>', fix_entities=True) == '<>'
    assert fix_text_segment('<>') == '<>'
    assert fix_text_segment('jednocześnie') == 'jednocześnie'
    assert fix_text_segment('JEDNOCZEŚNIE') == 'JEDNOCZEŚNIE'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...'
    assert fix_text_segment('broken') == 'broken\x81'
    assert unescape_html('euro €') == 'euro €'
    assert unescape_html('not an entity x6;') == 'not an entity x6;'

ftfy

Fixes mojibake and other problems with Unicode, after the fact

Apache-2.0
Latest version published 2 months ago

Package Health Score

75 / 100
Full package analysis

Popular ftfy functions

Similar packages