Spaces:

Maximax67
/

cefrpy-demo

Running

File size: 12,300 Bytes

import gradio as gr
import spacy
from spacy import displacy
from spacy.cli.download import download
from cefrpy import CEFRSpaCyAnalyzer, CEFRLevel

MODEL = "en_core_web_sm"

ALL_ENTS = [
    "CARDINAL",
    "DATE",
    "EVENT",
    "FAC",
    "GPE",
    "LANGUAGE",
    "LAW",
    "LOC",
    "MONEY",
    "NORP",
    "ORDINAL",
    "ORG",
    "PERCENT",
    "PERSON",
    "PRODUCT",
    "QUANTITY",
    "TIME",
    "WORK_OF_ART",
]

DEFAULT_ENTITY_ITEMS_TO_SKIP = [
    "QUANTITY",
    "MONEY",
    "LANGUAGE",
    "LAW",
    "WORK_OF_ART",
    "PRODUCT",
    "GPE",
    "ORG",
    "FAC",
    "PERSON",
]

TOKEN_ATTRIBUTES = ["Token", "POS", "Skipped", "Level", "Start", "End"]

WORDLIST_HEADER = ["Word", "Pos", "CEFR", "Level"]

DEFAULT_WORDLIST_SLIDER_LEVEL = 4.0

DEFAULT_TEXT = """The world's oldest known recipe is for beer. It dates back to around 5,000 BC and was found in ancient Sumeria (modern-day Iraq).

Due to thermal expansion, the iron structure of the Eiffel Tower can expand in hot weather, making the tower grow by up to 6 inches (15 centimeters) in height.

Did you know that the word "antidisestablishmentarianism" is often cited as one of the longest non-technical words in the English language? It originated in the 19th century in Britain during debates over the disestablishment of the Church of England, and it refers to the opposition to the withdrawal of state support for an established church. This word has gained notoriety for its length and has been used as a challenge for spelling bees and word enthusiasts alike.

In 2006, a Coca-Cola employee offered to sell Coca-Cola secrets to Pepsi. Pepsi responded by notifying Coca-Cola, and the FBI set up a sting operation to catch the culprit.

Like humans, cows form strong social bonds and often have "best friends" within their herds. They display complex social behaviors, including grooming, playing, and even grieving when separated from their friends."""

# Light-mode colors (used by displacy; must match the hex values in the CSS below)
DISPLACY_RENDER_OPTIONS = {
    "colors": {
        "A1": "#b0c4de",
        "A2": "#87ceeb",
        "B1": "#90ee90",
        "B2": "#adff2f",
        "C1": "#ffd700",
        "C2": "#ff9380",
        "SKIP": "#ffafed",
        "UNKNOWN": "#BCAAA4",
    }
}

ABBREVIATION_MAPPING = {
    "'m": "am",
    "'s": "is",
    "'re": "are",
    "'ve": "have",
    "'d": "had",
    "n't": "not",
    "'ll": "will",
}

# Minimal CSS: only the dual-theme entity colors. All other styling is unchanged.
CSS = """
h1 {
    padding-top: 5px;
    text-align: center;
    display:block;
}

.cefr-link-btn:hover {
    background: #f6f8fa;
    border-color: #999 !important;
}

.dark .cefr-link-btn:hover {
    background: #30363d;
    border-color: #888 !important;
}

/* Light-mode CEFR entity colors (matches DISPLACY_RENDER_OPTIONS above) */
:root {
    --cefr-a1:      #b0c4de;
    --cefr-a2:      #87ceeb;
    --cefr-b1:      #90ee90;
    --cefr-b2:      #adff2f;
    --cefr-c1:      #ffd700;
    --cefr-c2:      #ff9380;
    --cefr-skip:    #ffafed;
    --cefr-unknown: #BCAAA4;
}

/* Dark-mode overrides — Gradio adds class="dark" to <html> */
.dark {
    --cefr-a1:      #1e4e8c;
    --cefr-a2:      #0c6080;
    --cefr-b1:      #145c2e;
    --cefr-b2:      #4a7200;
    --cefr-c1:      #8a6400;
    --cefr-c2:      #952e1e;
    --cefr-skip:    #7a2070;
    --cefr-unknown: #4a3c38;
}

/* Override displacy's inline background colors using attribute selectors.
   Both cases (lower and upper) are covered since different browsers/displacy
   versions may render hex in either case. */
mark[style*="#b0c4de"], mark[style*="#B0C4DE"] { background: var(--cefr-a1)      !important; }
mark[style*="#87ceeb"], mark[style*="#87CEEB"] { background: var(--cefr-a2)      !important; }
mark[style*="#90ee90"], mark[style*="#90EE90"] { background: var(--cefr-b1)      !important; }
mark[style*="#adff2f"], mark[style*="#ADFF2F"] { background: var(--cefr-b2)      !important; }
mark[style*="#ffd700"], mark[style*="#FFD700"] { background: var(--cefr-c1)      !important; }
mark[style*="#ff9380"], mark[style*="#FF9380"] { background: var(--cefr-c2)      !important; }
mark[style*="#ffafed"], mark[style*="#FFAFED"] { background: var(--cefr-skip)    !important; }
mark[style*="#BCAAA4"], mark[style*="#bcaaa4"] { background: var(--cefr-unknown) !important; }
"""


try:
    nlp = spacy.load(MODEL)
except OSError:
    download(MODEL)
    nlp = spacy.load(MODEL)


def get_dict_ents(
    text: str, tokens: list[tuple[str, str, bool, float, int, int]]
) -> dict:
    ents = []

    for token in tokens:
        if token[3]:
            ents.append(
                {
                    "start": token[4],
                    "end": token[5],
                    "label": str(CEFRLevel(round(token[3]))),
                }
            )
        elif token[0].isalpha():
            ents.append(
                {
                    "start": token[4],
                    "end": token[5],
                    "label": "SKIP" if token[2] else "UNKNOWN",
                }
            )

    dict_ents = {"text": text, "ents": ents}

    return dict_ents


def get_cefr_tokens(
    text: str, ents_to_skip: list[str]
) -> list[tuple[str, str, bool, float, int, int]]:
    doc = nlp(text)
    text_analyzer = CEFRSpaCyAnalyzer(
        entity_types_to_skip=ents_to_skip, abbreviation_mapping=ABBREVIATION_MAPPING
    )
    tokens = text_analyzer.analyze_doc(doc)

    return tokens


def get_html_visualization(
    text: str, tokens: list[tuple[str, str, bool, float, int, int]]
) -> str:
    dict_ents = get_dict_ents(text, tokens)
    html = displacy.render(
        dict_ents, manual=True, style="ent", options=DISPLACY_RENDER_OPTIONS
    )

    return html


def get_wordlist_set(
    tokens: list[tuple[str, str, bool, float, int, int]], min_level: float
) -> set[tuple[str, str, bool, float, int, int]]:
    filtered_tokens = set()
    for word, pos, _, level, _, _ in tokens:
        if level and level >= min_level:
            filtered_tokens.add(
                (word.lower(), pos, str(CEFRLevel(round(level))), level)
            )

    return filtered_tokens


def get_wordlist(
    tokens: list[tuple[str, str, bool, float, int, int]], min_level: float
):
    wordlist_set = get_wordlist_set(tokens, min_level)
    wordlist = list(wordlist_set)
    wordlist.sort()

    return wordlist


def get_wordlist_from_dataframe(dataframe, min_level: float):
    return get_wordlist(dataframe.values, min_level)


def process_text(
    text: str,
    ents_to_skip: list[str] = DEFAULT_ENTITY_ITEMS_TO_SKIP,
    min_level: float = DEFAULT_WORDLIST_SLIDER_LEVEL,
) -> tuple[list[tuple], list[tuple], str]:
    tokens = get_cefr_tokens(text, ents_to_skip)
    html = get_html_visualization(text, tokens)
    wordlist = get_wordlist(tokens, min_level)

    return tokens, wordlist, html


initial_tokens, initial_wordlist, initial_html = process_text(DEFAULT_TEXT)

demo = gr.Blocks()

with demo:
    with gr.Row():
        with gr.Column():
            with gr.Column():
                gr.HTML("""
                    <div style="display:flex; align-items:center; justify-content:space-between; flex-wrap:wrap; gap:8px;">
                        <h1 style="margin:0; font-size:1.6rem; font-weight:700;">Gradio Demo: cefrpy</h1>
                        <div style="display:flex; gap:10px; align-items:center; flex-wrap:wrap;">
                            <a class="cefr-link-btn" href="https://github.com/Maximax67/cefrpy" target="_blank"
                            style="display:inline-flex; align-items:center; gap:6px; padding:5px 12px;
                                border-radius:6px; border:1px solid #d0d7de; text-decoration:none;
                                color:inherit; font-size:0.9rem; font-weight:500; transition:background 0.15s, border-color 0.15s;">
                                <svg width="18" height="18" viewBox="0 0 16 16" fill="currentColor">
                                    <path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38
                                    0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13
                                    -.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66
                                    .07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15
                                    -.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0
                                    1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82
                                    1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01
                                    1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"/>
                                </svg>
                                GitHub
                            </a>
                            <a class="cefr-link-btn" href="https://maximax67.github.io/cefrpy" target="_blank"
                            style="display:inline-flex; align-items:center; gap:6px; padding:5px 12px;
                                    border-radius:6px; border:1px solid #d0d7de; text-decoration:none;
                                    color:inherit; font-size:0.9rem; font-weight:500; transition:background 0.15s, border-color 0.15s;">
                                <svg width="18" height="18" viewBox="0 0 24 24" fill="none"
                                    stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
                                    <path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"/>
                                    <polyline points="14 2 14 8 20 8"/>
                                    <line x1="16" y1="13" x2="8" y2="13"/>
                                    <line x1="16" y1="17" x2="8" y2="17"/>
                                    <polyline points="10 9 9 9 8 9"/>
                                </svg>
                                Docs
                            </a>
                        </div>
                    </div>
                """)

                with gr.Row():
                    text_input = gr.TextArea(
                        value=DEFAULT_TEXT,
                        interactive=True,
                        max_lines=500,
                        label="Input Text",
                        buttons=["copy"],
                    )

                with gr.Row():
                    ent_input = gr.CheckboxGroup(
                        ALL_ENTS,
                        value=DEFAULT_ENTITY_ITEMS_TO_SKIP,
                        label="Entity types to skip CEFR",
                    )

                with gr.Row():
                    clear_button = gr.ClearButton(text_input)
                    render_button = gr.Button("Render", variant="primary")

        with gr.Column():
            with gr.Row():
                gr.Markdown("# Words CEFR level visualization", padding=True)

            with gr.Row():
                rendered_html = gr.HTML(initial_html)

    with gr.Row():
        with gr.Column():
            with gr.Row():
                tokens_output = gr.Dataframe(
                    headers=TOKEN_ATTRIBUTES, value=initial_tokens, interactive=False
                )

        with gr.Column():
            with gr.Row():
                min_level_slider = gr.Slider(
                    minimum=1.0,
                    maximum=6.0,
                    value=DEFAULT_WORDLIST_SLIDER_LEVEL,
                    step=0.02,
                    interactive=True,
                    label="Min level to generate word list",
                )

            with gr.Row():
                wordlist = gr.Dataframe(
                    headers=WORDLIST_HEADER, value=initial_wordlist, interactive=False
                )

    render_button.click(
        process_text,
        inputs=[text_input, ent_input],
        outputs=[tokens_output, wordlist, rendered_html],
        api_name="process_text",
    )

    min_level_slider.release(
        get_wordlist_from_dataframe,
        inputs=[tokens_output, min_level_slider],
        outputs=[wordlist],
    )

demo.launch(css=CSS)