santhosh commited on
Commit
9f7f7e6
·
verified ·
1 Parent(s): 498e00a

Deploy DYK card shuffler

Browse files
Files changed (3) hide show
  1. README.md +15 -5
  2. app.py +261 -0
  3. requirements.txt +3 -0
README.md CHANGED
@@ -1,13 +1,23 @@
1
  ---
2
- title: Wiki Dyk
3
- emoji: 🦀
4
  colorFrom: blue
5
- colorTo: pink
6
  sdk: gradio
7
  sdk_version: 6.20.0
8
- python_version: '3.13'
9
  app_file: app.py
10
  pinned: false
 
11
  ---
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: Wikipedia Did You Know
3
+ emoji: 🎲
4
  colorFrom: blue
5
+ colorTo: indigo
6
  sdk: gradio
7
  sdk_version: 6.20.0
 
8
  app_file: app.py
9
  pinned: false
10
+ license: cc-by-sa-4.0
11
  ---
12
 
13
+ # Wikipedia — Did You Know?
14
+
15
+ A card shuffler for the
16
+ [santhosh/wiki-dyk](https://huggingface.co/datasets/santhosh/wiki-dyk)
17
+ dataset. Each card shows one random "Did you know" hook: the featured
18
+ article's image, the fact itself, its topics, and a link to the article
19
+ exactly as it stood the day it was featured. Filter by language (en/fr/de) and
20
+ topic.
21
+
22
+ Text and images are from Wikipedia, licensed
23
+ [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/).
app.py ADDED
@@ -0,0 +1,261 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """A "Did You Know?" card shuffler for the wikimedia/wiki-dyk dataset.
2
+
3
+ Shows one random hook at a time: the featured article's image, the fact
4
+ (rendered from content_body), its topics, and a link to the article exactly
5
+ as it stood the day it was featured. Filter by language and topic.
6
+
7
+ Data is read straight from the published dataset on the Hub. Set DYK_LOCAL to
8
+ a directory of dyk-{lang}.parquet files to run against local outputs instead.
9
+ """
10
+
11
+ import html
12
+ import os
13
+ import random
14
+ import re
15
+ from collections import Counter
16
+ from urllib.parse import quote
17
+
18
+ import gradio as gr
19
+ import pandas as pd
20
+
21
+ DATASET = "wikimedia/wiki-dyk"
22
+ LANGS = ["en", "fr", "de"]
23
+ TOP_TOPICS = 200 # dropdown cap; the full unique topic list is unusably long
24
+ ALL_TOPICS = "All topics"
25
+
26
+ _cache: dict[str, dict] = {}
27
+
28
+
29
+ def _data_url(lang: str) -> str:
30
+ local = os.environ.get("DYK_LOCAL")
31
+ if local:
32
+ return f"{local}/dyk-{lang}.parquet"
33
+ return f"hf://datasets/{DATASET}/{lang}/dyk-{lang}.parquet"
34
+
35
+
36
+ def _as_list(value) -> list:
37
+ """linked_pages / article_topics arrive as numpy arrays or None."""
38
+ return [] if value is None else list(value)
39
+
40
+
41
+ def _clean_int(value):
42
+ return None if value is None or pd.isna(value) else int(value)
43
+
44
+
45
+ def load(lang: str) -> dict:
46
+ """Load and normalize one language, cached. Builds a frequency-ranked
47
+ topic -> row-indices index for O(1) filtered sampling."""
48
+ if lang in _cache:
49
+ return _cache[lang]
50
+
51
+ df = pd.read_parquet(
52
+ _data_url(lang),
53
+ columns=["day", "content_body", "linked_pages"],
54
+ )
55
+
56
+ records = []
57
+ counter: Counter = Counter()
58
+ for day, body, linked in zip(
59
+ df["day"], df["content_body"], df["linked_pages"], strict=True
60
+ ):
61
+ pages = []
62
+ topics_all: set[str] = set()
63
+ for page in _as_list(linked):
64
+ topics = [
65
+ topic["title"]
66
+ for topic in _as_list(page["article_topics"])
67
+ if topic is not None and topic["title"]
68
+ ]
69
+ pages.append(
70
+ {
71
+ "page_title": page["page_title"],
72
+ "short_description": page["short_description"],
73
+ "page_image": page["page_image"],
74
+ "rev_id": _clean_int(page["rev_id"]),
75
+ "prominent": bool(page["prominent"]),
76
+ "topics": topics,
77
+ }
78
+ )
79
+ topics_all.update(topics)
80
+ counter.update(topics_all)
81
+ records.append(
82
+ {"day": day, "content_body": body, "pages": pages, "topics": topics_all}
83
+ )
84
+
85
+ top = {title for title, _ in counter.most_common(TOP_TOPICS)}
86
+ index: dict[str, list[int]] = {title: [] for title in top}
87
+ for i, record in enumerate(records):
88
+ for title in record["topics"] & top:
89
+ index[title].append(i)
90
+ choices = sorted(
91
+ ((title.replace("_", " "), title) for title in top), key=lambda c: c[0].lower()
92
+ )
93
+
94
+ _cache[lang] = {
95
+ "records": records,
96
+ "all_idx": list(range(len(records))),
97
+ "topic_index": index,
98
+ "topic_choices": choices,
99
+ }
100
+ return _cache[lang]
101
+
102
+
103
+ def _featured(pages: list[dict]) -> dict | None:
104
+ """The article the card centres on: a prominent one, preferring one that
105
+ has an image; fall back to any imaged page, then the first page."""
106
+ prominent = [p for p in pages if p["prominent"]]
107
+ for pool in (prominent, pages):
108
+ imaged = [p for p in pool if p["page_image"]]
109
+ if imaged:
110
+ return imaged[0]
111
+ if prominent:
112
+ return prominent[0]
113
+ return pages[0] if pages else None
114
+
115
+
116
+ def _image_url(lang: str, filename: str) -> str:
117
+ return (
118
+ f"https://{lang}.wikipedia.org/wiki/Special:FilePath/"
119
+ f"{quote(filename)}?width=400"
120
+ )
121
+
122
+
123
+ def _wiki_url(lang: str, title: str) -> str:
124
+ return f"https://{lang}.wikipedia.org/wiki/{quote(title.replace(' ', '_'))}"
125
+
126
+
127
+ def _rewrite_links(lang: str, body: str) -> str:
128
+ """content_body links are relative (href="./Title")."""
129
+ return re.sub(
130
+ r'href="\./([^"]*)"',
131
+ lambda m: (
132
+ f'href="https://{lang}.wikipedia.org/wiki/{m.group(1)}" '
133
+ f'target="_blank" rel="noopener"'
134
+ ),
135
+ body,
136
+ )
137
+
138
+
139
+ def _fmt_day(day) -> str | None:
140
+ if day is None or pd.isna(day):
141
+ return None
142
+ d = pd.Timestamp(day)
143
+ return f"{d.day} {d:%B %Y}"
144
+
145
+
146
+ CARD_CSS = """
147
+ <style>
148
+ .dyk-card { max-width: 640px; margin: 0 auto; padding: 1.25rem;
149
+ border: 1px solid rgba(128,128,128,.3); border-radius: 16px; }
150
+ .dyk-head { display: flex; gap: 1rem; align-items: flex-start; }
151
+ .dyk-img { width: 120px; height: 120px; object-fit: cover; border-radius: 12px;
152
+ flex-shrink: 0; }
153
+ .dyk-title { font-size: 1.1rem; font-weight: 600; }
154
+ .dyk-desc { opacity: .7; font-size: .9rem; }
155
+ .dyk-fact { margin: 1rem 0; font-size: 1.15rem; line-height: 1.5; }
156
+ .dyk-chips { display: flex; flex-wrap: wrap; gap: .4rem; margin-bottom: .75rem; }
157
+ .dyk-chip { font-size: .8rem; padding: .15rem .6rem; border-radius: 999px;
158
+ border: 1px solid rgba(128,128,128,.3); opacity: .85; }
159
+ .dyk-prov { font-size: .85rem; opacity: .7; }
160
+ .dyk-prov a { color: inherit; }
161
+ </style>
162
+ """
163
+
164
+
165
+ def render_card(lang: str, record: dict | None) -> str:
166
+ if record is None:
167
+ return f"{CARD_CSS}<div class='dyk-card'>No facts to show.</div>"
168
+
169
+ featured = _featured(record["pages"])
170
+ head = ""
171
+ if featured:
172
+ img = ""
173
+ if featured["page_image"]:
174
+ img = (
175
+ f'<img class="dyk-img" alt="" '
176
+ f'src="{_image_url(lang, featured["page_image"])}">'
177
+ )
178
+ title = html.escape(featured["page_title"])
179
+ desc = html.escape(featured["short_description"] or "")
180
+ head = (
181
+ f'<div class="dyk-head">{img}<div>'
182
+ f'<div class="dyk-title">'
183
+ f'<a href="{_wiki_url(lang, featured["page_title"])}" '
184
+ f'target="_blank" rel="noopener" style="color:inherit">{title}</a></div>'
185
+ f'<div class="dyk-desc">{desc}</div></div></div>'
186
+ )
187
+
188
+ chips = ""
189
+ if featured and featured["topics"]:
190
+ chips = (
191
+ '<div class="dyk-chips">'
192
+ + "".join(
193
+ f'<span class="dyk-chip">#{html.escape(t.replace("_", " "))}</span>'
194
+ for t in featured["topics"]
195
+ )
196
+ + "</div>"
197
+ )
198
+
199
+ prov = ""
200
+ day = _fmt_day(record["day"])
201
+ if day:
202
+ prov = f"Featured on {day}"
203
+ if featured and featured["rev_id"]:
204
+ link = f"https://{lang}.wikipedia.org/w/index.php?oldid={featured['rev_id']}"
205
+ sep = " · " if prov else ""
206
+ prov += (
207
+ f'{sep}<a href="{link}" target="_blank" rel="noopener">'
208
+ f"see the article as it was then →</a>"
209
+ )
210
+ prov = f'<div class="dyk-prov">{prov}</div>' if prov else ""
211
+
212
+ fact = _rewrite_links(lang, record["content_body"])
213
+ return (
214
+ f'{CARD_CSS}<div class="dyk-card">{head}'
215
+ f'<div class="dyk-fact">{fact}</div>{chips}{prov}</div>'
216
+ )
217
+
218
+
219
+ def shuffle(lang: str, topic: str) -> str:
220
+ data = load(lang)
221
+ if topic and topic != ALL_TOPICS:
222
+ candidates = data["topic_index"].get(topic, [])
223
+ else:
224
+ candidates = data["all_idx"]
225
+ record = data["records"][random.choice(candidates)] if candidates else None
226
+ return render_card(lang, record)
227
+
228
+
229
+ def switch_language(lang: str):
230
+ data = load(lang)
231
+ choices = [(ALL_TOPICS, ALL_TOPICS), *data["topic_choices"]]
232
+ return (
233
+ gr.update(choices=choices, value=ALL_TOPICS),
234
+ shuffle(lang, ALL_TOPICS),
235
+ )
236
+
237
+
238
+ with gr.Blocks(title="Wikipedia — Did You Know?") as demo:
239
+ gr.Markdown(
240
+ "# 🎲 Wikipedia — Did You Know?\n"
241
+ "Random facts from the "
242
+ "[wikimedia/wiki-dyk](https://huggingface.co/datasets/wikimedia/wiki-dyk) "
243
+ "dataset — the *Did you know* snippets Wikipedia featured on its Main Page."
244
+ )
245
+ with gr.Row():
246
+ lang = gr.Radio(LANGS, value="en", label="Language")
247
+ topic = gr.Dropdown(
248
+ choices=[(ALL_TOPICS, ALL_TOPICS)],
249
+ value=ALL_TOPICS,
250
+ label="Topic",
251
+ filterable=True,
252
+ )
253
+ card = gr.HTML()
254
+ gr.Button("Another fact →", variant="primary").click(shuffle, [lang, topic], card)
255
+
256
+ lang.change(switch_language, lang, [topic, card])
257
+ topic.change(shuffle, [lang, topic], card)
258
+ demo.load(switch_language, lang, [topic, card])
259
+
260
+ if __name__ == "__main__":
261
+ demo.launch()
requirements.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ pandas>=2.2
2
+ pyarrow>=24.0.0
3
+ huggingface_hub>=0.35.0