From a1fad27b0e7bb5eefe5df8940398c3c7fec4f96c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 31 Jul 2025 09:36:07 -0400 Subject: [PATCH] embedded docs/wiki search Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + cli/docs.py | 144 ++++++++++++++ extensions-builtin/sdnext-modernui | 2 +- javascript/changelog.js | 7 - javascript/docs.js | 24 +++ javascript/sdnext.css | 82 +++++++- modules/ui.py | 11 +- modules/ui_docs.py | 306 +++++++++++++++++++++++++---- 8 files changed, 516 insertions(+), 61 deletions(-) create mode 100755 cli/docs.py create mode 100644 javascript/docs.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bfc90863..53082f631 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ in settings -> model options - prompt parser allow explict `BOS` and `EOS` tokens in prompt - **UI** + - new embedded docs/wiki search! - modernui checkbox/radio styling - **Fixes** - fix Wan 2.2-5B I2V workflow diff --git a/cli/docs.py b/cli/docs.py new file mode 100755 index 000000000..1295c3d8b --- /dev/null +++ b/cli/docs.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python +import os +import sys +import time +import logging + + +logging.basicConfig(level = logging.INFO, format = '%(asctime)s %(levelname)s: %(message)s') +log = logging.getLogger(__name__) + + +class Page(): + def __init__(self, fn, full: bool = True): + self.fn = fn + self.title = '' + self.size = 0 + self.mtime = 0 + self.h1 = [] + self.h2 = [] + self.h3 = [] + self.lines = [] + self.read(full=full) + + def read(self, full: bool = True): + try: + self.title = ' ' + os.path.basename(self.fn).replace('.md', '').replace('-', ' ') + ' ' + self.mtime = int(os.path.getmtime(self.fn)) + with open(self.fn, 'r', encoding='utf-8') as f: + content = f.read() + self.size = len(content) + self.lines = [line.strip().lower() + ' ' for line in content.splitlines() if len(line)>1] + self.h1 = [line[1:] for line in self.lines if line.startswith('# ')] + self.h2 = [line[2:] for line in self.lines if line.startswith('## ')] + self.h3 = [line[3:] for line in self.lines if line.startswith('### ')] + if not full: + self.lines.clear() + except Exception as e: + log.error(f'Wiki: page="{self.fn}" {e}') + + def search(self, text): + if not text or len(text) < 2: + return [] + text = text.lower() + if text.strip() == self.title.lower().strip(): + return 1.0 + if self.title.lower().startswith(f'{text} '): + return 0.99 + if f' {text} ' in self.title.lower(): + return 0.98 + if f' {text}' in self.title.lower(): + return 0.97 + + if any(f' {text} ' in h for h in self.h1): + return 0.89 + if any(f' {text}' in h for h in self.h1): + return 0.88 + + if any(f' {text} ' in h for h in self.h2): + return 0.79 + if any(f' {text}' in h for h in self.h2): + return 0.78 + + if any(f' {text} ' in h for h in self.h3): + return 0.69 + if any(f' {text}' in h for h in self.h3): + return 0.68 + + if f'{text}' in self.title.lower(): + return 0.59 + if any(f'{text}' in h for h in self.h1): + return 0.58 + if any(f'{text}' in h for h in self.h2): + return 0.57 + if any(f'{text}' in h for h in self.h3): + return 0.56 + + if any(text in line for line in self.lines): + return 0.50 + + return 0.0 + + def get(self): + try: + with open(self.fn, 'r', encoding='utf-8') as f: + content = f.read() + return content + except Exception as e: + log.error(f'Wiki: page="{self.fn}" {e}') + return '' + + def __str__(self): + return f'Page(title="{self.title.strip()}" fn="{self.fn}" mtime={self.mtime} h1={[h.strip() for h in self.h1]} h2={len(self.h2)} h3={len(self.h3)} lines={len(self.lines)} size={self.size})' + + +class Pages(): + def __init__(self): + self.time = time.time() + self.size = 0 + self.full = None + self.pages: list[Page] = [] + + def build(self, full: bool = True): + self.pages.clear() + self.full = full + with os.scandir('wiki') as entries: + for entry in entries: + if entry.is_file() and entry.name.endswith('.md'): + page = Page(entry.path, full=full) + self.pages.append(page) + self.size = sum(page.size for page in self.pages) + + def search(self, text: str, topk: int = 10, full: bool = True) -> list[Page]: + if not text: + return [] + if len(self.pages) == 0: + self.build(full=full) + text = text.lower() + scores = [page.search(text) for page in self.pages] + mtimes = [page.mtime for page in self.pages] + found = sorted(zip(scores, mtimes, self.pages), key=lambda x: (x[0], x[1]), reverse=True) + found = [item for item in found if item[0] > 0] + return [(item[0], item[2]) for item in found][:topk] + + +index = Pages() + + +if __name__ == "__main__": + sys.argv.pop(0) + if len(sys.argv) < 1: + log.error("Usage: python cli/docs.py ") + text = ' '.join(sys.argv) + topk = 10 + full = True + log.info(f'Search: "{text}" topk={topk}, full={full}') + t0 = time.time() + results = index.search(text, topk=topk, full=full) + t1 = time.time() + log.info(f'Results: pages={len(results)} size={index.size} time={t1-t0:.3f}') + for score, page in results: + log.info(f'Score: {score:.2f} {page}') + # if len(results) > 0: + # log.info('Top result:') + # log.info(results[0][1].get()) diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 9741e151b..008640274 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 9741e151b01dda2d2697c8ca8a369e50482e976e +Subproject commit 008640274856a56ed6901bba77ada14921725445 diff --git a/javascript/changelog.js b/javascript/changelog.js index 09a84382a..4cd9a11c7 100644 --- a/javascript/changelog.js +++ b/javascript/changelog.js @@ -79,10 +79,3 @@ async function initChangelog() { }; search.addEventListener('keyup', searchChangelog); } - -function wikiSearch(txt) { - log('wikiSearch', txt); - const url = `https://github.com/search?q=repo%3Avladmandic%2Fsdnext+${encodeURIComponent(txt)}&type=wikis`; - // window.open(url, '_blank').focus(); - return txt; -} diff --git a/javascript/docs.js b/javascript/docs.js new file mode 100644 index 000000000..d94068b28 --- /dev/null +++ b/javascript/docs.js @@ -0,0 +1,24 @@ +let lastGitHubSearch = ''; +let lastDocsSearch = ''; + +async function clickGitHubWikiPage(page) { + log(`clickGitHubWikiPage: page="${page}"`); + lastGitHubSearch = page; + const el = gradioApp().getElementById('github_md_btn'); + if (el) el.click(); +} + +function getGitHubWikiPage() { + return lastGitHubSearch; +} + +async function clickDocsPage(page) { + log(`clickDocsPage: page="${page}"`); + lastDocsSearch = page; + const el = gradioApp().getElementById('docs_md_btn'); + if (el) el.click(); +} + +function getDocsPage() { + return lastDocsSearch; +} diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 7eacd1978..3398fa6d8 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -1637,7 +1637,7 @@ div:has(>#tab-gallery-folders) { cursor: cell; padding: 8px; background-color: var(--input-background-fill); - border-radius: var(--sd-border-radius); + border-radius: var(--radius-lg); max-width: 100%; } @@ -1649,7 +1649,7 @@ div:has(>#tab-gallery-folders) { display: inline-block; transition: transform 0.2s ease-in-out; flex-shrink: 0; - color: var(--sd-input-text-color); + color: var(--block-title-text-color); } .gallery-separator-name { @@ -1811,6 +1811,84 @@ div:has(>#tab-gallery-folders) { border-top-color: var(--primary-300); } +.docs-search textarea { + height: 1em !important; + resize: none !important +} + +.github-result, #docs_result { + max-height: 38vh; + overflow-y: auto; +} + +.github-result a { + margin: 0; + padding: 0; + background-color: unset !important; +} + +.github-result h3, .github-md h3 { + margin: 0; + padding: 0; + font-size: 1.1em; +} + +.github-page { + background-color: var(--background-fill-primary); + margin: 1em 0 0.2em 0; + border-radius: var(--radius-lg); + padding: 4px; + font-size: 1em; + font-weight: 400; + cursor: help; +} + +.github-result li { + font-size: 0.9em; + display: ruby; + filter: brightness(0.5); +} + +.github-md, .docs-md { + padding: 0.2em; +} + +.docs-results { + background-color: var(--sd-group-background-color); +} + +.docs-card { + margin: 1em 0; + background-color: var(--background-fill-primary); + cursor: help; + padding: 0.5em; +} + +.docs-card-title { + font-size: 1.2em; + line-height: 1.6em; + color: var(--button-primary-background-fill) !important; +} + +.docs-card-h1 { + font-weight: bold; + font-size: 1.0; +} + +.docs-card-h2 { + font-size: 1.0; + max-height: 4em; + overflow: hidden; +} + +.docs-card-footer { + display: flex; + justify-content: space-between; + filter: brightness(0.5); + font-size: 0.9em; + margin-top: 0.2em; +} + @keyframes move { from { background-position-x: 0, -40px; diff --git a/modules/ui.py b/modules/ui.py index f3f942636..d87f02c70 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -123,14 +123,9 @@ def create_ui(startup_timer = None): timer.startup.record("ui-extensions") with gr.Blocks(analytics_enabled=False) as info_interface: - with gr.Tabs(elem_id="tabs_info"): - with gr.TabItem("Change log", id="change_log", elem_id="system_tab_changelog"): - from modules import ui_docs - ui_docs.create_ui_logs() - - with gr.TabItem("Wiki", id="wiki", elem_id="system_tab_wiki"): - from modules import ui_docs - ui_docs.create_ui_wiki() + from modules import ui_docs + ui_docs.create_ui() + timer.startup.record("ui-info") with gr.Blocks(analytics_enabled=False) as extensions_interface: from modules import ui_extensions diff --git a/modules/ui_docs.py b/modules/ui_docs.py index 7f371ab09..1b64f946c 100644 --- a/modules/ui_docs.py +++ b/modules/ui_docs.py @@ -1,5 +1,231 @@ +import os +import time import gradio as gr -from modules import ui_symbols, ui_components, shared +from modules import ui_symbols, ui_components +from installer import install, log + + +class Page(): + def __init__(self, fn, full: bool = True): + self.fn = fn + self.title = '' + self.size = 0 + self.mtime = 0 + self.h1 = [] + self.h2 = [] + self.h3 = [] + self.lines = [] + self.read(full=full) + + def read(self, full: bool = True): + try: + self.title = ' ' + os.path.basename(self.fn).replace('.md', '').replace('-', ' ') + ' ' + self.mtime = time.localtime(os.path.getmtime(self.fn)) + with open(self.fn, 'r', encoding='utf-8') as f: + content = f.read() + self.size = len(content) + self.lines = [line.strip().lower() + ' ' for line in content.splitlines() if len(line)>1] + self.h1 = [line[1:] for line in self.lines if line.startswith('# ')] + self.h2 = [line[2:] for line in self.lines if line.startswith('## ')] + self.h3 = [line[3:] for line in self.lines if line.startswith('### ')] + if not full: + self.lines.clear() + except Exception as e: + log.error(f'Search docs: page="{self.fn}" {e}') + + def search(self, text): + if not text or len(text) < 2: + return [] + text = text.lower() + if text.strip() == self.title.lower().strip(): + return 1.0 + if self.title.lower().startswith(f'{text} '): + return 0.99 + if f' {text} ' in self.title.lower(): + return 0.98 + if f' {text}' in self.title.lower(): + return 0.97 + + if any(f' {text} ' in h for h in self.h1): + return 0.89 + if any(f' {text}' in h for h in self.h1): + return 0.88 + + if any(f' {text} ' in h for h in self.h2): + return 0.79 + if any(f' {text}' in h for h in self.h2): + return 0.78 + + if any(f' {text} ' in h for h in self.h3): + return 0.69 + if any(f' {text}' in h for h in self.h3): + return 0.68 + + if f'{text}' in self.title.lower(): + return 0.59 + if any(f'{text}' in h for h in self.h1): + return 0.58 + if any(f'{text}' in h for h in self.h2): + return 0.57 + if any(f'{text}' in h for h in self.h3): + return 0.56 + + if any(text in line for line in self.lines): + return 0.50 + + return 0.0 + + def get(self): + if self.fn is None or not os.path.exists(self.fn): + log.error(f'Search docs: page="{self.fn}" does not exist') + return f'page="{self.fn}" does not exist' + try: + with open(self.fn, 'r', encoding='utf-8') as f: + content = f.read() + return content + except Exception as e: + log.error(f'Search docs: page="{self.fn}" {e}') + return '' + + def __str__(self): + return f'Page(title="{self.title.strip()}" fn="{self.fn}" mtime={self.mtime} h1={[h.strip() for h in self.h1]} h2={len(self.h2)} h3={len(self.h3)} lines={len(self.lines)} size={self.size})' + + +class Pages(): + def __init__(self): + self.time = time.time() + self.size = 0 + self.full = None + self.pages: list[Page] = [] + + def build(self, full: bool = True): + self.pages.clear() + self.full = full + with os.scandir('wiki') as entries: + for entry in entries: + if entry.is_file() and entry.name.endswith('.md'): + page = Page(entry.path, full=full) + self.pages.append(page) + self.size = sum(page.size for page in self.pages) + + def search(self, text: str, topk: int = 10, full: bool = True) -> list[Page]: + if not text or len(text) < 2: + return [] + if len(self.pages) == 0: + self.build(full=full) + try: + text = text.lower() + scores = [page.search(text) for page in self.pages] + mtimes = [page.mtime for page in self.pages] + found = sorted(zip(scores, mtimes, self.pages), key=lambda x: (x[0], x[1]), reverse=True) + found = [item for item in found if item[0] > 0] + return [(item[0], item[2]) for item in found][:topk] + except Exception as e: + log.error(f'Search docs: text="{text}" {e}') + return [] + + def get(self, title: str) -> Page: + if len(self.pages) == 0: + self.build(full=self.full) + for page in self.pages: + if page.title.lower().strip() == title.lower().strip(): + return page + return Page('') + + +index = Pages() + + +def get_docs_page(page_title: str) -> str: + if len(index.pages) == 0: + index.build(full=True) + page = index.get(page_title) + log.debug(f'Search docs: title="{page_title}" {page}') + content = page.get() + return content + + +def search_html(pages: list[Page]) -> str: + html = '' + for score, page in pages: + if score > 0.0: + html += f''' +
+
{page.title.strip()}
+
Heading | {' | '.join([h.strip() for h in page.h1])}
+
Topics | {' | '.join([h.strip() for h in page.h2])}
+
+ Score | {score} + Last modified | {time.strftime('%c', page.mtime)} +
+
''' + return html + + +def search_docs(search_term): + topk = 10 + full = True + t0 = time.time() + results = index.search(search_term, topk=topk, full=full) + t1 = time.time() + log.debug(f'Search results: search="{search_term}" topk={topk}, full={full} pages={len(results)} size={index.size} time={t1-t0:.3f}') + for score, page in results: + log.trace(f'Search results: score={score:.2f} {page}') + html = search_html(results) + return html + + +def get_github_page(page): + try: + with open(os.path.join('wiki', f'{page}.md'), 'r', encoding='utf-8') as f: + content = f.read() + log.debug(f'Search wiki: page="{page}" size={len(content)}') + except Exception as e: + log.error(f'Search wiki: page="{page}" {e}') + content = f'Error: {e}' + return content + + +def search_github(search_term): + import requests + from urllib.parse import quote + install('beautifulsoup4') + from bs4 import BeautifulSoup + + url = f'https://github.com/search?q=repo%3Avladmandic%2Fsdnext+{quote(search_term)}&type=wikis' + res = requests.get(url, timeout=10) + pages = [] + if res.status_code == 200: + html = res.content + soup = BeautifulSoup(html, 'html.parser') + + # remove header links + tags = soup.find_all(attrs={"data-hovercard-url": "/vladmandic/sdnext/hovercard"}) + for tag in tags: + tag.extract() + + # replace relative links with full links + tags = soup.find_all('a') + for tag in tags: + if tag.has_attr('href'): + if tag['href'].startswith('/vladmandic/sdnext/wiki/'): + page = tag['href'].replace('/vladmandic/sdnext/wiki/', '') + tag.name = 'div' + tag['class'] = 'github-page' + tag['onclick'] = f'clickGitHubWikiPage("{page}")' + pages.append(page) + elif tag['href'].startswith('/'): + tag['href'] = 'https://github.com' + tag['href'] + + # find result only + result = soup.find(attrs={"data-testid": "results-list"}) + if result is None: + return 'No results found' + html = str(result) + else: + html = f'Error: {res.status_code}' + log.debug(f'Search wiki: code={res.status_code} text="{search_term}" pages={pages}') + return html def create_ui_logs(): @@ -12,53 +238,47 @@ def create_ui_logs(): with gr.Column(): get_changelog_btn = gr.Button(value='Get Changelog', elem_id="get_changelog") with gr.Column(): - _changelog_search = gr.Textbox(label="Search Changelog", elem_id="changelog_search") + _changelog_search = gr.Textbox(label="Search Changelog", elem_id="changelog_search", elem_classes="docs-search") _changelog_result = gr.HTML(elem_id="changelog_result") changelog_markdown = gr.Markdown('', elem_id="changelog_markdown") get_changelog_btn.click(fn=get_changelog, outputs=[changelog_markdown], show_progress=True) -def create_ui_wiki(): - def search_github(search_term): - import requests - from urllib.parse import quote - from installer import install - - install('beautifulsoup4') - from bs4 import BeautifulSoup - - url = f'https://github.com/search?q=repo%3Avladmandic%2Fsdnext+{quote(search_term)}&type=wikis' - res = requests.get(url, timeout=10) - shared.log.debug(f'Search: wiki="{search_term}" code={res.status_code}') - if res.status_code == 200: - html = res.content - soup = BeautifulSoup(html, 'html.parser') - - # remove header links - tags = soup.find_all(attrs={"data-hovercard-url": "/vladmandic/sdnext/hovercard"}) - for tag in tags: - tag.extract() - - # replace relative links with full links - tags = soup.find_all('a') - for tag in tags: - if tag.has_attr('href') and tag['href'].startswith('/'): - tag['href'] = 'https://github.com' + tag['href'] - - # find result only - result = soup.find(attrs={"data-testid": "results-list"}) - if result is None: - return 'No results found' - html = str(result) - return html - else: - return f'Error: {res.status_code}' - +def create_ui_github(): with gr.Row(): - wiki_search = gr.Textbox(label="Search Wiki Pages", elem_id="wiki_search") - wiki_search_btn = ui_components.ToolButton(value=ui_symbols.search, elem_id="wiki_search_btn") + github_search = gr.Textbox(label="Search GitHub Wiki Pages", elem_id="github_search", elem_classes="docs-search") + github_search_btn = ui_components.ToolButton(value=ui_symbols.search, elem_id="github_search_btn") with gr.Row(): - wiki_result = gr.HTML(elem_id="wiki_result", value='') - wiki_search.submit(_js="wikiSearch", fn=search_github, inputs=[wiki_search], outputs=[wiki_result]) - wiki_search_btn.click(_js="wikiSearch", fn=search_github, inputs=[wiki_search], outputs=[wiki_result]) + github_result = gr.HTML(elem_id="github_result", value='', elem_classes="github-result") + with gr.Row(): + github_md_btn = gr.Button(value='html2md', elem_id="github_md_btn", visible=False) + github_md = gr.Markdown(elem_id="github_md", value='', elem_classes="github-md") + github_search.submit(fn=search_github, inputs=[github_search], outputs=[github_result], show_progress=True) + github_search_btn.click(fn=search_github, inputs=[github_search], outputs=[github_result], show_progress=True) + github_md_btn.click(fn=get_github_page, _js='getGitHubWikiPage', inputs=[github_search], outputs=[github_md], show_progress=True) + + +def create_ui_docs(): + with gr.Row(): + docs_search = gr.Textbox(label="Search Docs", elem_id="github_search", elem_classes="docs-search") + docs_search_btn = ui_components.ToolButton(value=ui_symbols.search, elem_id="github_search_btn") + with gr.Row(): + docs_result = gr.HTML(elem_id="docs_result", value='', elem_classes="docs-result") + with gr.Row(): + docs_md_btn = gr.Button(value='html2md', elem_id="docs_md_btn", visible=False) + docs_md = gr.Markdown(elem_id="docs_md", value='', elem_classes="docs-md") + docs_search.submit(fn=search_docs, inputs=[docs_search], outputs=[docs_result], show_progress=False) + docs_search.change(fn=search_docs, inputs=[docs_search], outputs=[docs_result], show_progress=False) + docs_search_btn.click(fn=search_docs, inputs=[docs_search], outputs=[docs_result], show_progress=False) + docs_md_btn.click(fn=get_docs_page, _js='getDocsPage', inputs=[docs_search], outputs=[docs_md], show_progress=False) + + +def create_ui(): + with gr.Tabs(elem_id="tabs_info"): + with gr.TabItem("Docs", id="docs", elem_id="system_tab_docs"): + create_ui_docs() + with gr.TabItem("Wiki", id="wiki", elem_id="system_tab_wiki"): + create_ui_github() + with gr.TabItem("Change log", id="change_log", elem_id="system_tab_changelog"): + create_ui_logs()