embedded docs/wiki search

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -7,6 +7,7 @@
     in settings -> model options  
   - prompt parser allow explict `BOS` and `EOS` tokens in prompt  
 - **UI**  
+  - new embedded docs/wiki search!  
   - modernui checkbox/radio styling  
 - **Fixes**  
   - fix Wan 2.2-5B I2V workflow  

cli/docs.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+import os
+import sys
+import time
+import logging
+
+
+logging.basicConfig(level = logging.INFO, format = '%(asctime)s %(levelname)s: %(message)s')
+log = logging.getLogger(__name__)
+
+
+class Page():
+    def __init__(self, fn, full: bool = True):
+        self.fn = fn
+        self.title = ''
+        self.size = 0
+        self.mtime = 0
+        self.h1 = []
+        self.h2 = []
+        self.h3 = []
+        self.lines = []
+        self.read(full=full)
+
+    def read(self, full: bool = True):
+        try:
+            self.title = ' ' + os.path.basename(self.fn).replace('.md', '').replace('-', ' ') + ' '
+            self.mtime = int(os.path.getmtime(self.fn))
+            with open(self.fn, 'r', encoding='utf-8') as f:
+                content = f.read()
+            self.size = len(content)
+            self.lines = [line.strip().lower() + ' ' for line in content.splitlines() if len(line)>1]
+            self.h1 = [line[1:] for line in self.lines if line.startswith('# ')]
+            self.h2 = [line[2:] for line in self.lines if line.startswith('## ')]
+            self.h3 = [line[3:] for line in self.lines if line.startswith('### ')]
+            if not full:
+                self.lines.clear()
+        except Exception as e:
+            log.error(f'Wiki: page="{self.fn}" {e}')
+
+    def search(self, text):
+        if not text or len(text) < 2:
+            return []
+        text = text.lower()
+        if text.strip() == self.title.lower().strip():
+            return 1.0
+        if self.title.lower().startswith(f'{text} '):
+            return 0.99
+        if f' {text} ' in self.title.lower():
+            return 0.98
+        if f' {text}' in self.title.lower():
+            return 0.97
+
+        if any(f' {text} ' in h for h in self.h1):
+            return 0.89
+        if any(f' {text}' in h for h in self.h1):
+            return 0.88
+
+        if any(f' {text} ' in h for h in self.h2):
+            return 0.79
+        if any(f' {text}' in h for h in self.h2):
+            return 0.78
+
+        if any(f' {text} ' in h for h in self.h3):
+            return 0.69
+        if any(f' {text}' in h for h in self.h3):
+            return 0.68
+
+        if f'{text}' in self.title.lower():
+            return 0.59
+        if any(f'{text}' in h for h in self.h1):
+            return 0.58
+        if any(f'{text}' in h for h in self.h2):
+            return 0.57
+        if any(f'{text}' in h for h in self.h3):
+            return 0.56
+
+        if any(text in line for line in self.lines):
+            return 0.50
+
+        return 0.0
+    
+    def get(self):
+        try:
+            with open(self.fn, 'r', encoding='utf-8') as f:
+                content = f.read()
+                return content
+        except Exception as e:
+            log.error(f'Wiki: page="{self.fn}" {e}')
+        return ''
+
+    def __str__(self):
+        return f'Page(title="{self.title.strip()}" fn="{self.fn}" mtime={self.mtime} h1={[h.strip() for h in self.h1]} h2={len(self.h2)} h3={len(self.h3)} lines={len(self.lines)} size={self.size})'
+
+
+class Pages():
+    def __init__(self):
+        self.time = time.time()
+        self.size = 0
+        self.full = None
+        self.pages: list[Page] = []
+
+    def build(self, full: bool = True):
+        self.pages.clear()
+        self.full = full
+        with os.scandir('wiki') as entries:
+            for entry in entries:
+                if entry.is_file() and entry.name.endswith('.md'):
+                    page = Page(entry.path, full=full)
+                    self.pages.append(page)
+        self.size = sum(page.size for page in self.pages)
+
+    def search(self, text: str, topk: int = 10, full: bool = True) -> list[Page]:
+        if not text:
+            return []
+        if len(self.pages) == 0:
+            self.build(full=full)
+        text = text.lower()
+        scores = [page.search(text) for page in self.pages]
+        mtimes = [page.mtime for page in self.pages]
+        found = sorted(zip(scores, mtimes, self.pages), key=lambda x: (x[0], x[1]), reverse=True)
+        found = [item for item in found if item[0] > 0]
+        return [(item[0], item[2]) for item in found][:topk]
+
+
+index = Pages()
+
+
+if __name__ == "__main__":
+    sys.argv.pop(0)
+    if len(sys.argv) < 1:
+        log.error("Usage: python cli/docs.py <search_term>")
+    text = ' '.join(sys.argv)
+    topk = 10
+    full = True
+    log.info(f'Search: "{text}" topk={topk}, full={full}')
+    t0 = time.time()
+    results = index.search(text, topk=topk, full=full)
+    t1 = time.time()
+    log.info(f'Results: pages={len(results)} size={index.size} time={t1-t0:.3f}')
+    for score, page in results:
+        log.info(f'Score: {score:.2f} {page}')
+    # if len(results) > 0:
+    #     log.info('Top result:')
+    #     log.info(results[0][1].get())

javascript/changelog.js

@@ -79,10 +79,3 @@ async function initChangelog() {
   };
   search.addEventListener('keyup', searchChangelog);
 }
-
-function wikiSearch(txt) {
-  log('wikiSearch', txt);
-  const url = `https://github.com/search?q=repo%3Avladmandic%2Fsdnext+${encodeURIComponent(txt)}&type=wikis`;
-  // window.open(url, '_blank').focus();
-  return txt;
-}

javascript/docs.js

@@ -0,0 +1,24 @@
+let lastGitHubSearch = '';
+let lastDocsSearch = '';
+
+async function clickGitHubWikiPage(page) {
+  log(`clickGitHubWikiPage: page="${page}"`);
+  lastGitHubSearch = page;
+  const el = gradioApp().getElementById('github_md_btn');
+  if (el) el.click();
+}
+
+function getGitHubWikiPage() {
+  return lastGitHubSearch;
+}
+
+async function clickDocsPage(page) {
+  log(`clickDocsPage: page="${page}"`);
+  lastDocsSearch = page;
+  const el = gradioApp().getElementById('docs_md_btn');
+  if (el) el.click();
+}
+
+function getDocsPage() {
+  return lastDocsSearch;
+}

javascript/sdnext.css

@@ -1637,7 +1637,7 @@ div:has(>#tab-gallery-folders) {
   cursor: cell;
   padding: 8px;
   background-color: var(--input-background-fill);
-  border-radius: var(--sd-border-radius);
+  border-radius: var(--radius-lg);
   max-width: 100%;
 }
 
@@ -1649,7 +1649,7 @@ div:has(>#tab-gallery-folders) {
   display: inline-block;
   transition: transform 0.2s ease-in-out;
   flex-shrink: 0;
-  color: var(--sd-input-text-color);
+  color: var(--block-title-text-color);
 }
 
 .gallery-separator-name {
@@ -1811,6 +1811,84 @@ div:has(>#tab-gallery-folders) {
   border-top-color: var(--primary-300);
 }
 
+.docs-search textarea {
+  height: 1em !important;
+  resize: none !important
+}
+
+.github-result, #docs_result {
+  max-height: 38vh;
+  overflow-y: auto;
+}
+
+.github-result a {
+  margin: 0;
+  padding: 0;
+  background-color: unset !important;
+}
+
+.github-result h3, .github-md h3 {
+  margin: 0;
+  padding: 0;
+  font-size: 1.1em;
+}
+
+.github-page {
+  background-color: var(--background-fill-primary);
+  margin: 1em 0 0.2em 0;
+  border-radius: var(--radius-lg);
+  padding: 4px;
+  font-size: 1em;
+  font-weight: 400;
+  cursor: help;
+}
+
+.github-result li {
+  font-size: 0.9em;
+  display: ruby;
+  filter: brightness(0.5);
+}
+
+.github-md, .docs-md {
+  padding: 0.2em;
+}
+
+.docs-results {
+  background-color: var(--sd-group-background-color);
+}
+
+.docs-card {
+  margin: 1em 0;
+  background-color: var(--background-fill-primary);
+  cursor: help;
+  padding: 0.5em;
+}
+
+.docs-card-title {
+  font-size: 1.2em;
+  line-height: 1.6em;
+  color: var(--button-primary-background-fill) !important;
+}
+
+.docs-card-h1 {
+  font-weight: bold;
+  font-size: 1.0;
+}
+
+.docs-card-h2 {
+  font-size: 1.0;
+  max-height: 4em;
+  overflow: hidden;
+}
+
+.docs-card-footer {
+  display: flex;
+  justify-content: space-between;
+  filter: brightness(0.5);
+  font-size: 0.9em;
+  margin-top: 0.2em;
+}
+
 @keyframes move {
   from {
     background-position-x: 0, -40px;

modules/ui.py

@@ -123,14 +123,9 @@ def create_ui(startup_timer = None):
         timer.startup.record("ui-extensions")
 
     with gr.Blocks(analytics_enabled=False) as info_interface:
-        with gr.Tabs(elem_id="tabs_info"):
-            with gr.TabItem("Change log", id="change_log", elem_id="system_tab_changelog"):
-                from modules import ui_docs
-                ui_docs.create_ui_logs()
-
-            with gr.TabItem("Wiki", id="wiki", elem_id="system_tab_wiki"):
-                from modules import ui_docs
-                ui_docs.create_ui_wiki()
+        from modules import ui_docs
+        ui_docs.create_ui()
+        timer.startup.record("ui-info")
 
     with gr.Blocks(analytics_enabled=False) as extensions_interface:
         from modules import ui_extensions

modules/ui_docs.py

@@ -1,5 +1,231 @@
+import os
+import time
 import gradio as gr
-from modules import ui_symbols, ui_components, shared
+from modules import ui_symbols, ui_components
+from installer import install, log
+
+
+class Page():
+    def __init__(self, fn, full: bool = True):
+        self.fn = fn
+        self.title = ''
+        self.size = 0
+        self.mtime = 0
+        self.h1 = []
+        self.h2 = []
+        self.h3 = []
+        self.lines = []
+        self.read(full=full)
+
+    def read(self, full: bool = True):
+        try:
+            self.title = ' ' + os.path.basename(self.fn).replace('.md', '').replace('-', ' ') + ' '
+            self.mtime = time.localtime(os.path.getmtime(self.fn))
+            with open(self.fn, 'r', encoding='utf-8') as f:
+                content = f.read()
+            self.size = len(content)
+            self.lines = [line.strip().lower() + ' ' for line in content.splitlines() if len(line)>1]
+            self.h1 = [line[1:] for line in self.lines if line.startswith('# ')]
+            self.h2 = [line[2:] for line in self.lines if line.startswith('## ')]
+            self.h3 = [line[3:] for line in self.lines if line.startswith('### ')]
+            if not full:
+                self.lines.clear()
+        except Exception as e:
+            log.error(f'Search docs: page="{self.fn}" {e}')
+
+    def search(self, text):
+        if not text or len(text) < 2:
+            return []
+        text = text.lower()
+        if text.strip() == self.title.lower().strip():
+            return 1.0
+        if self.title.lower().startswith(f'{text} '):
+            return 0.99
+        if f' {text} ' in self.title.lower():
+            return 0.98
+        if f' {text}' in self.title.lower():
+            return 0.97
+
+        if any(f' {text} ' in h for h in self.h1):
+            return 0.89
+        if any(f' {text}' in h for h in self.h1):
+            return 0.88
+
+        if any(f' {text} ' in h for h in self.h2):
+            return 0.79
+        if any(f' {text}' in h for h in self.h2):
+            return 0.78
+
+        if any(f' {text} ' in h for h in self.h3):
+            return 0.69
+        if any(f' {text}' in h for h in self.h3):
+            return 0.68
+
+        if f'{text}' in self.title.lower():
+            return 0.59
+        if any(f'{text}' in h for h in self.h1):
+            return 0.58
+        if any(f'{text}' in h for h in self.h2):
+            return 0.57
+        if any(f'{text}' in h for h in self.h3):
+            return 0.56
+
+        if any(text in line for line in self.lines):
+            return 0.50
+
+        return 0.0
+
+    def get(self):
+        if self.fn is None or not os.path.exists(self.fn):
+            log.error(f'Search docs: page="{self.fn}" does not exist')
+            return f'page="{self.fn}" does not exist'
+        try:
+            with open(self.fn, 'r', encoding='utf-8') as f:
+                content = f.read()
+                return content
+        except Exception as e:
+            log.error(f'Search docs: page="{self.fn}" {e}')
+        return ''
+
+    def __str__(self):
+        return f'Page(title="{self.title.strip()}" fn="{self.fn}" mtime={self.mtime} h1={[h.strip() for h in self.h1]} h2={len(self.h2)} h3={len(self.h3)} lines={len(self.lines)} size={self.size})'
+
+
+class Pages():
+    def __init__(self):
+        self.time = time.time()
+        self.size = 0
+        self.full = None
+        self.pages: list[Page] = []
+
+    def build(self, full: bool = True):
+        self.pages.clear()
+        self.full = full
+        with os.scandir('wiki') as entries:
+            for entry in entries:
+                if entry.is_file() and entry.name.endswith('.md'):
+                    page = Page(entry.path, full=full)
+                    self.pages.append(page)
+        self.size = sum(page.size for page in self.pages)
+
+    def search(self, text: str, topk: int = 10, full: bool = True) -> list[Page]:
+        if not text or len(text) < 2:
+            return []
+        if len(self.pages) == 0:
+            self.build(full=full)
+        try:
+            text = text.lower()
+            scores = [page.search(text) for page in self.pages]
+            mtimes = [page.mtime for page in self.pages]
+            found = sorted(zip(scores, mtimes, self.pages), key=lambda x: (x[0], x[1]), reverse=True)
+            found = [item for item in found if item[0] > 0]
+            return [(item[0], item[2]) for item in found][:topk]
+        except Exception as e:
+            log.error(f'Search docs: text="{text}" {e}')
+            return []
+
+    def get(self, title: str) -> Page:
+        if len(self.pages) == 0:
+            self.build(full=self.full)
+        for page in self.pages:
+            if page.title.lower().strip() == title.lower().strip():
+                return page
+        return Page('')
+
+
+index = Pages()
+
+
+def get_docs_page(page_title: str) -> str:
+    if len(index.pages) == 0:
+        index.build(full=True)
+    page = index.get(page_title)
+    log.debug(f'Search docs: title="{page_title}" {page}')
+    content = page.get()
+    return content
+
+
+def search_html(pages: list[Page]) -> str:
+    html = ''
+    for score, page in pages:
+        if score > 0.0:
+            html += f'''
+                <div class="docs-card" onclick="clickDocsPage('{page.title}')">
+                    <div class="docs-card-title">{page.title.strip()}</div>
+                    <div class="docs-card-h1">Heading | {' | '.join([h.strip() for h in page.h1])}</div>
+                    <div class="docs-card-h2"><b>Topics</b> | {' | '.join([h.strip() for h in page.h2])}</div>
+                    <div class="docs-card-footer">
+                        <span class="docs-card-score">Score | {score}</span>
+                        <span class="docs-card-mtime">Last modified | {time.strftime('%c', page.mtime)}</span>
+                    </div>
+                </div>'''
+    return html
+
+
+def search_docs(search_term):
+    topk = 10
+    full = True
+    t0 = time.time()
+    results = index.search(search_term, topk=topk, full=full)
+    t1 = time.time()
+    log.debug(f'Search results: search="{search_term}" topk={topk}, full={full} pages={len(results)} size={index.size} time={t1-t0:.3f}')
+    for score, page in results:
+        log.trace(f'Search results: score={score:.2f} {page}')
+    html = search_html(results)
+    return html
+
+
+def get_github_page(page):
+    try:
+        with open(os.path.join('wiki', f'{page}.md'), 'r', encoding='utf-8') as f:
+            content = f.read()
+            log.debug(f'Search wiki: page="{page}" size={len(content)}')
+    except Exception as e:
+        log.error(f'Search wiki: page="{page}" {e}')
+        content = f'Error: {e}'
+    return content
+
+
+def search_github(search_term):
+    import requests
+    from urllib.parse import quote
+    install('beautifulsoup4')
+    from bs4 import BeautifulSoup
+
+    url = f'https://github.com/search?q=repo%3Avladmandic%2Fsdnext+{quote(search_term)}&type=wikis'
+    res = requests.get(url, timeout=10)
+    pages = []
+    if res.status_code == 200:
+        html = res.content
+        soup = BeautifulSoup(html, 'html.parser')
+
+        # remove header links
+        tags = soup.find_all(attrs={"data-hovercard-url": "/vladmandic/sdnext/hovercard"})
+        for tag in tags:
+            tag.extract()
+
+        # replace relative links with full links
+        tags = soup.find_all('a')
+        for tag in tags:
+            if tag.has_attr('href'):
+                if tag['href'].startswith('/vladmandic/sdnext/wiki/'):
+                    page = tag['href'].replace('/vladmandic/sdnext/wiki/', '')
+                    tag.name = 'div'
+                    tag['class'] = 'github-page'
+                    tag['onclick'] = f'clickGitHubWikiPage("{page}")'
+                    pages.append(page)
+                elif tag['href'].startswith('/'):
+                    tag['href'] = 'https://github.com' + tag['href']
+
+        # find result only
+        result = soup.find(attrs={"data-testid": "results-list"})
+        if result is None:
+            return 'No results found'
+        html = str(result)
+    else:
+        html = f'Error: {res.status_code}'
+    log.debug(f'Search wiki: code={res.status_code} text="{search_term}" pages={pages}')
+    return html
 
 
 def create_ui_logs():
@@ -12,53 +238,47 @@ def create_ui_logs():
     with gr.Column():
         get_changelog_btn = gr.Button(value='Get Changelog', elem_id="get_changelog")
     with gr.Column():
-        _changelog_search = gr.Textbox(label="Search Changelog", elem_id="changelog_search")
+        _changelog_search = gr.Textbox(label="Search Changelog", elem_id="changelog_search", elem_classes="docs-search")
         _changelog_result = gr.HTML(elem_id="changelog_result")
 
     changelog_markdown = gr.Markdown('', elem_id="changelog_markdown")
     get_changelog_btn.click(fn=get_changelog, outputs=[changelog_markdown], show_progress=True)
 
 
-def create_ui_wiki():
-    def search_github(search_term):
-        import requests
-        from urllib.parse import quote
-        from installer import install
-
-        install('beautifulsoup4')
-        from bs4 import BeautifulSoup
-
-        url = f'https://github.com/search?q=repo%3Avladmandic%2Fsdnext+{quote(search_term)}&type=wikis'
-        res = requests.get(url, timeout=10)
-        shared.log.debug(f'Search: wiki="{search_term}" code={res.status_code}')
-        if res.status_code == 200:
-            html = res.content
-            soup = BeautifulSoup(html, 'html.parser')
-
-            # remove header links
-            tags = soup.find_all(attrs={"data-hovercard-url": "/vladmandic/sdnext/hovercard"})
-            for tag in tags:
-                tag.extract()
-
-            # replace relative links with full links
-            tags = soup.find_all('a')
-            for tag in tags:
-                if tag.has_attr('href') and tag['href'].startswith('/'):
-                    tag['href'] = 'https://github.com' + tag['href']
-
-            # find result only
-            result = soup.find(attrs={"data-testid": "results-list"})
-            if result is None:
-                return 'No results found'
-            html = str(result)
-            return html
-        else:
-            return f'Error: {res.status_code}'
-
+def create_ui_github():
     with gr.Row():
-        wiki_search = gr.Textbox(label="Search Wiki Pages", elem_id="wiki_search")
-        wiki_search_btn = ui_components.ToolButton(value=ui_symbols.search, elem_id="wiki_search_btn")
+        github_search = gr.Textbox(label="Search GitHub Wiki Pages", elem_id="github_search", elem_classes="docs-search")
+        github_search_btn = ui_components.ToolButton(value=ui_symbols.search, elem_id="github_search_btn")
     with gr.Row():
-        wiki_result = gr.HTML(elem_id="wiki_result", value='')
-    wiki_search.submit(_js="wikiSearch", fn=search_github, inputs=[wiki_search], outputs=[wiki_result])
-    wiki_search_btn.click(_js="wikiSearch", fn=search_github, inputs=[wiki_search], outputs=[wiki_result])
+        github_result = gr.HTML(elem_id="github_result", value='', elem_classes="github-result")
+    with gr.Row():
+        github_md_btn = gr.Button(value='html2md', elem_id="github_md_btn", visible=False)
+        github_md = gr.Markdown(elem_id="github_md", value='', elem_classes="github-md")
+    github_search.submit(fn=search_github, inputs=[github_search], outputs=[github_result], show_progress=True)
+    github_search_btn.click(fn=search_github, inputs=[github_search], outputs=[github_result], show_progress=True)
+    github_md_btn.click(fn=get_github_page, _js='getGitHubWikiPage', inputs=[github_search], outputs=[github_md], show_progress=True)
+
+
+def create_ui_docs():
+    with gr.Row():
+        docs_search = gr.Textbox(label="Search Docs", elem_id="github_search", elem_classes="docs-search")
+        docs_search_btn = ui_components.ToolButton(value=ui_symbols.search, elem_id="github_search_btn")
+    with gr.Row():
+        docs_result = gr.HTML(elem_id="docs_result", value='', elem_classes="docs-result")
+    with gr.Row():
+        docs_md_btn = gr.Button(value='html2md', elem_id="docs_md_btn", visible=False)
+        docs_md = gr.Markdown(elem_id="docs_md", value='', elem_classes="docs-md")
+    docs_search.submit(fn=search_docs, inputs=[docs_search], outputs=[docs_result], show_progress=False)
+    docs_search.change(fn=search_docs, inputs=[docs_search], outputs=[docs_result], show_progress=False)
+    docs_search_btn.click(fn=search_docs, inputs=[docs_search], outputs=[docs_result], show_progress=False)
+    docs_md_btn.click(fn=get_docs_page, _js='getDocsPage', inputs=[docs_search], outputs=[docs_md], show_progress=False)
+
+
+def create_ui():
+    with gr.Tabs(elem_id="tabs_info"):
+        with gr.TabItem("Docs", id="docs", elem_id="system_tab_docs"):
+            create_ui_docs()
+        with gr.TabItem("Wiki", id="wiki", elem_id="system_tab_wiki"):
+            create_ui_github()
+        with gr.TabItem("Change log", id="change_log", elem_id="system_tab_changelog"):
+            create_ui_logs()