Skip to content

Commit fed9a0b

Browse files
iwilligiwillig
authored
Merge pull request #7 from iwillig/ivan/improve-search-results
Improve search results
2 parents c25cd36 + 9c028db commit fed9a0b

File tree

5 files changed

+434
-624
lines changed

5 files changed

+434
-624
lines changed

‎bb.edn‎

Lines changed: 0 additions & 271 deletions
Original file line numberDiff line numberDiff line change
@@ -106,119 +106,6 @@
106106
"Remove ANSI color codes from string"
107107
(str/replace s (re-pattern "\033\\[[0-9;]*m") "")))
108108

109-
;; Build a specific prompt file by name
110-
build
111-
{:doc "Build a specific prompt file by name (without .md extension)\nUsage: bb build clojure_skill_builder"
112-
:task (let [name (require-arg "name")
113-
output-file (str "_build/" name ".md")]
114-
115-
(info-msg (str "Building " output-file "..."))
116-
(fs/create-dirs "_build")
117-
118-
;; Use CLI to generate prompt content
119-
(let [result (p/shell {:out :string
120-
:err :string
121-
:continue true}
122-
(str "bb main prompt show " name))]
123-
(if (zero? (:exit result))
124-
(let [content (:out result)
125-
cleaned (strip-ansi content)]
126-
(spit output-file cleaned)
127-
(success-msg (str "Build successful: " output-file)))
128-
(do
129-
(error-msg (str "Failed to build prompt: " name))
130-
(println (:err result))
131-
(System/exit 1)))))}
132-
133-
;; Build all prompt files
134-
build-all
135-
{:doc "Build all prompt files"
136-
:task (do
137-
(info-msg "Building all prompts...")
138-
139-
;; Build clojure_skill_builder
140-
(shell "bb build clojure_skill_builder")
141-
142-
;; Build clojure_build
143-
(shell "bb build clojure_build")
144-
(shell "bb build code_archivist")
145-
(shell "bb build plan_executor")
146-
147-
(shell "bb build shortcut_cli_builder")
148-
(shell "bb build software_alchemist_build")
149-
150-
(success-msg "All builds complete!"))}
151-
152-
;; List built prompts with token counts
153-
list-prompts
154-
{:doc "List all built prompts with file sizes and token counts"
155-
:task (let [build-dir "_build"]
156-
157-
(if-not (fs/exists? build-dir)
158-
(do
159-
(warning-msg "No _build directory found. Run 'bb build-all' first.")
160-
(System/exit 1))
161-
162-
(let [prompt-files (sort (filter (fn [f]
163-
(str/ends-with? (str f) ".md"))
164-
(fs/list-dir build-dir)))
165-
166-
;; Collect data for each prompt with metadata
167-
prompts (for [file prompt-files]
168-
(let [file-path (str file)
169-
filename (fs/file-name file)
170-
base-name (-> filename
171-
(str/replace ".compressed.md" "")
172-
(str/replace ".md" ""))
173-
source-prompt (str "prompts/" base-name ".md")
174-
metadata (if (fs/exists? source-prompt)
175-
(extract-metadata-with-pandoc source-prompt)
176-
(extract-metadata-with-pandoc file-path))
177-
content (slurp file-path)
178-
file-size (fs/size file)
179-
char-count (count content)
180-
token-count (estimate-tokens content)
181-
prompt-name (or (get metadata :name)
182-
(get metadata :title)
183-
base-name)
184-
description (strip-html-tags (get metadata :description ""))]
185-
{:filename filename
186-
:name prompt-name
187-
:description description
188-
:file file-path
189-
:size file-size
190-
:size-str (format-size file-size)
191-
:chars char-count
192-
:tokens (long token-count)}))
193-
194-
;; Calculate totals
195-
total-size (reduce + (map :size prompts))
196-
total-chars (reduce + (map :chars prompts))
197-
total-tokens (reduce + (map :tokens prompts))]
198-
199-
(info-msg "Built Prompts")
200-
201-
(println)
202-
(table/print-table [:filename :name :description :size :chars :tokens]
203-
(concat
204-
(for [p prompts]
205-
{:filename (:filename p)
206-
:name (:name p)
207-
:description (truncate-string (:description p) 60)
208-
:size (:size-str p)
209-
:chars (format-number (:chars p))
210-
:tokens (format-number (:tokens p))})
211-
[{:filename "TOTAL"
212-
:name ""
213-
:description ""
214-
:size (format-size total-size)
215-
:chars (format-number total-chars)
216-
:tokens (format-number total-tokens)}]))
217-
218-
(println)
219-
(println (bling/bling [:dim "Note: Token count is estimated (~4 chars per token)"]))
220-
(println (bling/bling [:dim " Anthropic's Claude models use this approximation."])))))}
221-
222109
;; Clean all build artifacts and temporary files
223110
clean
224111
{:doc "Clean all build artifacts and temporary files"
@@ -406,164 +293,6 @@
406293
(run 'test)
407294
(success-msg "CI pipeline completed successfully"))}
408295

409-
list-skills
410-
{:doc "List all skills with metadata in a table format"
411-
:task (let [skills-dir "skills"
412-
skill-files (sort (map str (fs/glob skills-dir "**/*.md")))
413-
414-
;; Parse all skills with pandoc metadata
415-
skills (for [file-path skill-files]
416-
(let [metadata (extract-metadata-with-pandoc file-path)
417-
file-size (fs/size file-path)
418-
content (slurp file-path)
419-
char-count (count content)
420-
token-count (long (estimate-tokens content))
421-
relative-path (str/replace file-path
422-
(str (fs/file skills-dir) "/")
423-
"")
424-
category (first (str/split relative-path (re-pattern "/")))
425-
skill-name (or (get metadata :name)
426-
(get metadata :title)
427-
(str/replace (fs/file-name file-path) ".md" ""))
428-
description (strip-html-tags (get metadata :description ""))]
429-
{:category category
430-
:name skill-name
431-
:path relative-path
432-
:size (format-size file-size)
433-
:tokens (format-number token-count)
434-
:description description}))
435-
436-
;; Calculate totals
437-
total-size (reduce + (map (fn [s] (fs/size (str "skills/" (:path s)))) skills))
438-
total-tokens (reduce + (map (fn [s] (long (estimate-tokens (slurp (str "skills/" (:path s)))))) skills))]
439-
440-
(info-msg "Available Skills")
441-
442-
(println)
443-
(table/print-table [:category :name :description :size :tokens]
444-
(concat
445-
(for [s (sort-by (juxt :category :name) skills)]
446-
(assoc s :description (truncate-string (:description s) 80)))
447-
[{:category "TOTAL"
448-
:name (str (count skills) " skills")
449-
:description ""
450-
:size (format-size total-size)
451-
:tokens (format-number total-tokens)}]))
452-
453-
(println)
454-
(println (bling/bling [:dim "Note: Token count is estimated (~4 chars per token)"])))}
455-
456-
watch
457-
{:doc "Watch for changes and rebuild prompts automatically\nUsage: bb watch [prompt-name]"
458-
:task (let [name (first *command-line-args*)]
459-
(info-msg (if name
460-
(str "Watching for changes to rebuild " name "...")
461-
"Watching for changes to rebuild all prompts..."))
462-
463-
;; Use babashka.fs/watch
464-
(let [watch-paths ["prompts" "skills" "prompt_configs"]]
465-
(apply fs/watch
466-
(for [path watch-paths
467-
:when (fs/exists? path)]
468-
path)
469-
(fn [{:keys [type path]}]
470-
(when (and (= :modify type)
471-
(str/ends-with? (str path) ".md"))
472-
(info-msg (str "Change detected: " path))
473-
(try
474-
(if name
475-
(shell (str "bb build " name))
476-
(shell "bb build-all"))
477-
(catch Exception e
478-
(error-msg (str "Build failed: " (.getMessage e))))))))))}
479-
480-
;; LLMLingua compression tasks
481-
482-
setup-python
483-
{:doc "Install Python dependencies using pipenv"
484-
:task (do
485-
(info-msg "Installing Python dependencies with pipenv...")
486-
487-
(shell "pipenv install")
488-
489-
(success-msg "Python environment ready!")
490-
491-
(println)
492-
(println "You can now use:")
493-
(println " bb compress <name> --ratio 2")
494-
(println " bb build-compressed <name> --ratio 3")
495-
(println " bb compress-skill skills/path/to/file.md --ratio 2"))}
496-
497-
compress
498-
{:doc "Compress a built prompt file using LLMLingua (default: 2x ratio)\nUsage: bb compress clojure_skill_builder [--ratio 2]"
499-
:task (let [args *command-line-args*
500-
name (first args)
501-
ratio (get-ratio-arg)
502-
503-
input (str "_build/" name ".md")
504-
output (str "_build/" name ".compressed.md")]
505-
506-
(when-not name
507-
(throw (ex-info "Usage: bb compress <name> [--ratio N]" {})))
508-
509-
(when-not (fs/exists? input)
510-
(error-msg (str "Built file not found: " input))
511-
(println "\nRun 'bb build " name "' first.")
512-
(System/exit 1))
513-
514-
(info-msg (str "Compressing " input " with " ratio "x ratio..."))
515-
516-
;; Run Python compression script via pipenv
517-
(let [result (shell {:out :string
518-
:err :string
519-
:continue true}
520-
(str "pipenv run python scripts/compress_prompt.py"
521-
" --input " input
522-
" --output " output
523-
" --ratio " ratio))]
524-
(if (zero? (:exit result))
525-
(do
526-
(println (:out result))
527-
(success-msg (str "Compression successful: " output)))
528-
(do
529-
(println (:err result))
530-
(error-msg "Compression failed")
531-
(System/exit 1)))))}
532-
533-
compress-skill
534-
{:doc "Compress a single skill file (default: 2x ratio)\nUsage: bb compress-skill skills/libraries/data_validation/malli.md [--ratio 2]"
535-
:task (let [args *command-line-args*
536-
input-file (first args)
537-
ratio (get-ratio-arg)
538-
539-
;; Create output path by adding .compressed before .md
540-
output-file (str/replace input-file (re-pattern "\\.md$") ".compressed.md")]
541-
542-
(when-not input-file
543-
(throw (ex-info "Usage: bb compress-skill <path/to/skill.md> [--ratio N]" {})))
544-
545-
(when-not (fs/exists? input-file)
546-
(error-msg (str "Skill file not found: " input-file))
547-
(System/exit 1))
548-
549-
(info-msg (str "Compressing " input-file " with " ratio "x ratio..."))
550-
551-
(let [result (shell {:out :string
552-
:err :string
553-
:continue true}
554-
(str "pipenv run python scripts/compress_prompt.py"
555-
" --input " input-file
556-
" --output " output-file
557-
" --ratio " ratio))]
558-
(if (zero? (:exit result))
559-
(do
560-
(println (:out result))
561-
(success-msg (str "Compressed: " output-file)))
562-
(do
563-
(println (:err result))
564-
(error-msg "Compression failed")
565-
(System/exit 1)))))}
566-
567296
migrate
568297
{:doc "Run database migrations"
569298
:task (do

‎deps.edn‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,14 @@
88
funcool/lentes {:mvn/version "1.3.3"}
99
clj-commons/clj-yaml {:mvn/version "1.0.29"}
1010
org.clj-commons/pretty {:mvn/version "3.6.7"}
11-
dev.weavejester/hashp {:mvn/version "0.5.1"}
1211
io.github.paintparty/bling {:mvn/version "0.8.8"}
13-
1412
mvxcvi/puget {:mvn/version "1.3.4"}
15-
13+
org.slf4j/slf4j-nop {:mvn/version "2.0.17"}
1614
;; SQL database dependencies
1715
com.github.seancorfield/next.jdbc {:mvn/version "1.3.955"}
1816
com.github.seancorfield/honeysql {:mvn/version "2.7.1350"}
1917
dev.weavejester/ragtime.next-jdbc {:mvn/version "0.9.4"}
20-
org.xerial/sqlite-jdbc {:mvn/version "3.47.1.0"}
21-
com.github.clojure-lsp/clojure-lsp {:mvn/version "2025.08.25-14.21.46"}}
18+
org.xerial/sqlite-jdbc {:mvn/version "3.47.1.0"}}
2219

2320
:paths ["src"
2421
"classes"

0 commit comments

Comments
 (0)