diff --git a/config_examples/application_specific_commands.jl b/config_examples/application_specific_commands.jl new file mode 100644 index 0000000..444ea9f --- /dev/null +++ b/config_examples/application_specific_commands.jl @@ -0,0 +1,47 @@ +# To run this do in the Julia REPL `include("path-to-file")` or simply copy paste it inside. +using JustSayIt + +# 1) Define reusable commands shared by several application contexts. +base_commands = Dict( + "sentence" => Keyboard.type_sentence, + "continuation" => Keyboard.type_sentence_lowercase, + "spell" => Keyboard.type_letters, + "type summary" => LLM.type_summary, + "answer" => LLM.read_text_answer, + "followup" => LLM.read_followup, + "read this" => TTS.read, + "pause" => TTS.pause, + "stop" => TTS.stop, +) + +# 2) Extend the base dictionary with browser-specific commands. +browser_commands = merge( + base_commands, + Dict( + "new tab" => (Key.ctrl, 't'), + "close tab" => (Key.ctrl, 'w'), + "jump address" => (Key.ctrl, 'l'), + "search web" => [(Key.ctrl, 'l'), Keyboard.type_lowercase, Key.enter], + ), +) + +# 3) Extend the base dictionary with coding-specific commands. +coding_commands = merge( + base_commands, + Dict( + "comment" => (Key.ctrl, '/'), + "format selection" => [(Key.ctrl, 'k'), (Key.ctrl, 'f')], + "workspace search" => (Key.ctrl, Key.shift, 'f'), + "jump line" => [(Key.ctrl, 'g'), Keyboard.type_digits, Key.enter], + ), +) + +# 4) Assemble top-level commands that activate nested application contexts. +commands = Dict( + "help" => Help.help, + "browser" => [`firefox`, browser_commands], + "coding" => [`code`, coding_commands], +) + +# 5) Start JustSayIt with the application-specific command dictionaries. +start(commands=commands) \ No newline at end of file diff --git a/config_examples/config_custom_function.jl b/config_examples/config_custom_function.jl index 21dc6a1..f56f638 100644 --- a/config_examples/config_custom_function.jl +++ b/config_examples/config_custom_function.jl @@ -1,6 +1,6 @@ # To run this do in the Julia REPL `include("path-to-file")` or simply copy paste it inside. using JustSayIt -using JustSayIt.API # Import JustSayIt API to write @voicearg functions +using JustSayIt.API # Import JustSayIt API to write @voiceargs functions using DefaultApplication # To install type: `]` and then `add DefaultApplication` # 1) Define a custom weather forecast search function. @@ -15,10 +15,10 @@ weather DefaultApplication.open("https://www.google.com/search?q=weather+$day") end -# 2) Define command name to function mapping, calling custom function -commands = Dict("help" => Help.help, - "weather" => weather, - ); +# 2) Define command name to function mapping, calling the custom function. +commands = Dict("help" => Help.help, + "weather" => weather, + ) # 3) Start JustSayIt with the custom commands. start(commands=commands) # If you say "help weather", it will show the documentation written above in the Julia REPL. diff --git a/config_examples/jupyter_llm_commands.jl b/config_examples/jupyter_llm_commands.jl new file mode 100644 index 0000000..0555dc7 --- /dev/null +++ b/config_examples/jupyter_llm_commands.jl @@ -0,0 +1,24 @@ +# To run this do in the Julia REPL `include("path-to-file")` or simply copy paste it inside. +using JustSayIt + +# 1) Define the Jupyter notebook LLM-assisted commands shown in the JuliaCon 2025 slides. +jupyter_llm_commands = Dict( + "generate magic" => () -> LLM.type_answer( + instruction_prefix="Generate the appropriate IPython magic command for the following task using proper syntax (%line magic or %%cell magic) with necessary options:" + ), + "explain cell" => () -> LLM.type_text_answer_to( + "Explain what the selected cell code does in simple terms. Include key functions, algorithms, and data transformations. Format as markdown with clear section headings." + ), + "parallelize code" => () -> LLM.type_text_answer_to( + "Convert the selected code to use parallel processing. Use appropriate IPython magic commands (%%px, %%parallel) and add necessary setup. Return only the parallelized code with essential comments." + ), + "visualize data" => () -> LLM.type_text_answer_to( + "Generate visualization code for the selected data/variables. Choose appropriate plotting libraries and create informative visualizations. Use %matplotlib inline for display. Return only executable code." + ), +) + +# 2) Combine them with the top-level help command. +commands = merge(Dict("help" => Help.help), jupyter_llm_commands) + +# 3) Start JustSayIt with the Jupyter notebook LLM-assisted commands. +start(commands=commands) \ No newline at end of file diff --git a/docs/Manifest.toml b/docs/Manifest.toml new file mode 100644 index 0000000..8143af7 --- /dev/null +++ b/docs/Manifest.toml @@ -0,0 +1,473 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.12.5" +manifest_format = "2.0" +project_hash = "c989af5d1ff44363d4d7aa4db72b974eb75b8b6a" + +[[deps.ANSIColoredPrinters]] +git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" +uuid = "a4c015fc-c6ff-483c-b24f-f7ea428134e9" +version = "0.0.1" + +[[deps.AbstractTrees]] +git-tree-sha1 = "2d9c9a55f9c93e8887ad391fbae72f8ef55e1177" +uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +version = "0.4.5" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.BitFlags]] +git-tree-sha1 = "0691e34b3bb8be9307330f88d1a3c3f25466c24d" +uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" +version = "0.1.9" + +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.8" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.3.0+1" + +[[deps.ConcurrentUtilities]] +deps = ["Serialization", "Sockets"] +git-tree-sha1 = "21d088c496ea22914fe80906eb5bce65755e5ec8" +uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" +version = "2.5.1" + +[[deps.Conda]] +deps = ["Downloads", "JSON", "VersionParsing"] +git-tree-sha1 = "8f06b0cfa4c514c7b9546756dbae91fcfbc92dc9" +uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d" +version = "1.10.3" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.DefaultApplication]] +deps = ["InteractiveUtils"] +git-tree-sha1 = "c0dfa5a35710a193d83f03124356eef3386688fc" +uuid = "3f0dd361-4fe0-5fc6-8523-80b14ec94d85" +version = "1.1.0" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" + +[[deps.DocExtensions]] +deps = ["Markdown"] +git-tree-sha1 = "bf4634dffad1ede92b9deca91262c75c000e47ca" +uuid = "cbdad009-89f1-4e05-85a0-06b07b50707d" +version = "0.2.0" + +[[deps.DocStringExtensions]] +git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.5" + +[[deps.Documenter]] +deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] +git-tree-sha1 = "56e9c37b5e7c3b4f080ab1da18d72d5c290e184a" +uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +version = "1.17.0" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.7.0" + +[[deps.ExceptionUnwrapping]] +deps = ["Test"] +git-tree-sha1 = "d36f682e590a83d63d1c7dbd287573764682d12a" +uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" +version = "0.1.11" + +[[deps.Expat_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "27af30de8b5445644e8ffe3bcb0d72049c089cf1" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.7.3+0" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.Git]] +deps = ["Git_LFS_jll", "Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "824a1890086880696fc908fe12a17bcf61738bd8" +uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" +version = "1.5.0" + +[[deps.Git_LFS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "bb8471f313ed941f299aa53d32a94ab3bee08844" +uuid = "020c3dae-16b3-5ae5-87b3-4cb189e250b2" +version = "3.7.0+0" + +[[deps.Git_jll]] +deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] +git-tree-sha1 = "dc34a3e3d96b4ed305b641e626dc14c12b7824b8" +uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" +version = "2.53.0+0" + +[[deps.HTTP]] +deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "PrecompileTools", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] +git-tree-sha1 = "51059d23c8bb67911a2e6fd5130229113735fc7e" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "1.11.0" + +[[deps.IOCapture]] +deps = ["Logging", "Random"] +git-tree-sha1 = "0ee181ec08df7d7c911901ea38baf16f755114dc" +uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" +version = "1.0.0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.7.1" + +[[deps.JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.4" + +[[deps.JSON3]] +deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] +git-tree-sha1 = "411eccfe8aba0814ffa0fdf4860913ed09c34975" +uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +version = "1.14.3" + + [deps.JSON3.extensions] + JSON3ArrowExt = ["ArrowTypes"] + + [deps.JSON3.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + +[[deps.JuliaSyntaxHighlighting]] +deps = ["StyledStrings"] +uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" +version = "1.12.0" + +[[deps.JustSayIt]] +deps = ["Conda", "DefaultApplication", "Downloads", "JSON", "MacroTools", "Pkg", "Preferences", "ProgressMeter", "PromptingTools", "PyCall"] +path = ".." +uuid = "d7252348-293e-4f11-8d42-c8418dda4315" +version = "0.3.1" + +[[deps.LazilyInitializedFields]] +git-tree-sha1 = "0f2da712350b020bc3957f269c9caad516383ee0" +uuid = "0e77f7df-68c5-4e49-93ce-4cd80f5598bf" +version = "1.3.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.15.0+0" + +[[deps.LibGit2]] +deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.9.0+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "OpenSSL_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.3+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "be484f5c92fad0bd8acfef35fe017900b0b73809" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.18.0+0" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.12.0" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.LoggingExtras]] +deps = ["Dates", "Logging"] +git-tree-sha1 = "f00544d95982ea270145636c181ceda21c4e2575" +uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" +version = "1.2.0" + +[[deps.MacroTools]] +git-tree-sha1 = "1e0228a030642014fe5cfe68c2c0a818f9e3f522" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.16" + +[[deps.Markdown]] +deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MarkdownAST]] +deps = ["AbstractTrees", "Markdown"] +git-tree-sha1 = "93c718d892e73931841089cdc0e982d6dd9cc87b" +uuid = "d0879d2d-cac2-40c8-9cee-1863dc0c7391" +version = "0.1.3" + +[[deps.MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] +git-tree-sha1 = "8785729fa736197687541f7053f6d8ab7fc44f92" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.1.10" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "ff69a2b1330bcb730b9ac1ab7dd680176f5896b8" +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.1010+0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2025.11.4" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.3.0" + +[[deps.OpenAI]] +deps = ["Dates", "HTTP", "JSON3"] +git-tree-sha1 = "fb6a407f3707daf513c4b88f25536dd3dbf94220" +uuid = "e9f21f70-7185-4079-aca2-91159181367c" +version = "0.9.1" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.29+0" + +[[deps.OpenSSH_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] +git-tree-sha1 = "57baa4b81a24c2910afbb6d853aa0685e4312bf7" +uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" +version = "10.3.1+0" + +[[deps.OpenSSL]] +deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "NetworkOptions", "OpenSSL_jll", "Sockets"] +git-tree-sha1 = "1d1aaa7d449b58415f97d2839c318b70ffb525a0" +uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" +version = "1.6.1" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.5.4+0" + +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.44.0+1" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "7d2f8f21da5db6a806faf7b9b292296da42b2810" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.8.3" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.12.1" +weakdeps = ["REPL"] + + [deps.Pkg.extensions] + REPLExt = "REPL" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "07a921781cab75691315adc645096ed5e370cb77" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.3.3" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.5.2" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.ProgressMeter]] +deps = ["Distributed", "Printf"] +git-tree-sha1 = "fbb92c6c56b34e1a2c4c36058f68f332bec840e7" +uuid = "92933f4c-e287-5a05-a399-4b506db050ca" +version = "1.11.0" + +[[deps.PromptingTools]] +deps = ["AbstractTrees", "Base64", "Dates", "HTTP", "JSON3", "Logging", "OpenAI", "Pkg", "PrecompileTools", "Preferences", "REPL", "Random", "StreamCallbacks", "Test"] +git-tree-sha1 = "9b75d7de372cfc175642f666024c6bdefff70ab8" +uuid = "670122d1-24a8-4d70-bfce-740807c42192" +version = "0.77.0" + + [deps.PromptingTools.extensions] + GoogleGenAIPromptingToolsExt = ["GoogleGenAI"] + MarkdownPromptingToolsExt = ["Markdown"] + + [deps.PromptingTools.weakdeps] + GoogleGenAI = "903d41d1-eaca-47dd-943b-fee3930375ab" + Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.PyCall]] +deps = ["Conda", "Dates", "Libdl", "LinearAlgebra", "MacroTools", "Serialization", "VersionParsing"] +git-tree-sha1 = "9816a3826b0ebf49ab4926e2b18842ad8b5c8f04" +uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" +version = "1.96.4" + +[[deps.REPL]] +deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.RegistryInstances]] +deps = ["LazilyInitializedFields", "Pkg", "TOML", "Tar"] +git-tree-sha1 = "ffd19052caf598b8653b99404058fce14828be51" +uuid = "2792f1a3-b283-48e8-9a74-f99dce5104f3" +version = "0.1.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.SimpleBufferStream]] +git-tree-sha1 = "f305871d2f381d21527c770d4788c06c097c9bc1" +uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" +version = "1.2.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.StreamCallbacks]] +deps = ["HTTP", "JSON3", "PrecompileTools"] +git-tree-sha1 = "0fd132a34025aa80f782cd93aed2eb800501f5e2" +uuid = "c1b9e933-98a0-46fc-8ea7-3b58b195fb0a" +version = "0.6.2" + +[[deps.StructTypes]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "159331b30e94d7b11379037feeb9b690950cace8" +uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" +version = "1.11.0" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.TranscodingStreams]] +git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.11.3" + +[[deps.URIs]] +git-tree-sha1 = "bef26fb046d031353ef97a82e3fdb6afe7f21b1a" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.6.1" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.VersionParsing]] +git-tree-sha1 = "58d6e80b4ee071f5efd07fda82cb9fbe17200868" +uuid = "81def892-9a0e-5fdd-b105-ffc91e053289" +version = "1.3.0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.3.1+2" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.15.0+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.64.0+1" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.7.0+0" diff --git a/docs/make.jl b/docs/make.jl index db854a5..6d930be 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -44,6 +44,8 @@ try "Usage" => "usage.md", "Examples" => [hide("..." => "examples.md"), "examples/config_custom_function.md", + "examples/application_specific_commands.md", + "examples/jupyter_llm_commands.md", ], "Software reference" => "software.md", "High-level API reference" => "api.md", diff --git a/docs/reflinks.jl b/docs/reflinks.jl index 38a775c..964bb46 100644 --- a/docs/reflinks.jl +++ b/docs/reflinks.jl @@ -6,7 +6,10 @@ reflinks = Dict( "[Julia REPL]" => "https://docs.julialang.org/en/v1/stdlib/REPL/", "[JustSayIt.jl]" => "https://github.com/omlins/JustSayIt.jl", "[MacroTools.jl]" => "https://github.com/FluxML/MacroTools.jl", + "[Ollama]" => "https://ollama.com/", + "[PromptingTools.jl]" => "https://github.com/svilupp/PromptingTools.jl", "[PyCall.jl]" => "https://github.com/JuliaPy/PyCall.jl", + "[Whisper]" => "https://github.com/openai/whisper", "[pynput]" => "https://github.com/moses-palmer/pynput", "[sounddevice]" => "https://github.com/spatialaudio/python-sounddevice", "[vosk]" => "https://github.com/alphacep/vosk-api", diff --git a/docs/src/api.MD b/docs/src/api.MD index a0203fa..c4570b8 100644 --- a/docs/src/api.MD +++ b/docs/src/api.MD @@ -4,7 +4,7 @@ CurrentModule = JustSayIt # High-level API reference -This is the official High-level API reference of `JustSayIt`. Note that it can also be queried interactively from the [Julia REPL] using the [help mode](https://docs.julialang.org/en/v1/stdlib/REPL/#Help-mode): +This is the official High-level API reference of `JustSayIt`. Start with the predefined command-module functions in the [Software reference](software.MD). If you need smaller building blocks, compose the root primitives `take`, `type`, `ask`, `ask!`, `grab`, `listen`, and `say`. When you need custom-function programming, reach for `@voiceargs` and `@voiceconfig`. The remaining entries on this page are the lower-level helper groups that support those custom functions. Note that this reference can also be queried interactively from the [Julia REPL] using the [help mode](https://docs.julialang.org/en/v1/stdlib/REPL/#Help-mode): ```julia-repl julia> using JustSayIt julia>? @@ -31,15 +31,13 @@ JustSayIt.API.@voiceinfo ## Functions #### Index -* [`JustSayIt.API.finalize_jsi`](@ref) -* [`JustSayIt.API.init_jsi`](@ref) * [`JustSayIt.API.controller`](@ref) +* [`JustSayIt.API.default_language`](@ref) +* [`JustSayIt.API.type_languages`](@ref) * [`JustSayIt.API.interpret_enum`](@ref) * [`JustSayIt.API.interpret_digit`](@ref) * [`JustSayIt.API.interpret_count`](@ref) * [`JustSayIt.API.interpret_language`](@ref) -* [`JustSayIt.API.default_language`](@ref) -* [`JustSayIt.API.type_languages`](@ref) * [`JustSayIt.API.ask_llm`](@ref) * [`JustSayIt.API.ask_llm!`](@ref) * [`JustSayIt.API.next_wordgroup`](@ref) @@ -60,14 +58,12 @@ JustSayIt.API.@voiceinfo #### Documentation ```@docs JustSayIt.API.controller -JustSayIt.API.finalize_jsi -JustSayIt.API.init_jsi +JustSayIt.API.default_language +JustSayIt.API.type_languages JustSayIt.API.interpret_enum JustSayIt.API.interpret_digit JustSayIt.API.interpret_count JustSayIt.API.interpret_language -JustSayIt.API.default_language -JustSayIt.API.type_languages JustSayIt.API.ask_llm JustSayIt.API.ask_llm! JustSayIt.API.next_wordgroup diff --git a/docs/src/assets/config_examples/application_specific_commands.jl b/docs/src/assets/config_examples/application_specific_commands.jl new file mode 100644 index 0000000..444ea9f --- /dev/null +++ b/docs/src/assets/config_examples/application_specific_commands.jl @@ -0,0 +1,47 @@ +# To run this do in the Julia REPL `include("path-to-file")` or simply copy paste it inside. +using JustSayIt + +# 1) Define reusable commands shared by several application contexts. +base_commands = Dict( + "sentence" => Keyboard.type_sentence, + "continuation" => Keyboard.type_sentence_lowercase, + "spell" => Keyboard.type_letters, + "type summary" => LLM.type_summary, + "answer" => LLM.read_text_answer, + "followup" => LLM.read_followup, + "read this" => TTS.read, + "pause" => TTS.pause, + "stop" => TTS.stop, +) + +# 2) Extend the base dictionary with browser-specific commands. +browser_commands = merge( + base_commands, + Dict( + "new tab" => (Key.ctrl, 't'), + "close tab" => (Key.ctrl, 'w'), + "jump address" => (Key.ctrl, 'l'), + "search web" => [(Key.ctrl, 'l'), Keyboard.type_lowercase, Key.enter], + ), +) + +# 3) Extend the base dictionary with coding-specific commands. +coding_commands = merge( + base_commands, + Dict( + "comment" => (Key.ctrl, '/'), + "format selection" => [(Key.ctrl, 'k'), (Key.ctrl, 'f')], + "workspace search" => (Key.ctrl, Key.shift, 'f'), + "jump line" => [(Key.ctrl, 'g'), Keyboard.type_digits, Key.enter], + ), +) + +# 4) Assemble top-level commands that activate nested application contexts. +commands = Dict( + "help" => Help.help, + "browser" => [`firefox`, browser_commands], + "coding" => [`code`, coding_commands], +) + +# 5) Start JustSayIt with the application-specific command dictionaries. +start(commands=commands) \ No newline at end of file diff --git a/docs/src/assets/config_examples/config_custom_function.jl b/docs/src/assets/config_examples/config_custom_function.jl index 21dc6a1..f56f638 100644 --- a/docs/src/assets/config_examples/config_custom_function.jl +++ b/docs/src/assets/config_examples/config_custom_function.jl @@ -1,6 +1,6 @@ # To run this do in the Julia REPL `include("path-to-file")` or simply copy paste it inside. using JustSayIt -using JustSayIt.API # Import JustSayIt API to write @voicearg functions +using JustSayIt.API # Import JustSayIt API to write @voiceargs functions using DefaultApplication # To install type: `]` and then `add DefaultApplication` # 1) Define a custom weather forecast search function. @@ -15,10 +15,10 @@ weather DefaultApplication.open("https://www.google.com/search?q=weather+$day") end -# 2) Define command name to function mapping, calling custom function -commands = Dict("help" => Help.help, - "weather" => weather, - ); +# 2) Define command name to function mapping, calling the custom function. +commands = Dict("help" => Help.help, + "weather" => weather, + ) # 3) Start JustSayIt with the custom commands. start(commands=commands) # If you say "help weather", it will show the documentation written above in the Julia REPL. diff --git a/docs/src/examples.MD b/docs/src/examples.MD index d841c7e..a3795d9 100644 --- a/docs/src/examples.MD +++ b/docs/src/examples.MD @@ -1,5 +1,5 @@ # Examples ```@contents -Pages = ["examples/config_custom_function.md"] +Pages = ["examples/config_custom_function.md", "examples/application_specific_commands.md", "examples/jupyter_llm_commands.md"] ``` diff --git a/docs/src/examples/application_specific_commands.MD b/docs/src/examples/application_specific_commands.MD new file mode 100644 index 0000000..486d36a --- /dev/null +++ b/docs/src/examples/application_specific_commands.MD @@ -0,0 +1,9 @@ +# Application-specific command dictionaries + +This example shows one larger executable configuration file that combines reusable base commands with browser and coding contexts. It keeps the code in one contiguous example, as it would actually be run, while the current [JustSayItRecipes.jl](https://github.com/omlins/JustSayItRecipes.jl) recipes remain the authoritative larger recipe collection. The top-level command dictionary includes `help` explicitly so spoken help stays available while switching application contexts. + +To run this example, type in the [Julia REPL] `include("path-to-file")` or simply copy-paste the code below inside (the corresponding file can be found [here](../assets/config_examples/application_specific_commands.jl)). + +```@eval +Main.mdinclude(joinpath(Main.EXAMPLEROOT, "application_specific_commands.jl")) +``` \ No newline at end of file diff --git a/docs/src/examples/config_custom_function.MD b/docs/src/examples/config_custom_function.MD index 2d99fa7..08ef824 100644 --- a/docs/src/examples/config_custom_function.MD +++ b/docs/src/examples/config_custom_function.MD @@ -3,9 +3,11 @@ This example shows how one can 1. program a weather forecast search function (`weather`), 2. define a command name to function mapping, and then -3. start JustSayIt using these customly defined commands. +3. start JustSayIt using these custom-defined commands. -The command `weather` programmed here allows to find out how the weather is `today` or `tomorrow` - just say "weather today" or "weather tomorrow". Furthermore, if you say "help weather", it will show in the [Julia REPL] the function documentation written here. To run this example, type in the [Julia REPL] `include("path-to-file")` or simply copy-paste the code below inside (the corresponding file can be found [here](../../../assets/config_examples/config_custom_function.jl)). +The command `weather` programmed here allows to find out how the weather is `today` or `tomorrow` - just say "weather today" or "weather tomorrow". The example also spells out the top-level `help` command explicitly, so if you say "help weather", it will show in the [Julia REPL] the function documentation written here. To run this example, type in the [Julia REPL] `include("path-to-file")` or simply copy-paste the code below inside (the corresponding file can be found [here](../assets/config_examples/config_custom_function.jl)). + +This is the focused companion for the custom-function section of the usage guide. For a larger nested command-dictionary example, see [Application-specific command dictionaries](application_specific_commands.MD). ```@eval Main.mdinclude(joinpath(Main.EXAMPLEROOT, "config_custom_function.jl")) diff --git a/docs/src/index.MD b/docs/src/index.MD index ad105ea..fcddd56 100644 --- a/docs/src/index.MD +++ b/docs/src/index.MD @@ -1,5 +1,9 @@ # [JustSayIt.jl] [![Star on GitHub](https://img.shields.io/github/stars/omlins/JustSayIt.jl.svg)](https://github.com/omlins/JustSayIt.jl/stargazers) -JustSayIt enables offline, low latency, highly accurate and secure speech to command translation on Linux, MacOS and Windows and is usable as software or high-level API. It implements a **novel algorithm for high performance context dependent recognition of spoken commands** which leverages the [Vosk Speech Recognition Toolkit]. Single word commands' latency (referring to the time elapsed between a command is spoken and executed) can be little as 5 miliseconds and does usually not exceed 30 miliseconds (measured on a regular notebook). +JustSayIt enables secure, offline, low-latency, highly accurate and secure speech to command translation for assembling personalized voice assistants on Linux, MacOS and Windows; it offers composable functionality using speech-to-text (STT), text-to-speech (TTS), large language models (LLMs) and more. It contains in addition an Application Programming Interface (API) for a fast and easy development of custom functions. + +A key practical strength of JustSayIt is the possibility to define application-specific commands through command dictionaries, so voice interaction can be tailored precisely to concrete workflows in editors, notebooks, browsers, terminals, and other software. These commands can be LLM-assisted and TTS-enhanced whenever useful, enabling focused voice workflows with intelligent text generation, explanation, or spoken output without losing the structure of a personalized voice assistant. + +It implements a **novel algorithm for high performance context dependent recognition of spoken commands** which leverages the [Vosk Speech Recognition Toolkit], while using a dual STT approach overall: [vosk] is used for constrained command recognition and a faster [Whisper]-based backend is used for free-speech recognition. Single word commands' latency on the command-recognition path (referring to the time elapsed between a command is spoken and executed) can be little as 5 miliseconds and does usually not exceed 30 miliseconds (measured on a regular notebook). Furthermore, JustSayIt provides an **unprecedented highly generic extension to the Julia programming language which allows to declare arguments in standard function definitions to be obtainable by voice**. For such functions, JustSayIt automatically generates a wrapper method that takes care of the complexity of retrieving the arguments from the speakers voice, including interpretation and conversion of the voice arguments to potentially any data type. JustSayIt commands are implemented with such *voice argument functions*, triggered by a user definable mapping of command names to functions. As a result, it empowers programmers without any knowledge of speech recognition to quickly write new commands that take their arguments from the speakers voice. **JustSayIt is ideally suited for development by the world-wide open source community**. @@ -12,29 +16,41 @@ Finally, JustSayIt puts a small load on the CPU, using only one core, and can th 2\. Install [Julia] if you do not have it yet. -3\. Execute the following in your shell to install and run JustSayIt: +3\. Execute the following in your shell to install JustSayIt, copy one maintained recipe into your JustSayIt config folder, edit it, and run it: ```julia-repl $> julia julia> ] - pkg> add JustSayIt + pkg> add JustSayIt, JustSayItRecipes pkg> -julia> using JustSayIt -julia> start() +julia> using JustSayIt, JustSayItRecipes +julia> recipe = normpath(joinpath(dirname(pathof(JustSayItRecipes)), "..", "recipes", "ubuntu_max.jl")) +julia> local_recipe = joinpath(homedir(), ".config", "JustSayIt", "my_commands.jl") +julia> mkpath(dirname(local_recipe)) +julia> cp(recipe, local_recipe; force=true) +julia> @edit("my_commands.jl") +julia> @include("my_commands.jl") ``` -4\. Say "help commands" and then, e.g., "help type". +4\. Say `help commands` and then, e.g., `help sentence`, and try some commands following the guidance of the help output. + +5\. Look through `my_commands.jl` itself to understand what commands are available and adjust them as desired. + +6\. Look through the rest of the documentation! -5\. Try some commands and then look through the rest of the documentation! +This recipe-copy workflow is the basic practical way to use JustSayIt properly, and it remains the standard recommended workflow for advanced users too. ## Dependencies -JustSayIt's primary dependencies are [vosk], [sounddevice], [pynput], [PyCall.jl], [Conda.jl] and [MacroTools.jl]. +JustSayIt combines Julia metaprogramming with selected Python packages and local AI runtimes. The current stack centers on constrained command recognition via [vosk], free-speech recognition via a faster [Whisper]-based backend, text-to-speech, desktop control via [pynput] and [sounddevice], and local LLM interaction via [PromptingTools.jl] and [Ollama], with [PyCall.jl] and [Conda.jl] bridging the Python side where needed. ## Support for Linux, MacOS and Windows JustSayIt is programmed in a highly portable manner relying exclusively on portable Julia and Python modules (see [Dependencies](@ref)). ## Your contributions -The only limit to the commands that can be programmed with JustSayIt is your imagination! Please open an issue to discuss your idea for a contribution beforehand. Furthermore, note that pull request should always address a significant issue in its completeness and new commands should be generic enough to be of interest for others. Moreover, pull requests should blend nicely into the existing JustSayIt project; common sense is the primary guide in this regard (community guideline documents, e.g. [ColPrac](https://github.com/SciML/ColPrac), can be consulted in addition for inspiration). We are looking forward to your contribution! +The only limit to the commands that can be defined with JustSayIt is your imagination! +The primary contribution path is [JustSayItRecipes.jl](https://github.com/omlins/JustSayItRecipes.jl), the shared collection of command dictionaries for assembling personalized voice assistants with JustSayIt.jl. Useful contributions there are not limited to programming: improved application-specific dictionaries, new command definitions, cross-platform adaptations, better organization of existing command sets, and clearer recipe comments or explanations are all valuable. +If you want to contribute broader functionality to JustSayIt.jl itself, please open an issue first to discuss the idea and scope. Pull requests should address a meaningful issue in its completeness and blend cleanly into the existing project; common sense is the primary guide in this regard (community guideline documents, e.g. [ColPrac](https://github.com/SciML/ColPrac) can be used for inspiration). +We are looking forward to your contribution! ## Contributors The principal contributors to [JustSayIt.jl] have been so far (ordered by the significance of the relative contributions): diff --git a/docs/src/software.MD b/docs/src/software.MD index 83f76af..2599cf9 100644 --- a/docs/src/software.MD +++ b/docs/src/software.MD @@ -11,11 +11,13 @@ julia>? help?> JustSayIt ``` + ## Root module ```@docs JustSayIt ``` + ## Functions to configure and start JustSayIt #### Index * [`JustSayIt.start`](@ref) @@ -29,7 +31,16 @@ start @edit ``` -## Root convenience reexports +## Keyboard shortcut namespace +The root `JustSayIt` module also re-exports `Key` for command mappings and keyboard shortcuts. +Use `Key.` completion in the Julia REPL to inspect the currently available key names. + + +## Composable primitives for quick command definition (reexported from the root module for convenient access) +These root primitives can be composed directly in command dictionaries when no wrapper function is needed. + +Use `type` here for low-level text typing when you already have text to send to the active application. For voice-input typing from speech, use the `Keyboard.type_*` family documented below. + #### Index * [`JustSayIt.take`](@ref) * [`JustSayIt.type`](@ref) @@ -50,22 +61,7 @@ listen say ``` -## Public helper utilities -#### Index -* [`JustSayIt.set_preferences!`](@ref) -* [`JustSayIt.load_preference`](@ref) -* [`JustSayIt.has_preference`](@ref) -* [`JustSayIt.delete_preferences!`](@ref) - -#### Documentation -```@docs -set_preferences! -load_preference -has_preference -delete_preferences! -``` - -## Software-facing modules +## Submodules for quick command definition with predefined, flexible and composable functions #### Index * [`JustSayIt.Clipboard`](@ref) * [`JustSayIt.Generic`](@ref) @@ -90,9 +86,19 @@ STT TTS ``` -## Keyboard shortcut namespace -The root `JustSayIt` module also re-exports `Key` for command mappings and keyboard shortcuts. -Use `Key.` completion in the Julia REPL to inspect the currently available key names. + +## Functions of [`Clipboard`](@ref) +#### Index +```@index +Modules = [JustSayIt.Clipboard] +Order = [:function] +``` + +#### Documentation +```@autodocs +Modules = [JustSayIt.Clipboard] +Order = [:function] +``` ## Functions of [`Generic`](@ref) @@ -164,6 +170,33 @@ Modules = [JustSayIt.Mouse] Order = [:function] ``` +## Functions of [`Selection`](@ref) +#### Index +```@index +Modules = [JustSayIt.Selection] +Order = [:function] +``` + +#### Documentation +```@autodocs +Modules = [JustSayIt.Selection] +Order = [:function] +``` + + +## Functions of [`STT`](@ref) +#### Index +```@index +Modules = [JustSayIt.STT] +Order = [:function] +``` + +#### Documentation +```@autodocs +Modules = [JustSayIt.STT] +Order = [:function] +``` + ## Functions of [`TTS`](@ref) #### Index @@ -177,3 +210,19 @@ Order = [:function] Modules = [JustSayIt.TTS] Order = [:function] ``` + + +## JustSayIt preference management wrappers +#### Index +* [`JustSayIt.set_preferences!`](@ref) +* [`JustSayIt.load_preference`](@ref) +* [`JustSayIt.has_preference`](@ref) +* [`JustSayIt.delete_preferences!`](@ref) + +#### Documentation +```@docs +set_preferences! +load_preference +has_preference +delete_preferences! +``` diff --git a/docs/src/usage.MD b/docs/src/usage.MD index dd90851..165b49c 100644 --- a/docs/src/usage.MD +++ b/docs/src/usage.MD @@ -6,141 +6,283 @@ After installing [Julia] - if not yet installed - JustSayIt can be installed dir julia>] pkg> add JustSayIt ``` -All dependencies are automatically installed. Python dependencies are automatically installed in a local Miniconda environment, set up by [Conda.jl]. Default language models are automatically downloaded at first usage of JustSayIt. +All dependencies are automatically installed (on some systems, the fully automatic installation might not work, and some manual installations are required). Python dependencies are automatically installed in a local Miniconda environment, set up by [Conda.jl]. Default speech-to-text, text-to-speech and large language models are acquired automatically at first use. -## Simple basic software usage -The basic usage of JustSayIt is as simple as launching the [Julia REPL] and calling the commands `using JustSayIt` followed by `start()`: +## Fast microphone and recorder setup +At startup, JustSayIt prints the available audio devices. In many setups the defaults are good enough, but you can also select devices explicitly: + +```julia +using JustSayIt + +start(microphone_name="USB", audiooutput_name="Headphones") +``` + +In practice, a headset or a well-positioned external microphone usually gives much better command recognition than a notebook microphone. Headphones are strongly recommended when you use asynchronous TTS, because spoken output can otherwise trigger commands by accident. + + +## Easy basic software usage +If you launch JustSayIt without passing a command dictionary, `start()` uses the default command dictionary, which contains only the "help" command. This is a good starting point to confirm that JustSayIt is listening and that spoken help works before moving on to an actual useful command set. To test that, just run: + ```julia-repl $> julia julia> using JustSayIt julia> start() ``` -More information on customization keywords of [`start`](@ref) is given in the following subsections and in the [Software reference](@ref). +If you then say `help commands`, the following is printed in the terminal: + +```text +[ Info: Starting command: help +┌ Info: +│ Your commands: +└ help => help +``` + +The next step is to define your own top-level command dictionary. If you want spoken help to remain available there as well, add the language-appropriate help command explicitly. + +One small flat command dictionary can then serve as a first illustrative example: + +```julia +using JustSayIt + +commands = Dict( + "help" => Help.help, + "sentence" => Keyboard.type_sentence, + "answer" => LLM.read_answer, + "read this" => TTS.read, + "stop" => TTS.stop, +) + +start(commands=commands) +``` + +This already gives you speech typing, one LLM-assisted workflow, spoken output, and spoken help, without introducing keyboard shortcuts, command sequences, or nested command dictionaries. + +It is still only an illustrative example. For actual day-to-day use, start from a recipe as shown in the quick start, run that file, and adapt it to your own applications and workflows. + +More information on customization keywords of [`start`](@ref) is given in the following sections and in the [Software reference](@ref). + + +## Incremental command-dictionary assembly +Command dictionaries are the main way to assemble personalized voice assistants with JustSayIt. + +A command name can map to a Julia function, a key or keyboard shortcut, a string to type literally, a command sequence stored as an array, or another command dictionary. + +The recommended place to start from is [JustSayItRecipes.jl](https://github.com/omlins/JustSayItRecipes.jl), the official repository of shared command-dictionary recipes. + + +## Command-name mappings to functions, shortcuts, and sequences +The keyword `commands` of [`start`](@ref) lets you define a mapping of command names to functions, keyboard shortcuts, literal strings, or sequences of those: -## User definable mapping of command names to functions or keyboard shortcuts -The keyword `commands` of [`start`](@ref) enables to freely define a mapping of command names to functions or keyboard shortcuts or sequences of those, e.g.: ```julia -# Define custom commands using JustSayIt -commands = Dict("help" => Help.help, - "type" => Keyboard.type, - "ma" => Mouse.click_left, - "select" => Mouse.press_left, - "okay" => Mouse.release_left, - "middle" => Mouse.click_middle, - "right" => Mouse.click_right, - "double" => Mouse.click_double, - "triple" => Mouse.click_triple, - "copy" => (Key.ctrl, 'c'), - "cut" => (Key.ctrl, 'x'), - "paste" => (Key.ctrl, 'v'), - "undo" => (Key.ctrl, 'z'), - "redo" => (Key.ctrl, Key.shift, 'z'), - "upwards" => Key.page_up, - "downwards" => Key.page_down, - "take" => [Mouse.click_double, (Key.ctrl, 'c')], - "replace" => [Mouse.click_double, (Key.ctrl, 'v')], - ); -start(commands=commands, max_speed_subset=["ma", "select", "okay", "middle", "right", "double", "triple", "copy", "upwards", "downwards", "take"]) + +commands = Dict( + "help" => Help.help, + "sentence" => Keyboard.type_sentence, + "copy" => (Key.ctrl, 'c'), + "paste" => (Key.ctrl, 'v'), + "take word" => [Mouse.click_double, (Key.ctrl, 'c')], + "replace" => [Mouse.click_double, (Key.ctrl, 'v')], +) + +start(commands=commands) ``` Note that keyboard shortcuts are given as either single keys (e.g., `Key.page_up`) or tuples of keys (e.g., `(Key.ctrl, 'c')`). Special keys are selected from those available in `Key`: type `Key.`+ `tab` in the [Julia REPL] to see the available keys (the full documentation on the available keys is available [here](https://pynput.readthedocs.io/en/latest/keyboard.html#pynput.keyboard.Key)). Character keys are simply given in single quotes (e.g., `'c'`). -A mapping of a command name to a sequence of commands can be defined by using as mapping target an array whose elements are functions or keyboard shortcuts (e.g., `[Mouse.click_double, (Key.ctrl, 'c')]`). This enables to express new complex actions without programming. In the above example, "take" double clicks the word below the mouse curser to select it and then copies it; "replace" double clicks the word below the mouse curser to select it and then replaces it. - -The keyword `subset` of [`start`](@ref) enables to activate only a subset of the default or user-defined commands. The following example selects a subset of the default commands: +A mapping target can also be an array whose elements are functions, strings, shell commands, or keyboard shortcuts. This is a practical way to express a small multi-step action without writing a wrapper function. + + +## Application-specific command assemblies +Once one application needs its own vocabulary, keep a reusable base dictionary and merge it into a nested application-specific dictionary: + ```julia -# Listen only to the commands "help" and "type". using JustSayIt -start(subset=["help", "type"]) + +base_commands = Dict( + "sentence" => Keyboard.type_sentence, + "summarize" => LLM.read_summary, + "read this" => TTS.read, +) + +coding_commands = merge( + base_commands, + Dict( + "comment" => (Key.ctrl, '/'), + "format selection" => [(Key.ctrl, 'k'), (Key.ctrl, 'f')], + "workspace search" => (Key.ctrl, Key.shift, 'f'), + ), +) + +commands = Dict( + "help" => Help.help, + "coding" => [`code`, coding_commands], +) + +start(commands=commands) ``` +Saying `coding` focuses or starts the application and activates the nested command dictionary. This pattern scales well for browser, editor, chat, or terminal specific vocabularies. See the [larger application-specific example](examples/application_specific_commands.MD) for a fuller version and consult JustSayItRecipes.jl for larger maintained assemblies. + -## Per command choice between maximum speed or accuracy +## Maximum-speed command selection +The keyword `max_speed_subset` of [`start`](@ref) lets you define a subset of command names that should be recognized with maximum speed rather than maximum confirmation latency: -The keyword `max_speed_subset` of [`start`](@ref) enables to define a subset of the `commands` for which the command names are to be recognized with maximum speed rather than with maximum accuracy, e.g.: ```julia -# Define custom commands using JustSayIt -commands = Dict("copy" => (Key.ctrl, 'c'), - "cut" => (Key.ctrl, 'x'), - "paste" => (Key.ctrl, 'v'), - "undo" => (Key.ctrl, 'z'), - "redo" => (Key.ctrl, Key.shift, 'z'), - "upwards" => Key.page_up, - "downwards" => Key.page_down, - ); -start(commands=commands, max_speed_subset=["upwards", "downwards", "copy"]) + +commands = Dict( + "help" => Help.help, + "copy" => (Key.ctrl, 'c'), + "cut" => (Key.ctrl, 'x'), + "paste" => (Key.ctrl, 'v'), + "undo" => (Key.ctrl, 'z'), + "redo" => (Key.ctrl, Key.shift, 'z'), + "upwards" => Key.page_up, + "downwards" => Key.page_down, +) + +start(commands=commands, max_speed_subset=["copy", "upwards", "downwards"]) ``` -Forcing maximum speed is usually desired for single word commands that map to functions or keyboard shortcuts that should trigger immediate actions as, e.g., mouse clicks or, as in the above example page up/down or copy (in general, actions that do not modify content and can therefore safely be triggered at maximum speed). However, it is typically not desired for "dangerous" actions, like "cut", "paste", "undo" and "redo" in this example. Note that forcing maximum speed means not to wait for a certain amount of silence after the end of a command as normally done for the full confirmation of a recognition. As a result, it enables a minimal latency between the saying of a command name and its execution. Note that it is usually possible to define very distinctive command names, which allow for a safe command name to shortcut mapping at maximum speed (to be tested case by case). + +This is best reserved for short, distinctive, low-risk commands that should trigger immediately. Leave destructive or easy-to-confuse commands such as `cut`, `paste`, `undo`, or `redo` out of the maximum-speed subset. Note furthermore that a good recording quality is important in order to achieve a good recognition accuracy. In particular, background noise might reduce recognition accuracy. Thus, a microphone integrated in a notebook or a webcam might potentially lead to unsatisfying accuracy, while a headset or an external microphone that is well set up should lead to good accuracy. JustSayIt relying on the [Vosk Speech Recognition Toolkit], it is the latter that dictates the requirements on recording quality for good recognition accuracy (for more information, have a look at the subsection [accuracy](https://alphacephei.com/vosk/accuracy) on their website). -## Help on commands callable by voice -Saying "help commands" lists your available commands in the [Julia REPL] leading to, e.g., the following output: -```julia-repl -┌ Info: -│ Your commands: -│ copy => ctrl + c -│ cut => ctrl + x -│ double => click_double -│ downwards => page_down -│ help => help -│ ma => click_left -│ middle => click_middle -│ okay => release_left -│ paste => ctrl + v -│ redo => ctrl + shift + z -│ right => click_right -│ select => press_left -│ triple => click_triple -│ type => type -│ undo => ctrl + z -└ upwards => page_up +## Spoken help +The `Help.help` function is a built-in command that is recommended to be included in the top-level command dictionary in order to have spoken help available. Assuming that is mapped with `"help" => Help.help`, then saying `help commands` lists the currently active commands in the [Julia REPL]. Saying `help ` shows the help for the corresponding command. + + +## Sleep and wake control by voice +Saying `sleep JustSayIt` puts JustSayIt to sleep. It will not execute commands until you wake it up again with `awake JustSayIt`. + + +## Speech typing +Use the `Keyboard.type_*` family when the command itself should capture text from speech. + +Typical mappings are for example: + +```julia +using JustSayIt + +commands = Dict( + "help" => Help.help, + "sentence" => Keyboard.type_sentence, + "continuation" => Keyboard.type_sentence_lowercase, + "spell" => Keyboard.type_letters, + "digits" => Keyboard.type_digits, + "camel case" => Keyboard.type_camelcase, +) + +start(commands=commands) ``` -Saying "help ", where `` is to be replaced with one of the available commands, shows the help for the corresponding command. Here is, e.g., the output produced when saying "help help": -```julia-repl -[ Info: Starting command: help (latency: 6 ms) -┌ Info: Command help -│ = -│ help() -│ -└ Show help for your commands or a spoken command or module. + +Use the root `type` primitive when you already have text and simply want to type it into the active application. Use the `Keyboard.type_*` family when speech itself should provide the text. + + +## Free-speech capture +The default speech models are acquired automatically when needed. + +For most practical workflows, the predefined typing commands such as `Keyboard.type_sentence`, `Keyboard.type_letters`, or `Keyboard.type_digits` are the most convenient speech-input entry points. + +When you need dictated text outside those fixed command patterns, the root `listen` primitive is the software-facing entry point for free-speech capture. For lower-level programming-facing capture helpers such as `next_wordgroup`, `next_letter`, or `next_digits`, see the [High-level API reference](@ref). + + +## TTS output and control +Text-to-speech is enabled by default. You can select a specific output device with `audiooutput_id` or `audiooutput_name`, and you can decide whether speech should normally be synchronous or asynchronous. + +The software-facing TTS commands stay simple: + +```julia +using JustSayIt + +commands = Dict( + "help" => Help.help, + "read this" => TTS.read, + "pause" => TTS.pause, + "resume" => TTS.resume, + "stop" => TTS.stop, + "talk mode" => TTS.set_async_default, +) + +start(commands=commands) ``` -The [Software reference](@ref) lists the current built-in command modules and root-level entry points. For custom workflows, you can also define your own command mappings to Julia functions, keyboard shortcuts, or command sequences. + +`TTS.read` reads the selected text or, if there is no selection, the clipboard text. The root `say` primitive is the natural spoken-output end of a composed command once you already have text that should be read aloud. -## Sleep and wake up by voice -Saying "sleep JustSayIt" puts JustSayIt to sleep. It will not execute any commands until it is awoken with the words "awake JustSayIt". +## Deterministic voice control with LLM-assisted commands +LLM integration is best treated as help for text transformation or question answering, while system operations themselves stay under deterministic command control. +At first use, JustSayIt prepares the local runtime it needs and automatically downloads the models it uses. -## Fast command programming with voice argument functions -JustSayIt commands map to regular Julia functions. Function arguments can be easily passed by voice thanks to the [`@voiceargs`](@ref) macro. It allows to declare arguments in standard function definitions to be directly obtainable by voice. It, furthermore, allows to define speech recognition parameters for each voice argument as, e.g., the valid speech input. The following shows some examples: +A few representative top-level mappings are for example: + +- `"summarize" => LLM.read_summary` +- `"type summary" => LLM.type_summary` +- `"answer" => LLM.read_text_answer` +- `"direct answer" => LLM.read_answer` +- `"followup" => LLM.read_followup` + +These are only examples of predefined generic LLM commands, not a complete overview of the available functionality. + +The more important pattern in practice is the definition of application-specific LLM-assisted commands. The JuliaCon 2025 talk used the following Jupyter notebook example: ```julia - @voiceargs (b, c) function f(a, b::String, c::String, d) - #(...) - return - end +jupyter_llm_commands = Dict( + "generate magic" => () -> LLM.type_answer( + instruction_prefix="Generate the appropriate IPython magic command for the following task using proper syntax (%line magic or %%cell magic) with necessary options:" + ), + "explain cell" => () -> LLM.type_text_answer_to( + "Explain what the selected cell code does in simple terms. Include key functions, algorithms, and data transformations. Format as markdown with clear section headings." + ), + "parallelize code" => () -> LLM.type_text_answer_to( + "Convert the selected code to use parallel processing. Use appropriate IPython magic commands (%%px, %%parallel) and add necessary setup. Return only the parallelized code with essential comments." + ), + "visualize data" => () -> LLM.type_text_answer_to( + "Generate visualization code for the selected data/variables. Choose appropriate plotting libraries and create informative visualizations. Use %matplotlib inline for display. Return only executable code." + ), +) ``` + +Here `LLM.type_answer` takes the spoken task after the command name and prepends a fixed instruction prefix, while `LLM.type_text_answer_to` works on selected text or clipboard text directly. That is the pattern to put in the center: define application-specific LLM-assisted commands that match the way you actually work. See [Jupyter notebook LLM-assisted commands](examples/jupyter_llm_commands.MD) for a runnable version of the same slide-based example. + + +## Custom voice functions with @voiceargs +JustSayIt commands map to regular Julia functions. Function arguments can be obtained by voice with [`@voiceargs`](@ref), so custom functions can join the same command dictionaries as the predefined commands. + +Here are some representative patterns: + +```julia +@voiceargs (b, c) function f(a, b::String, c::String, d) + #(...) + return +end +``` + ```julia - @voiceargs (b=>(use_max_speed=true), c) function f(a, b::String, c::String, d) - #(...) - return - end +@voiceargs (b=>(use_max_speed=true), c) function f(a, b::String, c::String, d) + #(...) + return +end ``` + ```julia - @enum TypeMode words formula - @voiceargs (mode=>(valid_input_auto=true), token=>(model=MODELNAME.TYPE.EN_US, vararg_timeout=2.0)) function type_tokens(mode::TypeMode, tokens::String...) - #(...) - return - end +@enum Day today tomorrow + +@voiceargs day=>(valid_input_auto=true) function weather(day::Day) + #(...) + return +end ``` + Detailed information on [`@voiceargs`](@ref) is found in the [High-level API reference](@ref). -While contributions to the JustSayIt command modules are very much encouraged, it is possible to quickly define and use custom [`@voiceargs`](@ref) functions thanks to the API of JustSayIt ([`JustSayIt.API`](@ref)). It is illustrated in [this example](@ref Quick-definition-and-usage-of-custom-function). More information on the JustSayIt API is given in the [High-level API reference](@ref). +[`@voiceconfig`](@ref) complements `@voiceargs` when you want to apply shared recognition settings to a call or to a function symbol without rewriting the function declaration itself. + +The smallest complete end-to-end example is shown in [Quick definition and usage of custom function](examples/config_custom_function.MD). For larger assemblies built around command dictionaries, see [Application-specific command dictionaries](examples/application_specific_commands.MD), [Jupyter notebook LLM-assisted commands](examples/jupyter_llm_commands.MD), and then continue with JustSayItRecipes.jl. Note that the JustSayIt application config folder (e.g., `~/.config/JustSayIt` on Unix systems) is an easily accessible storage for scripts to start JustSayIt and/or for custom command functions: [`JustSayIt.@include`](@ref) and [`JustSayIt.@edit`](@ref) permit to conveniently `include` and `edit` files from this folder. diff --git a/src/API.jl b/src/API.jl index 5b60896..018456b 100644 --- a/src/API.jl +++ b/src/API.jl @@ -3,13 +3,15 @@ Application Programming Interface (API) of JustSayIt. +Use this facade when the predefined software-facing command-module functions and the root composable primitives of `JustSayIt` are no longer sufficient for the command mapping you want to build. A typical progression is to start with the software-facing building blocks, then compose root primitives such as `take`, `type`, `ask`, `ask!`, `grab`, `listen`, and `say`, then define custom functions with [`@voiceargs`](@ref) and [`@voiceconfig`](@ref), and only then reach for the lower-level helper groups listed below. + #### Macros - [`@voiceargs`](@ref) - [`@voiceconfig`](@ref) - [`@voiceinfo`](@ref) #### Functions -- Lifecycle and device access: [`controller`](@ref), [`init_jsi`](@ref), [`finalize_jsi`](@ref), [`default_language`](@ref), [`type_languages`](@ref) +- Session and device access: [`controller`](@ref), [`default_language`](@ref), [`type_languages`](@ref) - Interpreters and STT helpers: [`interpret_enum`](@ref), [`interpret_digit`](@ref), [`interpret_count`](@ref), [`interpret_language`](@ref), [`next_wordgroup`](@ref), [`next_letter`](@ref), [`next_letters`](@ref), [`next_digit`](@ref), [`next_digits`](@ref), [`get_language`](@ref) - Text access: [`get_selection_content`](@ref), [`get_clipboard_content`](@ref) - TTS helpers: [`is_playing_tts`](@ref), [`pause_tts`](@ref), [`resume_tts`](@ref), [`stop_tts`](@ref), [`set_tts_async_default`](@ref), [`tts_async_default`](@ref) diff --git a/src/Commands/LLM.jl b/src/Commands/LLM.jl index 9aa28b0..2b6b23d 100644 --- a/src/Commands/LLM.jl +++ b/src/Commands/LLM.jl @@ -1,7 +1,9 @@ """ Module LLM -Provides functions for operations using an LLM. +Provides predefined generic LLM commands and helper functions that are mainly meant as building blocks for application-specific command dictionaries. + +In practical use, application-specific commands are often more powerful than the generic ones, because they can be designed around the way one actually works. # Functions @@ -10,12 +12,17 @@ Provides functions for operations using an LLM. - [`LLM.type_translation`](@ref) - [`LLM.type_answer`](@ref) - [`LLM.type_text_answer`](@ref) +- [`LLM.type_text_answer_to`](@ref) +- [`LLM.type_followup`](@ref) ###### Text reading - [`LLM.read_summary`](@ref) - [`LLM.read_translation`](@ref) - [`LLM.read_answer`](@ref) - [`LLM.read_text_answer`](@ref) +- [`LLM.read_text_answer_to`](@ref) +- [`LLM.read_followup`](@ref) +- [`LLM.chat_followup`](@ref) ###### Text generation (no typing or reading) - [`LLM.ask`](@ref) diff --git a/src/JustSayIt.jl b/src/JustSayIt.jl index 31ce478..4674966 100644 --- a/src/JustSayIt.jl +++ b/src/JustSayIt.jl @@ -1,7 +1,8 @@ """ Module JustSayIt -Enables offline, low latency, highly accurate speech to command translation. It contains in addition an Application Programming Interface (API) for a fast and easy development of functionalities using speech-to-text (STT), text-to-speech (TTS) and large language models (LLMs). +Enables secure, offline, low-latency, highly accurate speech to command translation for assembling personalized voice assistants on Linux, MacOS, and Windows; it offers composable functionality using speech-to-text (STT), text-to-speech (TTS), large language models (LLMs) and more. +It contains in addition an Application Programming Interface (API) for a fast and easy development of custom functions. # General overview and examples https://github.com/omlins/JustSayIt.jl @@ -10,9 +11,12 @@ https://github.com/omlins/JustSayIt.jl ``` > julia julia> using JustSayIt -julia> start() +julia> commands = Dict( + ... +) +julia> start(commands=commands, ) ``` -Type `?start` to learn about customization keywords. +Type `?start` to learn about customization keywords to pass a command dictionary and set different options. # API @@ -21,7 +25,7 @@ Type `?start` to learn about customization keywords. julia> import JustSayIt julia> using JustSayIt.API ``` -Type `?JustSayIt.API` to learn about the Application Programming Interface (API) of JustSayIt. +Type `?JustSayIt.API` to learn about the Application Programming Interface ([`JustSayIt.API`](@ref)) of JustSayIt. """ module JustSayIt diff --git a/src/start.jl b/src/start.jl index 70b38a6..f3e781d 100644 --- a/src/start.jl +++ b/src/start.jl @@ -7,13 +7,12 @@ const DEFAULT_COMMANDS = Dict( """ - start() start() Start offline, low latency, highly accurate and secure speech to command translation. # Keyword arguments -- `commands::Dict{String, <:Any}=DEFAULT_COMMANDS[default_language]`: the commands to be recognized with their mapping to a function or to a keyboard key or shortcut or a sequence of any of those. +- `commands::Dict{String, <:Any}=DEFAULT_COMMANDS[default_language]`: the commands to be recognized with their mapping to a function or to a keyboard key or shortcut or a sequence of any of those. If you define your own top-level command dictionary and want spoken help there as well, include the language-appropriate help command explicitly (for example `"help" => Help.help` for english-US). - `subset::AbstractArray{String}=nothing`: a subset of the `commands` to be recognised and executed (instead of the complete `commands` list). - `max_speed_subset::AbstractArray{String}=nothing`: a subset of the `commands` for which the command names (first word of a command) are to be recognised with maxium speed rather than with maximum accuracy. Forcing maximum speed is usually desired for single word commands that map to functions or keyboard shortcuts that should trigger immediate actions as, e.g., mouse clicks or page up/down (in general, actions that do not modify content and can therefore safely be triggered at maximum speed). Note that forcing maximum speed means not to wait for a certain amount of silence after the end of a command as normally done for the full confirmation of a recognition. As a result, it enables a minimal latency between the saying of a command name and its execution. Note that it is usually possible to define very distinctive command names, which allow for a safe command name to shortcut mapping at maximum speed (to be tested case by case). - `default_language::String="$(LANG.EN_US)"`: the default language, which is used for the command names, for the voice arguments and for typing when no other language is specified (noted with its IETF language tag https://en.wikipedia.org/wiki/IETF_language_tag). Currently supported are: english-US ("en-us"), German ("de"), French ("fr") and Spanish ("es"). @@ -45,30 +44,14 @@ Start offline, low latency, highly accurate and secure speech to command transla To see a description of a submodule, type `?`. -# Examples - -#### Define `default_language` (also used for typing if `type_languages` is not set) -``` -# Set command and type language to french: -using JustSayIt -start(default_language="$(LANG.FR)") -``` - -#### Define `default_language` and `type_languages` +# Default `commands` ``` -# Set command language to french and type languages to french and spanish: -using JustSayIt -start(default_language="$(LANG.FR)", type_languages=["$(LANG.FR)","$(LANG.ES)"]) +$(pretty_dict_string(DEFAULT_COMMANDS)) ``` -#### Define `subset` -``` -# Listen only to the commands "help" and "type". -using JustSayIt -start(subset=["help", "type"]) -``` +# Examples -#### Define custom `commands` - functions, keyboard shortcuts and sequences of those - and a `max_speed_subset` +#### Define `commands` - functions, keyboard shortcuts and sequences of those - and a `max_speed_subset` ``` using JustSayIt commands = Dict("help" => Help.help, @@ -92,9 +75,28 @@ commands = Dict("help" => Help.help, start(commands=commands, max_speed_subset=["ma", "select", "okay", "middle", "right", "double", "triple", "copy", "upwards", "downwards", "take"]) ``` -# Default `commands` +#### Define `commands` with nested subcommands - activated automatically when the parent command is said ``` -$(pretty_dict_string(DEFAULT_COMMANDS)) +using JustSayIt +vsc_commands = Dict("comment" => (Key.ctrl, '/'), + "save" => (Key.ctrl, 's')) +commands = Dict("help" => Help.help, + "coding" => [`code`, vsc_commands]) +start(commands=commands) +``` + +#### Define `default_language` (also used for typing if `type_languages` is not set) +``` +# Set command and type language to french: +using JustSayIt +start(default_language="$(LANG.FR)") +``` + +#### Define `default_language` and `type_languages` +``` +# Set command language to french and type languages to french and spanish: +using JustSayIt +start(default_language="$(LANG.FR)", type_languages=["$(LANG.FR)","$(LANG.ES)"]) ``` """