-
-
Notifications
You must be signed in to change notification settings - Fork 6
/
consult-web.el
1523 lines (1274 loc) · 68.8 KB
/
consult-web.el
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
;;; consult-web.el --- Consulting Web Search Engines -*- lexical-binding: t -*-
;; Copyright (C) 2024 Armin Darvish
;; Author: Armin Darvish
;; Maintainer: Armin Darvish
;; Created: 2024
;; Version: 0.1
;; Package-Requires: ((emacs "28.1") (consult "1.1"))
;; Homepage: https://github.com/armindarvish/consult-web
;; Keywords: convenience
;;; Commentary:
;;; Code:
;;; Requirements
(eval-when-compile
(require 'consult)
(require 'url)
(when (featurep 'json)
(require 'json))
(when (featurep 'request)
(require 'request))
)
;;; Group
(defgroup consult-web nil
"Consulting search engines and AI assistants"
:group 'convenience
:group 'minibuffer
:group 'consult
:group 'web
:group 'search
:prefix "consult-web-")
;;; Customization Variables
(defcustom consult-web-sources-modules-to-load (list)
"List of source modules/features to load.
This variable is a list of symbols;
each symbol being a source featue (e.g. consult-web-brave)"
:type '(repeat :tag "list of source modules/features to load" symbol))
(defcustom consult-web-default-browse-function #'browse-url
"consult-web default function when selecting a link"
:type '(choice (function :tag "(Default) Browse URL" #'browse-url)
(function :tag "Custom Function")))
(defcustom consult-web-alternate-browse-function #'eww-browse-url
"consult-web default function when selecting a link"
:type '(choice (function :tag "(Default) EWW" #'eww-browse-url)
(function :tag "Custom Function")))
(defcustom consult-web-default-preview-function #'eww-browse-url
"consult-web default function when previewing a link"
:type '(choice (function :tag "(Default) EWW" #'eww-browse-url)
(function :tag "Custom Function")))
(defcustom consult-web-show-preview nil
"Should`consult-web' show previews?
This turns previews on/off globally for all consult-web sources."
:type 'boolean)
(defcustom consult-web-preview-key consult-preview-key
"Preview key for consult-web.
This is similar to `consult-preview-key' but explicitly For consult-web."
:type '(choice (const :tag "Any Key" Any)
(List :tag "Debounced"
(const :Debounce)
(Float :tag "Seconds" 0.1)
(const Any))
(const :tag "No Preview" nil)
(Key :tag "Key")
(repeat :tag "List Of Keys" Key)))
(defcustom consult-web-default-count 5
"Number Of search results to retrieve."
:type 'integer)
(defcustom consult-web-default-page 0
"Offset of search results to retrieve.
If this is set to N, the first N “pages”
(or other first N entities, items for example,
depending On the source search engine capabilities)
of the search results are omitted and the rest are shown."
:type 'integer)
(defcustom consult-web-default-timeout 30
"Default timeout in seconds for
`consult-web--url-retrieve-synchronously."
:type 'integer)
(defcustom consult-web-log nil
"Default timeout in seconds for
`consult-web--url-retrieve-synchronously."
:type 'boolean)
(defcustom consult-web-log-buffer-name " *consult-web-log*"
"String for consult-web-log buffer name"
:type 'string)
(defcustom consult-web-log-level nil
"How to make logs for consult-web requests?
This can be set to nil, info or debug
nil: Does not log anything
info: Logs urls and response's http header
debug: Logs urls and the entire http response.
When non-nil, information is logged to `consult-web-log-buffer-name'."
:type '(choice
(const :tag "No Logging" nil)
(const :tag "Just HTTP Header" info)
(const :tag "Full Response" debug)))
(defcustom consult-web-group-by :domain
"What field to use to group the results in the minibuffer?
By default it is set to :domain. but can be any of:
:url group by URL
:domain group by the domain of the URL
:source group by source
"
:type '(radio (const :tag "url path" :url)
(const :tag "domain of url path":domain)
(const :tag "name of the search engine or source" :source)
(const :tag "custom other field (constant)" :any)
(const :tag "do not group" nil)))
(defcustom consult-web-multi-sources (list)
"List of sources used by `consult-web-multi'.
This variable is a list of strings, each string being name of a source.
The source name has to be a key from `consult-web-sources-alist'.
Sources can be made with the convinient macro `consult-web-define-source'."
:type '(choice (repeat :tag "list of source names" string)))
(defcustom consult-web-omni-sources (list)
"List of sources used by `consult-web-omni'.
This variable is a list of strings or symbols;
- strings can be name of a source, a key from `consult-web-sources-alist',
which can be made with the convinient macro `consult-web-define-source'
or by using `consult-web--make-source-from-consult-source'.
- symbols can be other consult sources
(see `consult-buffer-sources' for example.)"
:type '(repeat :tag "list of source names" (choice (string symbol))))
(defcustom consult-web-dynamic-omni-sources (list)
"List of sources used by `consult-web-dynamic-omni'.
This variable is a list of strings, each string being name of a source.
The source name has to be a key from `consult-web-sources-alist'.
Sources can be made with the convinient macro `consult-web-define-source'
or by using `consult-web--make-source-from-consult-source'."
:type '(choice (repeat :tag "list of source names" string)))
(defcustom consult-web-scholar-sources (list)
"List of sources used by `consult-web-scholar'.
This variable is a list of strings, each string being name of a source.
The source name has to be a key from `consult-web-sources-alist'.
Sources can be made with the convinient macro `consult-web-define-source'
or by using `consult-web--make-source-from-consult-source'."
:type '(choice (repeat :tag "list of source names" string)))
(defcustom consult-web-dynamic-sources (list)
"List of sources used by `consult-web-dynamic'.
This variable is a list of strings, each string being name of a source.
The source name has to be a key from `consult-web-sources-alist'.
Sources can be made with the convinient macro `consult-web-define-source'
or by using `consult-web--make-source-from-consult-source'."
:type '(choice (repeat :tag "list of source names" string)))
(defcustom consult-web-highlight-matches t
"Should `consult-web' highlight search queries in the minibuffer?"
:type 'boolean)
(defcustom consult-web-default-interactive-command #'consult-web-multi
"Which command should `consult-web' call?"
:type '(choice (function :tag "(Default) Search with dynamic completion (i.e. `consult-web-dynamic')" #'consult-web-dynamic)
(function :tag "Search without dynamic completion (i.e. `consult-web-multi')" #'consult-web-multi)
(function :tag "Search academic research literature (i.e. `consult-web-scholar')" #'consult-web-scholar)
(function :tag "Custom function")))
(defcustom consult-web-retrieve-backend #'consult-web-url-retrieve-sync
"Which command should `consult-web' use for url requests?"
:type '(choice (function :tag "(Default) url-retrieve backend" #'consult-web-url-retrieve-sync)
(function :tag "Emacs Request Backend" #'consult-web-request)))
(defcustom consult-web-default-autosuggest-command nil
"Which command should `consult-web' use for auto suggestion on search input?"
:type '(choice (function :tag "(default) use brave autosuggestion (i.e. `consult-web-dynamic-brave-autosuggest')" #'consult-web-dynamic-brave-autosuggest)
(function :tag "use google autosuggestion (i.e. `consult-web-dynamic-google-autosuggest')" #'consult-web-dynamic-google-autosuggest)
(function :tag "custom function")))
(defcustom consult-web-dynamic-input-debounce consult-async-input-debounce
"Input debounce for dynamic commands.
The dynamic collection process is started only when
there has not been new input for consult-web-dynamic-input-debounce seconds. This is similarto `consult-async-input-debounce' but
specifically for consult-web dynamic commands.
By default inherits from `consult-async-input-debounce'."
:type '(float :tag "delay in seconds"))
(defcustom consult-web-dynamic-input-throttle consult-async-input-throttle
"Input throttle for dynamic commands.
The dynamic collection process is started only every
`consult-web-dynamic-input-throttle' seconds. this is similar
to `consult-async-input-throttle' but specifically for
consult-web dynamic commands.
By default inherits from `consult-async-input-throttle'."
:type '(float :tag "delay in seconds"))
(defcustom consult-web-dynamic-refresh-delay consult-async-refresh-delay
"refreshing delay of the completion ui for dynamic commands.
The completion UI is only updated every
`consult-web-dynamic-refresh-delay' seconds.
This is similar to `consult-async-refresh-delay' but specifically
for consult-web dynamic commands.
By default inherits from `consult-async-refresh-delay'. "
:type '(float :tag "delay in seconds"))
;;; Other Variables
(defvar consult-web-sources--all-modules-list (list)
"List of all source modules.")
(defvar consult-web-category 'consult-web
"Category symbol for the `consult-web' package.")
(defvar consult-web-scholar-category 'consult-web-scholar
"Category symbol for the `consult-web' package.")
(defvar consult-web--selection-history (list)
"History variable that keeps selected items.")
(defvar consult-web--search-history (list)
"History variable that keeps search terms.")
(defvar consult-web-sources-alist (list)
"Alist of search engine or ai assistant sources.
This is an alist mapping source names to source property lists.
This alist is used to define how to process data form
a source (e.g. format data) or find what commands to run on
selecting candidates from a source, etc.
You can use the convinient macro `consult-web-define-source'
or the command `consult-web--make-source-from-consult-source'
to add to this alist.")
(defvar consult-web--override-group-by nil
"Override grouping in `consult-group' based on user input.
This is used in dynamic collection to change grouping.")
(defvar consult-web--current-sources (list)
"List of sources of the candidates in the current minibuffer.
This is used for defining narrow functions
(e.g. `consult-web--dynamic-narrow-function'."
)
;;; Faces
(defface consult-web-default-face
`((t :inherit 'default))
"Default face used for listing items in minibuffer.")
(defface consult-web-prompt-face
`((t :inherit 'font-lock-variable-use-face))
"The face used for prompts in minibuffer.")
(defface consult-web-engine-source-face
`((t :inherit 'font-lock-variable-use-face))
"The face for search engine source types in minibuffer.")
(defface consult-web-ai-source-face
`((t :inherit 'font-lock-operator-face))
"The face for AI assistant source types in minibuffer.")
(defface consult-web-files-source-face
`((t :inherit 'font-lock-number-face))
"The face for file source types in minibuffer.")
(defface consult-web-notes-source-face
`((t :inherit 'font-lock-warning-face))
"The face for notes source types in minibuffer.")
(defface consult-web-scholar-source-face
`((t :inherit 'font-lock-function-call-face))
"The face for academic literature source types in minibuffer.")
(defface consult-web-domain-face
`((t :inherit 'font-lock-variable-face))
"The face for domain annotation in minibuffer.")
(defface consult-web-path-face
`((t :inherit 'font-lock-warning-face))
"The face for path annotation in minibuffer.")
(defface consult-web-source-face
`((t :inherit 'font-lock-comment-face))
"The face for source annotation in minibuffer.")
(defface consult-web-highlight-match-face
`((t :inherit 'consult-highlight-match))
"Highlight match face for `consult-web'.")
(defface consult-web-preview-match-face
`((t :inherit 'consult-preview-match))
"Preview match face in `consult-web' preview buffers.")
;;; Bakcend Functions
(defun consult-web--set-string-width (string width &optional prepend)
"Sets the STRING width to a fixed value, WIDTH.
If the STRING is longer than WIDTH, it truncates the STRING
and adds ellipsis, \"...\". if the STRING is shorter,
it adds whitespace to the STRING.
If PREPEND is non-nil, it truncates or adds whitespace from
the beginning of STRING, instead of the end."
(let* ((string (format "%s" string))
(w (string-width string)))
(when (< w width)
(if prepend
(setq string (format "%s%s" (make-string (- width w) ?\s) (substring string)))
(setq string (format "%s%s" (substring string) (make-string (- width w) ?\s)))))
(when (> w width)
(if prepend
(setq string (format "...%s" (substring string (- w (- width 3)) w)))
(setq string (format "%s..." (substring string 0 (- width (+ w 3)))))))
string))
(defun consult-web--justify-left (string prefix maxwidth)
"Sets the width of STRING+PREFIX justified from left.
It uses `consult-web--set-string-width' and sets the width
of the concatenate of STRING+PREFIX
(e.g. `(concat PREFIX STRING)`) within MAXWIDTH.
This is used for aligning marginalia info in minibuffer."
(let ((s (string-width string))
(w (string-width prefix)))
(if (> maxwidth w)
(consult-web--set-string-width string (- maxwidth w) t)
string
)
))
(defun consult-web--highlight-match (regexp str ignore-case)
"Highlights REGEXP in STR.
If a regular expression contains capturing groups,
only these are highlighted.
If no capturing groups are used, highlight the whole match.
Case is ignored, if ignore-case is non-nil.
(This is adapted from `consult--highlight-regexps'.)"
(let ((i 0))
(while (and (let ((case-fold-search ignore-case))
(string-match regexp str i))
(> (match-end 0) i))
(let ((m (match-data)))
(setq i (cadr m)
m (or (cddr m) m))
(while m
(when (car m)
(add-face-text-property (car m) (cadr m)
'consult-web-highlight-match-face nil str)
)
(setq m (cddr m))))))
str)
(defun consult-web--overlay-match (match-str buffer ignore-case)
"Highlights MATCH-STR in BUFFER using an overlay.
If IGNORE-CASE is non-nil, it uses case-insensitive match.
This is provided for convinience,
if needed in formating candidates or preview buffers."
(with-current-buffer (or (get-buffer buffer) (current-buffer))
(remove-overlays (point-min) (point-max) 'consult-web-overlay t)
(goto-char (point-min))
(let ((case-fold-search ignore-case)
(consult-web-overlays (list)))
(while (search-forward match-str nil t)
(when-let* ((m (match-data))
(beg (car m))
(end (cadr m))
(overlay (make-overlay beg end))
)
(overlay-put overlay 'consult-web-overlay t)
(overlay-put overlay 'face 'consult-web-highlight-match-face)
)))))
(defun consult-web-overlays-toggle (&optional buffer)
"Toggles overlay highlights in consult-web view/preview buffers."
(interactive)
(let ((buffer (or buffer (current-buffer))))
(with-current-buffer buffer
(dolist (o (overlays-in (point-min) (point-max)))
(when (overlay-get o 'consult-web-overlay)
(if (and (overlay-get o 'face) (eq (overlay-get o 'face) 'consult-web-highlight-match-face))
(overlay-put o 'face nil)
(overlay-put o 'face 'consult-web-highlight-match-face))
)
))))
(defun consult-web--make-url-string (url params &optional ignore-keys)
"Adds key value pairs in PARAMS to URL as “&key=val”.
PARMAS should be an alist with keys and values to add to the URL.
Does not add keys for the key in IGNORE-KEYS list."
(let* ((url (if (equal (substring-no-properties url -1 nil) "?")
url
(concat url "?")))
(list (append (list url) (cl-loop for (key . value) in params
collect
(unless (member key ignore-keys)
(format "&%s=%s" key value))))))
(mapconcat #'identity list)))
(defun consult-web-properties-to-plist (string &optional ignore-keys)
"Returns a plist of the text properties of STRING.
Ommits keys in IGNORE-KEYs."
(let ((properties (text-properties-at 0 string))
(pl nil))
(cl-loop for k in properties
when (keywordp k)
collect (unless (member k ignore-keys) (push (list k (plist-get properties k)) pl)))
(apply #'append pl)))
(defun consult-web-hashtable-to-plist (hashtable &optional ignore-keys)
"Converts a HASHTABLE to a plist.
Ommits keys in IGNORE-KEYS."
(let ((pl nil))
(maphash
(lambda (k v)
(unless (member k ignore-keys)
(push (list k v) pl)))
hashtable)
(apply #'append pl)))
(defun consult-web-expand-variable-function (var)
"Call the function if VAR is a function"
(if (functionp var)
(funcall var)
var))
(defun consult-web--log (string)
"Logs the response from `consult-web-url-retrieve-sync' in `consult-web-log-buffer-name'."
(with-current-buffer (get-buffer-create consult-web-log-buffer-name)
(goto-char (point-min))
(insert "**********************************************\n")
(goto-char (point-min))
(insert (format-time-string "%F - %T%n" (current-time)))
(insert string)
(insert "\n")
(goto-char (point-min))
(insert "\n\n**********************************************\n")))
(defun consult-web--parse-http-response (&optional buffer)
"Parse the first header line such as \"HTTP/1.1 200 OK\"."
(with-current-buffer (or buffer (current-buffer))
(save-excursion
(goto-char (point-min))
(when (re-search-forward "\\=[ \t\n]*HTTP/\\(?1:[0-9\\.]+\\) +\\(?2:[0-9]+\\)" url-http-end-of-headers t)
`(:http-version ,(match-string 1) :code ,(string-to-number (match-string 2)))))))
(cl-defun consult-web--url-retrieve-synchronously (url &rest settings &key params headers parser data type error encoding timeout)
"Retrieves URL synchronously.
Passes all the arguments to url-retriev and fetches the results.
PARAMS are parameters added to the base url using `consult-web--make-url-string'.
HEADERS are headers passed to `url-request-extra-headers'.
DATA are http request data passed to `url-request-data'.
TYPE is the http request type (e.g. “GET”, “POST”)
ERROR
ENCODING
TIMEOUT
PARSER is a function that is executed in the url-retrieve response buffer and the results are returned s the output of this function.
"
(let* ((url-request-method type)
(url-request-extra-headers headers)
(url-request-data data)
(url-with-params (consult-web--make-url-string url params))
(response-data nil)
(buffer (if timeout
(with-timeout
(timeout
(setf response-data (plist-put response-data :status 'timeout))
nil)
(url-retrieve-synchronously url-with-params t))
(url-retrieve-synchronously url-with-params t))
))
(when buffer
(with-current-buffer buffer
(when consult-web-log-level
(save-excursion
(goto-char (point-min))
(cond
((eq consult-web-log-level 'info)
(consult-web--log (format "URL: %s\nRESPONSE: %s" url (buffer-substring (point-min) (pos-eol)))))
((eq consult-web-log-level 'debug)
(consult-web--log (format "URL: %s\n\nRESPONSE-HEADER:\n%s\n\nRESPONSE-BODY: %s\n" url (buffer-substring (point-min) url-http-end-of-headers) (buffer-substring url-http-end-of-headers (point-max))))))
))
(let* ((response-header (buffer-substring (point-min) url-http-end-of-headers))
(response-content (buffer-substring (+ url-http-end-of-headers 1) (point-max)))
(response-status (consult-web--parse-http-response))
)
(delete-region (point-min) (+ url-http-end-of-headers 1))
(when-let ((parsed-data (funcall parser)))
(setf response-data (plist-put response-data :data parsed-data))
)
(when response-header
(setf response-data (plist-put response-data :header response-header)))
(when response-status
(setf response-data (plist-put response-data :status response-status)))
(when response-content
(setf response-data (plist-put response-data :content response-content)))
)))
response-data
))
(defun consult-web--url-response-body (response-data)
"Extracts the response body from `url-retrieve'."
(plist-get response-data :data))
(cl-defun consult-web-url-retrieve-sync (url &key params headers parser data type error encoding timeout)
"Retrieves URL synchronously.
Passes all the arguments to `consult-web--url-retrieve-synchronously' and in trun to `url-retrieve' fetches the results.
PARAMS are parameters added to the base url using `consult-web--make-url-string'.
HEADERS are headers passed to `url-request-extra-headers'.
DATA are http request data passed to `url-request-data'.
TYPE is the http request type (e.g. “GET”, “POST”)
ERROR
ENCODING
TIMEOUT
PARSER is a function that is executed in the url-retrieve response buffer and the results are returned s the output of this function.
"
(let ((type (or type "GET"))
(encoding (or encoding 'utf8))
(timeout (or timeout consult-web-default-timeout))
)
(consult-web--url-response-body
(consult-web--url-retrieve-synchronously url :params params :headers headers :parser parser :data data :type type :error error :encoding encoding :timeout timeout))))
(cl-defun consult-web--error-handler (&rest args &key symbol-status error-thrown &allow-other-keys)
"Handles errors for `consult-web-request'."
(message "request: %s - %s" symbol-status error-thrown))
(cl-defun consult-web-request (url &rest args &key params headers data parser placeholder error &allow-other-keys)
"Convinient wrapper for `request'.
Passes all the arguments to request and fetches the results *synchronously*.
Refer to `request' documents for details."
(unless (functionp 'request)
(error "Request backend not available. Either install the package “emacs-request” or change the custom variable `consult-web-retrieve-backend'"))
(let (candidates)
(request
url
:sync t
:params params
:headers headers
:parser parser
:error (or error #'consult-web--error-handler)
:data data
:encoding 'utf-8
:success (cl-function (lambda (&key data &allow-other-keys)
(setq candidates data))))
candidates))
(defun consult-web-dynamic--split-thingatpt (thing &optional split-initial)
"Return THING at point.
If SPLIT-INITIAL is non-nil, use `consult--async-split-initial' to format the string."
(when-let (str (thing-at-point thing t))
(if split-initial
(consult--async-split-initial str)
str)))
(defun consult-web--table-to-formatted-candidate-simple (table &optional face &rest args)
"Returns a formatted candidate for TABLE.
TABLE is a hashtable that stores metadata for a consult-web candidate.
Returns a cons set of `key . value`;
The key is the value of :title key in the TABLE.
The value is all the (key value) pairs in the table as a plist.
"
(let* ((query (gethash :query table))
(title (format "%s" (gethash :title table)))
(title-str (consult-web--set-string-width title (floor (* (frame-width) 0.4))))
(pl (consult-web-hashtable-to-plist table))
)
(apply #'propertize title-str pl)
))
(defun consult-web--table-to-formatted-candidate-searchable (table &optional face &rest args)
"Formats a consult-web candidate.
TABLE is a hashtable with metadata for the candidate as (key value) pairs.
Returns a string (from :title field in TABLE) with text-properties that conatin
all the key value pairs in the table.
"
(let* ((pl (consult-web-hashtable-to-plist table))
(title (format "%s" (gethash :title table)))
(url (gethash :url table))
(urlobj (if url (url-generic-parse-url url)))
(domain (if (url-p urlobj) (url-domain urlobj)))
(domain (if (stringp domain) (propertize domain 'face 'consult-web-domain-face)))
(path (if (url-p urlobj) (url-filename urlobj)))
(path (if (stringp path) (propertize path 'face 'consult-web-path-face)))
(source (gethash :source table))
(source (if (stringp source) (propertize source 'face 'consult-web-source-face)))
(query (gethash :query table))
(snippet (gethash :snippet table))
(snippet (if (and snippet (stringp snippet) (> (string-width snippet) 25)) (concat (substring snippet 0 22) "...") snippet))
(match-str (if (stringp query) (consult--split-escaped (car (consult--command-split query))) nil))
(title-str (consult-web--set-string-width title (floor (* (frame-width) 0.4))))
(title-str (propertize title-str 'face (or face 'consult-web-default-face)))
(extra-args (consult-web-hashtable-to-plist table '(:title :url :search-url :query :source :snippet)))
(str (concat title-str (if domain (concat "\t" domain (if path path))) (if snippet (format "\s\s%s" snippet)) (if source (concat "\t" source)) (if extra-args (format "\s\s%s" extra-args))))
(str (apply #'propertize str pl))
)
(if consult-web-highlight-matches
(cond
((listp match-str)
(mapcar (lambda (match) (setq str (consult-web--highlight-match match str t))) match-str))
((stringp match-str)
(setq str (consult-web--highlight-match match-str str t)))))
str))
(defun consult-web--format-candidates-list (list &optional format-func face)
"Format a LIST of candidates.
LIST is a list of hashtables, each representing one candidate.
FORMAT-FUNC is a function that is used to format candidates if provided.
Returns a list of formatted candidates using either FORMAT-FUNC or otherwise uses default formating for the source retrieved from `consult-web-sources-alist'."
(mapcar (lambda (table)
(let* ((source (gethash :source table))
(format-func (or format-func
(plist-get (cdr (assoc source consult-web-sources-alist)) :format-func)
#'consult-web--table-to-formatted-candidate-searchable))
(face (or face
(plist-get (cdr (assoc source consult-web-sources-alist)) :face)
'consult-web-default-face))
)
(funcall format-func table face))) list))
(defun consult-web--annotate-function (cand)
"Annotates each candidate in the minibuffer.
This is provided for convinience to be passed as `:annotate' key when making sources using `consult-web-define-source'.
For more info on annotation refer to `consult' manual, particularly 'consult--read' and `consult--read-annotate' documentation."
(let* ((url (get-text-property 0 :url cand))
(urlobj (if url (url-generic-parse-url url)))
(domain (if (url-p urlobj) (url-domain urlobj) nil))
(path (if (url-p urlobj) (url-filename urlobj) nil))
(url-str nil)
(source (get-text-property 0 :source cand))
(snippet (get-text-property 0 :snippet cand))
(extra-args (consult-web-properties-to-plist cand '(:url :source :title :search-url :query :snippet :model :backend))))
(if domain (setq domain (propertize domain 'face 'consult-web-domain-face)))
(if path (setq path (propertize path 'face 'consult-web-path-face)))
(if (and snippet (stringp snippet) (> (string-width snippet) 25)) (setq snippet (concat (substring snippet 0 22) "...")))
(setq url-str (concat (if domain domain) (if path path)))
(unless (string-empty-p url-str) (setq url url-str))
(when (and url (> (string-width url) (floor (* (frame-width) 0.4))))
(setq url (consult-web--set-string-width url (floor (* (frame-width) 0.4)))))
(concat (if url (format "\s%s" url)) (if source (format "\t%s" source)) (if snippet (format "\s\s%s" snippet)) (if extra-args (format "\t%s" extra-args)))
))
(defun consult-web--group-function (group-by cand transform)
"Group candidates by GROUP-BY keyword.
This is passed as GROUP to `consult--read' on candidates and is used to define the grouping for CAND. "
(let* ((group-by (or consult-web--override-group-by group-by consult-web-group-by))
(group-by (if (not (keywordp group-by)) (intern (concat ":" (format "%s" group-by))) group-by))
(name (or (if group-by (get-text-property 0 group-by cand) "N/A"))))
(cond
((equal group-by :domain)
(when-let* ((url (get-text-property 0 :url cand))
(urlobj (if url (url-generic-parse-url url) nil))
(domain (if (url-p urlobj) (url-domain urlobj))))
(setq name domain))))
(if transform (substring cand) name)))
(defun consult-web--narrow-function (source)
"Make a narrowing (key . value) pair for the SOURCE string.
key is the first character, and value is the entire source STRING.
For example when “wikipedia” is passed as a source, it returns (w . “wikipedia”)."
`(,(string-to-char source) . ,source)
)
(defun consult-web--dynamic-narrow-function ()
"Dynamically makes a list of (key . value) for all the sources in the current list of candidates using `consult-web--narrow-function'."
(let* ((narrow-pred (lambda (cand)
(if-let ((source (get-text-property 0 :source (car cand))))
(equal (string-to-char source) consult--narrow)
)))
(narrow-keys (mapcar (lambda (c) (cons (string-to-char c) c))
consult-web--current-sources)))
`(:Predicate ,narrow-pred :keys ,narrow-keys)
))
(defun consult-web--lookup-function ()
"Lookup function for `consult-web' minibuffer candidates.
This is passed as LOOKUP to `consult--read' on candidates and is used to format the output when a candidate is selected."
(lambda (sel cands &rest args)
(let* ((info (or (car (member sel cands)) ""))
(title (get-text-property 0 :title info))
(url (get-text-property 0 :url info))
)
(apply #'propertize (or title url "nil") (or (text-properties-at 0 info) (list)))
)))
(defun consult-web--default-url-preview (cand)
"Default function to use for previewing CAND."
(when-let* ((url (cond
((listp cand)
(or (get-text-property 0 :url (car cand)) (car cand)))
(t
(or (get-text-property 0 :url cand) cand))))
(buff (funcall consult-web-default-preview-function url)))
(funcall (consult--buffer-preview) 'preview
buff
)
)
)
(cl-defun consult-web--make-state-function (&rest args &key setup preview exit return &allow-other-keys)
"Convinient wrapper for `consult-web' to make custom state functions.
This can be passed as STATE to `consult--read' on candidates and is
used to define actions when setting up, previewing or selecting a
candidate. Refer to `consult--read' documentation for more details."
(lambda (action cand &rest args)
(if cand
(pcase action
('setup
(funcall setup cand))
('preview
(funcall preview cand))
('exit
(funcall exit cand))
('return
(funcall return cand))
)))
)
(defun consult-web--dynamic-state-function ()
"State function for `consult-web' minibuffer candidates.
This is passed as STATE to `consult--read' on candidates and is used
to define actions that happen when a candidate is previewed or
selected.
The preview and retrun actions are retrieve from `consult-web-sources-alist'."
(lambda (action cand &rest args)
(if cand
(let* ((source (get-text-property 0 :source cand))
(state (plist-get (cdr (assoc source consult-web-sources-alist)) :state))
(preview (plist-get (cdr (assoc source consult-web-sources-alist)) :on-preview))
(return (plist-get (cdr (assoc source consult-web-sources-alist)) :on-return)))
(if state
(funcall state action cand args)
(pcase action
('preview
(if preview (funcall preview cand) (consult-web--default-url-preview cand)))
('return
(if return (funcall return cand) cand))
))
)))
)
(defun consult-web--default-callback (cand)
"Default CALLBACK for CAND.
The CALLBACK is called when a CAND is selected.
When making consult-web sources, if a CALLBACK is not provided, this
CALLBACK is used as a fall back."
(if-let ((url (get-text-property 0 :url cand)))
(funcall consult-web-default-browse-function url)))
(defun consult-web--read-search-string (&optional initial)
(consult--read nil
:prompt "Search: "
:initial initial
:category 'consult-web
:history 'consult-web--search-history
:add-history (delq nil
(cl-remove-duplicates
(append (mapcar (lambda (thing) (consult-web-dynamic--split-thingatpt thing nil))
(list 'number 'word 'sexp 'symbol 'url 'filename 'sentence 'line)) (list isearch-string))))
))
(defun consult-web--extract-opt-pair (opt opts ignore-opts)
"Extracts a pair of (OPT . value) from a list OPTS.
values is the next element after OPT in OPTS.
Excludes keys in IGNORE_OPTS.
This i suseful for example to extract key value pairs
from command-line options in alist of strings"
(let* ((key (cond
((string-match "--.*$" opt)
(intern (concat ":" (replace-regexp-in-string "--" "" opt))))
((string-match ":.*$" opt)
(intern opt))
(t nil)))
(val (or (nth (+ (cl-position opt opts :test 'equal) 1) opts) "nil"))
(val (cond
((string-match "--.*$\\|:.*$" val)
nil)
((stringp val)
(intern val)))))
(when (and key (not (member opt ignore-opts)))
(cons key val))
))
(defun consult-web--split-args (args)
"Splits ARGS to remaining args and input
input is the last element of ARGS
remaining args are turned into a plist"
(pcase-let* ((input (car (last args)))
(args (seq-difference (remove input args) '((nil nil) (nil)))) ;;this is hacky should find a better way
(`(,arg . ,opts) (consult--command-split input))
(remaining-opts (list)))
(cl-loop for opt in opts
do
(pcase-let* ((`(,key . ,val) (consult-web--extract-opt-pair opt opts (list "--group" ":group"))))
(when key
(setq args (append args (list key val)))
(setq remaining-opts (cl-delete-duplicates (append remaining-opts (list opt (format "%s" val))))))
))
(setq opts (seq-difference opts remaining-opts))
(when (member "-n" opts)
(setq args (append args `(:count ,(intern (or (nth (+ (cl-position "-n" opts :test 'equal) 1) opts) "nil"))))))
(when (member "-p" opts)
(setq args (append args `(:page ,(intern (or (nth (+ (cl-position "-p" opts :test 'equal) 1) opts) "nil")))))
)
(if (or (member "-g" opts) (member ":group" opts) (member "--group" opts))
(cond
((member "-g" opts)
(setq consult-web--override-group-by (intern (or (nth (+ (cl-position "-g" opts :test 'equal) 1) opts) "nil")))
)
((member "--group" opts)
(setq consult-web--override-group-by (intern (or (nth (+ (cl-position "--group" opts :test 'equal) 1) opts) "nil")))
)
((member ":group" opts)
(setq consult-web--override-group-by (intern (or (nth (+ (cl-position ":group" opts :test 'equal) 1) opts) "nil")))
))
(setq consult-web--override-group-by nil)
)
(list (or arg input) args)
))
(defun consult-web-dynamic--list-from-sources (sources &optional format-func face &rest args)
"Builds ARGS from user input and collects candidates from all
SOURCES."
(pcase-let* ((`(,input ,args) (consult-web--split-args args)))
(cond
((and (listp sources))
(apply 'append
(cl-loop for source in sources
collect
(consult-web--format-candidates-list
(apply source input args)))))
((functionp sources)
(consult-web--format-candidates-list
(apply sources input args) format-func face))
(t
(error "%s is not a consult-web-source!")))))
(defun consult-web-dynamic--collection (sources &optional format-func face &rest args)
"This is a wrapper using `consult--dynamic-collection' and
`consult-web-dynamic--list-from-sources'."
(consult--dynamic-collection (apply-partially #'consult-web-dynamic--list-from-sources sources format-func face args)))
(defun consult-web-dynamic--internal (prompt collection &optional initial category lookup history-var preview-key)
"internal function to run `consult--read'.
PROMPT COLLECTION and INITIAL are passed to `consult--read'."
(consult--read collection
:prompt prompt
:group (apply-partially #'consult-web--group-function :source)
:narrow (consult-web--dynamic-narrow-function)
:lookup (or lookup (consult-web--lookup-function))
:state (consult-web--dynamic-state-function)
:initial (consult--async-split-initial initial)
:category (or category 'consult-web)
:preview-key (and consult-web-show-preview (or preview-key consult-web-preview-key))
:history (cond
((eq history-var t)
t)
((eq history-var nil)
nil)
((and history-var (symbolp history-var))
`(:input ,history-var)))
:add-history (delq nil
(cl-remove-duplicates
(append (mapcar (lambda (thing) (consult-web-dynamic--split-thingatpt thing t))
(list 'number 'word 'sexp 'symbol 'url 'filename 'sentence 'line)) (list isearch-string))))
:sort t
)
)
(defun consult-web--source-name (source-name &optional suffix)
"Returns a symbol for SOURCE-NAME variable.
The variable is consult-web--source-%s (%s=source-name).
Adds suffix to the name if provided."
(intern (format "consult-web--source-%s" (concat (replace-regexp-in-string " " "-" (downcase source-name)) (if suffix (downcase suffix) nil)))))
(defun consult-web--source-generate-docstring (source-name)
"Makes a generic documentation string for SOURCE-NAME.
This is used in `consult-web-define-source' macro to make generic
docstrings for variables."
(format "consult-web source for %s.\n \nThis function was defined by the macro `consult-web-define-source'."
(capitalize source-name)))
(defun consult-web--func-name (source-name &optional prefix suffix)
"Make a function symbol for interactive command for SOURCE-NAME.
Adds prefix and suffix if non-nil."
(intern (concat "consult-web-" (if prefix prefix) (replace-regexp-in-string " " "-" (downcase source-name)) (if suffix suffix))))
(defun consult-web--func-generate-docstring (source-name &optional dynamic)
"Make a generic documentaion string for an interactive command.
This is used to make docstring for function made by `consult-web-define-source'."
(concat "consult-web's " (if dynamic "dynamic ") (format "interactive command to search %s."
(capitalize source-name))))
(defun consult-web--make-source-list (source-name request format annotate face narrow-char state preview-key category lookup selection-history input args)
"Internal function to make a source for `consult--multi'.
Do not use this function directly, use `consult-web-define-source' macro
instead."
`(:name ,source-name
,(if (and annotate face) :face)
,(if (and annotate face) (cond
((eq face t)
'consult-web-default-face)
(t face)))
:narrow ,narrow-char
:state ,(or state #'consult-web--dynamic-state-function)
:category ,(or category 'consult-web)
:history ,selection-history
:add-history (delq nil
(cl-remove-duplicates
(append (mapcar (lambda (thing) (consult-web-dynamic--split-thingatpt thing))
(list 'number 'word 'sexp 'symbol 'url 'filename 'sentence 'line)) (list isearch-string))))
:items ,(funcall #'consult-web--format-candidates-list (funcall request input args) format)
:annotate ,(cond
((and annotate (functionp annotate))
annotate)
((eq annotate t)
#'consult-web--annotate-function)
(t nil))
:lookup (if (and lookup (functionp lookup))
lookup
(consult-web--lookup-function))
:preview-key ,(and consult-web-show-preview (or preview-key consult-web-preview-key))
:sort t
)
)