vic-import-from-zotero-to-ebib.el 10.9 KB
Newer Older
1
2
3
4
;;;; My functions to cleanup a bibliography exported from zotero.
;;;  through the the Better Bibtex plugin (https://retorque.re/zotero-better-bibtex/)
;;;;
;;;; Author:  Victor A. Stoichita (victor.stoichita@cnrs.fr)
5
6
;;;;
;;;; Interactive functions (see their docstrings for details)
7
8
;;;; - `vic/ebib-convert-groups-to-keywords' :: Better Bibtex uses the Groups field in the bibtex export to indicate that an entry belongs
;;;;    to one or several collections. Use this function to move the contents of the Groups field to the Keywords field adding to the
9
;;;;    existing keywords
10
;;;; - `vic/ebib-fix-zotero-keywords' :: strip from keywords those characters not suited for org tags. Add space after comma separator because that's ebib's default. 
11
;;;; - `vic/ebib-import-zotero-file' :: bring zotero attachments of the entry at point into the default attachment folder.
12
;;;;   Uses Prince from https://www.princexml.com/ to convert html attachments to pdf 
13
14
15
16
17
18
19
;;;; - `vic/ebib-annotation-to-org' :: convert the latex annotation field of the entry at point to a note in org-mode format
;;;;
;;;; These functions operate on the entry at point in the ebib index window.
;;;; Suggestion for batch processing: bind one of these functions to an easily accessed key.
;;;; Ex: (define-key ebib-index-mode-map (kbd "C-$") 'vic/ebib-import-zotero-file)
;;;; Then use ebib to filter the database to entries which need to be edited with that function. Those which have
;;;; - a non-empty groups field
20
21
;;;; - the base zotero path in their file field
;;;; - a non-empty annotation field
22
;;;; Apply the function, check the results, move on to the next entry. Optionally, record a macro to just apply-move with one key. 
23
24
25

;; (define-key ebib-index-mode-map (kbd "C-$") 'vic/ebib-annotation-to-note)

26
27
(require 'ebib)
(require 'vic-bib-functions) ; Needed only for vic/ebib-create-org-noter-id
28

29
;;; IMPORT ATTACHMENTS FROM ZOTERO DATABASE
30

31
(defun vic/ebib-import-zotero-file ()
32
33
34
35
36
37
38
39
40
41
42
43
44
  "Scan ebib entry for references to zotero attachments. 
Bringing attachment into specified directory and add it to the entry.
If Zotero filename has the annotated file identifier, keep that identifier when renaming. 
The default identifier is \"_annotated\" (default of the Zotero zotfile plugin)
If file is html convert it to pdf (with Prince)
If file is neither pdf nor html, ask what to do.
If filename exists, ask for new filename."
  (interactive)
  (let* ((zot-file-base "/home/vic/.zotero/vic_library/storage/")
	 (annotated-filename-has "_annotated")
	 (db ebib--cur-db)
	 (key (ebib--get-key-at-point))
	 ;; (key "V-test-key")
45
	 (files-field (ebib-get-field-value "file" key db nil t))
46
47
48
49
50
51
52
53
54
55
56
57
	 ;; (files-field zotfileshtml)   ;
	 (newpaths ()) ;; list of paths to add to entry
	 (errors nil)) ;; holds the error status of an entry 

    ;; Make list with individual file names
    (if files-field
	(setq files-list (split-string files-field ";" t))
      (return (message (format "No files for entry %s" key))))

    ;; Iterate over file names
    (dolist (elm files-list)
      ;; Get paths
58
      (setq zotpath (vic/ebib-import-zotero-file-find-zotpath zot-file-base elm))
59
      (if zotpath
60
	  (setq targetpath (vic/ebib-import-zotero-file-build-targetpath key zotpath annotated-filename-has))
61
62
63
64
65
66
67
68
69
70
71
72
73
74
	(error (format "Could not find pdf file for %s" elm)))      
   
      (if (and zotpath targetpath (not (string= zotpath "DELETE")))
	  (progn
	    (copy-file zotpath targetpath)
	    (move-file-to-trash zotpath)
	    (message "File copied.")
	    (push targetpath newpaths))
	;; Else
	(unless (string= zotpath "DELETE")
	  (setq errors t))))
  
    (if errors
	(error (format "Can’t copy zotero path %s to target path %s for key %s" zotpath targetpath key))
75
      (vic/ebib-import-zotero-file-call-ebib newpaths key db))))
76

77
(defun vic/ebib-import-zotero-file-find-zotpath (zot-file-base elm)
78
79
80
81
82
83
84
  "Find a zotero path in an entry exported by Better bibtex for zotero."
  (let ((zot-regex-html "text/html")
	(zot-regex-pdf "application/pdf")
	(zotpath))
    (save-match-data
      ;; Convert html to pdf
      (when (string-match (format ":\\(%s.*?\\):%s" zot-file-base zot-regex-html) elm 0)
85
	(setq zotpath (vic/ebib-import-zotero-file-convert-html (match-string 1 elm))))
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
      ;; Normal files are pdf
      (unless zotpath ;; skip if html > pdf conversion  
	(string-match (format ":\\(%s.*?\\):%s" zot-file-base zot-regex-pdf) elm 0)
	(setq zotpath (match-string 1 elm)))
      (unless zotpath ;; Last resort, for files in docx ppt etc
	(string-match (format ":\\(%s.*?\\):" zot-file-base) elm 0)
    	(setq zotpath (match-string 1 elm))
	(unless (yes-or-no-p (format "Found %s. Should I keep it?" zotpath))
	  (setq zotpath "DELETE"))))      
    (when (and zotpath (not (file-exists-p zotpath)) (not (string= zotpath "DELETE")))
      (if (yes-or-no-p (format "%s not in zotero database. Delete reference to it in bib file?" elm))
	  (setq zotpath "DELETE")
	(error "Stopping here to let you fix this." )))
    zotpath))
  
101
(defun vic/ebib-import-zotero-file-convert-html (zotpath)
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
  "Convert zotpath from html to pdf. Uses Prince from https://www.princexml.com/download/"
  (let ((wait t))
    (while wait
      (let ((choice (read-char-choice (format "What shall I do with %s: (c)onvert to pdf / (v)iew / (d)elete / (q)uit? " zotpath )'(?c ?v ?d ?q))))
        (cl-case choice
          (?q (error "quit"))
          (?c (call-process-shell-command (format "prince --media=print --page-size=A4 --no-network %s -o %s" zotpath (concat zotpath ".pdf")))
	      (setq zotpath (concat zotpath ".pdf"))
	      (unless (file-exists-p zotpath)
		(error (format "Could not create file %s" zotpath)))
	      (setq wait nil))
	  (?v (call-process-shell-command (format "see %s" zotpath)))
	  (?d (setq wait nil) (setq zotpath "DELETE"))))))
  zotpath)

117
(defun vic/ebib-import-zotero-file-build-targetpath (key zotpath annotated-filename-has)
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
  "Build a target path to copy the file from zotero storage to new directory. Handle annotated files. Don’t overwrite existing files"
  (let ((targetdir (concat (nth 0 ebib-file-search-dirs)))
	(suffix (file-name-extension zotpath)))
    ;; Body
    (setq targetpath (concat targetdir "/" key))
    ;; If file is annotated we retain the
    (when (string-match-p annotated-filename-has zotpath)
      (setq targetpath (concat targetpath annotated-filename-has)))
    (setq targetpath (concat targetpath "." suffix))
    ;; If it exists, ask for other name 
    (while (file-exists-p targetpath)
      (setq targetpath (read-file-name (format "File %s already exists. Choose another file name for %s:" targetpath zotpath) targetdir)))
    ;; Return
    targetpath))

133
(defun vic/ebib-import-zotero-file-call-ebib (newpaths key db)
134
  "Call ebib to set new file paths"
135
136
137
138
139
140
141
  (let ((new-field))
    (dolist (felm newpaths)
      ;; Add filename separator if new-field already populated
      (when new-field
	(setq new-field (concat new-field ebib-filename-separator)))
      (setq new-field (concat new-field (ebib--transform-file-name-for-storing felm))))
    ;; store the field
142
    (ebib-set-field-value "file" new-field key db 'overwrite))
143
144

  ;; Set modified flag
145
  (vic/ebib-set-modified-and-refresh key db))
146
147
148

(defun vic/ebib-clean-zotero-latex-annotation ()
  "Strip zotero specifics from an annotation"
149
  ;; Remove zotero links (created by zotfile) Keep the textual reference
150
  (replace-regexp "\\\\href{zotero\:.*?}{\\(.*?\\)}" "\\1")
151
152
153
154
  ;; Remove empty lines with ~
  (replace-regexp "^~$" "")
  ;; Replace asterisk bullets with org-mode list sign
  (replace-regexp "^* " "- "))
155
156
157
158
159
160
161
162
163
164
165
166

(defun vic/ebib-latex-annotation-to-org (&optional key db)
  "If entry at point has an annotation, return its org-mode representation. Return empty string if not."
  (let* ((key (or key (ebib--get-key-at-point)))
	 (db (or db ebib--cur-db))
	 (annot (or (ebib-get-field-value "annotation" key db t t nil) "")))
    (with-temp-buffer 
      (insert annot)
      (goto-char (point-min))
      (vic/ebib-clean-zotero-latex-annotation)
      (vic/pandoc-convert-latex-to-org 2))))

167
168
;;; ANNOTATION TO NOTE

169
170
171
(defun vic/ebib-annotation-to-note (&optional nodelete)
  "Using ebib convert annotation in bibtex entry to standalone note. 
   Original annotation is deleted unless NODELETE is non-nil in which case ask for confirmation"
172
173
174
  (interactive)
  (let ((key (ebib--get-key-at-point))
	(db ebib--cur-db)
175
176
	(nodelete (or nodelete nil))
	;; Temporarily add %A to note template
177
178
	(ebib-notes-template-specifiers (cons '(65 . vic/ebib-latex-annotation-to-org) ebib-notes-template-specifiers))
	(ebib-notes-template (concat ebib-notes-template "%A")))
179
    (ebib-open-note key)
180
181
    (unless nodelete
      (vic/ebib-delete-annotation-after-convert key db))))
182

183
184
185
186
187
188
189
190
191
(defun vic/ebib-set-modified-and-refresh (key db)
  "Set modified flag on datbase and refresh display"
  (ebib--set-modified t db t (seq-filter (lambda (slave)
                                           (ebib-db-has-key key slave))
                                         (ebib--list-slaves db)))
  ;; Hackishly refresh display
  (ebib-next-entry)
  (ebib-prev-entry))

192
193
194
195
196
197
198
(defun vic/ebib-delete-annotation-after-convert (key db)
  "Select entry buffer and delete the annotation"
  ;; Go back to index. We don’t know its buffer name therefore go to ebib entry first.
  (switch-to-buffer "*Ebib-entry*")
  (ebib-quit-entry-buffer)
  (ebib-set-field-value "annotation" nil key db 'overwrite)
  ;; Set modified flag
199
200
201
  (vic/ebib-set-modified-and-refresh key db))


202
203
204
205
206
207
208
209
(defun vic/ebib-convert-groups-to-keywords ()
  "Copy content of entry's Groups field to Keywords field. Delete Groups field afterwards."
  (interactive)
  (let* ((key (ebib--get-key-at-point))
	 (db ebib--cur-db)
	 (groups (ebib-get-field-value "groups" key db 'noerror 'unbraced 'xref))
	 (keywords (ebib-get-field-value "keywords" key db 'noerror 'unbraced 'xref)))
    (unless (not (or groups keywords))
210
211
212
213
214
215
      ;; concatenate
      (setq keywords (replace-regexp-in-string "^,\\|,$" "" (concat keywords "," groups)))
      ;; cleanup
      (setq keywords (vic/ebib-fix-zotero-keywords keywords))
      ;; set
      (ebib-set-field-value "keywords" keywords key db 'overwrite)
216
217
      (ebib-set-field-value "groups" nil key db 'overwrite)
      (vic/ebib-set-modified-and-refresh key db))))
218
219
220
221
222
223
224
225

(defun vic/ebib-fix-zotero-keywords (&optional keywords)
  "Add space after keyword separator (comma). Remove spaces and other characters improper for org tags"
  (interactive)
  (let* ((key (ebib--get-key-at-point))
	 (db ebib--cur-db)
	 (keywords (or keywords (ebib-get-field-value "keywords" key db 'noerror 'unbraced 'xref))))
    (when keywords
226
227
      ;; remove spaces. for consistency with org tags
      (setq keywords (replace-regexp-in-string "[  ]" "_" keywords))
228
229
230
231
232
233
234
235
236
      ;; remove special characters
      (setq keywords (replace-regexp-in-string "[?.\!]" "" keywords))
      ;; add space after comma
      (setq keywords (replace-regexp-in-string "," ", " keywords)))
    ;; Set the field if called interactively
    (when (called-interactively-p 'any)
      (ebib-set-field-value "keywords" keywords key db 'overwrite)
      (vic/ebib-set-modified-and-refresh key db))
    keywords))