NeoMutt  2024-02-01-23-g345d7b
Teaching an old dog new tricks
DOXYGEN
Loading...
Searching...
No Matches
lib.h File Reference

Conversion between different character encodings. More...

#include <wchar.h>
+ Include dependency graph for lib.h:
+ This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Functions

size_t mutt_convert_file_from_to (FILE *fp, const struct Slist *fromcodes, const struct Slist *tocodes, char **fromcode, char **tocode, struct Content *info)
 Convert a file between encodings.
 
size_t mutt_convert_file_to (FILE *fp, const char *fromcode, struct Slist const *const tocodes, int *tocode, struct Content *info)
 Change the encoding of a file.
 
struct Contentmutt_get_content_info (const char *fname, struct Body *b, struct ConfigSubset *sub)
 Analyze file to determine MIME encoding to use.
 
void mutt_update_content_info (struct Content *info, struct ContentState *s, char *buf, size_t buflen)
 Cache some info about an email.
 

Detailed Description

Conversion between different character encodings.

Authors
  • Michal Siedlaczek
  • Richard Russon

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.

You should have received a copy of the GNU General Public License along with this program. If not, see http://www.gnu.org/licenses/.

Definition in file lib.h.

Function Documentation

◆ mutt_convert_file_from_to()

size_t mutt_convert_file_from_to ( FILE *  fp,
const struct Slist fromcodes,
const struct Slist tocodes,
char **  fromcode,
char **  tocode,
struct Content info 
)

Convert a file between encodings.

Parameters
[in]fpFile to read from
[in]fromcodesCharsets to try converting FROM
[in]tocodesCharsets to try converting TO
[out]fromcodeFrom charset selected
[out]tocodeTo charset selected
[out]infoInfo about the file
Return values
numCharacters converted
ICONV_ILLEGAL_SEQError (as a size_t)

Find the first of the fromcodes that gives a valid conversion and the best charset conversion of the file into one of the tocodes. If successful, set *fromcode and *tocode to dynamically allocated strings, set Content *info, and return the number of characters converted inexactly. If no conversion was possible, return -1.

Definition at line 215 of file convert.c.

218{
219 char **tcode = NULL;
220 size_t rc;
221 int cn;
222 struct ListNode *np = NULL;
223
224 /* Copy them */
225 tcode = mutt_mem_calloc(tocodes->count, sizeof(char *));
226 np = NULL;
227 cn = 0;
228 STAILQ_FOREACH(np, &tocodes->head, entries)
229 {
230 tcode[cn++] = mutt_str_dup(np->data);
231 }
232
234 np = NULL;
235 cn = 0;
236 STAILQ_FOREACH(np, &fromcodes->head, entries)
237 {
238 /* Try each fromcode in turn */
239 rc = mutt_convert_file_to(fp, np->data, tocodes, &cn, info);
240 if (rc != ICONV_ILLEGAL_SEQ)
241 {
242 *fromcode = np->data;
243 *tocode = tcode[cn];
244 tcode[cn] = 0;
245 break;
246 }
247 }
248
249 /* Free memory */
250 for (cn = 0; cn < tocodes->count; cn++)
251 FREE(&tcode[cn]);
252
253 FREE(&tcode);
254
255 return rc;
256}
size_t mutt_convert_file_to(FILE *fp, const char *fromcode, struct Slist const *const tocodes, int *tocode, struct Content *info)
Change the encoding of a file.
Definition: convert.c:64
void * mutt_mem_calloc(size_t nmemb, size_t size)
Allocate zeroed memory on the heap.
Definition: memory.c:50
#define FREE(x)
Definition: memory.h:45
#define ICONV_ILLEGAL_SEQ
Error value for iconv() - Illegal sequence.
Definition: charset.h:104
char * mutt_str_dup(const char *str)
Copy a string, safely.
Definition: string.c:253
#define STAILQ_FOREACH(var, head, field)
Definition: queue.h:352
A List node for strings.
Definition: list.h:35
char * data
String.
Definition: list.h:36
struct ListHead head
List containing values.
Definition: slist.h:38
size_t count
Number of values in list.
Definition: slist.h:39
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ mutt_convert_file_to()

size_t mutt_convert_file_to ( FILE *  fp,
const char *  fromcode,
struct Slist const *const  tocodes,
int *  tocode,
struct Content info 
)

Change the encoding of a file.

Parameters
[in]fpFile to convert
[in]fromcodeOriginal encoding
[in]tocodesList of target encodings
[out]tocodeChosen encoding
[out]infoEncoding information
Return values
-1Error, no conversion was possible
>0Success, number of bytes converted

Find the best charset conversion of the file from fromcode into one of the tocodes. If successful, set *tocode and Content *info and return the number of characters converted inexactly.

We convert via UTF-8 in order to avoid the condition -1(EINVAL), which would otherwise prevent us from knowing the number of inexact conversions. Where the candidate target charset is UTF-8 we avoid doing the second conversion because iconv_open("UTF-8", "UTF-8") fails with some libraries.

We assume that the output from iconv is never more than 4 times as long as the input for any pair of charsets we might be interested in.

Definition at line 64 of file convert.c.

66{
67 char bufi[256] = { 0 };
68 char bufu[512] = { 0 };
69 char bufo[4 * sizeof(bufi)] = { 0 };
70 size_t rc;
71
72 const iconv_t cd1 = mutt_ch_iconv_open("utf-8", fromcode, MUTT_ICONV_NO_FLAGS);
73 if (!iconv_t_valid(cd1))
74 return -1;
75
76 int ncodes = tocodes->count;
77 iconv_t *cd = mutt_mem_calloc(ncodes, sizeof(iconv_t));
78 size_t *score = mutt_mem_calloc(ncodes, sizeof(size_t));
79 struct ContentState *states = mutt_mem_calloc(ncodes, sizeof(struct ContentState));
80 struct Content *infos = mutt_mem_calloc(ncodes, sizeof(struct Content));
81
82 struct ListNode *np = NULL;
83 int ni = 0;
84 STAILQ_FOREACH(np, &tocodes->head, entries)
85 {
86 if (!mutt_istr_equal(np->data, "utf-8"))
87 {
88 cd[ni] = mutt_ch_iconv_open(np->data, "utf-8", MUTT_ICONV_NO_FLAGS);
89 }
90 else
91 {
92 /* Special case for conversion to UTF-8 */
93 cd[ni] = ICONV_T_INVALID;
94 score[ni] = ICONV_ILLEGAL_SEQ;
95 }
96 ni += 1;
97 }
98
99 rewind(fp);
100 size_t ibl = 0;
101 while (true)
102 {
103 /* Try to fill input buffer */
104 size_t n = fread(bufi + ibl, 1, sizeof(bufi) - ibl, fp);
105 ibl += n;
106
107 /* Convert to UTF-8 */
108 const char *ib = bufi;
109 char *ob = bufu;
110 size_t obl = sizeof(bufu);
111 n = iconv(cd1, (ICONV_CONST char **) ((ibl != 0) ? &ib : 0), &ibl, &ob, &obl);
112 if ((n == ICONV_ILLEGAL_SEQ) && (((errno != EINVAL) && (errno != E2BIG)) || (ib == bufi)))
113 {
115 break;
116 }
117 const size_t ubl1 = ob - bufu;
118
119 /* Convert from UTF-8 */
120 for (int i = 0; i < ncodes; i++)
121 {
122 if (iconv_t_valid(cd[i]) && (score[i] != ICONV_ILLEGAL_SEQ))
123 {
124 const char *ub = bufu;
125 size_t ubl = ubl1;
126 ob = bufo;
127 obl = sizeof(bufo);
128 n = iconv(cd[i], (ICONV_CONST char **) ((ibl || ubl) ? &ub : 0), &ubl, &ob, &obl);
129 if (n == ICONV_ILLEGAL_SEQ)
130 {
131 score[i] = ICONV_ILLEGAL_SEQ;
132 }
133 else
134 {
135 score[i] += n;
136 mutt_update_content_info(&infos[i], &states[i], bufo, ob - bufo);
137 }
138 }
139 else if (!iconv_t_valid(cd[i]) && (score[i] == ICONV_ILLEGAL_SEQ))
140 {
141 /* Special case for conversion to UTF-8 */
142 mutt_update_content_info(&infos[i], &states[i], bufu, ubl1);
143 }
144 }
145
146 if (ibl)
147 {
148 /* Save unused input */
149 memmove(bufi, ib, ibl);
150 }
151 else if (!ubl1 && (ib < bufi + sizeof(bufi)))
152 {
153 rc = 0;
154 break;
155 }
156 }
157
158 if (rc == 0)
159 {
160 /* Find best score */
162 for (int i = 0; i < ncodes; i++)
163 {
164 if (!iconv_t_valid(cd[i]) && (score[i] == ICONV_ILLEGAL_SEQ))
165 {
166 /* Special case for conversion to UTF-8 */
167 *tocode = i;
168 rc = 0;
169 break;
170 }
171 else if (!iconv_t_valid(cd[i]) || (score[i] == ICONV_ILLEGAL_SEQ))
172 {
173 continue;
174 }
175 else if ((rc == ICONV_ILLEGAL_SEQ) || (score[i] < rc))
176 {
177 *tocode = i;
178 rc = score[i];
179 if (rc == 0)
180 break;
181 }
182 }
183 if (rc != ICONV_ILLEGAL_SEQ)
184 {
185 memcpy(info, &infos[*tocode], sizeof(struct Content));
186 mutt_update_content_info(info, &states[*tocode], 0, 0); /* EOF */
187 }
188 }
189
190 FREE(&cd);
191 FREE(&infos);
192 FREE(&score);
193 FREE(&states);
194
195 return rc;
196}
void mutt_update_content_info(struct Content *info, struct ContentState *s, char *buf, size_t buflen)
Cache some info about an email.
Definition: content_info.c:49
iconv_t mutt_ch_iconv_open(const char *tocode, const char *fromcode, uint8_t flags)
Set up iconv for conversions.
Definition: charset.c:593
#define ICONV_T_INVALID
Error value for iconv functions.
Definition: charset.h:101
#define MUTT_ICONV_NO_FLAGS
No flags are set.
Definition: charset.h:72
static bool iconv_t_valid(const iconv_t cd)
Is the conversion descriptor valid?
Definition: charset.h:113
bool mutt_istr_equal(const char *a, const char *b)
Compare two strings, ignoring case.
Definition: string.c:721
Info about the body of an email.
Definition: content.h:57
Info about an attachment.
Definition: content.h:36
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ mutt_get_content_info()

struct Content * mutt_get_content_info ( const char *  fname,
struct Body b,
struct ConfigSubset sub 
)

Analyze file to determine MIME encoding to use.

Parameters
fnameFile to examine
bBody to update
subConfig Subset
Return values
ptrNewly allocated Content

Also set the body charset, sometimes, or not.

Definition at line 188 of file content_info.c.

190{
191 struct Content *info = NULL;
192 struct ContentState cstate = { 0 };
193 FILE *fp = NULL;
194 char *fromcode = NULL;
195 char *tocode = NULL;
196 char buf[100] = { 0 };
197 size_t r;
198
199 struct stat st = { 0 };
200
201 if (b && !fname)
202 fname = b->filename;
203 if (!fname)
204 return NULL;
205
206 if (stat(fname, &st) == -1)
207 {
208 mutt_error(_("Can't stat %s: %s"), fname, strerror(errno));
209 return NULL;
210 }
211
212 if (!S_ISREG(st.st_mode))
213 {
214 mutt_error(_("%s isn't a regular file"), fname);
215 return NULL;
216 }
217
218 fp = mutt_file_fopen(fname, "r");
219 if (!fp)
220 {
221 mutt_debug(LL_DEBUG1, "%s: %s (errno %d)\n", fname, strerror(errno), errno);
222 return NULL;
223 }
224
225 info = mutt_mem_calloc(1, sizeof(struct Content));
226
227 const char *const c_charset = cc_charset();
228 if (b && (b->type == TYPE_TEXT) && (!b->noconv && !b->force_charset))
229 {
230 const struct Slist *const c_attach_charset = cs_subset_slist(sub, "attach_charset");
231 const struct Slist *const c_send_charset = cs_subset_slist(sub, "send_charset");
232 struct Slist *c_charset_slist = slist_parse(c_charset, D_SLIST_SEP_COLON);
233
234 const struct Slist *fchs = b->use_disp ?
235 (c_attach_charset ? c_attach_charset : c_charset_slist) :
236 c_charset_slist;
237
238 struct Slist *chs = slist_parse(mutt_param_get(&b->parameter, "charset"), D_SLIST_SEP_COLON);
239
240 if (c_charset && (chs || c_send_charset) &&
241 (mutt_convert_file_from_to(fp, fchs, chs ? chs : c_send_charset, &fromcode,
242 &tocode, info) != ICONV_ILLEGAL_SEQ))
243 {
244 if (!chs)
245 {
246 char chsbuf[256] = { 0 };
247 mutt_ch_canonical_charset(chsbuf, sizeof(chsbuf), tocode);
248 mutt_param_set(&b->parameter, "charset", chsbuf);
249 }
250 FREE(&b->charset);
251 b->charset = mutt_str_dup(fromcode);
252 FREE(&tocode);
253 mutt_file_fclose(&fp);
254 slist_free(&c_charset_slist);
255 slist_free(&chs);
256 return info;
257 }
258
259 slist_free(&c_charset_slist);
260 slist_free(&chs);
261 }
262
263 rewind(fp);
264 while ((r = fread(buf, 1, sizeof(buf), fp)))
265 mutt_update_content_info(info, &cstate, buf, r);
266 mutt_update_content_info(info, &cstate, 0, 0);
267
268 mutt_file_fclose(&fp);
269
270 if (b && (b->type == TYPE_TEXT) && (!b->noconv && !b->force_charset))
271 {
272 mutt_param_set(&b->parameter, "charset",
273 (!info->hibin ? "us-ascii" :
274 c_charset && !mutt_ch_is_us_ascii(c_charset) ? c_charset :
275 "unknown-8bit"));
276 }
277
278 return info;
279}
const struct Slist * cs_subset_slist(const struct ConfigSubset *sub, const char *name)
Get a string-list config item by name.
Definition: helpers.c:243
const char * cc_charset(void)
Get the cached value of $charset.
Definition: config_cache.c:116
size_t mutt_convert_file_from_to(FILE *fp, const struct Slist *fromcodes, const struct Slist *tocodes, char **fromcode, char **tocode, struct Content *info)
Convert a file between encodings.
Definition: convert.c:215
#define mutt_file_fclose(FP)
Definition: file.h:148
#define mutt_file_fopen(PATH, MODE)
Definition: file.h:147
#define mutt_error(...)
Definition: logging2.h:92
#define mutt_debug(LEVEL,...)
Definition: logging2.h:89
@ LL_DEBUG1
Log at debug level 1.
Definition: logging2.h:43
@ TYPE_TEXT
Type: 'text/*'.
Definition: mime.h:38
void mutt_ch_canonical_charset(char *buf, size_t buflen, const char *name)
Canonicalise the charset of a string.
Definition: charset.c:374
#define mutt_ch_is_us_ascii(str)
Definition: charset.h:98
#define _(a)
Definition: message.h:28
struct Slist * slist_parse(const char *str, uint32_t flags)
Parse a list of strings into a list.
Definition: slist.c:179
void slist_free(struct Slist **ptr)
Free an Slist object.
Definition: slist.c:126
char * mutt_param_get(const struct ParameterList *pl, const char *s)
Find a matching Parameter.
Definition: parameter.c:85
void mutt_param_set(struct ParameterList *pl, const char *attribute, const char *value)
Set a Parameter.
Definition: parameter.c:111
bool noconv
Don't do character set conversion.
Definition: body.h:46
char * charset
Send mode: charset of attached file as stored on disk.
Definition: body.h:78
struct ParameterList parameter
Parameters of the content-type.
Definition: body.h:62
bool use_disp
Content-Disposition uses filename= ?
Definition: body.h:47
bool force_charset
Send mode: don't adjust the character set when in send-mode.
Definition: body.h:44
unsigned int type
content-type primary type, ContentType
Definition: body.h:40
char * filename
When sending a message, this is the file to which this structure refers.
Definition: body.h:58
long hibin
8-bit characters
Definition: content.h:37
String list.
Definition: slist.h:37
#define D_SLIST_SEP_COLON
Slist items are colon-separated.
Definition: types.h:109
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ mutt_update_content_info()

void mutt_update_content_info ( struct Content info,
struct ContentState s,
char *  buf,
size_t  buflen 
)

Cache some info about an email.

Parameters
infoInfo about an Attachment
sInfo about the Body of an email
bufBuffer for the result
buflenLength of the buffer

Definition at line 49 of file content_info.c.

51{
52 bool from = s->from;
53 int whitespace = s->whitespace;
54 bool dot = s->dot;
55 int linelen = s->linelen;
56 bool was_cr = s->was_cr;
57
58 if (!buf) /* This signals EOF */
59 {
60 if (was_cr)
61 info->binary = true;
62 if (linelen > info->linemax)
63 info->linemax = linelen;
64
65 return;
66 }
67
68 for (; buflen; buf++, buflen--)
69 {
70 char ch = *buf;
71
72 if (was_cr)
73 {
74 was_cr = false;
75 if (ch == '\n')
76 {
77 if (whitespace)
78 info->space = true;
79 if (dot)
80 info->dot = true;
81 if (linelen > info->linemax)
82 info->linemax = linelen;
83 whitespace = 0;
84 dot = false;
85 linelen = 0;
86 continue;
87 }
88
89 info->binary = true;
90 }
91
92 linelen++;
93 if (ch == '\n')
94 {
95 info->crlf++;
96 if (whitespace)
97 info->space = true;
98 if (dot)
99 info->dot = true;
100 if (linelen > info->linemax)
101 info->linemax = linelen;
102 whitespace = 0;
103 linelen = 0;
104 dot = false;
105 }
106 else if (ch == '\r')
107 {
108 info->crlf++;
109 info->cr = true;
110 was_cr = true;
111 continue;
112 }
113 else if (ch & 0x80)
114 {
115 info->hibin++;
116 }
117 else if ((ch == '\t') || (ch == '\f'))
118 {
119 info->ascii++;
120 whitespace++;
121 }
122 else if (ch == 0)
123 {
124 info->nulbin++;
125 info->lobin++;
126 }
127 else if ((ch < 32) || (ch == 127))
128 {
129 info->lobin++;
130 }
131 else
132 {
133 if (linelen == 1)
134 {
135 if ((ch == 'F') || (ch == 'f'))
136 from = true;
137 else
138 from = false;
139 if (ch == '.')
140 dot = true;
141 else
142 dot = false;
143 }
144 else if (from)
145 {
146 if ((linelen == 2) && (ch != 'r'))
147 {
148 from = false;
149 }
150 else if ((linelen == 3) && (ch != 'o'))
151 {
152 from = false;
153 }
154 else if (linelen == 4)
155 {
156 if (ch == 'm')
157 info->from = true;
158 from = false;
159 }
160 }
161 if (ch == ' ')
162 whitespace++;
163 info->ascii++;
164 }
165
166 if (linelen > 1)
167 dot = false;
168 if ((ch != ' ') && (ch != '\t'))
169 whitespace = 0;
170 }
171
172 s->from = from;
173 s->whitespace = whitespace;
174 s->dot = dot;
175 s->linelen = linelen;
176 s->was_cr = was_cr;
177}
bool was_cr
Was the last character CR?
Definition: content.h:62
int whitespace
Number of trailing whitespaces.
Definition: content.h:59
bool from
Is the current line a prefix of "From "?
Definition: content.h:58
int linelen
Length of the current line.
Definition: content.h:61
bool dot
Was the last character a dot?
Definition: content.h:60
long crlf
\r and \n characters
Definition: content.h:40
bool cr
Has CR, even when in a CRLF pair.
Definition: content.h:47
bool space
Whitespace at the end of lines?
Definition: content.h:43
long ascii
Number of ascii chars.
Definition: content.h:41
bool binary
Long lines, or CR not in CRLF pair.
Definition: content.h:44
bool from
Has a line beginning with "From "?
Definition: content.h:45
long nulbin
Null characters (0x0)
Definition: content.h:39
long linemax
Length of the longest line in the file.
Definition: content.h:42
long lobin
Unprintable 7-bit chars (eg., control chars)
Definition: content.h:38
bool dot
Has a line consisting of a single dot?
Definition: content.h:46
+ Here is the caller graph for this function: