Sql.mysql: Clean up character set handling.

Adds helper functions fix_{query,result}_charset() that contain
code that used to be in the QUERY_BODY() macro. This makes the
code quite a bit easier to read.

Adds charset wrappers to list_{dbs,tables,fields}() that previously
changed behavior depending on what the send_charset was.

Remove some now obsoleted character set handling from Mysql.SqlTable.

Potential fix for intermittent failures for Mysql.SqlTable to handle
tables with wide characters in the table name.

lib/modules/Mysql.pmod/SqlTable.pike

@@ -215,12 +215,11 @@ protected void create (function(void:Sql.Sql) get_db,
     col_types = ([]);
     timestamp_cols = ([]);
     datetime_cols = ([]);
-    array(mapping(string:mixed)) col_list =
-      conn->list_fields (string_to_utf8 (table));
+    array(mapping(string:mixed)) col_list = conn->list_fields(table);
     mapping(string:mixed) prop_col_info;
 
     foreach (col_list, mapping(string:mixed) col_info) {
-      string name = utf8_to_string (col_info->name);
+      string name = col_info->name;
       if (String.width (name) > 8) query_charset = 0;
       if (col_types[name])
 	error ("Strange duplicate column %O in %O\n", name, col_list);

lib/modules/Sql.pmod/mysql.pike

@@ -653,68 +653,112 @@ int decode_datetime (string timestr)
 #define HAVE_MYSQL_FIELD_CHARSETNR_IFELSE(TRUE, FALSE) FALSE
 #endif
 
+//! @returns
+//!   Returns an array:
+//!   @array
+//!     @item string 0
+//!       The adjusted query.
+//!     @item string|zero 1
+//!       The charset to restore after performing the query (if any).
+//!       This is only set when @[charset] has been set.
+//!   @endarray
+protected array(string(8bit)) fix_query_charset(string query,
+						string|void charset)
+{
+  if (charset) {
+    string current_charset = send_charset || get_charset();
+    if (charset != current_charset) {
+      CH_DEBUG ("Switching charset from %O to %O (due to charset arg).\n",
+		current_charset, charset);
+      ::big_query ("SET character_set_client=" + charset);
+      /* Can't be changed automatically - has side effects. /mast */
+      /* ::big_query("SET character_set_connection=" + charset); */
+      return ({ query, current_charset });
+    }
+    return ({ query, 0 });
+  }
+  if (!send_charset) return ({ query, 0 });
+
+  string new_send_charset = send_charset;
+  if (utf8_mode & LATIN1_UNICODE_ENCODE_MODE) {
+    if (String.width (query) == 8)
+      new_send_charset = "latin1";
+    else {
+      CH_DEBUG ("Converting (mysql-)latin1 query to utf8.\n");
+      query = utf8_encode_query (query, latin1_to_utf8, 2);
+      new_send_charset = "utf8";
+    }
+  }
+  else {  /* utf8_mode & UTF8_UNICODE_ENCODE_MODE */
+    /* NB: The send_charset may only be upgraded from
+     * "latin1" to "utf8", not the other way around.
+     * This is to avoid extraneous charset changes
+     * where the charset is changed from query to query.
+     */
+    if ((send_charset == "utf8") || !_can_send_as_latin1(query)) {
+      CH_DEBUG ("Converting query to utf8.\n");
+      query = utf8_encode_query (query, string_to_utf8, 2);
+      new_send_charset = "utf8";
+    }
+  }
+
+  if (new_send_charset != send_charset) {
+    CH_DEBUG ("Switching charset from %O to %O.\n",
+	      send_charset, new_send_charset);
+    if (mixed err = catch {
+	::big_query ("SET character_set_client=" + new_send_charset);
+	/* Can't be changed automatically - has side effects. /mast */
+	/* ::big_query("SET character_set_connection=" +
+	   new_send_charset); */
+      }) {
+      if (new_send_charset == "utf8")
+	predef::error ("The query is a wide string "
+		       "and the MySQL server doesn't support UTF-8: %s\n",
+		       describe_error (err));
+      else
+	throw (err);
+    }
+    send_charset = new_send_charset;
+  }
+
+  return ({ query, 0 });
+}
+
+protected array|object|mapping|string|int fix_result_charset(array|object|mapping|string|int res)
+{
+  if (!res) return UNDEFINED;
+
+  if (!(utf8_mode & UNICODE_DECODE_MODE)) return res;
+
+  if (objectp(res)) {
+    CH_DEBUG ("Using unicode wrapper for result.\n");
+    return
+      HAVE_MYSQL_FIELD_CHARSETNR_IFELSE (
+	.sql_util.MySQLUnicodeWrapper(res),
+	.sql_util.MySQLBrokenUnicodeWrapper(res));
+  }
+
+  if (arrayp(res)) {
+    return map(res, fix_result_charset);
+  }
+
+  if (mappingp(res)) {
+    return mkmapping(fix_result_charset(indices(res)),
+		     fix_result_charset(values(res)));
+  }
+
+  if (stringp(res)) {
+    return utf8_to_string(res);
+  }
+
+  return res;
+}
+
 #define QUERY_BODY(do_query)						\
   if (bindings)								\
     query = .sql_util.emulate_bindings(query,bindings,this);		\
 									\
-  string restore_charset;						\
-  if (charset) {							\
-    restore_charset = send_charset || get_charset();			\
-    if (charset != restore_charset) {					\
-      CH_DEBUG ("Switching charset from %O to %O (due to charset arg).\n", \
-		restore_charset, charset);				\
-      ::big_query ("SET character_set_client=" + charset);		\
-      /* Can't be changed automatically - has side effects. /mast */	\
-      /* ::big_query("SET character_set_connection=" + charset); */	\
-    } else								\
-      restore_charset = 0;						\
-  }									\
-									\
-  else if (send_charset) {						\
-    string new_send_charset = send_charset;				\
-									\
-    if (utf8_mode & LATIN1_UNICODE_ENCODE_MODE) {			\
-      if (String.width (query) == 8)					\
-	new_send_charset = "latin1";					\
-      else {								\
-	CH_DEBUG ("Converting (mysql-)latin1 query to utf8.\n");	\
-	query = utf8_encode_query (query, latin1_to_utf8, 2);		\
-	new_send_charset = "utf8";					\
-      }									\
-    }									\
-									\
-    else {  /* utf8_mode & UTF8_UNICODE_ENCODE_MODE */			\
-      /* NB: The send_charset may only be upgraded from			\
-       * "latin1" to "utf8", not the other way around.			\
-       * This is to avoid extraneous charset changes			\
-       * where the charset is changed from query to query.		\
-       */								\
-      if ((send_charset == "utf8") || !_can_send_as_latin1(query)) {	\
-	CH_DEBUG ("Converting query to utf8.\n");			\
-	query = utf8_encode_query (query, string_to_utf8, 2);		\
-	new_send_charset = "utf8";					\
-      }									\
-    }									\
-									\
-    if (new_send_charset != send_charset) {				\
-      CH_DEBUG ("Switching charset from %O to %O.\n",			\
-		send_charset, new_send_charset);			\
-      if (mixed err = catch {						\
-	  ::big_query ("SET character_set_client=" + new_send_charset);	\
-	  /* Can't be changed automatically - has side effects. /mast */ \
-	  /* ::big_query("SET character_set_connection=" +		\
-	     new_send_charset); */					\
-	  }) {								\
-	if (new_send_charset == "utf8")					\
-	  predef::error ("The query is a wide string "			\
-			 "and the MySQL server doesn't support UTF-8: %s\n", \
-			 describe_error (err));				\
-	else								\
-	  throw (err);							\
-      }									\
-      send_charset = new_send_charset;					\
-    }									\
-  }									\
+  [query, string restore_charset] = fix_query_charset(query, charset);	\
 									\
   CH_DEBUG ("Sending query with charset %O: %s.\n",			\
 	    charset || send_charset,					\
@@ -722,7 +766,7 @@ int decode_datetime (string timestr)
 	     sprintf ("%O...", query[..200]) :				\
 	     sprintf ("%O", query)));					\
 									\
-  int|object res = ::do_query(query);					\
+  int|object res = fix_result_charset(::do_query(query));		\
 									\
   if (restore_charset) {						\
     if (send_charset && (<"latin1", "utf8">)[charset])			\
@@ -735,15 +779,6 @@ int decode_datetime (string timestr)
     }									\
   }									\
 									\
-  if (!objectp(res)) return res;					\
-									\
-  if (utf8_mode & UNICODE_DECODE_MODE) {				\
-    CH_DEBUG ("Using unicode wrapper for result.\n");			\
-    return								\
-      HAVE_MYSQL_FIELD_CHARSETNR_IFELSE (				\
-	.sql_util.MySQLUnicodeWrapper(res),				\
-	.sql_util.MySQLBrokenUnicodeWrapper (res));			\
-  }									\
   return res;
 
 Mysql.mysql_result big_query (string query,
@@ -835,6 +870,37 @@ Mysql.mysql_result streaming_typed_query (string query,
   QUERY_BODY (streaming_typed_query);
 }
 
+array(string) list_dbs(string|void wild)
+{
+  return fix_result_charset(::list_dbs(wild && fix_query_charset(wild)[0]));
+}
+
+array(string) list_tables(string|void wild)
+{
+  return fix_result_charset(::list_tables(wild && fix_query_charset(wild)[0]));
+}
+
+array(mapping(string:mixed)) list_fields(string table, string|void wild)
+{
+  if (!wild) {
+    // Very common case.
+    return fix_result_charset(::list_fields(fix_query_charset(table)[0]));
+  }
+
+  string table_and_wild = table + "\0\0PIKE\0\0" + wild;
+  table_and_wild = fix_query_charset(table_and_wild)[0];
+  array(string) a = table_and_wild / "\0\0PIKE\0\0";
+  if (sizeof(a) == 2) {
+    // Common case.
+    return fix_result_charset(::list_fields(@a));
+  }
+
+  // Very uncommon case, but...
+  //
+  // Assume that the table name can not contain NUL characters.
+  return fix_result_charset(::list_fields(a[0], a[1..] * "\0\0PIKE\0\0"));
+}
+
 int(0..1) is_keyword( string name )
 //! Return 1 if the argument @[name] is a mysql keyword that needs to
 //! be quoted in a query. The list is currently up-to-date with MySQL