std::string *UTF8::convert(const std::string &incoming) { #ifdef HAVE_ERROR_CALL_ICONV_CONST char *in, *in_copy; #else const char *in_copy; #endif char *out, *out_copy; size_t in_size, in_size_copy, out_size, out_size_copy, irreversible_conversions; std::string *retval; if (_conversion_descriptor == (iconv_t) -1) { printf("conversion descriptor was NULL"); return NULL; } /* * If there were leftover bytes from a previous conversion, we'll need * to allocate a new string, and prepend those bytes to the incoming * string. * * Create a copy of the incoming string pointer because iconv is going * to update one of those pointers as its converting. We keep the other * around for reference. * * The ifdefs below are a hack to make iconv callable across multiple * platforms. On the Mac, iconv builds with the interface that it * actually exposes (i.e., the second argument is a const char ** where * the underlying string is never touched). Unfortunately, on Linux, * the interface is built with a char **, and it's a tremendous pain in * the ass to make that go away even though the string is never touched. * Yay, legacy code. * * Other alternatives: strdup every incoming line (cpu expensive with no * benefit), autoconf macro (this looks doable, but not in the small * number of minutes I have available for this), patching libiconv (just * plain gross). */ _leftovers.append(incoming); #ifdef HAVE_ERROR_CALL_ICONV_CONST /** TODO: handle no mem */ in_copy = in = (char *)malloc(_leftovers.size()); memcpy(in, _leftovers.c_str(), _leftovers.size()); #else in_copy = _leftovers.c_str(); #endif in_size_copy = in_size = _leftovers.size(); /* * For conversions of any of the common character sets today to UTF-8, * the output string will never be more than twice the size of the input * string. We allocate this much storage and add a little extra because * there might be leftover bytes from the conversion of the previous * block of data from this source. * * We keep an extra copy of the size because iconv is going to update * the output size based on the number of characters it has successfully * converted. * * If it's not possible to allocate the output storage, there's nothing we * can do. Passing the data down the pipeline would probably just cause * various parts of the architecture to crash, so we log the unfortunate * situation and drop the data out of the pipeline. */ out_size_copy = out_size = in_size * 2; out_copy = out = (char *)malloc(out_size); if (out == NULL) { #ifdef HAVE_ERROR_CALL_ICONV_CONST free(in); #endif printf("Could not allocate memory, line 157\n"); return NULL; } /* * Re-init the conversion descriptor because it might be in an * intermediate state from the previous run. This might not actually be * necessary, but some testing is required to verify that it works well * enough to be relied on. */ iconv(_conversion_descriptor, NULL, NULL, NULL, NULL); /* * Now try to convert the data. If the conversion returns non-negative, * then all characters were successfully converted and we happily move * along. If the conversion returns -1, then we've run into one of * three situations. Either we didn't have enough space to store the * results (unlikely), we ran across an unconvertible character (not in * the source encoding), or we have a partial character. */ tryagain: irreversible_conversions = iconv(_conversion_descriptor, &in_copy, &in_size_copy, &out_copy, &out_size_copy); if (irreversible_conversions == (size_t) -1) { switch(errno) { case E2BIG: /* * the conversion required more bytes in the * output than we had available. */ iconv(_conversion_descriptor, NULL, NULL, NULL, NULL); out_size_copy = out_size *= 2; out_copy = out = (char *)realloc(out, out_size); if (out == NULL) { printf("Could not reallocate memory, line 191\n"); return NULL; } #ifdef HAVE_ERROR_CALL_ICONV_CONST in_copy = in; #else in_copy = _leftovers.c_str(); #endif in_size_copy = in_size; goto tryagain; case EILSEQ: /* * an illegal sequence of characters was * enountered in the input. */ sprintf(out_copy, "\\x%2x", *(const unsigned char *)in_copy); in_copy++; out_copy += 4; out_size_copy -= 4; in_size_copy--; iconv(_conversion_descriptor, NULL, NULL, NULL, NULL); goto tryagain; case EINVAL: /* * an incomplete sequence of characters was * encountered in the input, these are the ones * we need to save off for the next go around. */ _leftovers.assign(in_copy, in_size_copy); break; case ENOENT: /* i hope this means nothing to convert? */ break; default: #ifdef HAVE_ERROR_CALL_ICONV_CONST free(in); #endif free(out); printf("Unexpected return value: %d\n", errno); return NULL; } } else _leftovers.clear(); retval = new std::string(out, out_size - out_size_copy); #ifdef HAVE_ERROR_CALL_ICONV_CONST free(in); #endif free(out); if (!retval) printf("newing the std::string failed: %s %d %d", out, out_size, out_size_copy); return retval; }