rewrite the endian conversion to fix a big-endian host

Client: py

Change htolell(dub) to htolell(transfer.t) in compact.h, because the
conversion takes a long long, not a double.  This matters on a
big-endian host, where the conversion must swap bytes.

The big-endian letohll(n) failed because ntohl(n) is just n, doesn't
swap n.  Rewrite ntohll(n) and letohll(n) to use the same code with
both big-endian and little-endian hosts.  Expect the compiler to
optimize away one of ntohll(n) or letohll(n) when it doesn't swap n.

Author: kernigh

lib/py/src/ext/compact.h

@@ -49,7 +49,8 @@ class CompactProtocol : public ProtocolBase<CompactProtocol> {
       double f;
       int64_t t;
     } transfer;
-    transfer.f = htolell(dub);
+    transfer.f = dub;
+    transfer.t = htolell(transfer.t);
     writeBuffer(reinterpret_cast<char*>(&transfer.t), sizeof(int64_t));
   }
 

lib/py/src/ext/endian.h

@@ -27,70 +27,41 @@
 #else
 #include <winsock2.h>
 #pragma comment(lib, "ws2_32.lib")
-#define BIG_ENDIAN (4321)
-#define LITTLE_ENDIAN (1234)
-#define BYTE_ORDER LITTLE_ENDIAN
 #define inline __inline
 #endif
 
-/* Fix endianness issues on Solaris */
-#if defined(__SVR4) && defined(__sun)
-#if defined(__i386) && !defined(__i386__)
-#define __i386__
-#endif
+static inline unsigned long long ntohll(unsigned long long n) {
+  union {
+    unsigned long long f;
+    unsigned char t[8];
+  } u;
+  u.f = n;
+  return static_cast<unsigned long long>(u.t[0]) << 56
+         | static_cast<unsigned long long>(u.t[1]) << 48
+         | static_cast<unsigned long long>(u.t[2]) << 40
+         | static_cast<unsigned long long>(u.t[3]) << 32
+         | static_cast<unsigned long long>(u.t[4]) << 24
+         | static_cast<unsigned long long>(u.t[5]) << 16
+         | static_cast<unsigned long long>(u.t[6]) << 8 | static_cast<unsigned long long>(u.t[7]);
+}
 
-#ifndef BIG_ENDIAN
-#define BIG_ENDIAN (4321)
-#endif
-#ifndef LITTLE_ENDIAN
-#define LITTLE_ENDIAN (1234)
-#endif
+#define htonll(n) ntohll(n)
 
-/* I386 is LE, even on Solaris */
-#if !defined(BYTE_ORDER) && defined(__i386__)
-#define BYTE_ORDER LITTLE_ENDIAN
-#endif
-#endif
+static inline unsigned long long letohll(unsigned long long n) {
+  union {
+    unsigned long long f;
+    unsigned char t[8];
+  } u;
+  u.f = n;
+  return static_cast<unsigned long long>(u.t[0]) | static_cast<unsigned long long>(u.t[1]) << 8
+         | static_cast<unsigned long long>(u.t[2]) << 16
+         | static_cast<unsigned long long>(u.t[3]) << 24
+         | static_cast<unsigned long long>(u.t[4]) << 32
+         | static_cast<unsigned long long>(u.t[5]) << 40
+         | static_cast<unsigned long long>(u.t[6]) << 48
+         | static_cast<unsigned long long>(u.t[7]) << 56;
+}
 
-#ifndef __BYTE_ORDER
-#if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN)
-#define __BYTE_ORDER BYTE_ORDER
-#define __LITTLE_ENDIAN LITTLE_ENDIAN
-#define __BIG_ENDIAN BIG_ENDIAN
-#else
-#error "Cannot determine endianness"
-#endif
-#endif
-
-// Same comment as the enum.  Sorry.
-#if __BYTE_ORDER == __BIG_ENDIAN
-#define ntohll(n) (n)
-#define htonll(n) (n)
-#if defined(__GNUC__) && defined(__GLIBC__)
-#include <byteswap.h>
-#define letohll(n) bswap_64(n)
-#define htolell(n) bswap_64(n)
-#else /* GNUC & GLIBC */
-#define letohll(n) ((((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32))
-#define htolell(n) ((((unsigned long long)htonl(n)) << 32) + htonl(n >> 32))
-#endif
-#elif __BYTE_ORDER == __LITTLE_ENDIAN
-#if defined(__GNUC__) && defined(__GLIBC__)
-#include <byteswap.h>
-#define ntohll(n) bswap_64(n)
-#define htonll(n) bswap_64(n)
-#elif defined(_MSC_VER)
-#include <stdlib.h>
-#define ntohll(n) _byteswap_uint64(n)
-#define htonll(n) _byteswap_uint64(n)
-#else /* GNUC & GLIBC */
-#define ntohll(n) ((((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32))
-#define htonll(n) ((((unsigned long long)htonl(n)) << 32) + htonl(n >> 32))
-#endif /* GNUC & GLIBC */
-#define letohll(n) (n)
-#define htolell(n) (n)
-#else /* __BYTE_ORDER */
-#error "Can't define htonll or ntohll!"
-#endif
+#define htolell(n) letohll(n)
 
 #endif // THRIFT_PY_ENDIAN_H