diff --git a/.travis.yml b/.travis.yml
index 221ded6d59334b68aed69094e1619ec451626585..f9d43a0cc30892c52724c64c2afed76ad830a222 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,12 +3,14 @@ python: 2.7
 env:
   - TOXENV=py26
   - TOXENV=py27
+  - TOXENV=py32
   - TOXENV=py33
   - TOXENV=pypy
 
 install:
   # Add the PyPy repository
   - "if [[ $TOXENV == 'pypy' ]]; then sudo add-apt-repository -y ppa:pypy/ppa; fi"
+  - "if [[ $TOXENV == 'pypy' ]]; then sudo apt-get update; fi"
   # Upgrade PyPy
   - "if [[ $TOXENV == 'pypy' ]]; then sudo apt-get -y install pypy; fi"
   # This is required because we need to get rid of the Travis installed PyPy
@@ -29,9 +31,3 @@ notifications:
       - "irc.freenode.org#cryptography-dev"
     use_notice: true
     skip_join: true
-
-
-matrix:
-  allow_failures:
-    # There is a segfault issue with PyPy
-    - env: TOXENV=pypy
diff --git a/MANIFEST.in b/MANIFEST.in
index bd6f93709bccac485ab119c791a6b56dde31b540..f7c92c539524cb53c6c9a23880df80a714efd8a8 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -2,29 +2,19 @@ include tox.ini
 include LICENSE
 
 # libsodium files
-include src/libsodium/AUTHORS
-include src/libsodium/ChangeLog
-include src/libsodium/LICENSE
-include src/libsodium/README
-include src/libsodium/THANKS
-include src/libsodium/compile
-include src/libsodium/configure
-include src/libsodium/depcomp
-include src/libsodium/install-sh
-include src/libsodium/missing
-include src/libsodium/test-driver
-recursive-include src *.S
-recursive-include src *.ac
-recursive-include src *.am
-recursive-include src *.c
-recursive-include src *.exp
-recursive-include src *.guess
-recursive-include src *.h
-recursive-include src *.in
-recursive-include src *.m4
-recursive-include src *.markdown
-recursive-include src *.sh
-recursive-include src *.sub
+recursive-include src/libsodium *
+recursive-include src/libsodium *.S
+recursive-include src/libsodium *.ac
+recursive-include src/libsodium *.am
+recursive-include src/libsodium *.c
+recursive-include src/libsodium *.exp
+recursive-include src/libsodium *.guess
+recursive-include src/libsodium *.h
+recursive-include src/libsodium *.in
+recursive-include src/libsodium *.m4
+recursive-include src/libsodium *.markdown
+recursive-include src/libsodium *.sh
+recursive-include src/libsodium *.sub
 
 # test files
 recursive-include docs *.png
@@ -35,4 +25,4 @@ recursive-include tests *.py
 recursive-include tests/data *
 
 # Remove CFFI files
-recursive-exclude nacl *.c
+global-exclude __pycache__/*
diff --git a/setup.py b/setup.py
index 7d1e25f343619d4a3cf608fa38a007536cf93b89..9ae0c3e21139a39b4855df9b3dc96f98537de084 100644
--- a/setup.py
+++ b/setup.py
@@ -16,13 +16,13 @@ import functools
 import glob
 import os
 import os.path
-import shlex
 import subprocess
 import sys
 
 from distutils.command.build_clib import build_clib as _build_clib
+from distutils.command.build_ext import build_ext as _build_ext
 
-from setuptools import setup
+from setuptools import Distribution, setup
 
 
 def here(*paths):
@@ -37,9 +37,6 @@ sys.path.append(here("src"))
 import nacl
 
 
-SODIUM_VERSION = "0.4.3"
-
-
 def which(name, flags=os.X_OK):  # Taken from twisted
     result = []
     exts = filter(None, os.environ.get('PATHEXT', '').split(os.pathsep))
@@ -60,7 +57,6 @@ def which(name, flags=os.X_OK):  # Taken from twisted
 # This hack exists so that we can import nacl here
 sys.path += glob.glob("*.egg")
 
-
 try:
     import nacl.nacl
 except ImportError:
@@ -69,102 +65,77 @@ except ImportError:
 else:
     # building bdist - cffi is here!
     ext_modules = [nacl.nacl.ffi.verifier.get_extension()]
-    ext_modules[0].include_dirs.append(sodium("include"))
+
+
+class Distribution(Distribution):
+
+    def has_c_libraries(self):
+        return True
 
 
 class build_clib(_build_clib):
 
+    def get_source_files(self):
+        files = glob.glob(here("src/libsodium/*"))
+        files += glob.glob(here("src/libsodium/*/*"))
+        files += glob.glob(here("src/libsodium/*/*/*"))
+        files += glob.glob(here("src/libsodium/*/*/*/*"))
+        files += glob.glob(here("src/libsodium/*/*/*/*/*"))
+        files += glob.glob(here("src/libsodium/*/*/*/*/*/*"))
+
+        return files
+
+    def build_libraries(self, libraries):
+        raise Exception("build_libraries")
+
+    def check_library_list(self, libraries):
+        raise Exception("check_library_list")
+
+    def get_library_names(self):
+        return ["sodium"]
+
     def run(self):
+        build_temp = os.path.abspath(self.build_temp)
+
+        # Ensure our temporary build directory exists
+        try:
+            os.makedirs(build_temp)
+        except IOError:
+            pass
+
+        # Locate our configure script
+        configure = here("src/libsodium/configure")
+
         # Run ./configure
         subprocess.check_call(
-            "./configure --disable-debug --disable-dependency-tracking",
-            cwd=here("src/libsodium"),
-            shell=True,
+            [
+                configure, "--disable-shared", "--enable-static",
+                "--disable-debug", "--disable-dependency-tracking",
+                "--with-pic", "--prefix", os.path.abspath(self.build_clib),
+            ],
+            cwd=build_temp,
         )
 
-        # Parse the Makefile to determine what macros to define
-        with open(here("src/libsodium/Makefile")) as makefile:
-            for line in makefile:
-                if line.startswith("DEFS"):
-                    defines = [
-                        tuple(shlex.split(i)[0][2:].split("=", 1))
-                        for i in shlex.split(line)
-                        if i.startswith("-D")
-                    ]
-
-        # Configure libsodium using the Makefile defines
-        libraries = []
-        for libname, build_info in self.libraries:
-            if libname == "sodium":
-                # Store the define macros inside the build info
-                macros = dict(build_info.get("macros", []))
-                macros.update(dict(defines))
-                build_info["macros"] = list(macros.items())
-
-                sources = build_info["sources"]
-
-                # Dynamically modify the implementation based on if we have
-                #   TIMODE or not
-                if "HAVE_TI_MODE" in macros:
-                    sources.extend([
-                        sodium("crypto_scalarmult/curve25519/donna_c64/base_curve25519_donna_c64.c"),
-                        sodium("crypto_scalarmult/curve25519/donna_c64/smult_curve25519_donna_c64.c"),
-                    ])
-                else:
-                    sources.extend([
-                        sodium("crypto_scalarmult/curve25519/ref/base_curve25519_ref.c"),
-                        sodium("crypto_scalarmult/curve25519/ref/smult_curve25519_ref.c"),
-                    ])
-
-                # Dynamically modify the implementation based on if we have
-                #   AMD64 ASM or not.
-                if "HAVE_AMD64_ASM" in macros:
-                    sources.extend([
-                        sodium("crypto_stream/salsa20/amd64_xmm6/stream_salsa20_amd64_xmm6.S"),
-                    ])
-
-                    self._include_asm = True
-                else:
-                    sources.extend([
-                        sodium("crypto_stream/salsa20/ref/stream_salsa20_ref.c"),
-                        sodium("crypto_stream/salsa20/ref/xor_salsa20_ref.c"),
-                    ])
-
-                    self._include_asm = False
-
-                build_info["sources"] = sources
-
-            libraries.append((libname, build_info))
-
-        self.libraries = libraries
-
-        # Call our normal run
-        return _build_clib.run(self)
+        # Build the library
+        subprocess.check_call(["make"], cwd=build_temp)
 
-    def build_libraries(self, libraries):
-        # This is a convenient place to modify the compiler so that we can add
-        #   the .S extension
-        if self._include_asm and not ".S" in self.compiler.src_extensions:
-            self.compiler.src_extensions.append(".S")
+        # Check the build library
+        subprocess.check_call(["make", "check"], cwd=build_temp)
 
-        # If we have a unix compiler see if it's gcc so we can enable certain
-        #   flags for it.
-        if self.compiler.compiler_type == "unix":
-            cc = which(self.compiler.executables["compiler"][0])[0]
-            cc = os.path.realpath(cc)
+        # Install the built library
+        subprocess.check_call(["make", "install"], cwd=build_temp)
 
-            if "gcc" in os.path.basename(cc):
-                real_compile = self.compiler._compile
 
-                def _compile(obj, src, ext, cc_args, extra_postargs, pp_opts):
-                    if "-std=c99" not in cc_args:
-                        cc_args += ["-std=c99"]
-                    return real_compile(
-                        obj, src, ext, cc_args, extra_postargs, pp_opts)
+class build_ext(_build_ext):
 
-                self.compiler._compile = _compile
+    def run(self):
+        build_clib = self.get_finalized_command("build_clib")
+        self.include_dirs.append(
+            os.path.join(build_clib.build_clib, "include")
+        )
+        self.library_dirs.append(os.path.join(build_clib.build_clib, "lib"))
 
-        return _build_clib.build_libraries(self, libraries)
+        return _build_ext.run(self)
 
 
 setup(
@@ -199,146 +170,12 @@ setup(
     ext_package="nacl",
     ext_modules=ext_modules,
 
-    libraries=[
-        ("sodium", {
-            "include_dirs": [
-                sodium("include/sodium"),
-            ],
-            "sources": [sodium(s) for s in [
-                "crypto_auth/crypto_auth.c",
-                "crypto_auth/hmacsha256/auth_hmacsha256_api.c",
-                "crypto_auth/hmacsha256/ref/hmac_hmacsha256.c",
-                "crypto_auth/hmacsha256/ref/verify_hmacsha256.c",
-                "crypto_auth/hmacsha512256/auth_hmacsha512256_api.c",
-                "crypto_auth/hmacsha512256/ref/hmac_hmacsha512256.c",
-                "crypto_auth/hmacsha512256/ref/verify_hmacsha512256.c",
-                "crypto_box/crypto_box.c",
-                "crypto_box/curve25519xsalsa20poly1305/box_curve25519xsalsa20poly1305_api.c",
-                "crypto_box/curve25519xsalsa20poly1305/ref/after_curve25519xsalsa20poly1305.c",
-                "crypto_box/curve25519xsalsa20poly1305/ref/before_curve25519xsalsa20poly1305.c",
-                "crypto_box/curve25519xsalsa20poly1305/ref/box_curve25519xsalsa20poly1305.c",
-                "crypto_box/curve25519xsalsa20poly1305/ref/keypair_curve25519xsalsa20poly1305.c",
-                "crypto_core/hsalsa20/ref2/core_hsalsa20.c",
-                "crypto_core/hsalsa20/core_hsalsa20_api.c",
-                "crypto_core/salsa20/ref/core_salsa20.c",
-                "crypto_core/salsa20/core_salsa20_api.c",
-                "crypto_core/salsa2012/ref/core_salsa2012.c",
-                "crypto_core/salsa2012/core_salsa2012_api.c",
-                "crypto_core/salsa208/ref/core_salsa208.c",
-                "crypto_core/salsa208/core_salsa208_api.c",
-                "crypto_generichash/crypto_generichash.c",
-                "crypto_generichash/blake2/generichash_blake2_api.c",
-                "crypto_generichash/blake2/ref/blake2b-ref.c",
-                "crypto_generichash/blake2/ref/generichash_blake2b.c",
-                "crypto_hash/crypto_hash.c",
-                "crypto_hash/sha256/hash_sha256_api.c",
-                "crypto_hash/sha256/ref/hash_sha256.c",
-                "crypto_hash/sha512/hash_sha512_api.c",
-                "crypto_hash/sha512/ref/hash_sha512.c",
-                "crypto_hashblocks/sha256/ref/blocks_sha256.c",
-                "crypto_hashblocks/sha256/hashblocks_sha256_api.c",
-                "crypto_hashblocks/sha512/ref/blocks_sha512.c",
-                "crypto_hashblocks/sha512/hashblocks_sha512_api.c",
-                "crypto_onetimeauth/crypto_onetimeauth.c",
-                "crypto_onetimeauth/poly1305/onetimeauth_poly1305.c",
-                "crypto_onetimeauth/poly1305/onetimeauth_poly1305_api.c",
-                "crypto_onetimeauth/poly1305/onetimeauth_poly1305_try.c",
-                "crypto_onetimeauth/poly1305/53/auth_poly1305_53.c",
-                "crypto_onetimeauth/poly1305/53/verify_poly1305_53.c",
-                "crypto_onetimeauth/poly1305/donna/auth_poly1305_donna.c",
-                "crypto_onetimeauth/poly1305/donna/verify_poly1305_donna.c",
-                "crypto_scalarmult/crypto_scalarmult.c",
-                "crypto_secretbox/crypto_secretbox.c",
-                "crypto_secretbox/xsalsa20poly1305/secretbox_xsalsa20poly1305_api.c",
-                "crypto_secretbox/xsalsa20poly1305/ref/box_xsalsa20poly1305.c",
-                "crypto_shorthash/crypto_shorthash.c",
-                "crypto_shorthash/siphash24/shorthash_siphash24_api.c",
-                "crypto_shorthash/siphash24/ref/shorthash_siphash24.c",
-                "crypto_sign/crypto_sign.c",
-                "crypto_sign/ed25519/sign_ed25519_api.c",
-                "crypto_sign/ed25519/ref10/fe_0.c",
-                "crypto_sign/ed25519/ref10/fe_1.c",
-                "crypto_sign/ed25519/ref10/fe_add.c",
-                "crypto_sign/ed25519/ref10/fe_cmov.c",
-                "crypto_sign/ed25519/ref10/fe_copy.c",
-                "crypto_sign/ed25519/ref10/fe_frombytes.c",
-                "crypto_sign/ed25519/ref10/fe_invert.c",
-                "crypto_sign/ed25519/ref10/fe_isnegative.c",
-                "crypto_sign/ed25519/ref10/fe_isnonzero.c",
-                "crypto_sign/ed25519/ref10/fe_mul.c",
-                "crypto_sign/ed25519/ref10/fe_neg.c",
-                "crypto_sign/ed25519/ref10/fe_pow22523.c",
-                "crypto_sign/ed25519/ref10/fe_sq.c",
-                "crypto_sign/ed25519/ref10/fe_sq2.c",
-                "crypto_sign/ed25519/ref10/fe_sub.c",
-                "crypto_sign/ed25519/ref10/fe_tobytes.c",
-                "crypto_sign/ed25519/ref10/ge_add.c",
-                "crypto_sign/ed25519/ref10/ge_double_scalarmult.c",
-                "crypto_sign/ed25519/ref10/ge_frombytes.c",
-                "crypto_sign/ed25519/ref10/ge_madd.c",
-                "crypto_sign/ed25519/ref10/ge_msub.c",
-                "crypto_sign/ed25519/ref10/ge_p1p1_to_p2.c",
-                "crypto_sign/ed25519/ref10/ge_p1p1_to_p3.c",
-                "crypto_sign/ed25519/ref10/ge_p2_0.c",
-                "crypto_sign/ed25519/ref10/ge_p2_dbl.c",
-                "crypto_sign/ed25519/ref10/ge_p3_0.c",
-                "crypto_sign/ed25519/ref10/ge_p3_dbl.c",
-                "crypto_sign/ed25519/ref10/ge_p3_to_cached.c",
-                "crypto_sign/ed25519/ref10/ge_p3_to_p2.c",
-                "crypto_sign/ed25519/ref10/ge_p3_tobytes.c",
-                "crypto_sign/ed25519/ref10/ge_precomp_0.c",
-                "crypto_sign/ed25519/ref10/ge_scalarmult_base.c",
-                "crypto_sign/ed25519/ref10/ge_sub.c",
-                "crypto_sign/ed25519/ref10/ge_tobytes.c",
-                "crypto_sign/ed25519/ref10/keypair.c",
-                "crypto_sign/ed25519/ref10/open.c",
-                "crypto_sign/ed25519/ref10/sc_muladd.c",
-                "crypto_sign/ed25519/ref10/sc_reduce.c",
-                "crypto_sign/ed25519/ref10/sign.c",
-                "crypto_sign/edwards25519sha512batch/sign_edwards25519sha512batch_api.c",
-                "crypto_sign/edwards25519sha512batch/ref/fe25519_edwards25519sha512batch.c",
-                "crypto_sign/edwards25519sha512batch/ref/ge25519_edwards25519sha512batch.c",
-                "crypto_sign/edwards25519sha512batch/ref/sc25519_edwards25519sha512batch.c",
-                "crypto_sign/edwards25519sha512batch/ref/sign_edwards25519sha512batch.c",
-                "crypto_stream/crypto_stream.c",
-                "crypto_stream/aes128ctr/portable/afternm_aes128ctr.c",
-                "crypto_stream/aes128ctr/stream_aes128ctr_api.c",
-                "crypto_stream/aes128ctr/portable/beforenm_aes128ctr.c",
-                "crypto_stream/aes128ctr/portable/common_aes128ctr.c",
-                "crypto_stream/aes128ctr/portable/consts_aes128ctr.c",
-                "crypto_stream/aes128ctr/portable/int128_aes128ctr.c",
-                "crypto_stream/aes128ctr/portable/stream_aes128ctr.c",
-                "crypto_stream/aes128ctr/portable/xor_afternm_aes128ctr.c",
-                "crypto_stream/aes256estream/hongjun/aes256-ctr.c",
-                "crypto_stream/aes256estream/stream_aes256estream_api.c",
-                "crypto_stream/salsa2012/stream_salsa2012_api.c",
-                "crypto_stream/salsa2012/ref/stream_salsa2012.c",
-                "crypto_stream/salsa2012/ref/xor_salsa2012.c",
-                "crypto_stream/salsa208/stream_salsa208_api.c",
-                "crypto_stream/salsa208/ref/stream_salsa208.c",
-                "crypto_stream/salsa208/ref/xor_salsa208.c",
-                "crypto_stream/xsalsa20/stream_xsalsa20_api.c",
-                "crypto_stream/xsalsa20/ref/stream_xsalsa20.c",
-                "crypto_stream/xsalsa20/ref/xor_xsalsa20.c",
-                "crypto_verify/16/verify_16_api.c",
-                "crypto_verify/16/ref/verify_16.c",
-                "crypto_verify/32/verify_32_api.c",
-                "crypto_verify/32/ref/verify_32.c",
-                "randombytes/randombytes.c",
-                "randombytes/salsa20/randombytes_salsa20_random.c",
-                "randombytes/sysrandom/randombytes_sysrandom.c",
-                "sodium/compat.c",
-                "sodium/core.c",
-                "sodium/utils.c",
-                "sodium/version.c",
-            ]],
-        }),
-    ],
-
-    zip_safe=False,
     cmdclass={
         "build_clib": build_clib,
+        "build_ext": build_ext,
     },
+    distclass=Distribution,
+    zip_safe=False,
 
     classifiers=[
         "Programming Language :: Python :: Implementation :: CPython",
diff --git a/src/nacl/_cffi_fix.py b/src/nacl/_cffi_fix.py
new file mode 100644
index 0000000000000000000000000000000000000000..c6ecc486f41c2970e967d494654cc069c111923d
--- /dev/null
+++ b/src/nacl/_cffi_fix.py
@@ -0,0 +1,72 @@
+# Copyright 2013 Donald Stufft and individual contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import imp
+import os.path
+import sys
+
+import cffi.vengine_cpy
+import cffi.vengine_gen
+
+
+def _get_so_suffixes():
+    suffixes = []
+    for suffix, mode, type in imp.get_suffixes():
+        if type == imp.C_EXTENSION:
+            suffixes.append(suffix)
+
+    if not suffixes:
+        # bah, no C_EXTENSION available.  Occurs on pypy without cpyext
+        if sys.platform == 'win32':
+            suffixes = [".pyd"]
+        else:
+            suffixes = [".so"]
+
+    return suffixes
+
+
+def vengine_cpy_find_module(self, module_name, path, so_suffix):
+    # We will ignore so_suffix and get it ourselves
+    so_suffixes = _get_so_suffixes()
+
+    try:
+        f, filename, descr = imp.find_module(module_name, path)
+    except ImportError:
+        return None
+    if f is not None:
+        f.close()
+
+    # Note that after a setuptools installation, there are both .py
+    # and .so files with the same basename.  The code here relies on
+    # imp.find_module() locating the .so in priority.
+    if descr[0] not in so_suffixes:
+        return None
+    return filename
+
+
+def vengine_gen_find_module(self, module_name, path, so_suffixes):
+    # We will ignore so_suffix and get it ourselves
+    so_suffixes = _get_so_suffixes()
+
+    for so_suffix in so_suffixes:
+        basename = module_name + so_suffix
+        if path is None:
+            path = sys.path
+        for dirname in path:
+            filename = os.path.join(dirname, basename)
+            if os.path.isfile(filename):
+                return filename
+
+
+cffi.vengine_cpy.VCPythonEngine.find_module = vengine_cpy_find_module
+cffi.vengine_gen.VGenericEngine.find_module = vengine_gen_find_module
diff --git a/src/nacl/nacl.py b/src/nacl/nacl.py
index 16991d32e2496bb0e46bf00bf8774ee2cd34055a..35ee3010c095b2669e3b59fbbb4d2bccbe1b9518 100644
--- a/src/nacl/nacl.py
+++ b/src/nacl/nacl.py
@@ -18,9 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 
 import functools
-import platform
 
-from distutils.sysconfig import get_config_vars
+# We need to patch cffi before importing it
+from nacl import _cffi_fix
 
 import cffi.verifier
 
@@ -30,15 +30,6 @@ from cffi import FFI
 __all__ = ["ffi", "lib"]
 
 
-# Monkeypatch cffi.verifier._get_so_suffix to return the same as distutils
-# See: https://bitbucket.org/cffi/cffi/issue/110/
-def _get_so_suffix():
-    return get_config_vars().get("EXT_SUFFIX", ".so")
-
-if not platform.python_implementation().lower() == "pypy":
-    cffi.verifier._get_so_suffix = _get_so_suffix
-
-
 ffi = FFI()
 ffi.cdef(
     # Secret Key Encryption
diff --git a/tox.ini b/tox.ini
index de0282668faa0112fe3653cc287504665cf3204f..ab32c94218f8ea7770e6d3671f81acb6bf139c1f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py26,py27,pypy,py33
+envlist = py26,py27,pypy,py32,py33
 
 [testenv]
 deps =