diff --git a/nacl/nacl.py b/nacl/nacl.py
index 8fb06b256d69985b6987dfafd383ae80b8db5316..e3ae1aa47498621347f52cd3ec9091f3f1a3d16e 100644
--- a/nacl/nacl.py
+++ b/nacl/nacl.py
@@ -5,8 +5,10 @@ from __future__ import absolute_import
 from __future__ import division
 
 import functools
+import os.path
 
 from cffi import FFI
+from cffi.verifier import Verifier
 
 
 __all__ = ["ffi", "lib"]
@@ -74,11 +76,23 @@ ffi.cdef(
 )
 
 
-lib = ffi.verify("#include <sodium.h>", libraries=["sodium"])
+#lib = ffi.verify("#include <sodium.h>", libraries=["sodium"])
+
+ffi.verifier = Verifier(ffi,
+    "#include <sodium.h>",
+
+    # We need to set a tmp directory otherwise when build_ext is run it'll get
+    # built in nacl/*.so but when ffi.verifier.load_library() is run it'll
+    # look (and ultimately build again) in nacl/__pycache__/*.so
+    tmpdir=os.path.abspath(os.path.dirname(__file__)),
+
+    # We need to link to the sodium library
+    libraries=["sodium"],
+)
 
 
 # This works around a bug in PyPy where CFFI exposed functions do not have a
-#   __name__ attribute. See https://bugs.pypy.org/issue1452
+# __name__ attribute. See https://bugs.pypy.org/issue1452
 def wraps(wrapped):
     def inner(func):
         if hasattr(wrapped, "__name__"):
@@ -89,9 +103,9 @@ def wraps(wrapped):
 
 
 # A lot of the functions in nacl return 0 for success and a negative integer
-#   for failure. This is inconvenient in Python as 0 is a falsey value while
-#   negative integers are truthy. This wrapper has them return True/False as
-#   you'd expect in Python
+# for failure. This is inconvenient in Python as 0 is a falsey value while
+# negative integers are truthy. This wrapper has them return True/False as
+# you'd expect in Python
 def wrap_nacl_function(func):
     @wraps(func)
     def wrapper(*args, **kwargs):
@@ -99,20 +113,48 @@ def wrap_nacl_function(func):
         return ret == 0
     return wrapper
 
-lib.crypto_secretbox = wrap_nacl_function(lib.crypto_secretbox)
-lib.crypto_secretbox_open = wrap_nacl_function(lib.crypto_secretbox_open)
 
-lib.crypto_sign_seed_keypair = wrap_nacl_function(lib.crypto_sign_seed_keypair)
-lib.crypto_sign = wrap_nacl_function(lib.crypto_sign)
-lib.crypto_sign_open = wrap_nacl_function(lib.crypto_sign_open)
+class Library(object):
+
+    wrap = [
+        "crypto_secretbox",
+        "crypto_secretbox_open",
+
+        "crypto_sign_seed_keypair",
+        "crypto_sign",
+        "crypto_sign_open",
+
+        "crypto_box_keypair",
+        "crypto_box_afternm",
+        "crypto_box_open_afternm",
+        "crypto_box_beforenm",
+
+        "crypto_hash",
+        "crypto_hash_sha256",
+        "crypto_hash_sha512",
+
+        "crypto_scalarmult_curve25519_base",
+    ]
+
+    def __init__(self, ffi):
+        self._ffi = ffi
+        self._initalized = False
+
+    def __getattr__(self, name):
+        if not self._initalized:
+            self._lib = self._ffi.verifier.load_library()
+
+        # redirect attribute access to the underlying lib
+        attr = getattr(self._lib, name)
+
+        # If this is a function that we're wrapping do the actual wrapping
+        if name in self.wrap:
+            attr = wrap_nacl_function(attr)
 
-lib.crypto_box_keypair = wrap_nacl_function(lib.crypto_box_keypair)
-lib.crypto_box_afternm = wrap_nacl_function(lib.crypto_box_afternm)
-lib.crypto_box_open_afternm = wrap_nacl_function(lib.crypto_box_open_afternm)
-lib.crypto_box_beforenm = wrap_nacl_function(lib.crypto_box_beforenm)
+        # Go ahead and assign the returned value to this class so we don't
+        # need to do this lookup again
+        setattr(self, name, attr)
 
-lib.crypto_hash = wrap_nacl_function(lib.crypto_hash)
-lib.crypto_hash_sha256 = wrap_nacl_function(lib.crypto_hash_sha256)
-lib.crypto_hash_sha512 = wrap_nacl_function(lib.crypto_hash_sha512)
+        return attr
 
-lib.crypto_scalarmult_curve25519_base = wrap_nacl_function(lib.crypto_scalarmult_curve25519_base)
+lib = Library(ffi)