diff --git a/.travis.yml b/.travis.yml
index f9d43a0cc30892c52724c64c2afed76ad830a222..ae70f2fb827b040aef79d53efbd98260ae806915 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,11 +1,16 @@
 language: python
 python: 2.7
 env:
-  - TOXENV=py26
-  - TOXENV=py27
-  - TOXENV=py32
-  - TOXENV=py33
-  - TOXENV=pypy
+  - TOXENV=py26 SODIUM_INSTALL=bundled
+  - TOXENV=py27 SODIUM_INSTALL=bundled
+  - TOXENV=py32 SODIUM_INSTALL=bundled
+  - TOXENV=py33 SODIUM_INSTALL=bundled
+  - TOXENV=pypy SODIUM_INSTALL=bundled
+  - TOXENV=py26 SODIUM_INSTALL=system
+  - TOXENV=py27 SODIUM_INSTALL=system
+  - TOXENV=py32 SODIUM_INSTALL=system
+  - TOXENV=py33 SODIUM_INSTALL=system
+  - TOXENV=pypy SODIUM_INSTALL=system
 
 install:
   # Add the PyPy repository
@@ -16,6 +21,18 @@ install:
   # This is required because we need to get rid of the Travis installed PyPy
   # or it'll take precedence over the PPA installed one.
   - "if [[ $TOXENV == 'pypy' ]]; then sudo rm -rf /usr/local/pypy/bin; fi"
+
+  # Install Sodium if we need too
+  - "if [[ $SODIUM_INSTALL == 'system' ]]; then wget https://download.libsodium.org/libsodium/releases/LATEST.tar.gz; fi"
+  - "if [[ $SODIUM_INSTALL == 'system' ]]; then tar zxvf LATEST.tar.gz; fi"
+  - "if [[ $SODIUM_INSTALL == 'system' ]]; then cd libsodium-*; fi"
+  - "if [[ $SODIUM_INSTALL == 'system' ]]; then ./configure; fi"
+  - "if [[ $SODIUM_INSTALL == 'system' ]]; then make; fi"
+  - "if [[ $SODIUM_INSTALL == 'system' ]]; then make check; fi"
+  - "if [[ $SODIUM_INSTALL == 'system' ]]; then sudo make install; fi"
+  - "if [[ $SODIUM_INSTALL == 'system' ]]; then sudo ldconfig; fi"
+
+  # Install tox and coveralls so we can run our tests
   - pip install tox coveralls
 
 
diff --git a/setup.py b/setup.py
index 9ae0c3e21139a39b4855df9b3dc96f98537de084..7aa269b87ed76e1f9dc51f17515271e217ec95c2 100644
--- a/setup.py
+++ b/setup.py
@@ -25,6 +25,10 @@ from distutils.command.build_ext import build_ext as _build_ext
 from setuptools import Distribution, setup
 
 
+SODIUM_MAJOR = 4
+SODIUM_MINOR = 3
+
+
 def here(*paths):
     return os.path.abspath(os.path.join(os.path.dirname(__file__), *paths))
 
@@ -67,10 +71,45 @@ else:
     ext_modules = [nacl.nacl.ffi.verifier.get_extension()]
 
 
+def use_system():
+    install_type = os.environ.get("SODIUM_INSTALL")
+
+    if install_type == "system":
+        # If we are forcing system installs, don't compile the bundled one
+        return True
+    elif install_type == "bundled":
+        # If we are forcing bundled installs, compile it
+        return False
+
+    # Detect if we have libsodium available
+    import cffi
+
+    ffi = cffi.FFI()
+    ffi.cdef("""
+        int sodium_library_version_major();
+        int sodium_library_version_minor();
+    """)
+
+    try:
+        system = ffi.dlopen("libsodium")
+    except OSError:
+        # We couldn't locate libsodium so we'll use the bundled one
+        return False
+
+    if system.sodium_library_version_major() != SODIUM_MAJOR:
+        return False
+
+    if system.sodium_library_version_minor() < SODIUM_MINOR:
+        return False
+
+    # If we got this far then the system library should be good enough
+    return True
+
+
 class Distribution(Distribution):
 
     def has_c_libraries(self):
-        return True
+        return not use_system()
 
 
 class build_clib(_build_clib):
@@ -95,6 +134,9 @@ class build_clib(_build_clib):
         return ["sodium"]
 
     def run(self):
+        if use_system():
+            return
+
         build_temp = os.path.abspath(self.build_temp)
 
         # Ensure our temporary build directory exists
@@ -129,11 +171,14 @@ class build_clib(_build_clib):
 class build_ext(_build_ext):
 
     def run(self):
-        build_clib = self.get_finalized_command("build_clib")
-        self.include_dirs.append(
-            os.path.join(build_clib.build_clib, "include")
-        )
-        self.library_dirs.append(os.path.join(build_clib.build_clib, "lib"))
+        if self.distribution.has_c_libraries():
+            build_clib = self.get_finalized_command("build_clib")
+            self.include_dirs.append(
+                os.path.join(build_clib.build_clib, "include"),
+            )
+            self.library_dirs.append(
+                os.path.join(build_clib.build_clib, "lib"),
+            )
 
         return _build_ext.run(self)