diff --git a/setup.py b/setup.py
index d3fb5de411cdbf3ff8e4a65e2b2322545250a8ee..f132c6951c5c01243542ef623bf875410bbb97c9 100644
--- a/setup.py
+++ b/setup.py
@@ -46,7 +46,7 @@ class build_clib(_build_clib):
             sourcefile.extractall(tmpdir)
 
             # Copy our installed directory into the build location
-            shutil.rmtree(here("build/sodium"))
+            shutil.rmtree(here("build/sodium"), ignore_errors=True)
             shutil.copytree(
                 os.path.join(tmpdir, "libsodium-%s" % SODIUM_VERSION),
                 here("build/sodium")