Can't get Theano to link against CUDNN on OSX - macos

Question

After a big battle I was finally able to get Theano to use the GPU in OSX.
But now, I've tried everything I can remember and Theano still can't use CuDNN.
I installed CUDA version 7 and CUDNN version 3.
I tried copying the libraries to /usr/local/cuda/lib and also to /usr/local/cuda/lib64, the include file was copied to /usr/local/cuda/include
My .theanorc is
[global]
floatX = float32
device = gpu
force_device = True
allow_gc = False
optimizer_including=cudnn
# warn_float64 = warn
# openmp = True
[nvcc]
fastmath = True
flags = -L/Users/morgado/anaconda/lib/
[cuda]
root = /usr/local/cuda
[gcc]
cxxflags = -L/usr/local/cuda/lib64
And my .profile has the relevant parts:
# CUDA
CUDA_ROOT=/usr/local/cuda
export PATH=$CUDA_ROOT/bin:$PATH
export DYLD_LIBRARY_PATH=$CUDA_ROOT/lib:$DYLD_LIBRARY_PATH
export LD_LIBRARY_PATH=$CUDA_ROOT/lib:$CUDA_ROOT/lib64:$LD_LIBRARY_PATH
But still when I try to get Theano to use CUDNN the further I get (with the files in lib64) gives me the error:
Using gpu device 0: GeForce GT 750M
1 #include <Python.h>
2 #include <iostream>
3 #include "cudnn.h"
4 //////////////////////
5 //// Support Code
6 //////////////////////
7
8 #if PY_MAJOR_VERSION >= 3
9 #define PyInt_FromLong PyLong_FromLong
10 #endif
11
12
13 namespace {
14 struct __struct_compiled_op_m086964a2c5561b842ea4c5d4a203a715 {
15 PyObject* __ERROR;
16
17 PyObject* storage_V1;
18
19
20 __struct_compiled_op_m086964a2c5561b842ea4c5d4a203a715() {}
21 ~__struct_compiled_op_m086964a2c5561b842ea4c5d4a203a715(void) {
22 cleanup();
23 }
24
25 int init(PyObject* __ERROR, PyObject* storage_V1) {
26 Py_XINCREF(storage_V1);
27 this->storage_V1 = storage_V1;
28
29
30
31 this->__ERROR = __ERROR;
32 return 0;
33 }
34 void cleanup(void) {
35 __label_1:
36
37 double __DUMMY_1;
38 __label_4:
39
40 double __DUMMY_4;
41
42 Py_XDECREF(this->storage_V1);
43 }
44 int run(void) {
45 int __failure = 0;
46
47 PyObject* py_V1;
48
49 PyObject* V1;
50
51 {
52
53 py_V1 = Py_None;
54 {Py_XINCREF(py_V1);}
55
56 V1 = NULL;
57
58 {
59 // Op class DnnVersion
60
61 #if defined(CUDNN_VERSION)
62 V1 = PyTuple_Pack(2, PyInt_FromLong(CUDNN_VERSION), PyInt_FromLong(cudnnGetVersion()));
63 #else
64 V1 = PyInt_FromLong(-1);
65 #endif
66 __label_3:
67
68 double __DUMMY_3;
69
70 }
71 __label_2:
72
73 if (!__failure) {
74
75 assert(py_V1->ob_refcnt > 1);
76 Py_DECREF(py_V1);
77 py_V1 = V1 ? V1 : Py_None;
78 Py_INCREF(py_V1);
79
80 PyObject* old = PyList_GET_ITEM(storage_V1, 0);
81 {Py_XINCREF(py_V1);}
82 PyList_SET_ITEM(storage_V1, 0, py_V1);
83 {Py_XDECREF(old);}
84 }
85
86 Py_XDECREF(V1);
87
88 {Py_XDECREF(py_V1);}
89
90 double __DUMMY_2;
91
92 }
93
94
95 if (__failure) {
96 // When there is a failure, this code puts the exception
97 // in __ERROR.
98 PyObject* err_type = NULL;
99 PyObject* err_msg = NULL;
100 PyObject* err_traceback = NULL;
101 PyErr_Fetch(&err_type, &err_msg, &err_traceback);
102 if (!err_type) {err_type = Py_None;Py_INCREF(Py_None);}
103 if (!err_msg) {err_msg = Py_None; Py_INCREF(Py_None);}
104 if (!err_traceback) {err_traceback = Py_None; Py_INCREF(Py_None);}
105 PyObject* old_err_type = PyList_GET_ITEM(__ERROR, 0);
106 PyObject* old_err_msg = PyList_GET_ITEM(__ERROR, 1);
107 PyObject* old_err_traceback = PyList_GET_ITEM(__ERROR, 2);
108 PyList_SET_ITEM(__ERROR, 0, err_type);
109 PyList_SET_ITEM(__ERROR, 1, err_msg);
110 PyList_SET_ITEM(__ERROR, 2, err_traceback);
111 {Py_XDECREF(old_err_type);}
112 {Py_XDECREF(old_err_msg);}
113 {Py_XDECREF(old_err_traceback);}
114 }
115 // The failure code is returned to index what code block failed.
116 return __failure;
117
118 }
119 };
120 }
121
122
123 static int __struct_compiled_op_m086964a2c5561b842ea4c5d4a203a715_executor(__struct_compiled_op_m086964a2c5561b842ea4c5d4a203a715 *self) {
124 return self->run();
125 }
126
127 static void __struct_compiled_op_m086964a2c5561b842ea4c5d4a203a715_destructor(PyObject *capsule) {
128 __struct_compiled_op_m086964a2c5561b842ea4c5d4a203a715 *self = (__struct_compiled_op_m086964a2c5561b842ea4c5d4a203a715 *)PyCapsule_GetContext(capsule);
129 delete self;
130 }
131
132 //////////////////////
133 //// Functions
134 //////////////////////
135 static PyObject * instantiate(PyObject * self, PyObject *argtuple) {
136 assert(PyTuple_Check(argtuple));
137 if (2 != PyTuple_Size(argtuple)){
138 PyErr_Format(PyExc_TypeError, "Wrong number of arguments, expected 2, got %i", (int)PyTuple_Size(argtuple));
139 return NULL;
140 }
141 __struct_compiled_op_m086964a2c5561b842ea4c5d4a203a715* struct_ptr = new __struct_compiled_op_m086964a2c5561b842ea4c5d4a203a715();
142 if (struct_ptr->init( PyTuple_GET_ITEM(argtuple, 0),PyTuple_GET_ITEM(argtuple, 1) ) != 0) {
143 delete struct_ptr;
144 return NULL;
145 }
146 PyObject* thunk = PyCapsule_New((void*)(&__struct_compiled_op_m086964a2c5561b842ea4c5d4a203a715_executor), NULL, __struct_compiled_op_m086964a2c5561b842ea4c5d4a203a715_destructor);
147 if (thunk != NULL && PyCapsule_SetContext(thunk, struct_ptr) != 0) {
148 PyErr_Clear();
149 Py_DECREF(thunk);
150 thunk = NULL;
151 }
152
153 return thunk; }
154
155 //////////////////////
156 //// Module init
157 //////////////////////
158 static PyMethodDef MyMethods[] = {
159 {"instantiate", instantiate, METH_VARARGS, "undocumented"} ,
160 {NULL, NULL, 0, NULL}
161 };
162 static struct PyModuleDef moduledef = {
163 PyModuleDef_HEAD_INIT,
164 "m086964a2c5561b842ea4c5d4a203a715",
165 NULL,
166 -1,
167 MyMethods,
168 };
169
170 PyMODINIT_FUNC PyInit_m086964a2c5561b842ea4c5d4a203a715(void) {
171 PyObject *m = PyModule_Create(&moduledef);
172 return m;
173 }
174
===============================
clang: warning: argument unused during compilation: '-undefined dynamic_lookup'
clang: warning: argument unused during compilation: '-undefined dynamic_lookup'
clang: warning: argument unused during compilation: '-undefined dynamic_lookup'
clang: warning: argument unused during compilation: '-undefined dynamic_lookup'
clang: warning: argument unused during compilation: '-undefined dynamic_lookup'
clang: warning: argument unused during compilation: '-undefined dynamic_lookup'
clang: warning: argument unused during compilation: '-undefined dynamic_lookup'
ld: library not found for -lcudnn
clang: error: linker command failed with exit code 1 (use -v to see invocation)
['nvcc', '-shared', '-O3', '-L/Users/morgado/anaconda/lib/', '-use_fast_math', '-arch=sm_30', '-m64', '-Xcompiler', '-fno-math-errno,-Wno-unused-label,-Wno-unused-variable,-Wno-write-strings,-DCUDA_NDARRAY_CUH=m11b90075e2397c684f9dc0f7276eab8f,-D NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION,-fPIC', '-Xlinker', '-rpath,/Users/morgado/.theano/compiledir_Darwin-14.4.0-x86_64-i386-64bit-i386-3.4.3-64/cuda_ndarray', '-Xlinker', '-rpath,/usr/local/cuda/lib', '-I/Users/morgado/anaconda/lib/python3.4/site-packages/numpy/core/include', '-I/Users/morgado/anaconda/include/python3.4m', '-I/Users/morgado/anaconda/lib/python3.4/site-packages/theano/sandbox/cuda', '-o', '/Users/morgado/.theano/compiledir_Darwin-14.4.0-x86_64-i386-64bit-i386-3.4.3-64/tmp578d03tw/m086964a2c5561b842ea4c5d4a203a715.so', 'mod.cu', '-lcudnn', '-lcudart', '-Xcompiler', '-undefined,dynamic_lookup', '-Xlinker', '-pie']
ERROR (theano.gof.opt): SeqOptimizer apply <theano.sandbox.cuda.dnn.NoCuDNNRaise object at 0x10ebb14e0>
ERROR (theano.gof.opt): Traceback:
ERROR (theano.gof.opt): Traceback (most recent call last):
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/gof/opt.py", line 196, in apply
sub_prof = optimizer.optimize(fgraph)
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/gof/opt.py", line 82, in optimize
ret = self.apply(fgraph, *args, **kwargs)
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/sandbox/cuda/dnn.py", line 1682, in apply
if not dnn_available():
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/sandbox/cuda/dnn.py", line 75, in dnn_available
v = version()
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/sandbox/cuda/dnn.py", line 206, in version
theano.Mode(optimizer=None))
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/compile/function.py", line 266, in function
profile=profile)
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/compile/pfunc.py", line 511, in pfunc
on_unused_input=on_unused_input)
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/compile/function_module.py", line 1466, in orig_function
defaults)
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/compile/function_module.py", line 1324, in create
input_storage=input_storage_lists)
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/gof/link.py", line 519, in make_thunk
output_storage=output_storage)[:3]
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/gof/vm.py", line 897, in make_all
no_recycling))
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/sandbox/cuda/__init__.py", line 259, in make_thunk
compute_map, no_recycling)
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/gof/op.py", line 739, in make_thunk
output_storage=node_output_storage)
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/gof/cc.py", line 1073, in make_thunk
keep_lock=keep_lock)
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/gof/cc.py", line 1015, in __compile__
keep_lock=keep_lock)
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/gof/cc.py", line 1442, in cthunk_factory
key=key, lnk=self, keep_lock=keep_lock)
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/gof/cmodule.py", line 1076, in module_from_key
module = lnk.compile_cmodule(location)
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/gof/cc.py", line 1354, in compile_cmodule
preargs=preargs)
File "/Users/morgado/anaconda/lib/python3.4/site-packages/theano/sandbox/cuda/nvcc_compiler.py", line 423, in compile_str
'for cmd', ' '.join(cmd))
Exception: ('The following error happened while compiling the node', <theano.sandbox.cuda.dnn.DnnVersion object at 0x11549af98>(), '\n', 'nvcc return status', 1, 'for cmd', 'nvcc -shared -O3 -L/Users/morgado/anaconda/lib/ -use_fast_math -arch=sm_30 -m64 -Xcompiler -fno-math-errno,-Wno-unused-label,-Wno-unused-variable,-Wno-write-strings,-DCUDA_NDARRAY_CUH=m11b90075e2397c684f9dc0f7276eab8f,-D NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION,-fPIC -Xlinker -rpath,/Users/morgado/.theano/compiledir_Darwin-14.4.0-x86_64-i386-64bit-i386-3.4.3-64/cuda_ndarray -Xlinker -rpath,/usr/local/cuda/lib -I/Users/morgado/anaconda/lib/python3.4/site-packages/numpy/core/include -I/Users/morgado/anaconda/include/python3.4m -I/Users/morgado/anaconda/lib/python3.4/site-packages/theano/sandbox/cuda -o /Users/morgado/.theano/compiledir_Darwin-14.4.0-x86_64-i386-64bit-i386-3.4.3-64/tmp578d03tw/m086964a2c5561b842ea4c5d4a203a715.so mod.cu -lcudnn -lcudart -Xcompiler -undefined,dynamic_lookup -Xlinker -pie', '[<theano.sandbox.cuda.dnn.DnnVersion object at 0x11549af98>()]'
Seems like clang is not getting the cudnn library even when I specifically told it to check that path.

Answer 1

In your profile add:
export LIBRARY_PATH=$CUDA_ROOT/lib:$CUDA_ROOT/lib64:$LIBRARY_PATH
Including the lib on LIBRARY_PATH worked for me.

Answer 2

I had the same problem and could fix it by setting
export DYLD_LIBRARY_PATH=/Developer/NVIDIA/CUDA-7.5/lib:$DYLD_LIBRARY_PATH
in my .bashrc and running
sudo update_dyld_shared_cache
afterwards.