diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c
index 2873760..486d29c 100644
--- a/drivers/dma-buf/sync_file.c
+++ b/drivers/dma-buf/sync_file.c
@@ -86,8 +86,6 @@
 		 fence->ops->get_timeline_name(fence), fence->context,
 		 fence->seqno);
 
-	fence_add_callback(fence, &sync_file->cb, fence_check_cb_func);
-
 	return sync_file;
 }
 EXPORT_SYMBOL(sync_file_create);
@@ -274,9 +272,6 @@
 		goto err;
 	}
 
-	fence_add_callback(sync_file->fence, &sync_file->cb,
-			   fence_check_cb_func);
-
 	strlcpy(sync_file->name, name, sizeof(sync_file->name));
 	return sync_file;
 
@@ -291,7 +286,8 @@
 	struct sync_file *sync_file = container_of(kref, struct sync_file,
 						     kref);
 
-	fence_remove_callback(sync_file->fence, &sync_file->cb);
+	if (test_bit(POLL_ENABLED, &sync_file->fence->flags))
+		fence_remove_callback(sync_file->fence, &sync_file->cb);
 	fence_put(sync_file->fence);
 	kfree(sync_file);
 }
@@ -307,17 +303,16 @@
 static unsigned int sync_file_poll(struct file *file, poll_table *wait)
 {
 	struct sync_file *sync_file = file->private_data;
-	int status;
 
 	poll_wait(file, &sync_file->wq, wait);
 
-	status = fence_is_signaled(sync_file->fence);
+	if (!test_and_set_bit(POLL_ENABLED, &sync_file->fence->flags)) {
+		if (fence_add_callback(sync_file->fence, &sync_file->cb,
+				       fence_check_cb_func) < 0)
+			wake_up_all(&sync_file->wq);
+	}
 
-	if (status)
-		return POLLIN;
-	if (status < 0)
-		return POLLERR;
-	return 0;
+	return fence_is_signaled(sync_file->fence) ? POLLIN : 0;
 }
 
 static long sync_file_ioctl_merge(struct sync_file *sync_file,
diff --git a/include/linux/sync_file.h b/include/linux/sync_file.h
index f7de5a0..aa17ccf 100644
--- a/include/linux/sync_file.h
+++ b/include/linux/sync_file.h
@@ -45,6 +45,8 @@
 	struct fence_cb cb;
 };
 
+#define POLL_ENABLED FENCE_FLAG_USER_BITS
+
 struct sync_file *sync_file_create(struct fence *fence);
 struct fence *sync_file_get_fence(int fd);