| // SPDX-License-Identifier: GPL-2.0 |
| /* |
| * Support for Clovertrail PNW Camera Imaging ISP subsystem. |
| * |
| * Copyright (c) 2012 Intel Corporation. All Rights Reserved. |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License version |
| * 2 as published by the Free Software Foundation. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * |
| */ |
| |
| /* |
| * This file implements loadable acceleration firmware API, |
| * including ioctls to map and unmap acceleration parameters and buffers. |
| */ |
| |
| #include <linux/init.h> |
| #include <media/v4l2-event.h> |
| |
| #include "hmm.h" |
| |
| #include "atomisp_acc.h" |
| #include "atomisp_internal.h" |
| #include "atomisp_compat.h" |
| #include "atomisp_cmd.h" |
| |
| #include "ia_css.h" |
| |
| static const struct { |
| unsigned int flag; |
| enum ia_css_pipe_id pipe_id; |
| } acc_flag_to_pipe[] = { |
| { ATOMISP_ACC_FW_LOAD_FL_PREVIEW, IA_CSS_PIPE_ID_PREVIEW }, |
| { ATOMISP_ACC_FW_LOAD_FL_COPY, IA_CSS_PIPE_ID_COPY }, |
| { ATOMISP_ACC_FW_LOAD_FL_VIDEO, IA_CSS_PIPE_ID_VIDEO }, |
| { ATOMISP_ACC_FW_LOAD_FL_CAPTURE, IA_CSS_PIPE_ID_CAPTURE }, |
| { ATOMISP_ACC_FW_LOAD_FL_ACC, IA_CSS_PIPE_ID_ACC } |
| }; |
| |
| /* |
| * Allocate struct atomisp_acc_fw along with space for firmware. |
| * The returned struct atomisp_acc_fw is cleared (firmware region is not). |
| */ |
| static struct atomisp_acc_fw *acc_alloc_fw(unsigned int fw_size) |
| { |
| struct atomisp_acc_fw *acc_fw; |
| |
| acc_fw = kzalloc(sizeof(*acc_fw), GFP_KERNEL); |
| if (!acc_fw) |
| return NULL; |
| |
| acc_fw->fw = vmalloc(fw_size); |
| if (!acc_fw->fw) { |
| kfree(acc_fw); |
| return NULL; |
| } |
| |
| return acc_fw; |
| } |
| |
| static void acc_free_fw(struct atomisp_acc_fw *acc_fw) |
| { |
| vfree(acc_fw->fw); |
| kfree(acc_fw); |
| } |
| |
| static struct atomisp_acc_fw * |
| acc_get_fw(struct atomisp_sub_device *asd, unsigned int handle) |
| { |
| struct atomisp_acc_fw *acc_fw; |
| |
| list_for_each_entry(acc_fw, &asd->acc.fw, list) |
| if (acc_fw->handle == handle) |
| return acc_fw; |
| |
| return NULL; |
| } |
| |
| static struct atomisp_map *acc_get_map(struct atomisp_sub_device *asd, |
| unsigned long css_ptr, size_t length) |
| { |
| struct atomisp_map *atomisp_map; |
| |
| list_for_each_entry(atomisp_map, &asd->acc.memory_maps, list) { |
| if (atomisp_map->ptr == css_ptr && |
| atomisp_map->length == length) |
| return atomisp_map; |
| } |
| return NULL; |
| } |
| |
| static int acc_stop_acceleration(struct atomisp_sub_device *asd) |
| { |
| int ret; |
| |
| ret = atomisp_css_stop_acc_pipe(asd); |
| atomisp_css_destroy_acc_pipe(asd); |
| |
| return ret; |
| } |
| |
| void atomisp_acc_cleanup(struct atomisp_device *isp) |
| { |
| int i; |
| |
| for (i = 0; i < isp->num_of_streams; i++) |
| ida_destroy(&isp->asd[i].acc.ida); |
| } |
| |
| void atomisp_acc_release(struct atomisp_sub_device *asd) |
| { |
| struct atomisp_acc_fw *acc_fw, *ta; |
| struct atomisp_map *atomisp_map, *tm; |
| |
| /* Stop acceleration if already running */ |
| if (asd->acc.pipeline) |
| acc_stop_acceleration(asd); |
| |
| /* Unload all loaded acceleration binaries */ |
| list_for_each_entry_safe(acc_fw, ta, &asd->acc.fw, list) { |
| list_del(&acc_fw->list); |
| ida_free(&asd->acc.ida, acc_fw->handle); |
| acc_free_fw(acc_fw); |
| } |
| |
| /* Free all mapped memory blocks */ |
| list_for_each_entry_safe(atomisp_map, tm, &asd->acc.memory_maps, list) { |
| list_del(&atomisp_map->list); |
| hmm_free(atomisp_map->ptr); |
| kfree(atomisp_map); |
| } |
| } |
| |
| int atomisp_acc_load_to_pipe(struct atomisp_sub_device *asd, |
| struct atomisp_acc_fw_load_to_pipe *user_fw) |
| { |
| static const unsigned int pipeline_flags = |
| ATOMISP_ACC_FW_LOAD_FL_PREVIEW | ATOMISP_ACC_FW_LOAD_FL_COPY | |
| ATOMISP_ACC_FW_LOAD_FL_VIDEO | |
| ATOMISP_ACC_FW_LOAD_FL_CAPTURE | ATOMISP_ACC_FW_LOAD_FL_ACC; |
| |
| struct atomisp_acc_fw *acc_fw; |
| int handle; |
| |
| if (!user_fw->data || user_fw->size < sizeof(*acc_fw->fw)) |
| return -EINVAL; |
| |
| /* Binary has to be enabled at least for one pipeline */ |
| if (!(user_fw->flags & pipeline_flags)) |
| return -EINVAL; |
| |
| /* We do not support other flags yet */ |
| if (user_fw->flags & ~pipeline_flags) |
| return -EINVAL; |
| |
| if (user_fw->type < ATOMISP_ACC_FW_LOAD_TYPE_OUTPUT || |
| user_fw->type > ATOMISP_ACC_FW_LOAD_TYPE_STANDALONE) |
| return -EINVAL; |
| |
| if (asd->acc.pipeline || asd->acc.extension_mode) |
| return -EBUSY; |
| |
| acc_fw = acc_alloc_fw(user_fw->size); |
| if (!acc_fw) |
| return -ENOMEM; |
| |
| if (copy_from_user(acc_fw->fw, user_fw->data, user_fw->size)) { |
| acc_free_fw(acc_fw); |
| return -EFAULT; |
| } |
| |
| handle = ida_alloc(&asd->acc.ida, GFP_KERNEL); |
| if (handle < 0) { |
| acc_free_fw(acc_fw); |
| return -ENOSPC; |
| } |
| |
| user_fw->fw_handle = handle; |
| acc_fw->handle = handle; |
| acc_fw->flags = user_fw->flags; |
| acc_fw->type = user_fw->type; |
| acc_fw->fw->handle = handle; |
| |
| /* |
| * correct isp firmware type in order ISP firmware can be appended |
| * to correct pipe properly |
| */ |
| if (acc_fw->fw->type == ia_css_isp_firmware) { |
| static const int type_to_css[] = { |
| [ATOMISP_ACC_FW_LOAD_TYPE_OUTPUT] = |
| IA_CSS_ACC_OUTPUT, |
| [ATOMISP_ACC_FW_LOAD_TYPE_VIEWFINDER] = |
| IA_CSS_ACC_VIEWFINDER, |
| [ATOMISP_ACC_FW_LOAD_TYPE_STANDALONE] = |
| IA_CSS_ACC_STANDALONE, |
| }; |
| acc_fw->fw->info.isp.type = type_to_css[acc_fw->type]; |
| } |
| |
| list_add_tail(&acc_fw->list, &asd->acc.fw); |
| return 0; |
| } |
| |
| int atomisp_acc_load(struct atomisp_sub_device *asd, |
| struct atomisp_acc_fw_load *user_fw) |
| { |
| struct atomisp_acc_fw_load_to_pipe ltp = {0}; |
| int r; |
| |
| ltp.flags = ATOMISP_ACC_FW_LOAD_FL_ACC; |
| ltp.type = ATOMISP_ACC_FW_LOAD_TYPE_STANDALONE; |
| ltp.size = user_fw->size; |
| ltp.data = user_fw->data; |
| r = atomisp_acc_load_to_pipe(asd, <p); |
| user_fw->fw_handle = ltp.fw_handle; |
| return r; |
| } |
| |
| int atomisp_acc_unload(struct atomisp_sub_device *asd, unsigned int *handle) |
| { |
| struct atomisp_acc_fw *acc_fw; |
| |
| if (asd->acc.pipeline || asd->acc.extension_mode) |
| return -EBUSY; |
| |
| acc_fw = acc_get_fw(asd, *handle); |
| if (!acc_fw) |
| return -EINVAL; |
| |
| list_del(&acc_fw->list); |
| ida_free(&asd->acc.ida, acc_fw->handle); |
| acc_free_fw(acc_fw); |
| |
| return 0; |
| } |
| |
| int atomisp_acc_start(struct atomisp_sub_device *asd, unsigned int *handle) |
| { |
| struct atomisp_device *isp = asd->isp; |
| struct atomisp_acc_fw *acc_fw; |
| int ret; |
| unsigned int nbin; |
| |
| if (asd->acc.pipeline || asd->acc.extension_mode) |
| return -EBUSY; |
| |
| /* Invalidate caches. FIXME: should flush only necessary buffers */ |
| wbinvd(); |
| |
| ret = atomisp_css_create_acc_pipe(asd); |
| if (ret) |
| return ret; |
| |
| nbin = 0; |
| list_for_each_entry(acc_fw, &asd->acc.fw, list) { |
| if (*handle != 0 && *handle != acc_fw->handle) |
| continue; |
| |
| if (acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_STANDALONE) |
| continue; |
| |
| /* Add the binary into the pipeline */ |
| ret = atomisp_css_load_acc_binary(asd, acc_fw->fw, nbin); |
| if (ret < 0) { |
| dev_err(isp->dev, "acc_load_binary failed\n"); |
| goto err_stage; |
| } |
| |
| ret = atomisp_css_set_acc_parameters(acc_fw); |
| if (ret < 0) { |
| dev_err(isp->dev, "acc_set_parameters failed\n"); |
| goto err_stage; |
| } |
| nbin++; |
| } |
| if (nbin < 1) { |
| /* Refuse creating pipelines with no binaries */ |
| dev_err(isp->dev, "%s: no acc binary available\n", __func__); |
| ret = -EINVAL; |
| goto err_stage; |
| } |
| |
| ret = atomisp_css_start_acc_pipe(asd); |
| if (ret) { |
| dev_err(isp->dev, "%s: atomisp_acc_start_acc_pipe failed\n", |
| __func__); |
| goto err_stage; |
| } |
| |
| return 0; |
| |
| err_stage: |
| atomisp_css_destroy_acc_pipe(asd); |
| return ret; |
| } |
| |
| int atomisp_acc_wait(struct atomisp_sub_device *asd, unsigned int *handle) |
| { |
| struct atomisp_device *isp = asd->isp; |
| int ret; |
| |
| if (!asd->acc.pipeline) |
| return -ENOENT; |
| |
| if (*handle && !acc_get_fw(asd, *handle)) |
| return -EINVAL; |
| |
| ret = atomisp_css_wait_acc_finish(asd); |
| if (acc_stop_acceleration(asd) == -EIO) { |
| atomisp_reset(isp); |
| return -EINVAL; |
| } |
| |
| return ret; |
| } |
| |
| void atomisp_acc_done(struct atomisp_sub_device *asd, unsigned int handle) |
| { |
| struct v4l2_event event = { 0 }; |
| |
| event.type = V4L2_EVENT_ATOMISP_ACC_COMPLETE; |
| event.u.frame_sync.frame_sequence = atomic_read(&asd->sequence); |
| event.id = handle; |
| |
| v4l2_event_queue(asd->subdev.devnode, &event); |
| } |
| |
| int atomisp_acc_map(struct atomisp_sub_device *asd, struct atomisp_acc_map *map) |
| { |
| struct atomisp_map *atomisp_map; |
| ia_css_ptr cssptr; |
| int pgnr; |
| |
| if (map->css_ptr) |
| return -EINVAL; |
| |
| if (asd->acc.pipeline) |
| return -EBUSY; |
| |
| if (map->user_ptr) { |
| /* Buffer to map must be page-aligned */ |
| if ((unsigned long)map->user_ptr & ~PAGE_MASK) { |
| dev_err(asd->isp->dev, |
| "%s: mapped buffer address %p is not page aligned\n", |
| __func__, map->user_ptr); |
| return -EINVAL; |
| } |
| |
| pgnr = DIV_ROUND_UP(map->length, PAGE_SIZE); |
| if (pgnr < ((PAGE_ALIGN(map->length)) >> PAGE_SHIFT)) { |
| dev_err(asd->isp->dev, |
| "user space memory size is less than the expected size..\n"); |
| return -ENOMEM; |
| } else if (pgnr > ((PAGE_ALIGN(map->length)) >> PAGE_SHIFT)) { |
| dev_err(asd->isp->dev, |
| "user space memory size is large than the expected size..\n"); |
| return -ENOMEM; |
| } |
| |
| cssptr = hmm_alloc(map->length, HMM_BO_USER, 0, map->user_ptr, |
| map->flags & ATOMISP_MAP_FLAG_CACHED); |
| |
| } else { |
| /* Allocate private buffer. */ |
| cssptr = hmm_alloc(map->length, HMM_BO_PRIVATE, 0, NULL, |
| map->flags & ATOMISP_MAP_FLAG_CACHED); |
| } |
| |
| if (!cssptr) |
| return -ENOMEM; |
| |
| atomisp_map = kmalloc(sizeof(*atomisp_map), GFP_KERNEL); |
| if (!atomisp_map) { |
| hmm_free(cssptr); |
| return -ENOMEM; |
| } |
| atomisp_map->ptr = cssptr; |
| atomisp_map->length = map->length; |
| list_add(&atomisp_map->list, &asd->acc.memory_maps); |
| |
| dev_dbg(asd->isp->dev, "%s: userptr %p, css_address 0x%x, size %d\n", |
| __func__, map->user_ptr, cssptr, map->length); |
| map->css_ptr = cssptr; |
| return 0; |
| } |
| |
| int atomisp_acc_unmap(struct atomisp_sub_device *asd, |
| struct atomisp_acc_map *map) |
| { |
| struct atomisp_map *atomisp_map; |
| |
| if (asd->acc.pipeline) |
| return -EBUSY; |
| |
| atomisp_map = acc_get_map(asd, map->css_ptr, map->length); |
| if (!atomisp_map) |
| return -EINVAL; |
| |
| list_del(&atomisp_map->list); |
| hmm_free(atomisp_map->ptr); |
| kfree(atomisp_map); |
| return 0; |
| } |
| |
| int atomisp_acc_s_mapped_arg(struct atomisp_sub_device *asd, |
| struct atomisp_acc_s_mapped_arg *arg) |
| { |
| struct atomisp_acc_fw *acc_fw; |
| |
| if (arg->memory >= ATOMISP_ACC_NR_MEMORY) |
| return -EINVAL; |
| |
| if (asd->acc.pipeline) |
| return -EBUSY; |
| |
| acc_fw = acc_get_fw(asd, arg->fw_handle); |
| if (!acc_fw) |
| return -EINVAL; |
| |
| if (arg->css_ptr != 0 || arg->length != 0) { |
| /* Unless the parameter is cleared, check that it exists */ |
| if (!acc_get_map(asd, arg->css_ptr, arg->length)) |
| return -EINVAL; |
| } |
| |
| acc_fw->args[arg->memory].length = arg->length; |
| acc_fw->args[arg->memory].css_ptr = arg->css_ptr; |
| |
| dev_dbg(asd->isp->dev, "%s: mem %d, address %p, size %ld\n", |
| __func__, arg->memory, (void *)arg->css_ptr, |
| (unsigned long)arg->length); |
| return 0; |
| } |
| |
| /* |
| * Appends the loaded acceleration binary extensions to the |
| * current ISP mode. Must be called just before sh_css_start(). |
| */ |
| int atomisp_acc_load_extensions(struct atomisp_sub_device *asd) |
| { |
| struct atomisp_acc_fw *acc_fw; |
| bool ext_loaded = false; |
| bool continuous = asd->continuous_mode->val && |
| asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW; |
| int ret = 0, i = -1; |
| struct atomisp_device *isp = asd->isp; |
| |
| if (asd->acc.pipeline || asd->acc.extension_mode) |
| return -EBUSY; |
| |
| /* Invalidate caches. FIXME: should flush only necessary buffers */ |
| wbinvd(); |
| |
| list_for_each_entry(acc_fw, &asd->acc.fw, list) { |
| if (acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_OUTPUT && |
| acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_VIEWFINDER) |
| continue; |
| |
| for (i = 0; i < ARRAY_SIZE(acc_flag_to_pipe); i++) { |
| /* QoS (ACC pipe) acceleration stages are currently |
| * allowed only in continuous mode. Skip them for |
| * all other modes. */ |
| if (!continuous && |
| acc_flag_to_pipe[i].flag == |
| ATOMISP_ACC_FW_LOAD_FL_ACC) |
| continue; |
| |
| if (acc_fw->flags & acc_flag_to_pipe[i].flag) { |
| ret = atomisp_css_load_acc_extension(asd, |
| acc_fw->fw, |
| acc_flag_to_pipe[i].pipe_id, |
| acc_fw->type); |
| if (ret) |
| goto error; |
| |
| ext_loaded = true; |
| } |
| } |
| |
| ret = atomisp_css_set_acc_parameters(acc_fw); |
| if (ret < 0) |
| goto error; |
| } |
| |
| if (!ext_loaded) |
| return ret; |
| |
| ret = atomisp_css_update_stream(asd); |
| if (ret) { |
| dev_err(isp->dev, "%s: update stream failed.\n", __func__); |
| goto error; |
| } |
| |
| asd->acc.extension_mode = true; |
| return 0; |
| |
| error: |
| while (--i >= 0) { |
| if (acc_fw->flags & acc_flag_to_pipe[i].flag) { |
| atomisp_css_unload_acc_extension(asd, acc_fw->fw, |
| acc_flag_to_pipe[i].pipe_id); |
| } |
| } |
| |
| list_for_each_entry_continue_reverse(acc_fw, &asd->acc.fw, list) { |
| if (acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_OUTPUT && |
| acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_VIEWFINDER) |
| continue; |
| |
| for (i = ARRAY_SIZE(acc_flag_to_pipe) - 1; i >= 0; i--) { |
| if (!continuous && |
| acc_flag_to_pipe[i].flag == |
| ATOMISP_ACC_FW_LOAD_FL_ACC) |
| continue; |
| if (acc_fw->flags & acc_flag_to_pipe[i].flag) { |
| atomisp_css_unload_acc_extension(asd, |
| acc_fw->fw, |
| acc_flag_to_pipe[i].pipe_id); |
| } |
| } |
| } |
| return ret; |
| } |
| |
| void atomisp_acc_unload_extensions(struct atomisp_sub_device *asd) |
| { |
| struct atomisp_acc_fw *acc_fw; |
| int i; |
| |
| if (!asd->acc.extension_mode) |
| return; |
| |
| list_for_each_entry_reverse(acc_fw, &asd->acc.fw, list) { |
| if (acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_OUTPUT && |
| acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_VIEWFINDER) |
| continue; |
| |
| for (i = ARRAY_SIZE(acc_flag_to_pipe) - 1; i >= 0; i--) { |
| if (acc_fw->flags & acc_flag_to_pipe[i].flag) { |
| atomisp_css_unload_acc_extension(asd, |
| acc_fw->fw, |
| acc_flag_to_pipe[i].pipe_id); |
| } |
| } |
| } |
| |
| asd->acc.extension_mode = false; |
| } |
| |
| int atomisp_acc_set_state(struct atomisp_sub_device *asd, |
| struct atomisp_acc_state *arg) |
| { |
| struct atomisp_acc_fw *acc_fw; |
| bool enable = (arg->flags & ATOMISP_STATE_FLAG_ENABLE) != 0; |
| struct ia_css_pipe *pipe; |
| int r; |
| int i; |
| |
| if (!asd->acc.extension_mode) |
| return -EBUSY; |
| |
| if (arg->flags & ~ATOMISP_STATE_FLAG_ENABLE) |
| return -EINVAL; |
| |
| acc_fw = acc_get_fw(asd, arg->fw_handle); |
| if (!acc_fw) |
| return -EINVAL; |
| |
| if (enable) |
| wbinvd(); |
| |
| for (i = 0; i < ARRAY_SIZE(acc_flag_to_pipe); i++) { |
| if (acc_fw->flags & acc_flag_to_pipe[i].flag) { |
| pipe = asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]. |
| pipes[acc_flag_to_pipe[i].pipe_id]; |
| r = ia_css_pipe_set_qos_ext_state(pipe, acc_fw->handle, |
| enable); |
| if (r) |
| return -EBADRQC; |
| } |
| } |
| |
| if (enable) |
| acc_fw->flags |= ATOMISP_ACC_FW_LOAD_FL_ENABLE; |
| else |
| acc_fw->flags &= ~ATOMISP_ACC_FW_LOAD_FL_ENABLE; |
| |
| return 0; |
| } |
| |
| int atomisp_acc_get_state(struct atomisp_sub_device *asd, |
| struct atomisp_acc_state *arg) |
| { |
| struct atomisp_acc_fw *acc_fw; |
| |
| if (!asd->acc.extension_mode) |
| return -EBUSY; |
| |
| acc_fw = acc_get_fw(asd, arg->fw_handle); |
| if (!acc_fw) |
| return -EINVAL; |
| |
| arg->flags = acc_fw->flags; |
| |
| return 0; |
| } |