blob: 94413a32c99a862ef1901877584a6fbeaf5511f4 [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0
/*
* MTD layer for NAND Mapped-block Management (NMBM)
*
* Copyright (C) 2020 MediaTek Inc. All Rights Reserved.
*
* Author: Weijie Gao <weijie.gao@mediatek.com>
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
#include <linux/wait.h>
#include <linux/spinlock.h>
#include <linux/mtd/mtd.h>
#include <linux/mtd/flashchip.h>
#include <linux/mtd/partitions.h>
#include <linux/of_platform.h>
#include <linux/kern_levels.h>
#include "nmbm-private.h"
#include "nmbm-debug.h"
#define NMBM_MAX_RATIO_DEFAULT 1
#define NMBM_MAX_BLOCKS_DEFAULT 256
struct nmbm_mtd {
struct mtd_info upper;
struct mtd_info *lower;
struct nmbm_instance *ni;
uint8_t *page_cache;
flstate_t state;
spinlock_t lock;
wait_queue_head_t wq;
struct device *dev;
struct list_head node;
};
struct list_head nmbm_devs;
static DEFINE_MUTEX(nmbm_devs_lock);
static int nmbm_lower_read_page(void *arg, uint64_t addr, void *buf, void *oob,
enum nmbm_oob_mode mode)
{
struct nmbm_mtd *nm = arg;
struct mtd_oob_ops ops;
int ret;
memset(&ops, 0, sizeof(ops));
switch (mode) {
case NMBM_MODE_PLACE_OOB:
ops.mode = MTD_OPS_PLACE_OOB;
break;
case NMBM_MODE_AUTO_OOB:
ops.mode = MTD_OPS_AUTO_OOB;
break;
case NMBM_MODE_RAW:
ops.mode = MTD_OPS_RAW;
break;
default:
pr_debug("%s: unsupported NMBM mode: %u\n", __func__, mode);
return -ENOTSUPP;
}
if (buf) {
ops.datbuf = buf;
ops.len = nm->lower->writesize;
}
if (oob) {
ops.oobbuf = oob;
ops.ooblen = mtd_oobavail(nm->lower, &ops);
}
ret = mtd_read_oob(nm->lower, addr, &ops);
nm->upper.ecc_stats.corrected = nm->lower->ecc_stats.corrected;
nm->upper.ecc_stats.failed = nm->lower->ecc_stats.failed;
/* Report error on failure (including ecc error) */
if (ret < 0 && ret != -EUCLEAN)
return ret;
/*
* Since mtd_read_oob() won't report exact bitflips, what we can know
* is whether bitflips exceeds the threshold.
* We want the -EUCLEAN to be passed to the upper layer, but not the
* error value itself. To achieve this, report bitflips above the
* threshold.
*/
if (ret == -EUCLEAN) {
return min_t(u32, nm->lower->bitflip_threshold + 1,
nm->lower->ecc_strength);
}
/* For bitflips less than the threshold, return 0 */
return 0;
}
static int nmbm_lower_write_page(void *arg, uint64_t addr, const void *buf,
const void *oob, enum nmbm_oob_mode mode)
{
struct nmbm_mtd *nm = arg;
struct mtd_oob_ops ops;
memset(&ops, 0, sizeof(ops));
switch (mode) {
case NMBM_MODE_PLACE_OOB:
ops.mode = MTD_OPS_PLACE_OOB;
break;
case NMBM_MODE_AUTO_OOB:
ops.mode = MTD_OPS_AUTO_OOB;
break;
case NMBM_MODE_RAW:
ops.mode = MTD_OPS_RAW;
break;
default:
pr_debug("%s: unsupported NMBM mode: %u\n", __func__, mode);
return -ENOTSUPP;
}
if (buf) {
ops.datbuf = (uint8_t *)buf;
ops.len = nm->lower->writesize;
}
if (oob) {
ops.oobbuf = (uint8_t *)oob;
ops.ooblen = mtd_oobavail(nm->lower, &ops);
}
return mtd_write_oob(nm->lower, addr, &ops);
}
static int nmbm_lower_panic_write_page(void *arg, uint64_t addr,
const void *buf)
{
struct nmbm_mtd *nm = arg;
size_t retlen;
return mtd_panic_write(nm->lower, addr, nm->lower->writesize, &retlen,
buf);
}
static int nmbm_lower_erase_block(void *arg, uint64_t addr)
{
struct nmbm_mtd *nm = arg;
struct erase_info ei;
memset(&ei, 0, sizeof(ei));
ei.addr = addr;
ei.len = nm->lower->erasesize;
return mtd_erase(nm->lower, &ei);
}
static int nmbm_lower_is_bad_block(void *arg, uint64_t addr)
{
struct nmbm_mtd *nm = arg;
return mtd_block_isbad(nm->lower, addr);
}
static int nmbm_lower_mark_bad_block(void *arg, uint64_t addr)
{
struct nmbm_mtd *nm = arg;
return mtd_block_markbad(nm->lower, addr);
}
static void nmbm_lower_log(void *arg, enum nmbm_log_category level,
const char *fmt, va_list ap)
{
struct nmbm_mtd *nm = arg;
char *msg;
char *kl;
msg = kvasprintf(GFP_KERNEL, fmt, ap);
if (!msg) {
dev_warn(nm->dev, "unable to print log\n");
return;
}
switch (level) {
case NMBM_LOG_DEBUG:
kl = KERN_DEBUG;
break;
case NMBM_LOG_WARN:
kl = KERN_WARNING;
break;
case NMBM_LOG_ERR:
kl = KERN_ERR;
break;
case NMBM_LOG_EMERG:
kl = KERN_EMERG;
break;
default:
kl = KERN_INFO ;
}
dev_printk(kl, nm->dev, "%s", msg);
kfree(msg);
}
static int nmbm_get_device(struct nmbm_mtd *nm, int new_state)
{
DECLARE_WAITQUEUE(wait, current);
retry:
spin_lock(&nm->lock);
if (nm->state == FL_READY) {
nm->state = new_state;
spin_unlock(&nm->lock);
return 0;
}
if (new_state == FL_PM_SUSPENDED) {
if (nm->state == FL_PM_SUSPENDED) {
spin_unlock(&nm->lock);
return 0;
}
}
set_current_state(TASK_UNINTERRUPTIBLE);
add_wait_queue(&nm->wq, &wait);
spin_unlock(&nm->lock);
schedule();
remove_wait_queue(&nm->wq, &wait);
goto retry;
}
static void nmbm_release_device(struct nmbm_mtd *nm)
{
spin_lock(&nm->lock);
nm->state = FL_READY;
wake_up(&nm->wq);
spin_unlock(&nm->lock);
}
static int nmbm_mtd_erase(struct mtd_info *mtd, struct erase_info *instr)
{
struct nmbm_mtd *nm = container_of(mtd, struct nmbm_mtd, upper);
int ret;
nmbm_get_device(nm, FL_ERASING);
ret = nmbm_erase_block_range(nm->ni, instr->addr, instr->len,
&instr->fail_addr);
nmbm_release_device(nm);
if (!ret)
return 0;
return -EIO;
}
static int nmbm_mtd_read_data(struct nmbm_mtd *nm, uint64_t addr,
struct mtd_oob_ops *ops, enum nmbm_oob_mode mode)
{
size_t len, ooblen, maxooblen, chklen;
uint32_t col, ooboffs;
uint8_t *datcache, *oobcache;
bool has_ecc_err = false;
int ret, max_bitflips = 0;
col = addr & nm->lower->writesize_mask;
addr &= ~nm->lower->writesize_mask;
maxooblen = mtd_oobavail(nm->lower, ops);
ooboffs = ops->ooboffs;
ooblen = ops->ooblen;
len = ops->len;
datcache = len ? nm->page_cache : NULL;
oobcache = ooblen ? nm->page_cache + nm->lower->writesize : NULL;
ops->oobretlen = 0;
ops->retlen = 0;
while (len || ooblen) {
ret = nmbm_read_single_page(nm->ni, addr, datcache, oobcache,
mode);
if (ret < 0 && ret != -EBADMSG)
return ret;
/* Continue reading on ecc error */
if (ret == -EBADMSG)
has_ecc_err = true;
/* Record the maximum bitflips between pages */
if (ret > max_bitflips)
max_bitflips = ret;
if (len) {
/* Move data */
chklen = nm->lower->writesize - col;
if (chklen > len)
chklen = len;
memcpy(ops->datbuf + ops->retlen, datcache + col,
chklen);
len -= chklen;
col = 0; /* (col + chklen) % */
ops->retlen += chklen;
}
if (ooblen) {
/* Move oob */
chklen = maxooblen - ooboffs;
if (chklen > ooblen)
chklen = ooblen;
memcpy(ops->oobbuf + ops->oobretlen, oobcache + ooboffs,
chklen);
ooblen -= chklen;
ooboffs = 0; /* (ooboffs + chklen) % maxooblen; */
ops->oobretlen += chklen;
}
addr += nm->lower->writesize;
}
if (has_ecc_err)
return -EBADMSG;
return max_bitflips;
}
static int nmbm_mtd_read_oob(struct mtd_info *mtd, loff_t from,
struct mtd_oob_ops *ops)
{
struct nmbm_mtd *nm = container_of(mtd, struct nmbm_mtd, upper);
uint32_t maxooblen;
enum nmbm_oob_mode mode;
int ret;
if (!ops->oobbuf && !ops->datbuf) {
if (ops->ooblen || ops->len)
return -EINVAL;
return 0;
}
switch (ops->mode) {
case MTD_OPS_PLACE_OOB:
mode = NMBM_MODE_PLACE_OOB;
break;
case MTD_OPS_AUTO_OOB:
mode = NMBM_MODE_AUTO_OOB;
break;
case MTD_OPS_RAW:
mode = NMBM_MODE_RAW;
break;
default:
pr_debug("%s: unsupported oob mode: %u\n", __func__, ops->mode);
return -ENOTSUPP;
}
maxooblen = mtd_oobavail(mtd, ops);
/* Do not allow read past end of device */
if (ops->datbuf && (from + ops->len) > mtd->size) {
pr_debug("%s: attempt to read beyond end of device\n",
__func__);
return -EINVAL;
}
if (!ops->oobbuf) {
nmbm_get_device(nm, FL_READING);
/* Optimized for reading data only */
ret = nmbm_read_range(nm->ni, from, ops->len, ops->datbuf,
mode, &ops->retlen);
nmbm_release_device(nm);
return ret;
}
if (unlikely(ops->ooboffs >= maxooblen)) {
pr_debug("%s: attempt to start read outside oob\n",
__func__);
return -EINVAL;
}
if (unlikely(from >= mtd->size ||
ops->ooboffs + ops->ooblen > ((mtd->size >> mtd->writesize_shift) -
(from >> mtd->writesize_shift)) * maxooblen)) {
pr_debug("%s: attempt to read beyond end of device\n",
__func__);
return -EINVAL;
}
nmbm_get_device(nm, FL_READING);
ret = nmbm_mtd_read_data(nm, from, ops, mode);
nmbm_release_device(nm);
return ret;
}
static int nmbm_mtd_write_data(struct nmbm_mtd *nm, uint64_t addr,
struct mtd_oob_ops *ops, enum nmbm_oob_mode mode)
{
size_t len, ooblen, maxooblen, chklen;
uint32_t col, ooboffs;
uint8_t *datcache, *oobcache;
int ret;
col = addr & nm->lower->writesize_mask;
addr &= ~nm->lower->writesize_mask;
maxooblen = mtd_oobavail(nm->lower, ops);
ooboffs = ops->ooboffs;
ooblen = ops->ooblen;
len = ops->len;
datcache = len ? nm->page_cache : NULL;
oobcache = ooblen ? nm->page_cache + nm->lower->writesize : NULL;
ops->oobretlen = 0;
ops->retlen = 0;
while (len || ooblen) {
if (len) {
/* Move data */
chklen = nm->lower->writesize - col;
if (chklen > len)
chklen = len;
memset(datcache, 0xff, col);
memcpy(datcache + col, ops->datbuf + ops->retlen,
chklen);
memset(datcache + col + chklen, 0xff,
nm->lower->writesize - col - chklen);
len -= chklen;
col = 0; /* (col + chklen) % */
ops->retlen += chklen;
}
if (ooblen) {
/* Move oob */
chklen = maxooblen - ooboffs;
if (chklen > ooblen)
chklen = ooblen;
memset(oobcache, 0xff, ooboffs);
memcpy(oobcache + ooboffs,
ops->oobbuf + ops->oobretlen, chklen);
memset(oobcache + ooboffs + chklen, 0xff,
nm->lower->oobsize - ooboffs - chklen);
ooblen -= chklen;
ooboffs = 0; /* (ooboffs + chklen) % maxooblen; */
ops->oobretlen += chklen;
}
ret = nmbm_write_single_page(nm->ni, addr, datcache, oobcache,
mode);
if (ret)
return ret;
addr += nm->lower->writesize;
}
return 0;
}
static int nmbm_mtd_write_oob(struct mtd_info *mtd, loff_t to,
struct mtd_oob_ops *ops)
{
struct nmbm_mtd *nm = container_of(mtd, struct nmbm_mtd, upper);
enum nmbm_oob_mode mode;
uint32_t maxooblen;
int ret;
if (!ops->oobbuf && !ops->datbuf) {
if (ops->ooblen || ops->len)
return -EINVAL;
return 0;
}
switch (ops->mode) {
case MTD_OPS_PLACE_OOB:
mode = NMBM_MODE_PLACE_OOB;
break;
case MTD_OPS_AUTO_OOB:
mode = NMBM_MODE_AUTO_OOB;
break;
case MTD_OPS_RAW:
mode = NMBM_MODE_RAW;
break;
default:
pr_debug("%s: unsupported oob mode: %u\n", __func__,
ops->mode);
return -ENOTSUPP;
}
maxooblen = mtd_oobavail(mtd, ops);
/* Do not allow write past end of device */
if (ops->datbuf && (to + ops->len) > mtd->size) {
pr_debug("%s: attempt to write beyond end of device\n",
__func__);
return -EINVAL;
}
if (!ops->oobbuf) {
nmbm_get_device(nm, FL_WRITING);
/* Optimized for writing data only */
ret = nmbm_write_range(nm->ni, to, ops->len, ops->datbuf,
mode, &ops->retlen);
nmbm_release_device(nm);
return ret;
}
if (unlikely(ops->ooboffs >= maxooblen)) {
pr_debug("%s: attempt to start write outside oob\n",
__func__);
return -EINVAL;
}
if (unlikely(to >= mtd->size ||
ops->ooboffs + ops->ooblen > ((mtd->size >> mtd->writesize_shift) -
(to >> mtd->writesize_shift)) * maxooblen)) {
pr_debug("%s: attempt to write beyond end of device\n",
__func__);
return -EINVAL;
}
nmbm_get_device(nm, FL_WRITING);
ret = nmbm_mtd_write_data(nm, to, ops, mode);
nmbm_release_device(nm);
return ret;
}
static int nmbm_mtd_panic_write(struct mtd_info *mtd, loff_t to, size_t len,
size_t *retlen, const u_char *buf)
{
struct nmbm_mtd *nm = container_of(mtd, struct nmbm_mtd, upper);
size_t chklen, wrlen = 0;
uint32_t col;
int ret;
col = to & nm->lower->writesize_mask;
to &= ~nm->lower->writesize_mask;
while (len) {
/* Move data */
chklen = nm->lower->writesize - col;
if (chklen > len)
chklen = len;
if (chklen < nm->lower->writesize)
memset(nm->page_cache, 0xff, nm->lower->writesize);
memcpy(nm->page_cache + col, buf + wrlen, chklen);
len -= chklen;
col = 0; /* (col + chklen) % */
wrlen += chklen;
ret = nmbm_panic_write_single_page(nm->ni, to, nm->page_cache);
if (ret)
break;
to += nm->lower->writesize;
}
if (retlen)
*retlen = wrlen;
return 0;
}
static int nmbm_mtd_block_isbad(struct mtd_info *mtd, loff_t offs)
{
struct nmbm_mtd *nm = container_of(mtd, struct nmbm_mtd, upper);
int ret;
nmbm_get_device(nm, FL_READING);
ret = nmbm_check_bad_block(nm->ni, offs);
nmbm_release_device(nm);
return ret;
}
static int nmbm_mtd_block_markbad(struct mtd_info *mtd, loff_t offs)
{
struct nmbm_mtd *nm = container_of(mtd, struct nmbm_mtd, upper);
int ret;
nmbm_get_device(nm, FL_WRITING);
ret = nmbm_mark_bad_block(nm->ni, offs);
nmbm_release_device(nm);
return ret;
}
static void nmbm_mtd_shutdown(struct mtd_info *mtd)
{
struct nmbm_mtd *nm = container_of(mtd, struct nmbm_mtd, upper);
nmbm_get_device(nm, FL_PM_SUSPENDED);
}
static int nmbm_probe(struct platform_device *pdev)
{
struct device_node *mtd_np, *np = pdev->dev.of_node;
uint32_t max_ratio, max_reserved_blocks, alloc_size;
bool forced_create, empty_page_ecc_ok;
struct nmbm_lower_device nld;
struct mtd_info *lower, *mtd;
struct nmbm_mtd *nm;
const char *mtdname;
int ret;
mtd_np = of_parse_phandle(np, "lower-mtd-device", 0);
if (mtd_np) {
lower = get_mtd_device_by_node(mtd_np);
if (!IS_ERR(lower))
goto do_attach_mtd;
dev_dbg(&pdev->dev, "failed to find mtd device by phandle\n");
return -EPROBE_DEFER;
}
ret = of_property_read_string(np, "lower-mtd-name", &mtdname);
if (!ret) {
lower = get_mtd_device_nm(mtdname);
if (!IS_ERR(lower))
goto do_attach_mtd;
dev_dbg(&pdev->dev, "failed to find mtd device by name '%s'\n",
mtdname);
return -EPROBE_DEFER;
}
do_attach_mtd:
if (of_property_read_u32(np, "max-ratio", &max_ratio))
max_ratio = NMBM_MAX_RATIO_DEFAULT;
if (of_property_read_u32(np, "max-reserved-blocks",
&max_reserved_blocks))
max_reserved_blocks = NMBM_MAX_BLOCKS_DEFAULT;
forced_create = of_property_read_bool(np, "forced-create");
empty_page_ecc_ok = of_property_read_bool(np,
"empty-page-ecc-protected");
memset(&nld, 0, sizeof(nld));
nld.flags = 0;
if (forced_create)
nld.flags |= NMBM_F_CREATE;
if (empty_page_ecc_ok)
nld.flags |= NMBM_F_EMPTY_PAGE_ECC_OK;
nld.max_ratio = max_ratio;
nld.max_reserved_blocks = max_reserved_blocks;
nld.size = lower->size;
nld.erasesize = lower->erasesize;
nld.writesize = lower->writesize;
nld.oobsize = lower->oobsize;
nld.oobavail = lower->oobavail;
nld.read_page = nmbm_lower_read_page;
nld.write_page = nmbm_lower_write_page;
nld.panic_write_page = nmbm_lower_panic_write_page;
nld.erase_block = nmbm_lower_erase_block;
nld.is_bad_block = nmbm_lower_is_bad_block;
nld.mark_bad_block = nmbm_lower_mark_bad_block;
nld.logprint = nmbm_lower_log;
alloc_size = nmbm_calc_structure_size(&nld);
nm = devm_kzalloc(&pdev->dev, sizeof(*nm) + alloc_size +
lower->writesize + lower->oobsize, GFP_KERNEL);
if (!nm) {
ret = -ENOMEM;
goto out;
}
nm->ni = (void *)nm + sizeof(*nm);
nm->page_cache = (uint8_t *)nm->ni + alloc_size;
nm->lower = lower;
nm->dev = &pdev->dev;
INIT_LIST_HEAD(&nm->node);
spin_lock_init(&nm->lock);
init_waitqueue_head(&nm->wq);
nld.arg = nm;
ret = nmbm_attach(&nld, nm->ni);
if (ret)
goto out;
/* Initialize upper mtd */
mtd = &nm->upper;
mtd->owner = THIS_MODULE;
mtd->dev.parent = &pdev->dev;
mtd->type = lower->type;
mtd->flags = lower->flags;
mtd->size = (uint64_t)nm->ni->data_block_count * lower->erasesize;
mtd->erasesize = lower->erasesize;
mtd->writesize = lower->writesize;
mtd->writebufsize = lower->writesize;
mtd->oobsize = lower->oobsize;
mtd->oobavail = lower->oobavail;
mtd->erasesize_shift = lower->erasesize_shift;
mtd->writesize_shift = lower->writesize_shift;
mtd->erasesize_mask = lower->erasesize_mask;
mtd->writesize_mask = lower->writesize_mask;
mtd->bitflip_threshold = lower->bitflip_threshold;
mtd->ooblayout = lower->ooblayout;
mtd->ecc_step_size = lower->ecc_step_size;
mtd->ecc_strength = lower->ecc_strength;
mtd->numeraseregions = lower->numeraseregions;
mtd->eraseregions = lower->eraseregions;
mtd->_erase = nmbm_mtd_erase;
mtd->_read_oob = nmbm_mtd_read_oob;
mtd->_write_oob = nmbm_mtd_write_oob;
mtd->_panic_write = nmbm_mtd_panic_write;
mtd->_block_isbad = nmbm_mtd_block_isbad;
mtd->_block_markbad = nmbm_mtd_block_markbad;
mtd->_reboot = nmbm_mtd_shutdown;
mtd_set_of_node(mtd, np);
ret = mtd_device_register(mtd, NULL, 0);
if (ret) {
dev_err(&pdev->dev, "failed to register mtd device\n");
nmbm_detach(nm->ni);
goto out;
}
platform_set_drvdata(pdev, nm);
mutex_lock(&nmbm_devs_lock);
list_add_tail(&nm->node, &nmbm_devs);
mutex_unlock(&nmbm_devs_lock);
return 0;
out:
if (nm)
devm_kfree(&pdev->dev, nm);
put_mtd_device(lower);
return ret;
}
static int nmbm_remove(struct platform_device *pdev)
{
struct nmbm_mtd *nm = platform_get_drvdata(pdev);
struct mtd_info *lower = nm->lower;
int ret;
ret = mtd_device_unregister(&nm->upper);
if (ret)
return ret;
nmbm_detach(nm->ni);
mutex_lock(&nmbm_devs_lock);
list_add_tail(&nm->node, &nmbm_devs);
mutex_unlock(&nmbm_devs_lock);
devm_kfree(&pdev->dev, nm);
put_mtd_device(lower);
platform_set_drvdata(pdev, NULL);
return 0;
}
static const struct of_device_id nmbm_ids[] = {
{ .compatible = "generic,nmbm" },
{ },
};
MODULE_DEVICE_TABLE(of, nmbm_ids);
static struct platform_driver nmbm_driver = {
.probe = nmbm_probe,
.remove = nmbm_remove,
.driver = {
.name = "nmbm",
.of_match_table = nmbm_ids,
},
};
static int __init nmbm_init(void)
{
int ret;
INIT_LIST_HEAD(&nmbm_devs);
ret = platform_driver_register(&nmbm_driver);
if (ret) {
pr_err("failed to register nmbm driver\n");
return ret;
}
return 0;
}
module_init(nmbm_init);
static void __exit nmbm_exit(void)
{
platform_driver_unregister(&nmbm_driver);
}
module_exit(nmbm_exit);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Weijie Gao <weijie.gao@mediatek.com>");
MODULE_DESCRIPTION("NAND mapping block management");