1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */
3 #include <linux/libnvdimm.h>
4 #include <asm/unaligned.h>
5 #include <linux/device.h>
6 #include <linux/module.h>
7 #include <linux/ndctl.h>
8 #include <linux/async.h>
9 #include <linux/slab.h>
10 #include <linux/nd.h>
11 #include "cxlmem.h"
12 #include "cxl.h"
13
14 extern const struct nvdimm_security_ops *cxl_security_ops;
15
16 static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
17
clear_exclusive(void * cxlds)18 static void clear_exclusive(void *cxlds)
19 {
20 clear_exclusive_cxl_commands(cxlds, exclusive_cmds);
21 }
22
unregister_nvdimm(void * nvdimm)23 static void unregister_nvdimm(void *nvdimm)
24 {
25 nvdimm_delete(nvdimm);
26 }
27
provider_show(struct device * dev,struct device_attribute * attr,char * buf)28 static ssize_t provider_show(struct device *dev, struct device_attribute *attr, char *buf)
29 {
30 struct nvdimm *nvdimm = to_nvdimm(dev);
31 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
32
33 return sysfs_emit(buf, "%s\n", dev_name(&cxl_nvd->dev));
34 }
35 static DEVICE_ATTR_RO(provider);
36
id_show(struct device * dev,struct device_attribute * attr,char * buf)37 static ssize_t id_show(struct device *dev, struct device_attribute *attr, char *buf)
38 {
39 struct nvdimm *nvdimm = to_nvdimm(dev);
40 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
41 struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds;
42
43 return sysfs_emit(buf, "%lld\n", cxlds->serial);
44 }
45 static DEVICE_ATTR_RO(id);
46
47 static struct attribute *cxl_dimm_attributes[] = {
48 &dev_attr_id.attr,
49 &dev_attr_provider.attr,
50 NULL
51 };
52
53 static const struct attribute_group cxl_dimm_attribute_group = {
54 .name = "cxl",
55 .attrs = cxl_dimm_attributes,
56 };
57
58 static const struct attribute_group *cxl_dimm_attribute_groups[] = {
59 &cxl_dimm_attribute_group,
60 NULL
61 };
62
cxl_nvdimm_probe(struct device * dev)63 static int cxl_nvdimm_probe(struct device *dev)
64 {
65 struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
66 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
67 struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;
68 unsigned long flags = 0, cmd_mask = 0;
69 struct cxl_dev_state *cxlds = cxlmd->cxlds;
70 struct nvdimm *nvdimm;
71 int rc;
72
73 set_exclusive_cxl_commands(cxlds, exclusive_cmds);
74 rc = devm_add_action_or_reset(dev, clear_exclusive, cxlds);
75 if (rc)
76 return rc;
77
78 set_bit(NDD_LABELING, &flags);
79 set_bit(NDD_REGISTER_SYNC, &flags);
80 set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
81 set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
82 set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
83 nvdimm = __nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd,
84 cxl_dimm_attribute_groups, flags,
85 cmd_mask, 0, NULL, cxl_nvd->dev_id,
86 cxl_security_ops, NULL);
87 if (!nvdimm)
88 return -ENOMEM;
89
90 dev_set_drvdata(dev, nvdimm);
91 return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
92 }
93
94 static struct cxl_driver cxl_nvdimm_driver = {
95 .name = "cxl_nvdimm",
96 .probe = cxl_nvdimm_probe,
97 .id = CXL_DEVICE_NVDIMM,
98 .drv = {
99 .suppress_bind_attrs = true,
100 },
101 };
102
cxl_pmem_get_config_size(struct cxl_dev_state * cxlds,struct nd_cmd_get_config_size * cmd,unsigned int buf_len)103 static int cxl_pmem_get_config_size(struct cxl_dev_state *cxlds,
104 struct nd_cmd_get_config_size *cmd,
105 unsigned int buf_len)
106 {
107 if (sizeof(*cmd) > buf_len)
108 return -EINVAL;
109
110 *cmd = (struct nd_cmd_get_config_size) {
111 .config_size = cxlds->lsa_size,
112 .max_xfer = cxlds->payload_size - sizeof(struct cxl_mbox_set_lsa),
113 };
114
115 return 0;
116 }
117
cxl_pmem_get_config_data(struct cxl_dev_state * cxlds,struct nd_cmd_get_config_data_hdr * cmd,unsigned int buf_len)118 static int cxl_pmem_get_config_data(struct cxl_dev_state *cxlds,
119 struct nd_cmd_get_config_data_hdr *cmd,
120 unsigned int buf_len)
121 {
122 struct cxl_mbox_get_lsa get_lsa;
123 struct cxl_mbox_cmd mbox_cmd;
124 int rc;
125
126 if (sizeof(*cmd) > buf_len)
127 return -EINVAL;
128 if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
129 return -EINVAL;
130
131 get_lsa = (struct cxl_mbox_get_lsa) {
132 .offset = cpu_to_le32(cmd->in_offset),
133 .length = cpu_to_le32(cmd->in_length),
134 };
135 mbox_cmd = (struct cxl_mbox_cmd) {
136 .opcode = CXL_MBOX_OP_GET_LSA,
137 .payload_in = &get_lsa,
138 .size_in = sizeof(get_lsa),
139 .size_out = cmd->in_length,
140 .payload_out = cmd->out_buf,
141 };
142
143 rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
144 cmd->status = 0;
145
146 return rc;
147 }
148
cxl_pmem_set_config_data(struct cxl_dev_state * cxlds,struct nd_cmd_set_config_hdr * cmd,unsigned int buf_len)149 static int cxl_pmem_set_config_data(struct cxl_dev_state *cxlds,
150 struct nd_cmd_set_config_hdr *cmd,
151 unsigned int buf_len)
152 {
153 struct cxl_mbox_set_lsa *set_lsa;
154 struct cxl_mbox_cmd mbox_cmd;
155 int rc;
156
157 if (sizeof(*cmd) > buf_len)
158 return -EINVAL;
159
160 /* 4-byte status follows the input data in the payload */
161 if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len)
162 return -EINVAL;
163
164 set_lsa =
165 kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
166 if (!set_lsa)
167 return -ENOMEM;
168
169 *set_lsa = (struct cxl_mbox_set_lsa) {
170 .offset = cpu_to_le32(cmd->in_offset),
171 };
172 memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
173 mbox_cmd = (struct cxl_mbox_cmd) {
174 .opcode = CXL_MBOX_OP_SET_LSA,
175 .payload_in = set_lsa,
176 .size_in = struct_size(set_lsa, data, cmd->in_length),
177 };
178
179 rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
180
181 /*
182 * Set "firmware" status (4-packed bytes at the end of the input
183 * payload.
184 */
185 put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
186 kvfree(set_lsa);
187
188 return rc;
189 }
190
cxl_pmem_nvdimm_ctl(struct nvdimm * nvdimm,unsigned int cmd,void * buf,unsigned int buf_len)191 static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
192 void *buf, unsigned int buf_len)
193 {
194 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
195 unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
196 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
197 struct cxl_dev_state *cxlds = cxlmd->cxlds;
198
199 if (!test_bit(cmd, &cmd_mask))
200 return -ENOTTY;
201
202 switch (cmd) {
203 case ND_CMD_GET_CONFIG_SIZE:
204 return cxl_pmem_get_config_size(cxlds, buf, buf_len);
205 case ND_CMD_GET_CONFIG_DATA:
206 return cxl_pmem_get_config_data(cxlds, buf, buf_len);
207 case ND_CMD_SET_CONFIG_DATA:
208 return cxl_pmem_set_config_data(cxlds, buf, buf_len);
209 default:
210 return -ENOTTY;
211 }
212 }
213
cxl_pmem_ctl(struct nvdimm_bus_descriptor * nd_desc,struct nvdimm * nvdimm,unsigned int cmd,void * buf,unsigned int buf_len,int * cmd_rc)214 static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
215 struct nvdimm *nvdimm, unsigned int cmd, void *buf,
216 unsigned int buf_len, int *cmd_rc)
217 {
218 /*
219 * No firmware response to translate, let the transport error
220 * code take precedence.
221 */
222 *cmd_rc = 0;
223
224 if (!nvdimm)
225 return -ENOTTY;
226 return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
227 }
228
detach_nvdimm(struct device * dev,void * data)229 static int detach_nvdimm(struct device *dev, void *data)
230 {
231 struct cxl_nvdimm *cxl_nvd;
232 bool release = false;
233
234 if (!is_cxl_nvdimm(dev))
235 return 0;
236
237 device_lock(dev);
238 if (!dev->driver)
239 goto out;
240
241 cxl_nvd = to_cxl_nvdimm(dev);
242 if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data)
243 release = true;
244 out:
245 device_unlock(dev);
246 if (release)
247 device_release_driver(dev);
248 return 0;
249 }
250
unregister_nvdimm_bus(void * _cxl_nvb)251 static void unregister_nvdimm_bus(void *_cxl_nvb)
252 {
253 struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;
254 struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus;
255
256 bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb, detach_nvdimm);
257
258 cxl_nvb->nvdimm_bus = NULL;
259 nvdimm_bus_unregister(nvdimm_bus);
260 }
261
cxl_nvdimm_bridge_probe(struct device * dev)262 static int cxl_nvdimm_bridge_probe(struct device *dev)
263 {
264 struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
265
266 cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
267 .provider_name = "CXL",
268 .module = THIS_MODULE,
269 .ndctl = cxl_pmem_ctl,
270 };
271
272 cxl_nvb->nvdimm_bus =
273 nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);
274
275 if (!cxl_nvb->nvdimm_bus)
276 return -ENOMEM;
277
278 return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb);
279 }
280
281 static struct cxl_driver cxl_nvdimm_bridge_driver = {
282 .name = "cxl_nvdimm_bridge",
283 .probe = cxl_nvdimm_bridge_probe,
284 .id = CXL_DEVICE_NVDIMM_BRIDGE,
285 .drv = {
286 .suppress_bind_attrs = true,
287 },
288 };
289
unregister_nvdimm_region(void * nd_region)290 static void unregister_nvdimm_region(void *nd_region)
291 {
292 nvdimm_region_delete(nd_region);
293 }
294
cxlr_pmem_remove_resource(void * res)295 static void cxlr_pmem_remove_resource(void *res)
296 {
297 remove_resource(res);
298 }
299
300 struct cxl_pmem_region_info {
301 u64 offset;
302 u64 serial;
303 };
304
cxl_pmem_region_probe(struct device * dev)305 static int cxl_pmem_region_probe(struct device *dev)
306 {
307 struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];
308 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
309 struct cxl_region *cxlr = cxlr_pmem->cxlr;
310 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
311 struct cxl_pmem_region_info *info = NULL;
312 struct nd_interleave_set *nd_set;
313 struct nd_region_desc ndr_desc;
314 struct cxl_nvdimm *cxl_nvd;
315 struct nvdimm *nvdimm;
316 struct resource *res;
317 int rc, i = 0;
318
319 memset(&mappings, 0, sizeof(mappings));
320 memset(&ndr_desc, 0, sizeof(ndr_desc));
321
322 res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
323 if (!res)
324 return -ENOMEM;
325
326 res->name = "Persistent Memory";
327 res->start = cxlr_pmem->hpa_range.start;
328 res->end = cxlr_pmem->hpa_range.end;
329 res->flags = IORESOURCE_MEM;
330 res->desc = IORES_DESC_PERSISTENT_MEMORY;
331
332 rc = insert_resource(&iomem_resource, res);
333 if (rc)
334 return rc;
335
336 rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);
337 if (rc)
338 return rc;
339
340 ndr_desc.res = res;
341 ndr_desc.provider_data = cxlr_pmem;
342
343 ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);
344 ndr_desc.target_node = phys_to_target_node(res->start);
345 if (ndr_desc.target_node == NUMA_NO_NODE) {
346 ndr_desc.target_node = ndr_desc.numa_node;
347 dev_dbg(&cxlr->dev, "changing target node from %d to %d",
348 NUMA_NO_NODE, ndr_desc.target_node);
349 }
350
351 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
352 if (!nd_set)
353 return -ENOMEM;
354
355 ndr_desc.memregion = cxlr->id;
356 set_bit(ND_REGION_CXL, &ndr_desc.flags);
357 set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);
358
359 info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL);
360 if (!info)
361 return -ENOMEM;
362
363 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
364 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
365 struct cxl_memdev *cxlmd = m->cxlmd;
366 struct cxl_dev_state *cxlds = cxlmd->cxlds;
367
368 cxl_nvd = cxlmd->cxl_nvd;
369 nvdimm = dev_get_drvdata(&cxl_nvd->dev);
370 if (!nvdimm) {
371 dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,
372 dev_name(&cxlmd->dev));
373 rc = -ENODEV;
374 goto out_nvd;
375 }
376
377 m->cxl_nvd = cxl_nvd;
378 mappings[i] = (struct nd_mapping_desc) {
379 .nvdimm = nvdimm,
380 .start = m->start,
381 .size = m->size,
382 .position = i,
383 };
384 info[i].offset = m->start;
385 info[i].serial = cxlds->serial;
386 }
387 ndr_desc.num_mappings = cxlr_pmem->nr_mappings;
388 ndr_desc.mapping = mappings;
389
390 /*
391 * TODO enable CXL labels which skip the need for 'interleave-set cookie'
392 */
393 nd_set->cookie1 =
394 nd_fletcher64(info, sizeof(*info) * cxlr_pmem->nr_mappings, 0);
395 nd_set->cookie2 = nd_set->cookie1;
396 ndr_desc.nd_set = nd_set;
397
398 cxlr_pmem->nd_region =
399 nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);
400 if (!cxlr_pmem->nd_region) {
401 rc = -ENOMEM;
402 goto out_nvd;
403 }
404
405 rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,
406 cxlr_pmem->nd_region);
407 out_nvd:
408 kfree(info);
409
410 return rc;
411 }
412
413 static struct cxl_driver cxl_pmem_region_driver = {
414 .name = "cxl_pmem_region",
415 .probe = cxl_pmem_region_probe,
416 .id = CXL_DEVICE_PMEM_REGION,
417 .drv = {
418 .suppress_bind_attrs = true,
419 },
420 };
421
cxl_pmem_init(void)422 static __init int cxl_pmem_init(void)
423 {
424 int rc;
425
426 set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
427 set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);
428
429 rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
430 if (rc)
431 return rc;
432
433 rc = cxl_driver_register(&cxl_nvdimm_driver);
434 if (rc)
435 goto err_nvdimm;
436
437 rc = cxl_driver_register(&cxl_pmem_region_driver);
438 if (rc)
439 goto err_region;
440
441 return 0;
442
443 err_region:
444 cxl_driver_unregister(&cxl_nvdimm_driver);
445 err_nvdimm:
446 cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
447 return rc;
448 }
449
cxl_pmem_exit(void)450 static __exit void cxl_pmem_exit(void)
451 {
452 cxl_driver_unregister(&cxl_pmem_region_driver);
453 cxl_driver_unregister(&cxl_nvdimm_driver);
454 cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
455 }
456
457 MODULE_LICENSE("GPL v2");
458 module_init(cxl_pmem_init);
459 module_exit(cxl_pmem_exit);
460 MODULE_IMPORT_NS(CXL);
461 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
462 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
463 MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);
464