source: mainline/uspace/srv/bd/hr/raid5.c@ bd51105

Last change on this file since bd51105 was da0570a, checked in by Miroslav Cimerman <mc@…>, 9 months ago

hr: RAID5 states, degraded R/W
  • Property mode set to 100644
File size: 12.8 KB
Line 
1/*
2 * Copyright (c) 2024 Miroslav Cimerman
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * - Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * - Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * - The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/** @addtogroup hr
30 * @{
31 */
32/**
33 * @file
34 */
35
36#include <abi/ipc/ipc.h>
37#include <bd_srv.h>
38#include <block.h>
39#include <errno.h>
40#include <hr.h>
41#include <io/log.h>
42#include <ipc/hr.h>
43#include <ipc/services.h>
44#include <loc.h>
45#include <mem.h>
46#include <task.h>
47#include <stdio.h>
48#include <stdlib.h>
49#include <str_error.h>
50
51#include "superblock.h"
52#include "util.h"
53#include "var.h"
54
55extern loc_srv_t *hr_srv;
56
57static errno_t hr_raid5_bd_open(bd_srvs_t *, bd_srv_t *);
58static errno_t hr_raid5_bd_close(bd_srv_t *);
59static errno_t hr_raid5_bd_read_blocks(bd_srv_t *, aoff64_t, size_t, void *,
60 size_t);
61static errno_t hr_raid5_bd_sync_cache(bd_srv_t *, aoff64_t, size_t);
62static errno_t hr_raid5_bd_write_blocks(bd_srv_t *, aoff64_t, size_t,
63 const void *, size_t);
64static errno_t hr_raid5_bd_get_block_size(bd_srv_t *, size_t *);
65static errno_t hr_raid5_bd_get_num_blocks(bd_srv_t *, aoff64_t *);
66
67static errno_t hr_raid5_write_parity(hr_volume_t *, uint64_t, uint64_t,
68 uint64_t, const void *, size_t);
69
70static bd_ops_t hr_raid5_bd_ops = {
71 .open = hr_raid5_bd_open,
72 .close = hr_raid5_bd_close,
73 .sync_cache = hr_raid5_bd_sync_cache,
74 .read_blocks = hr_raid5_bd_read_blocks,
75 .write_blocks = hr_raid5_bd_write_blocks,
76 .get_block_size = hr_raid5_bd_get_block_size,
77 .get_num_blocks = hr_raid5_bd_get_num_blocks
78};
79
80static errno_t hr_raid5_vol_usable(hr_volume_t *vol)
81{
82 if (vol->status == HR_VOL_ONLINE ||
83 vol->status == HR_VOL_DEGRADED)
84 return EOK;
85 return EINVAL;
86}
87
88/*
89 * Returns (-1) if all extents are online,
90 * else returns index of first bad one.
91 */
92static ssize_t hr_raid5_get_bad_ext(hr_volume_t *vol)
93{
94 for (size_t i = 0; i < vol->dev_no; i++)
95 if (vol->extents[i].status != HR_EXT_ONLINE)
96 return i;
97 return -1;
98}
99
100static errno_t hr_raid5_update_vol_status(hr_volume_t *vol)
101{
102 hr_vol_status_t old_state = vol->status;
103 size_t bad = 0;
104 for (size_t i = 0; i < vol->dev_no; i++)
105 if (vol->extents[i].status != HR_EXT_ONLINE)
106 bad++;
107
108 switch (bad) {
109 case 0:
110 if (old_state != HR_VOL_ONLINE) {
111 log_msg(LOG_DEFAULT, LVL_ERROR,
112 "RAID 5 has all extents online, "
113 "marking \"%s\" (%lu) as ONLINE",
114 vol->devname, vol->svc_id);
115 vol->status = HR_VOL_ONLINE;
116 }
117 return EOK;
118 case 1:
119 if (old_state != HR_VOL_DEGRADED) {
120 log_msg(LOG_DEFAULT, LVL_ERROR,
121 "RAID 5 array \"%s\" (%lu) has 1 extent inactive, "
122 "marking as DEGRADED",
123 vol->devname, vol->svc_id);
124 vol->status = HR_VOL_DEGRADED;
125 }
126 return EOK;
127 default:
128 if (old_state != HR_VOL_FAULTY) {
129 log_msg(LOG_DEFAULT, LVL_ERROR,
130 "RAID 5 array \"%s\" (%lu) has more than one 1 "
131 "extent inactive, marking as FAULTY",
132 vol->devname, vol->svc_id);
133 vol->status = HR_VOL_FAULTY;
134 }
135 return EINVAL;
136 }
137}
138
139static void xor(void *dst, const void *src, size_t size)
140{
141 size_t i;
142 uint64_t *d = dst;
143 const uint64_t *s = src;
144
145 for (i = 0; i < size / sizeof(uint64_t); ++i)
146 *d++ ^= *s++;
147}
148
149static errno_t hr_raid5_read_degraded(hr_volume_t *vol, uint64_t bad,
150 uint64_t block, void *data, size_t cnt)
151{
152 errno_t rc;
153 size_t i;
154 void *xorbuf;
155 void *buf;
156 uint64_t len = vol->bsize * cnt;
157
158 xorbuf = malloc(len);
159 if (xorbuf == NULL)
160 return ENOMEM;
161
162 buf = malloc(len);
163 if (buf == NULL) {
164 free(xorbuf);
165 return ENOMEM;
166 }
167
168 /* read all other extents in the stripe */
169 memset(xorbuf, 0, len);
170 for (i = 0; i < vol->dev_no; i++) {
171 if (i == bad) {
172 continue;
173 } else {
174 rc = block_read_direct(vol->extents[i].svc_id, block,
175 cnt, buf);
176 if (rc != EOK)
177 goto end;
178 xor(xorbuf, buf, len);
179 }
180 }
181
182 memcpy(data, xorbuf, len);
183end:
184 free(xorbuf);
185 free(buf);
186 return rc;
187}
188
189static errno_t hr_raid5_write(hr_volume_t *vol, uint64_t p_extent,
190 uint64_t extent, aoff64_t ba, const void *data, size_t cnt)
191{
192 errno_t rc;
193 size_t i;
194 void *xorbuf;
195 void *buf;
196 uint64_t len = vol->bsize * cnt;
197
198 ssize_t bad = hr_raid5_get_bad_ext(vol);
199 if (bad == -1 || (size_t)bad == p_extent) {
200 rc = block_write_direct(vol->extents[extent].svc_id, ba, cnt,
201 data);
202 if (rc != EOK)
203 return rc;
204 /*
205 * DEGRADED parity - skip parity write
206 */
207 if ((size_t)bad == p_extent)
208 return EOK;
209
210 rc = hr_raid5_write_parity(vol, p_extent, extent, ba, data,
211 cnt);
212 return rc;
213 }
214
215 xorbuf = malloc(len);
216 if (xorbuf == NULL)
217 return ENOMEM;
218
219 buf = malloc(len);
220 if (buf == NULL) {
221 free(xorbuf);
222 return ENOMEM;
223 }
224
225 if (extent == (size_t) bad) {
226 /*
227 * new parity = read other and xor in new data
228 *
229 * write new parity
230 */
231 memset(xorbuf, 0, len);
232 for (i = 1; i < vol->dev_no; i++) {
233 if (i == (size_t) bad) {
234 continue;
235 } else {
236 rc = block_read_direct(vol->extents[i].svc_id,
237 ba, cnt, buf);
238 if (rc != EOK)
239 goto end;
240 xor(xorbuf, buf, len);
241 }
242 }
243 xor(xorbuf, data, len);
244 rc = block_write_direct(vol->extents[p_extent].svc_id, ba, cnt,
245 xorbuf);
246 if (rc != EOK)
247 goto end;
248 } else {
249 /*
250 * new parity = xor original data and old parity and new data
251 *
252 * write parity, new data
253 */
254 rc = block_read_direct(vol->extents[extent].svc_id, ba, cnt,
255 xorbuf);
256 if (rc != EOK)
257 goto end;
258 rc = block_read_direct(vol->extents[p_extent].svc_id, ba, cnt,
259 buf);
260 if (rc != EOK)
261 goto end;
262
263 xor(xorbuf, buf, len);
264
265 xor(xorbuf, data, len);
266
267 rc = block_write_direct(vol->extents[p_extent].svc_id, ba, cnt,
268 xorbuf);
269 if (rc != EOK)
270 goto end;
271 rc = block_write_direct(vol->extents[extent].svc_id, ba, cnt,
272 data);
273 if (rc != EOK)
274 goto end;
275 }
276end:
277 free(xorbuf);
278 free(buf);
279 return rc;
280}
281
282static errno_t hr_raid5_write_parity(hr_volume_t *vol, uint64_t p_extent,
283 uint64_t extent, uint64_t block, const void *data, size_t cnt)
284{
285 errno_t rc;
286 size_t i;
287 void *xorbuf;
288 void *buf;
289 uint64_t len = vol->bsize * cnt;
290
291 xorbuf = malloc(len);
292 if (xorbuf == NULL)
293 return ENOMEM;
294
295 buf = malloc(len);
296 if (buf == NULL) {
297 free(xorbuf);
298 return ENOMEM;
299 }
300
301 memset(xorbuf, 0, len);
302 for (i = 0; i < vol->dev_no; i++) {
303 if (i == p_extent)
304 continue;
305 if (i == extent) {
306 xor(xorbuf, data, vol->bsize);
307 } else {
308 rc = block_read_direct(vol->extents[i].svc_id,
309 block, cnt, buf);
310 if (rc != EOK)
311 goto end;
312 xor(xorbuf, buf, vol->bsize);
313 }
314 }
315
316 rc = block_write_direct(vol->extents[p_extent].svc_id, block, cnt,
317 xorbuf);
318end:
319 free(xorbuf);
320 free(buf);
321 return rc;
322}
323
324static errno_t hr_raid5_bd_open(bd_srvs_t *bds, bd_srv_t *bd)
325{
326 log_msg(LOG_DEFAULT, LVL_NOTE, "hr_bd_open()");
327 return EOK;
328}
329
330static errno_t hr_raid5_bd_close(bd_srv_t *bd)
331{
332 log_msg(LOG_DEFAULT, LVL_NOTE, "hr_bd_close()");
333 return EOK;
334}
335
336static errno_t hr_raid5_bd_op(hr_bd_op_type_t type, bd_srv_t *bd, aoff64_t ba,
337 size_t cnt, void *dst, const void *src, size_t size)
338{
339 hr_volume_t *vol = bd->srvs->sarg;
340 errno_t rc;
341 uint64_t phys_block, len;
342 size_t left;
343 const uint8_t *data_write = src;
344 uint8_t *data_read = dst;
345
346 /* propagate sync */
347 if (type == HR_BD_SYNC && ba == 0 && cnt == 0) {
348 hr_sync_all_extents(vol);
349 rc = hr_raid5_update_vol_status(vol);
350 return rc;
351 }
352
353 if (type == HR_BD_READ || type == HR_BD_WRITE)
354 if (size < cnt * vol->bsize)
355 return EINVAL;
356
357 rc = hr_check_ba_range(vol, cnt, ba);
358 if (rc != EOK)
359 return rc;
360
361 uint64_t strip_size = vol->strip_size / vol->bsize; /* in blocks */
362 uint64_t stripe = (ba / strip_size); /* stripe number */
363 uint64_t p_extent = (stripe / (vol->dev_no - 1)) % vol->dev_no; /* parity extent */
364 uint64_t extent;
365 if ((stripe % (vol->dev_no - 1)) < p_extent)
366 extent = (stripe % (vol->dev_no - 1));
367 else
368 extent = ((stripe % (vol->dev_no - 1)) + 1);
369 uint64_t ext_stripe = stripe / (vol->dev_no - 1); /* stripe level */
370 uint64_t strip_off = ba % strip_size; /* strip offset */
371
372 fibril_mutex_lock(&vol->lock);
373
374 rc = hr_raid5_vol_usable(vol);
375 if (rc != EOK) {
376 fibril_mutex_unlock(&vol->lock);
377 return EIO;
378 }
379
380 left = cnt;
381 while (left != 0) {
382 phys_block = ext_stripe * strip_size + strip_off;
383 cnt = min(left, strip_size - strip_off);
384 len = vol->bsize * cnt;
385 hr_add_ba_offset(vol, &phys_block);
386 switch (type) {
387 case HR_BD_SYNC:
388 if (vol->extents[extent].status != HR_EXT_ONLINE)
389 break;
390 rc = block_sync_cache(vol->extents[extent].svc_id,
391 phys_block, cnt);
392 /* allow unsupported sync */
393 if (rc == ENOTSUP)
394 rc = EOK;
395 break;
396 case HR_BD_READ:
397 retry_read:
398 ssize_t bad = hr_raid5_get_bad_ext(vol);
399 if (bad > 0 && extent == (size_t) bad) {
400 rc = hr_raid5_read_degraded(vol, bad,
401 phys_block, data_read, cnt);
402 } else {
403 rc = block_read_direct(vol->extents[extent].svc_id,
404 phys_block, cnt, data_read);
405 }
406 data_read += len;
407 break;
408 case HR_BD_WRITE:
409 retry_write:
410 rc = hr_raid5_write(vol, p_extent, extent, phys_block,
411 data_write, cnt);
412 data_write += len;
413 break;
414 default:
415 rc = EINVAL;
416 goto error;
417 }
418
419 if (rc == ENOMEM)
420 goto error;
421
422 if (rc == ENOENT)
423 hr_update_ext_status(vol, extent, HR_EXT_MISSING);
424 else if (rc != EOK)
425 hr_update_ext_status(vol, extent, HR_EXT_FAILED);
426
427 if (rc != EOK) {
428 rc = hr_raid5_update_vol_status(vol);
429 if (rc == EOK) {
430 /*
431 * State changed from ONLINE -> DEGRADED,
432 * rewind and retry
433 */
434 if (type == HR_BD_WRITE) {
435 data_write -= len;
436 goto retry_write;
437 } else if (type == HR_BD_WRITE) {
438 data_read -= len;
439 goto retry_read;
440 }
441 } else {
442 rc = EIO;
443 goto error;
444 }
445 }
446
447 left -= cnt;
448 strip_off = 0;
449 if (extent + 1 >= vol->dev_no ||
450 (extent + 1 == p_extent && p_extent + 1 >= vol->dev_no))
451 ext_stripe++;
452 stripe++;
453 p_extent = (stripe / (vol->dev_no - 1)) % vol->dev_no; /* parity extent */
454 if ((stripe % (vol->dev_no - 1)) < p_extent)
455 extent = (stripe % (vol->dev_no - 1));
456 else
457 extent = ((stripe % (vol->dev_no - 1)) + 1);
458 }
459
460error:
461 (void) hr_raid5_update_vol_status(vol);
462 fibril_mutex_unlock(&vol->lock);
463 return rc;
464}
465
466static errno_t hr_raid5_bd_sync_cache(bd_srv_t *bd, aoff64_t ba, size_t cnt)
467{
468 return hr_raid5_bd_op(HR_BD_SYNC, bd, ba, cnt, NULL, NULL, 0);
469}
470
471static errno_t hr_raid5_bd_read_blocks(bd_srv_t *bd, aoff64_t ba, size_t cnt,
472 void *buf, size_t size)
473{
474 return hr_raid5_bd_op(HR_BD_READ, bd, ba, cnt, buf, NULL, size);
475}
476
477static errno_t hr_raid5_bd_write_blocks(bd_srv_t *bd, aoff64_t ba, size_t cnt,
478 const void *data, size_t size)
479{
480 return hr_raid5_bd_op(HR_BD_WRITE, bd, ba, cnt, NULL, data, size);
481}
482
483static errno_t hr_raid5_bd_get_block_size(bd_srv_t *bd, size_t *rsize)
484{
485 hr_volume_t *vol = bd->srvs->sarg;
486
487 *rsize = vol->bsize;
488 return EOK;
489}
490
491static errno_t hr_raid5_bd_get_num_blocks(bd_srv_t *bd, aoff64_t *rnb)
492{
493 hr_volume_t *vol = bd->srvs->sarg;
494
495 *rnb = vol->data_blkno;
496 return EOK;
497}
498
499errno_t hr_raid5_create(hr_volume_t *new_volume)
500{
501 errno_t rc;
502
503 assert(new_volume->level == HR_LVL_5);
504
505 if (new_volume->dev_no < 3) {
506 log_msg(LOG_DEFAULT, LVL_ERROR,
507 "RAID 5 array needs at least 3 devices");
508 return EINVAL;
509 }
510
511 rc = hr_raid5_update_vol_status(new_volume);
512 if (rc != EOK)
513 return rc;
514
515 bd_srvs_init(&new_volume->hr_bds);
516 new_volume->hr_bds.ops = &hr_raid5_bd_ops;
517 new_volume->hr_bds.sarg = new_volume;
518
519 rc = hr_register_volume(new_volume);
520
521 return rc;
522}
523
524errno_t hr_raid5_init(hr_volume_t *vol)
525{
526 errno_t rc;
527 size_t bsize;
528 uint64_t total_blkno;
529
530 assert(vol->level == HR_LVL_5);
531
532 rc = hr_check_devs(vol, &total_blkno, &bsize);
533 if (rc != EOK)
534 return rc;
535
536 vol->nblocks = total_blkno;
537 vol->bsize = bsize;
538 vol->data_offset = HR_DATA_OFF;
539 vol->data_blkno = vol->nblocks - (vol->data_offset * vol->dev_no) -
540 (vol->nblocks / vol->dev_no);
541 vol->strip_size = HR_STRIP_SIZE;
542
543 return EOK;
544}
545
546/** @}
547 */
Note: See TracBrowser for help on using the repository browser.