Changeset 5b320ac in mainline

Timestamp:

2024-11-22T18:38:49Z (6 months ago)

Author:

Miroslav Cimerman <mc@…>

Children:

d0f0744

Parents:

06f2762

Message:

hr: hotspares + RAID1 rebuild

Location:

uspace

Files:

• app/hrctl/hrctl.c (modified) (5 diffs)
• lib/device/include/hr.h (modified) (5 diffs)
• lib/device/include/ipc/hr.h (modified) (1 diff)
• lib/device/src/hr.c (modified) (4 diffs)
• srv/bd/hr/hr.c (modified) (4 diffs)
• srv/bd/hr/raid1.c (modified) (6 diffs)
• srv/bd/hr/var.h (modified) (3 diffs)

uspace/app/hrctl/hrctl.c

@@ -69 +69 @@
     "  -4                        parity on one extent\n"
     "  -5                        distributed parity\n"
+    "  -H, --hotspare=DEV        add hotspare extent\n"
     "\n"
     "When specifying name for creation or assembly, the device name\n"
@@ -80 +81 @@
     "    - assembles RAID device named /hr0 consisting of 2 drives,\n"
     "      that were previously in an array\n"
+    "  hrctl devices/hr0 --hotspare=devices/disk10\n"
+    "    - adds \"devices/disk10\" as hotspare extent\n"
     "Limitations:\n"
     "  - device name must be less than 32 characters in size\n";
@@ -93 +96 @@
         { "destroy", required_argument, 0, 'D' },
         { "fail-extent", required_argument, 0, 'F' },
+        { "hotspare", required_argument, 0, 'H' },
         { 0, 0, 0, 0 }
 };
@@ -247 +251 @@
 
         while (c != -1) {
-                c = getopt_long(argc, argv, "hsC:c:A:a:l:0145Ln:D:F:",
+                c = getopt_long(argc, argv, "hsC:c:A:a:l:0145Ln:D:F:H:",
                     long_options, NULL);
                 switch (c) {
@@ -345 +349 @@
                         }
                         break;
+                case 'H':
+                        if (optind != 3 && argc != 4)
+                                goto bad;
+
+                        service_id_t hotspare;
+                        service_id_t vol_svc_id;
+
+                        rc = loc_service_get_id(argv[1], &vol_svc_id, 0);
+                        if (rc != EOK) {
+                                printf("hrctl: error resolving volume \"%s\", "
+                                    "aborting extent addition\n", argv[1]);
+                                goto bad;
+                        }
+
+                        rc = loc_service_get_id(optarg, &hotspare, 0);
+                        if (rc != EOK) {
+                                printf("hrctl: error resolving device \"%s\", "
+                                    "aborting extent addition\n", optarg);
+                                goto bad;
+                        }
+
+                        rc = hr_add_hotspare(vol_svc_id, hotspare);
+                        if (rc != EOK)
+                                printf("hrctl: hr_add_hotspare() rc: %s\n",
+                                    str_error(rc));
+
+                        free(cfg);
+                        if (rc != EOK)
+                                return 1;
+                        else
+                                return 0;
                 }
         }

uspace/lib/device/include/hr.h

@@ -43 +43 @@
 /* for now */
 #define HR_MAX_EXTENTS 4
+#define HR_MAX_HOTSPARES HR_MAX_EXTENTS
 
 #define HR_DEVNAME_LEN 32
@@ -57 +58 @@
         HR_VOL_ONLINE,  /* OK, OPTIMAL */
         HR_VOL_FAULTY,
-        HR_VOL_DEGRADED /* also used for partial, but usable mirror */
+        HR_VOL_DEGRADED, /* also used for partial, but usable mirror */
+        HR_VOL_REBUILD
 } hr_vol_status_t;
 
@@ -63 +65 @@
         HR_EXT_ONLINE,  /* OK */
         HR_EXT_MISSING,
-        HR_EXT_FAILED
+        HR_EXT_FAILED,
+        HR_EXT_REBUILD,
+        HR_EXT_HOTSPARE
 } hr_ext_status_t;
 
@@ -84 +88 @@
 typedef struct hr_vol_info {
         hr_extent_t extents[HR_MAX_EXTENTS];
+        hr_extent_t hotspares[HR_MAX_HOTSPARES];
         size_t extent_no;
+        size_t hotspare_no;
         service_id_t svc_id;
         hr_level_t level;
@@ -98 +104 @@
 extern errno_t hr_create(hr_t *, hr_config_t *, bool);
 extern errno_t hr_stop(const char *, long);
+extern errno_t hr_add_hotspare(service_id_t, service_id_t);
 extern errno_t hr_print_status(void);
 

uspace/lib/device/include/ipc/hr.h

@@ -42 +42 @@
         HR_ASSEMBLE,
         HR_STOP,
+        HR_ADD_HOTSPARE,
         HR_STATUS
 } hr_request_t;

uspace/lib/device/src/hr.c

@@ -166 +166 @@
                         printf("          %s    %zu       %s\n", hr_get_ext_status_msg(ext->status), i, devname);
         }
+
+        if (vol_info->hotspare_no == 0)
+                return EOK;
+
+        printf("hotspares: [status] [index] [devname]\n");
+        for (i = 0; i < vol_info->hotspare_no; i++) {
+                ext = &vol_info->hotspares[i];
+                if (ext->status == HR_EXT_MISSING) {
+                        devname = (char *) "MISSING-devname";
+                } else {
+                        rc = loc_service_get_name(ext->svc_id, &devname);
+                        if (rc != EOK)
+                                return rc;
+                }
+                printf("            %s   %zu     %s\n",
+                    hr_get_ext_status_msg(ext->status), i, devname);
+        }
+
         return EOK;
 }
@@ -194 +212 @@
         if (rc != EOK)
                 goto error;
+error:
+        hr_sess_destroy(hr);
+        return rc;
+}
+
+errno_t hr_add_hotspare(service_id_t vol_svc_id, service_id_t hs_svc_id)
+{
+        hr_t *hr;
+        errno_t rc;
+        async_exch_t *exch;
+
+        rc = hr_sess_init(&hr);
+        if (rc != EOK)
+                return rc;
+
+        exch = async_exchange_begin(hr->sess);
+        if (exch == NULL) {
+                rc = EINVAL;
+                goto error;
+        }
+
+        rc = async_req_2_0(exch, HR_ADD_HOTSPARE, vol_svc_id, hs_svc_id);
+        async_exchange_end(exch);
 error:
         hr_sess_destroy(hr);
@@ -277 +318 @@
         case HR_VOL_DEGRADED:
                 return "DEGRADED";
+        case HR_VOL_REBUILD:
+                return "REBUILD";
         default:
                 return "UNKNOWN";
@@ -291 +334 @@
         case HR_EXT_FAILED:
                 return "FAILED";
+        case HR_EXT_REBUILD:
+                return "REBUILD";
+        case HR_EXT_HOTSPARE:
+                return "HOTSPARE";
         default:
                 return "UNKNOWN";

uspace/srv/bd/hr/hr.c

@@ -191 +191 @@
                 new_volume->hr_ops.init = hr_raid1_init;
                 new_volume->hr_ops.status_event = hr_raid1_status_event;
+                new_volume->hr_ops.add_hotspare = hr_raid1_add_hotspare;
                 break;
         case HR_LVL_0:
@@ -287 +288 @@
 }
 
+static void hr_add_hotspare_srv(ipc_call_t *icall)
+{
+        HR_DEBUG("hr_add_hotspare()\n");
+
+        errno_t rc = EOK;
+        service_id_t vol_svc_id;
+        service_id_t hotspare;
+        hr_volume_t *vol;
+
+        vol_svc_id = ipc_get_arg1(icall);
+        hotspare = ipc_get_arg2(icall);
+
+        vol = hr_get_volume(vol_svc_id);
+        if (vol == NULL) {
+                async_answer_0(icall, ENOENT);
+                return;
+        }
+
+        if (vol->hr_ops.add_hotspare == NULL) {
+                HR_DEBUG("hr_add_hotspare(): not supported on RAID level %d\n",
+                    vol->level);
+                async_answer_0(icall, ENOTSUP);
+                return;
+        }
+
+        rc = vol->hr_ops.add_hotspare(vol, hotspare);
+
+        async_answer_0(icall, rc);
+}
+
 static void hr_print_status_srv(ipc_call_t *icall)
 {
@@ -318 +349 @@
                 memcpy(info.extents, vol->extents,
                     sizeof(hr_extent_t) * HR_MAX_EXTENTS);
+                memcpy(info.hotspares, vol->hotspares,
+                    sizeof(hr_extent_t) * HR_MAX_HOTSPARES);
                 info.svc_id = vol->svc_id;
                 info.extent_no = vol->dev_no;
+                info.hotspare_no = vol->hotspare_no;
                 info.level = vol->level;
                 /* print usable number of blocks */
@@ -377 +411 @@
                         hr_stop_srv(&call);
                         break;
+                case HR_ADD_HOTSPARE:
+                        hr_add_hotspare_srv(&call);
+                        break;
                 case HR_STATUS:
                         hr_print_status_srv(&call);

uspace/srv/bd/hr/raid1.c

@@ -58 +58 @@
 static errno_t hr_raid1_bd_op(hr_bd_op_type_t, bd_srv_t *, aoff64_t, size_t,
     void *, const void *, size_t);
+static errno_t hr_raid1_rebuild(void *);
 
 /* bdops */
@@ -132 +133 @@
 }
 
+errno_t hr_raid1_add_hotspare(hr_volume_t *vol, service_id_t hotspare)
+{
+        HR_DEBUG("hr_raid1_add_hotspare()\n");
+
+        fibril_mutex_lock(&vol->lock);
+
+        if (vol->hotspare_no >= HR_MAX_HOTSPARES) {
+                HR_ERROR("hr_raid1_add_hotspare(): cannot add more hotspares "
+                    "to \"%s\"\n", vol->devname);
+                fibril_mutex_unlock(&vol->lock);
+                return ELIMIT;
+        }
+
+        vol->hotspares[vol->hotspare_no].svc_id = hotspare;
+        vol->hotspares[vol->hotspare_no].status = HR_EXT_HOTSPARE;
+        vol->hotspare_no++;
+
+        /*
+         * If the volume is degraded, start rebuild right away.
+         */
+        if (vol->status == HR_VOL_DEGRADED) {
+                HR_DEBUG("hr_raid1_add_hotspare(): volume in DEGRADED state, "
+                    "spawning new rebuild fibril\n");
+                fid_t fib = fibril_create(hr_raid1_rebuild, vol);
+                if (fib == 0)
+                        return EINVAL;
+                fibril_start(fib);
+                fibril_detach(fib);
+        }
+
+        fibril_mutex_unlock(&vol->lock);
+
+        return EOK;
+}
+
 static errno_t hr_raid1_bd_open(bd_srvs_t *bds, bd_srv_t *bd)
 {
@@ -180 +216 @@
 {
         if (vol->status == HR_VOL_ONLINE ||
-            vol->status == HR_VOL_DEGRADED)
+            vol->status == HR_VOL_DEGRADED ||
+            vol->status == HR_VOL_REBUILD)
                 return EOK;
         return EINVAL;
@@ -192 +229 @@
 {
         hr_vol_status_t old_state = vol->status;
-        size_t healthy = 0;
-        for (size_t i = 0; i < vol->dev_no; i++)
-                if (vol->extents[i].status == HR_EXT_ONLINE)
-                        healthy++;
+        size_t healthy = hr_count_extents(vol, HR_EXT_ONLINE);
 
         if (healthy == 0) {
@@ -206 +240 @@
                 return EINVAL;
         } else if (healthy < vol->dev_no) {
-                if (old_state != HR_VOL_DEGRADED) {
+                if (old_state != HR_VOL_DEGRADED &&
+                    old_state != HR_VOL_REBUILD) {
                         HR_WARN("RAID 1 array \"%s\" (%lu) has some "
-                            "inactive extent(s), marking volume as DEGRADED",
+                            "unusable extent(s), marking volume as DEGRADED",
                             vol->devname, vol->svc_id);
                         vol->status = HR_VOL_DEGRADED;
+                        if (vol->hotspare_no > 0) {
+                                fid_t fib = fibril_create(hr_raid1_rebuild,
+                                    vol);
+                                if (fib == 0) {
+                                        return EINVAL;
+                                }
+                                fibril_start(fib);
+                                fibril_detach(fib);
+                        }
                 }
                 return EOK;
@@ -309 +353 @@
 }
 
+/*
+ * Put the last HOTSPARE extent in place
+ * of first DEGRADED, and start the rebuild.
+ */
+static errno_t hr_raid1_rebuild(void *arg)
+{
+        HR_DEBUG("hr_raid1_rebuild()\n");
+
+        hr_volume_t *vol = arg;
+        void *buf = NULL;
+        errno_t rc = EOK;
+
+        fibril_mutex_lock(&vol->lock);
+
+        if (vol->hotspare_no == 0) {
+                HR_WARN("hr_raid1_rebuild(): no free hotspares on \"%s\", "
+                    "aborting rebuild\n", vol->devname);
+                /* retval isn't checked for now */
+                goto end;
+        }
+
+        size_t bad = vol->dev_no;
+        for (size_t i = 0; i < vol->dev_no; i++) {
+                if (vol->extents[i].status == HR_EXT_FAILED) {
+                        bad = i;
+                        break;
+                }
+        }
+
+        if (bad == vol->dev_no) {
+                HR_WARN("hr_raid1_rebuild(): no bad extent on \"%s\", "
+                    "aborting rebuild\n", vol->devname);
+                /* retval isn't checked for now */
+                goto end;
+        }
+
+        block_fini(vol->extents[bad].svc_id);
+
+        size_t hotspare_idx = vol->hotspare_no - 1;
+
+        vol->extents[bad].svc_id = vol->hotspares[hotspare_idx].svc_id;
+        hr_update_ext_status(vol, bad, HR_EXT_REBUILD);
+
+        vol->hotspares[hotspare_idx].svc_id = 0;
+        vol->hotspares[hotspare_idx].status = HR_EXT_MISSING;
+        vol->hotspare_no--;
+
+        HR_WARN("hr_raid1_rebuild(): changing volume \"%s\" (%lu) state "
+            "from %s to %s\n", vol->devname, vol->svc_id,
+            hr_get_vol_status_msg(vol->status),
+            hr_get_vol_status_msg(HR_VOL_REBUILD));
+        vol->status = HR_VOL_REBUILD;
+
+        hr_extent_t *hotspare = &vol->extents[bad];
+
+        HR_DEBUG("hr_raid1_rebuild(): initing (%lu)\n", hotspare->svc_id);
+
+        rc = block_init(hotspare->svc_id);
+        if (rc != EOK) {
+                HR_ERROR("hr_raid1_rebuild(): initing (%lu) failed, "
+                    "aborting rebuild\n", hotspare->svc_id);
+                goto end;
+        }
+
+        size_t left = vol->data_blkno;
+        size_t max_blks = DATA_XFER_LIMIT / vol->bsize;
+        buf = malloc(max_blks * vol->bsize);
+
+        hr_extent_t *ext;
+        size_t rebuild_ext_idx = bad;
+
+        size_t cnt;
+        uint64_t ba = 0;
+        hr_add_ba_offset(vol, &ba);
+        while (left != 0) {
+                cnt = min(max_blks, left);
+                for (size_t i = 0; i < vol->dev_no; i++) {
+                        ext = &vol->extents[i];
+                        if (ext->status == HR_EXT_ONLINE) {
+                                rc = block_read_direct(ext->svc_id, ba, cnt,
+                                    buf);
+                                if (rc != EOK) {
+                                        handle_extent_error(vol, i, rc);
+                                        if (i + 1 < vol->dev_no) {
+                                                /* still might have one ONLINE */
+                                                continue;
+                                        } else {
+                                                HR_ERROR("rebuild on \"%s\" (%lu), failed due to "
+                                                    "too many failed extents\n",
+                                                    vol->devname, vol->svc_id);
+                                                goto end;
+                                        }
+                                }
+                                break;
+                        }
+                }
+
+                rc = block_write_direct(hotspare->svc_id, ba, cnt, buf);
+                if (rc != EOK) {
+                        handle_extent_error(vol, rebuild_ext_idx, rc);
+                        HR_ERROR("rebuild on \"%s\" (%lu), extent number %lu\n",
+                            vol->devname, vol->svc_id, rebuild_ext_idx);
+                        goto end;
+
+                }
+
+                ba += cnt;
+                left -= cnt;
+        }
+
+        HR_DEBUG("hr_raid1_rebuild(): rebuild finished on \"%s\" (%lu), "
+            "extent number %lu\n", vol->devname, vol->svc_id, hotspare_idx);
+
+        hr_update_ext_status(vol, hotspare_idx, HR_EXT_ONLINE);
+        /*
+         * For now write metadata at the end, because
+         * we don't sync metada accross extents yet.
+         */
+        hr_write_meta_to_ext(vol, bad);
+end:
+        (void) hr_raid1_update_vol_status(vol);
+
+        fibril_mutex_unlock(&vol->lock);
+
+        if (buf != NULL)
+                free(buf);
+
+        /* retval isn't checked anywhere for now */
+        return rc;
+}
+
 /** @}
  */

uspace/srv/bd/hr/var.h

@@ -51 +51 @@
         errno_t (*init)(hr_volume_t *);
         void    (*status_event)(hr_volume_t *);
+        errno_t (*add_hotspare)(hr_volume_t *, service_id_t);
 } hr_ops_t;
 
@@ -56 +57 @@
         hr_ops_t hr_ops;
         bd_srvs_t hr_bds;
+
         link_t lvolumes;
         fibril_mutex_t lock;
-        char devname[HR_DEVNAME_LEN];
+
+        size_t dev_no;
         hr_extent_t extents[HR_MAX_EXTENTS];
+
+        size_t hotspare_no;
+        hr_extent_t hotspares[HR_MAX_EXTENTS];
+
+        size_t bsize;
         uint64_t nblocks;
         uint64_t data_blkno;
         uint32_t data_offset; /* in blocks */
         uint32_t strip_size;
+
         service_id_t svc_id;
-        size_t bsize;
-        size_t dev_no;
+        hr_vol_status_t status;
         hr_level_t level;
-        hr_vol_status_t status;
+        char devname[HR_DEVNAME_LEN];
 } hr_volume_t;
 
@@ -95 +103 @@
 extern void hr_raid5_status_event(hr_volume_t *);
 
+extern errno_t hr_raid1_add_hotspare(hr_volume_t *, service_id_t);
+
 #endif